mirror of
https://github.com/letic/terraform-provider-google.git
synced 2024-07-05 17:52:38 +00:00
Don't allow destroying backend buckets in dataproc (#2224)
Removing the delete_autogen_bucket property Buckets will not be deleted when a dataproc resource is destroyed
This commit is contained in:
parent
6670f0cccc
commit
a93c57e76a
|
@ -4,19 +4,15 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
|
||||||
"regexp"
|
"regexp"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/hashicorp/errwrap"
|
|
||||||
"github.com/hashicorp/terraform/helper/resource"
|
|
||||||
"github.com/hashicorp/terraform/helper/schema"
|
"github.com/hashicorp/terraform/helper/schema"
|
||||||
"github.com/hashicorp/terraform/helper/validation"
|
"github.com/hashicorp/terraform/helper/validation"
|
||||||
|
|
||||||
"google.golang.org/api/dataproc/v1"
|
"google.golang.org/api/dataproc/v1"
|
||||||
"google.golang.org/api/googleapi"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func resourceDataprocCluster() *schema.Resource {
|
func resourceDataprocCluster() *schema.Resource {
|
||||||
|
@ -92,16 +88,6 @@ func resourceDataprocCluster() *schema.Resource {
|
||||||
Elem: &schema.Resource{
|
Elem: &schema.Resource{
|
||||||
Schema: map[string]*schema.Schema{
|
Schema: map[string]*schema.Schema{
|
||||||
|
|
||||||
"delete_autogen_bucket": {
|
|
||||||
Type: schema.TypeBool,
|
|
||||||
Optional: true,
|
|
||||||
Default: false,
|
|
||||||
Deprecated: "autogenerated buckets are shared by all clusters in the same region, " +
|
|
||||||
"so deleting this bucket could adversely harm other dataproc clusters. " +
|
|
||||||
"If you need a bucket that can be deleted, please create a new one and set the " +
|
|
||||||
"`staging_bucket` field",
|
|
||||||
},
|
|
||||||
|
|
||||||
"staging_bucket": {
|
"staging_bucket": {
|
||||||
Type: schema.TypeString,
|
Type: schema.TypeString,
|
||||||
Optional: true,
|
Optional: true,
|
||||||
|
@ -722,8 +708,7 @@ func resourceDataprocClusterRead(d *schema.ResourceData, meta interface{}) error
|
||||||
func flattenClusterConfig(d *schema.ResourceData, cfg *dataproc.ClusterConfig) ([]map[string]interface{}, error) {
|
func flattenClusterConfig(d *schema.ResourceData, cfg *dataproc.ClusterConfig) ([]map[string]interface{}, error) {
|
||||||
|
|
||||||
data := map[string]interface{}{
|
data := map[string]interface{}{
|
||||||
"delete_autogen_bucket": d.Get("cluster_config.0.delete_autogen_bucket").(bool),
|
"staging_bucket": d.Get("cluster_config.0.staging_bucket").(string),
|
||||||
"staging_bucket": d.Get("cluster_config.0.staging_bucket").(string),
|
|
||||||
|
|
||||||
"bucket": cfg.ConfigBucket,
|
"bucket": cfg.ConfigBucket,
|
||||||
"gce_cluster_config": flattenGceClusterConfig(d, cfg.GceClusterConfig),
|
"gce_cluster_config": flattenGceClusterConfig(d, cfg.GceClusterConfig),
|
||||||
|
@ -866,121 +851,7 @@ func resourceDataprocClusterDelete(d *schema.ResourceData, meta interface{}) err
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Printf("[INFO] Dataproc cluster %s has been deleted", d.Id())
|
log.Printf("[INFO] Dataproc cluster %s has been deleted", d.Id())
|
||||||
|
|
||||||
if d.Get("cluster_config.0.delete_autogen_bucket").(bool) {
|
|
||||||
if err := deleteAutogenBucketIfExists(d, meta); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
d.SetId("")
|
d.SetId("")
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func deleteAutogenBucketIfExists(d *schema.ResourceData, meta interface{}) error {
|
|
||||||
config := meta.(*Config)
|
|
||||||
|
|
||||||
// If the user did not specify a specific override staging bucket, then GCP
|
|
||||||
// creates one automatically. Clean this up to avoid dangling resources.
|
|
||||||
if v, ok := d.GetOk("cluster_config.0.staging_bucket"); ok {
|
|
||||||
log.Printf("[DEBUG] staging bucket %s (for dataproc cluster) has explicitly been set, leaving it...", v)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
bucket := d.Get("cluster_config.0.bucket").(string)
|
|
||||||
|
|
||||||
log.Printf("[DEBUG] Attempting to delete autogenerated bucket %s (for dataproc cluster)", bucket)
|
|
||||||
return emptyAndDeleteStorageBucket(config, bucket)
|
|
||||||
}
|
|
||||||
|
|
||||||
func emptyAndDeleteStorageBucket(config *Config, bucket string) error {
|
|
||||||
err := deleteStorageBucketContents(config, bucket)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = deleteEmptyBucket(config, bucket)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func deleteEmptyBucket(config *Config, bucket string) error {
|
|
||||||
// remove empty bucket
|
|
||||||
err := resource.Retry(1*time.Minute, func() *resource.RetryError {
|
|
||||||
err := config.clientStorage.Buckets.Delete(bucket).Do()
|
|
||||||
if err == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
gerr, ok := err.(*googleapi.Error)
|
|
||||||
if gerr.Code == http.StatusNotFound {
|
|
||||||
// Bucket may be gone already ignore
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if ok && gerr.Code == http.StatusTooManyRequests {
|
|
||||||
return resource.RetryableError(gerr)
|
|
||||||
}
|
|
||||||
return resource.NonRetryableError(err)
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
fmt.Printf("[ERROR] Attempting to delete autogenerated bucket (for dataproc cluster): Error deleting bucket %s: %v\n\n", bucket, err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster): Deleted bucket %v\n\n", bucket)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
func deleteStorageBucketContents(config *Config, bucket string) error {
|
|
||||||
|
|
||||||
res, err := config.clientStorage.Objects.List(bucket).Do()
|
|
||||||
if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == http.StatusNotFound {
|
|
||||||
// Bucket is already gone ...
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("[DEBUG] Attempting to delete autogenerated bucket %s (for dataproc cluster). Error Objects.List failed: %v", bucket, err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(res.Items) > 0 {
|
|
||||||
// purge the bucket...
|
|
||||||
log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster). \n\n")
|
|
||||||
|
|
||||||
for _, object := range res.Items {
|
|
||||||
log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster). Found %s", object.Name)
|
|
||||||
|
|
||||||
err := config.clientStorage.Objects.Delete(bucket, object.Name).Do()
|
|
||||||
if err != nil {
|
|
||||||
if gerr, ok := err.(*googleapi.Error); ok && gerr.Code != http.StatusNotFound {
|
|
||||||
log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster): Error trying to delete object: %s %s\n\n", object.Name, err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster): Object deleted: %s \n\n", object.Name)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Wait until they're actually deleted
|
|
||||||
refreshFunc := func() (interface{}, string, error) {
|
|
||||||
res, err := config.clientStorage.Objects.List(bucket).Do()
|
|
||||||
if err != nil {
|
|
||||||
return nil, "error", err
|
|
||||||
}
|
|
||||||
return res.Items, strconv.FormatBool(len(res.Items) == 0), nil
|
|
||||||
}
|
|
||||||
conf := &resource.StateChangeConf{
|
|
||||||
Pending: []string{"false"},
|
|
||||||
Target: []string{"true"},
|
|
||||||
Refresh: refreshFunc,
|
|
||||||
Timeout: 5 * time.Minute,
|
|
||||||
ContinuousTargetOccurence: 3,
|
|
||||||
}
|
|
||||||
_, err := conf.WaitForState()
|
|
||||||
if err != nil {
|
|
||||||
return errwrap.Wrapf(fmt.Sprintf("Error waiting for all items to be deleted from bucket %q: {{err}}", bucket), err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -124,12 +124,6 @@ The `cluster_config` block supports:
|
||||||
with other clusters in the same region/zone also choosing to use the auto generation
|
with other clusters in the same region/zone also choosing to use the auto generation
|
||||||
option.
|
option.
|
||||||
|
|
||||||
* `delete_autogen_bucket` (Optional, Deprecated) If this is set to true, upon destroying the cluster,
|
|
||||||
if no explicit `staging_bucket` was specified (i.e. an auto generated bucket was relied
|
|
||||||
upon) then this auto generated bucket will also be deleted as part of the cluster destroy.
|
|
||||||
By default this is set to false. This value is deprecated: autogenerated buckets are shared by
|
|
||||||
all clusters in the same region, so deleting the bucket could adversely harm other dataproc clusters.
|
|
||||||
|
|
||||||
* `gce_cluster_config` (Optional) Common config settings for resources of Google Compute Engine cluster
|
* `gce_cluster_config` (Optional) Common config settings for resources of Google Compute Engine cluster
|
||||||
instances, applicable to all instances in the cluster. Structure defined below.
|
instances, applicable to all instances in the cluster. Structure defined below.
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue
Block a user