From a93c57e76a7eee591acf80b9bf99d56be2b83ee6 Mon Sep 17 00:00:00 2001 From: Chris Stephens Date: Thu, 11 Oct 2018 12:57:59 -0700 Subject: [PATCH] Don't allow destroying backend buckets in dataproc (#2224) Removing the delete_autogen_bucket property Buckets will not be deleted when a dataproc resource is destroyed --- google/resource_dataproc_cluster.go | 131 +----------------- website/docs/r/dataproc_cluster.html.markdown | 6 - 2 files changed, 1 insertion(+), 136 deletions(-) diff --git a/google/resource_dataproc_cluster.go b/google/resource_dataproc_cluster.go index 092cba12..c30d09b5 100644 --- a/google/resource_dataproc_cluster.go +++ b/google/resource_dataproc_cluster.go @@ -4,19 +4,15 @@ import ( "errors" "fmt" "log" - "net/http" "regexp" "strconv" "strings" "time" - "github.com/hashicorp/errwrap" - "github.com/hashicorp/terraform/helper/resource" "github.com/hashicorp/terraform/helper/schema" "github.com/hashicorp/terraform/helper/validation" "google.golang.org/api/dataproc/v1" - "google.golang.org/api/googleapi" ) func resourceDataprocCluster() *schema.Resource { @@ -92,16 +88,6 @@ func resourceDataprocCluster() *schema.Resource { Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ - "delete_autogen_bucket": { - Type: schema.TypeBool, - Optional: true, - Default: false, - Deprecated: "autogenerated buckets are shared by all clusters in the same region, " + - "so deleting this bucket could adversely harm other dataproc clusters. " + - "If you need a bucket that can be deleted, please create a new one and set the " + - "`staging_bucket` field", - }, - "staging_bucket": { Type: schema.TypeString, Optional: true, @@ -722,8 +708,7 @@ func resourceDataprocClusterRead(d *schema.ResourceData, meta interface{}) error func flattenClusterConfig(d *schema.ResourceData, cfg *dataproc.ClusterConfig) ([]map[string]interface{}, error) { data := map[string]interface{}{ - "delete_autogen_bucket": d.Get("cluster_config.0.delete_autogen_bucket").(bool), - "staging_bucket": d.Get("cluster_config.0.staging_bucket").(string), + "staging_bucket": d.Get("cluster_config.0.staging_bucket").(string), "bucket": cfg.ConfigBucket, "gce_cluster_config": flattenGceClusterConfig(d, cfg.GceClusterConfig), @@ -866,121 +851,7 @@ func resourceDataprocClusterDelete(d *schema.ResourceData, meta interface{}) err } log.Printf("[INFO] Dataproc cluster %s has been deleted", d.Id()) - - if d.Get("cluster_config.0.delete_autogen_bucket").(bool) { - if err := deleteAutogenBucketIfExists(d, meta); err != nil { - return err - } - } - d.SetId("") - return nil -} - -func deleteAutogenBucketIfExists(d *schema.ResourceData, meta interface{}) error { - config := meta.(*Config) - - // If the user did not specify a specific override staging bucket, then GCP - // creates one automatically. Clean this up to avoid dangling resources. - if v, ok := d.GetOk("cluster_config.0.staging_bucket"); ok { - log.Printf("[DEBUG] staging bucket %s (for dataproc cluster) has explicitly been set, leaving it...", v) - return nil - } - bucket := d.Get("cluster_config.0.bucket").(string) - - log.Printf("[DEBUG] Attempting to delete autogenerated bucket %s (for dataproc cluster)", bucket) - return emptyAndDeleteStorageBucket(config, bucket) -} - -func emptyAndDeleteStorageBucket(config *Config, bucket string) error { - err := deleteStorageBucketContents(config, bucket) - if err != nil { - return err - } - - err = deleteEmptyBucket(config, bucket) - if err != nil { - return err - } - return nil -} - -func deleteEmptyBucket(config *Config, bucket string) error { - // remove empty bucket - err := resource.Retry(1*time.Minute, func() *resource.RetryError { - err := config.clientStorage.Buckets.Delete(bucket).Do() - if err == nil { - return nil - } - gerr, ok := err.(*googleapi.Error) - if gerr.Code == http.StatusNotFound { - // Bucket may be gone already ignore - return nil - } - if ok && gerr.Code == http.StatusTooManyRequests { - return resource.RetryableError(gerr) - } - return resource.NonRetryableError(err) - }) - if err != nil { - fmt.Printf("[ERROR] Attempting to delete autogenerated bucket (for dataproc cluster): Error deleting bucket %s: %v\n\n", bucket, err) - return err - } - log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster): Deleted bucket %v\n\n", bucket) - - return nil - -} - -func deleteStorageBucketContents(config *Config, bucket string) error { - - res, err := config.clientStorage.Objects.List(bucket).Do() - if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == http.StatusNotFound { - // Bucket is already gone ... - return nil - } - if err != nil { - log.Fatalf("[DEBUG] Attempting to delete autogenerated bucket %s (for dataproc cluster). Error Objects.List failed: %v", bucket, err) - return err - } - - if len(res.Items) > 0 { - // purge the bucket... - log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster). \n\n") - - for _, object := range res.Items { - log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster). Found %s", object.Name) - - err := config.clientStorage.Objects.Delete(bucket, object.Name).Do() - if err != nil { - if gerr, ok := err.(*googleapi.Error); ok && gerr.Code != http.StatusNotFound { - log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster): Error trying to delete object: %s %s\n\n", object.Name, err) - return err - } - } - log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster): Object deleted: %s \n\n", object.Name) - } - - // Wait until they're actually deleted - refreshFunc := func() (interface{}, string, error) { - res, err := config.clientStorage.Objects.List(bucket).Do() - if err != nil { - return nil, "error", err - } - return res.Items, strconv.FormatBool(len(res.Items) == 0), nil - } - conf := &resource.StateChangeConf{ - Pending: []string{"false"}, - Target: []string{"true"}, - Refresh: refreshFunc, - Timeout: 5 * time.Minute, - ContinuousTargetOccurence: 3, - } - _, err := conf.WaitForState() - if err != nil { - return errwrap.Wrapf(fmt.Sprintf("Error waiting for all items to be deleted from bucket %q: {{err}}", bucket), err) - } - } return nil } diff --git a/website/docs/r/dataproc_cluster.html.markdown b/website/docs/r/dataproc_cluster.html.markdown index 19443ae0..84d2b2b8 100644 --- a/website/docs/r/dataproc_cluster.html.markdown +++ b/website/docs/r/dataproc_cluster.html.markdown @@ -124,12 +124,6 @@ The `cluster_config` block supports: with other clusters in the same region/zone also choosing to use the auto generation option. -* `delete_autogen_bucket` (Optional, Deprecated) If this is set to true, upon destroying the cluster, - if no explicit `staging_bucket` was specified (i.e. an auto generated bucket was relied - upon) then this auto generated bucket will also be deleted as part of the cluster destroy. - By default this is set to false. This value is deprecated: autogenerated buckets are shared by - all clusters in the same region, so deleting the bucket could adversely harm other dataproc clusters. - * `gce_cluster_config` (Optional) Common config settings for resources of Google Compute Engine cluster instances, applicable to all instances in the cluster. Structure defined below.