public/terraform-provider-google
0
0
Fork 0
mirror of https://github.com/letic/terraform-provider-google.git synced 2024-07-03 08:42:39 +00:00
Code Issues Projects Releases Wiki Activity

Don't allow destroying backend buckets in dataproc (#2224)

Browse Source 
Removing the delete_autogen_bucket property
Buckets will not be deleted when a dataproc resource is destroyed
This commit is contained in:
Chris Stephens 2018-10-11 12:57:59 -07:00 committed by Nathan McKinley
parent 6670f0cccc
commit a93c57e76a
2 changed files with 1 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

131
google/resource_dataproc_cluster.go
View File

@ -4,19 +4,15 @@ import (
"errors"
"fmt"
"log"
"net/http"
"regexp"
"strconv"
"strings"
"time"
"github.com/hashicorp/errwrap"
"github.com/hashicorp/terraform/helper/resource"
"github.com/hashicorp/terraform/helper/schema"
"github.com/hashicorp/terraform/helper/validation"
"google.golang.org/api/dataproc/v1"
"google.golang.org/api/googleapi"
)
func resourceDataprocCluster() *schema.Resource {
@ -92,16 +88,6 @@ func resourceDataprocCluster() *schema.Resource {
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"delete_autogen_bucket": {
Type: schema.TypeBool,
Optional: true,
Default: false,
Deprecated: "autogenerated buckets are shared by all clusters in the same region, " +
"so deleting this bucket could adversely harm other dataproc clusters. " +
"If you need a bucket that can be deleted, please create a new one and set the " +
"`staging_bucket` field",
},
"staging_bucket": {
Type: schema.TypeString,
Optional: true,
@ -722,8 +708,7 @@ func resourceDataprocClusterRead(d *schema.ResourceData, meta interface{}) error
func flattenClusterConfig(d *schema.ResourceData, cfg *dataproc.ClusterConfig) ([]map[string]interface{}, error) {
data := map[string]interface{}{
"delete_autogen_bucket": d.Get("cluster_config.0.delete_autogen_bucket").(bool),
"staging_bucket": d.Get("cluster_config.0.staging_bucket").(string),
"staging_bucket": d.Get("cluster_config.0.staging_bucket").(string),
"bucket": cfg.ConfigBucket,
"gce_cluster_config": flattenGceClusterConfig(d, cfg.GceClusterConfig),
@ -866,121 +851,7 @@ func resourceDataprocClusterDelete(d *schema.ResourceData, meta interface{}) err
}
log.Printf("[INFO] Dataproc cluster %s has been deleted", d.Id())
if d.Get("cluster_config.0.delete_autogen_bucket").(bool) {
if err := deleteAutogenBucketIfExists(d, meta); err != nil {
return err
}
}
d.SetId("")
return nil
}
func deleteAutogenBucketIfExists(d *schema.ResourceData, meta interface{}) error {
config := meta.(*Config)
// If the user did not specify a specific override staging bucket, then GCP
// creates one automatically. Clean this up to avoid dangling resources.
if v, ok := d.GetOk("cluster_config.0.staging_bucket"); ok {
log.Printf("[DEBUG] staging bucket %s (for dataproc cluster) has explicitly been set, leaving it...", v)
return nil
}
bucket := d.Get("cluster_config.0.bucket").(string)
log.Printf("[DEBUG] Attempting to delete autogenerated bucket %s (for dataproc cluster)", bucket)
return emptyAndDeleteStorageBucket(config, bucket)
}
func emptyAndDeleteStorageBucket(config *Config, bucket string) error {
err := deleteStorageBucketContents(config, bucket)
if err != nil {
return err
}
err = deleteEmptyBucket(config, bucket)
if err != nil {
return err
}
return nil
}
func deleteEmptyBucket(config *Config, bucket string) error {
// remove empty bucket
err := resource.Retry(1*time.Minute, func() *resource.RetryError {
err := config.clientStorage.Buckets.Delete(bucket).Do()
if err == nil {
return nil
}
gerr, ok := err.(*googleapi.Error)
if gerr.Code == http.StatusNotFound {
// Bucket may be gone already ignore
return nil
}
if ok && gerr.Code == http.StatusTooManyRequests {
return resource.RetryableError(gerr)
}
return resource.NonRetryableError(err)
})
if err != nil {
fmt.Printf("[ERROR] Attempting to delete autogenerated bucket (for dataproc cluster): Error deleting bucket %s: %v\n\n", bucket, err)
return err
}
log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster): Deleted bucket %v\n\n", bucket)
return nil
}
func deleteStorageBucketContents(config *Config, bucket string) error {
res, err := config.clientStorage.Objects.List(bucket).Do()
if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == http.StatusNotFound {
// Bucket is already gone ...
return nil
}
if err != nil {
log.Fatalf("[DEBUG] Attempting to delete autogenerated bucket %s (for dataproc cluster). Error Objects.List failed: %v", bucket, err)
return err
}
if len(res.Items) > 0 {
// purge the bucket...
log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster). \n\n")
for _, object := range res.Items {
log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster). Found %s", object.Name)
err := config.clientStorage.Objects.Delete(bucket, object.Name).Do()
if err != nil {
if gerr, ok := err.(*googleapi.Error); ok && gerr.Code != http.StatusNotFound {
log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster): Error trying to delete object: %s %s\n\n", object.Name, err)
return err
}
}
log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster): Object deleted: %s \n\n", object.Name)
}
// Wait until they're actually deleted
refreshFunc := func() (interface{}, string, error) {
res, err := config.clientStorage.Objects.List(bucket).Do()
if err != nil {
return nil, "error", err
}
return res.Items, strconv.FormatBool(len(res.Items) == 0), nil
}
conf := &resource.StateChangeConf{
Pending: []string{"false"},
Target: []string{"true"},
Refresh: refreshFunc,
Timeout: 5 * time.Minute,
ContinuousTargetOccurence: 3,
}
_, err := conf.WaitForState()
if err != nil {
return errwrap.Wrapf(fmt.Sprintf("Error waiting for all items to be deleted from bucket %q: {{err}}", bucket), err)
}
}
return nil
}

6
website/docs/r/dataproc_cluster.html.markdown
View File

@ -124,12 +124,6 @@ The `cluster_config` block supports:
with other clusters in the same region/zone also choosing to use the auto generation
option.
* `delete_autogen_bucket` (Optional, Deprecated) If this is set to true, upon destroying the cluster,
if no explicit `staging_bucket` was specified (i.e. an auto generated bucket was relied
upon) then this auto generated bucket will also be deleted as part of the cluster destroy.
By default this is set to false. This value is deprecated: autogenerated buckets are shared by
all clusters in the same region, so deleting the bucket could adversely harm other dataproc clusters.
* `gce_cluster_config` (Optional) Common config settings for resources of Google Compute Engine cluster
instances, applicable to all instances in the cluster. Structure defined below.