mirror of
https://github.com/letic/terraform-provider-google.git
synced 2024-07-05 17:52:38 +00:00
Don't allow destroying backend buckets in dataproc (#2224)
Removing the delete_autogen_bucket property Buckets will not be deleted when a dataproc resource is destroyed
This commit is contained in:
parent
6670f0cccc
commit
a93c57e76a
|
@ -4,19 +4,15 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/errwrap"
|
||||
"github.com/hashicorp/terraform/helper/resource"
|
||||
"github.com/hashicorp/terraform/helper/schema"
|
||||
"github.com/hashicorp/terraform/helper/validation"
|
||||
|
||||
"google.golang.org/api/dataproc/v1"
|
||||
"google.golang.org/api/googleapi"
|
||||
)
|
||||
|
||||
func resourceDataprocCluster() *schema.Resource {
|
||||
|
@ -92,16 +88,6 @@ func resourceDataprocCluster() *schema.Resource {
|
|||
Elem: &schema.Resource{
|
||||
Schema: map[string]*schema.Schema{
|
||||
|
||||
"delete_autogen_bucket": {
|
||||
Type: schema.TypeBool,
|
||||
Optional: true,
|
||||
Default: false,
|
||||
Deprecated: "autogenerated buckets are shared by all clusters in the same region, " +
|
||||
"so deleting this bucket could adversely harm other dataproc clusters. " +
|
||||
"If you need a bucket that can be deleted, please create a new one and set the " +
|
||||
"`staging_bucket` field",
|
||||
},
|
||||
|
||||
"staging_bucket": {
|
||||
Type: schema.TypeString,
|
||||
Optional: true,
|
||||
|
@ -722,7 +708,6 @@ func resourceDataprocClusterRead(d *schema.ResourceData, meta interface{}) error
|
|||
func flattenClusterConfig(d *schema.ResourceData, cfg *dataproc.ClusterConfig) ([]map[string]interface{}, error) {
|
||||
|
||||
data := map[string]interface{}{
|
||||
"delete_autogen_bucket": d.Get("cluster_config.0.delete_autogen_bucket").(bool),
|
||||
"staging_bucket": d.Get("cluster_config.0.staging_bucket").(string),
|
||||
|
||||
"bucket": cfg.ConfigBucket,
|
||||
|
@ -866,121 +851,7 @@ func resourceDataprocClusterDelete(d *schema.ResourceData, meta interface{}) err
|
|||
}
|
||||
|
||||
log.Printf("[INFO] Dataproc cluster %s has been deleted", d.Id())
|
||||
|
||||
if d.Get("cluster_config.0.delete_autogen_bucket").(bool) {
|
||||
if err := deleteAutogenBucketIfExists(d, meta); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
d.SetId("")
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteAutogenBucketIfExists(d *schema.ResourceData, meta interface{}) error {
|
||||
config := meta.(*Config)
|
||||
|
||||
// If the user did not specify a specific override staging bucket, then GCP
|
||||
// creates one automatically. Clean this up to avoid dangling resources.
|
||||
if v, ok := d.GetOk("cluster_config.0.staging_bucket"); ok {
|
||||
log.Printf("[DEBUG] staging bucket %s (for dataproc cluster) has explicitly been set, leaving it...", v)
|
||||
return nil
|
||||
}
|
||||
bucket := d.Get("cluster_config.0.bucket").(string)
|
||||
|
||||
log.Printf("[DEBUG] Attempting to delete autogenerated bucket %s (for dataproc cluster)", bucket)
|
||||
return emptyAndDeleteStorageBucket(config, bucket)
|
||||
}
|
||||
|
||||
func emptyAndDeleteStorageBucket(config *Config, bucket string) error {
|
||||
err := deleteStorageBucketContents(config, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = deleteEmptyBucket(config, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteEmptyBucket(config *Config, bucket string) error {
|
||||
// remove empty bucket
|
||||
err := resource.Retry(1*time.Minute, func() *resource.RetryError {
|
||||
err := config.clientStorage.Buckets.Delete(bucket).Do()
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
gerr, ok := err.(*googleapi.Error)
|
||||
if gerr.Code == http.StatusNotFound {
|
||||
// Bucket may be gone already ignore
|
||||
return nil
|
||||
}
|
||||
if ok && gerr.Code == http.StatusTooManyRequests {
|
||||
return resource.RetryableError(gerr)
|
||||
}
|
||||
return resource.NonRetryableError(err)
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Printf("[ERROR] Attempting to delete autogenerated bucket (for dataproc cluster): Error deleting bucket %s: %v\n\n", bucket, err)
|
||||
return err
|
||||
}
|
||||
log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster): Deleted bucket %v\n\n", bucket)
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func deleteStorageBucketContents(config *Config, bucket string) error {
|
||||
|
||||
res, err := config.clientStorage.Objects.List(bucket).Do()
|
||||
if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == http.StatusNotFound {
|
||||
// Bucket is already gone ...
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
log.Fatalf("[DEBUG] Attempting to delete autogenerated bucket %s (for dataproc cluster). Error Objects.List failed: %v", bucket, err)
|
||||
return err
|
||||
}
|
||||
|
||||
if len(res.Items) > 0 {
|
||||
// purge the bucket...
|
||||
log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster). \n\n")
|
||||
|
||||
for _, object := range res.Items {
|
||||
log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster). Found %s", object.Name)
|
||||
|
||||
err := config.clientStorage.Objects.Delete(bucket, object.Name).Do()
|
||||
if err != nil {
|
||||
if gerr, ok := err.(*googleapi.Error); ok && gerr.Code != http.StatusNotFound {
|
||||
log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster): Error trying to delete object: %s %s\n\n", object.Name, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
log.Printf("[DEBUG] Attempting to delete autogenerated bucket (for dataproc cluster): Object deleted: %s \n\n", object.Name)
|
||||
}
|
||||
|
||||
// Wait until they're actually deleted
|
||||
refreshFunc := func() (interface{}, string, error) {
|
||||
res, err := config.clientStorage.Objects.List(bucket).Do()
|
||||
if err != nil {
|
||||
return nil, "error", err
|
||||
}
|
||||
return res.Items, strconv.FormatBool(len(res.Items) == 0), nil
|
||||
}
|
||||
conf := &resource.StateChangeConf{
|
||||
Pending: []string{"false"},
|
||||
Target: []string{"true"},
|
||||
Refresh: refreshFunc,
|
||||
Timeout: 5 * time.Minute,
|
||||
ContinuousTargetOccurence: 3,
|
||||
}
|
||||
_, err := conf.WaitForState()
|
||||
if err != nil {
|
||||
return errwrap.Wrapf(fmt.Sprintf("Error waiting for all items to be deleted from bucket %q: {{err}}", bucket), err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -124,12 +124,6 @@ The `cluster_config` block supports:
|
|||
with other clusters in the same region/zone also choosing to use the auto generation
|
||||
option.
|
||||
|
||||
* `delete_autogen_bucket` (Optional, Deprecated) If this is set to true, upon destroying the cluster,
|
||||
if no explicit `staging_bucket` was specified (i.e. an auto generated bucket was relied
|
||||
upon) then this auto generated bucket will also be deleted as part of the cluster destroy.
|
||||
By default this is set to false. This value is deprecated: autogenerated buckets are shared by
|
||||
all clusters in the same region, so deleting the bucket could adversely harm other dataproc clusters.
|
||||
|
||||
* `gce_cluster_config` (Optional) Common config settings for resources of Google Compute Engine cluster
|
||||
instances, applicable to all instances in the cluster. Structure defined below.
|
||||
|
||||
|
|
Loading…
Reference in New Issue
Block a user