From d77c0adbb2299d6354be6c8ca17127fa215d59e3 Mon Sep 17 00:00:00 2001 From: Nathan McKinley Date: Fri, 17 Aug 2018 14:33:27 -0700 Subject: [PATCH] Add support for bucket encryption (#1876) * Support encryption in Storage bucket. * Add self links to crypto keys and key ring ids * Update ID documentation to talk about self link instead. --- google/resource_kms_crypto_key.go | 5 ++ google/resource_kms_crypto_key_test.go | 6 +- google/resource_kms_key_ring.go | 31 +++++++-- google/resource_storage_bucket.go | 54 ++++++++++++++- google/resource_storage_bucket_test.go | 65 +++++++++++++++++++ .../r/google_kms_crypto_key.html.markdown | 4 +- .../docs/r/google_kms_key_ring.html.markdown | 2 +- website/docs/r/storage_bucket.html.markdown | 8 +++ 8 files changed, 160 insertions(+), 15 deletions(-) diff --git a/google/resource_kms_crypto_key.go b/google/resource_kms_crypto_key.go index 08dcbff6..abc8e676 100644 --- a/google/resource_kms_crypto_key.go +++ b/google/resource_kms_crypto_key.go @@ -39,6 +39,10 @@ func resourceKmsCryptoKey() *schema.Resource { Optional: true, ValidateFunc: validateKmsCryptoKeyRotationPeriod, }, + "self_link": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, }, } } @@ -163,6 +167,7 @@ func resourceKmsCryptoKeyRead(d *schema.ResourceData, meta interface{}) error { d.Set("key_ring", cryptoKeyId.KeyRingId.terraformId()) d.Set("name", cryptoKeyId.Name) d.Set("rotation_period", cryptoKey.RotationPeriod) + d.Set("self_link", cryptoKey.Name) d.SetId(cryptoKeyId.cryptoKeyId()) diff --git a/google/resource_kms_crypto_key_test.go b/google/resource_kms_crypto_key_test.go index 8d2fa268..abf9b726 100644 --- a/google/resource_kms_crypto_key_test.go +++ b/google/resource_kms_crypto_key_test.go @@ -345,7 +345,7 @@ resource "google_kms_key_ring" "key_ring" { resource "google_kms_crypto_key" "crypto_key" { name = "%s" - key_ring = "${google_kms_key_ring.key_ring.id}" + key_ring = "${google_kms_key_ring.key_ring.self_link}" rotation_period = "1000000s" } `, projectId, projectId, projectOrg, projectBillingAccount, keyRingName, cryptoKeyName) @@ -376,7 +376,7 @@ resource "google_kms_key_ring" "key_ring" { resource "google_kms_crypto_key" "crypto_key" { name = "%s" - key_ring = "${google_kms_key_ring.key_ring.id}" + key_ring = "${google_kms_key_ring.key_ring.self_link}" rotation_period = "%s" } `, projectId, projectId, projectOrg, projectBillingAccount, keyRingName, cryptoKeyName, rotationPeriod) @@ -407,7 +407,7 @@ resource "google_kms_key_ring" "key_ring" { resource "google_kms_crypto_key" "crypto_key" { name = "%s" - key_ring = "${google_kms_key_ring.key_ring.id}" + key_ring = "${google_kms_key_ring.key_ring.self_link}" } `, projectId, projectId, projectOrg, projectBillingAccount, keyRingName, cryptoKeyName) } diff --git a/google/resource_kms_key_ring.go b/google/resource_kms_key_ring.go index 1d905162..1d86b4de 100644 --- a/google/resource_kms_key_ring.go +++ b/google/resource_kms_key_ring.go @@ -5,6 +5,7 @@ import ( "log" "regexp" "strings" + "time" "github.com/hashicorp/terraform/helper/schema" "google.golang.org/api/cloudkms/v1" @@ -36,6 +37,10 @@ func resourceKmsKeyRing() *schema.Resource { Computed: true, ForceNew: true, }, + "self_link": &schema.Schema{ + Type: schema.TypeString, + Computed: true, + }, }, } } @@ -72,16 +77,27 @@ func resourceKmsKeyRingCreate(d *schema.ResourceData, meta interface{}) error { Name: d.Get("name").(string), } - keyRing, err := config.clientKms.Projects.Locations.KeyRings.Create(keyRingId.parentId(), &cloudkms.KeyRing{}).KeyRingId(keyRingId.Name).Do() + // This resource is often created just after a project, and requires + // billing support, which is eventually consistent. We attempt to + // wait on billing support in the project resource, but we can't + // always get it right - this retry fixes a lot of flaky tests we were + // noticing. + err = retryTimeDuration(func() error { + keyRing, err := config.clientKms.Projects.Locations.KeyRings.Create(keyRingId.parentId(), &cloudkms.KeyRing{}).KeyRingId(keyRingId.Name).Do() + if err != nil { + return fmt.Errorf("Error creating KeyRing: %s", err) + } + + log.Printf("[DEBUG] Created KeyRing %s", keyRing.Name) + + d.SetId(keyRingId.keyRingId()) + return nil + }, time.Duration(30*time.Second)) if err != nil { - return fmt.Errorf("Error creating KeyRing: %s", err) + return err } - log.Printf("[DEBUG] Created KeyRing %s", keyRing.Name) - - d.SetId(keyRingId.keyRingId()) - return resourceKmsKeyRingRead(d, meta) } @@ -100,13 +116,14 @@ func resourceKmsKeyRingRead(d *schema.ResourceData, meta interface{}) error { log.Printf("[DEBUG] Executing read for KMS KeyRing %s", keyRingId.keyRingId()) - _, err = config.clientKms.Projects.Locations.KeyRings.Get(keyRingId.keyRingId()).Do() + keyRing, err := config.clientKms.Projects.Locations.KeyRings.Get(keyRingId.keyRingId()).Do() if err != nil { return fmt.Errorf("Error reading KeyRing: %s", err) } d.Set("project", project) + d.Set("self_link", keyRing.Name) return nil } diff --git a/google/resource_storage_bucket.go b/google/resource_storage_bucket.go index 38ede2a7..1f94a6bb 100644 --- a/google/resource_storage_bucket.go +++ b/google/resource_storage_bucket.go @@ -34,6 +34,20 @@ func resourceStorageBucket() *schema.Resource { ForceNew: true, }, + "encryption": &schema.Schema{ + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "default_kms_key_name": &schema.Schema{ + Type: schema.TypeString, + Required: true, + }, + }, + }, + }, + "force_destroy": &schema.Schema{ Type: schema.TypeBool, Optional: true, @@ -244,11 +258,11 @@ func resourceStorageBucketCreate(d *schema.ResourceData, meta interface{}) error return err } - // Get the bucket and acl + // Get the bucket and location bucket := d.Get("name").(string) location := d.Get("location").(string) - // Create a bucket, setting the acl, location and name. + // Create a bucket, setting the labels, location and name. sb := &storage.Bucket{ Name: bucket, Labels: expandLabels(d), @@ -295,6 +309,10 @@ func resourceStorageBucketCreate(d *schema.ResourceData, meta interface{}) error sb.Logging = expandBucketLogging(v.([]interface{})) } + if v, ok := d.GetOk("encryption"); ok { + sb.Encryption = expandBucketEncryption(v.([]interface{})) + } + var res *storage.Bucket err = retry(func() error { @@ -373,6 +391,14 @@ func resourceStorageBucketUpdate(d *schema.ResourceData, meta interface{}) error } } + if d.HasChange("encryption") { + if v, ok := d.GetOk("encryption"); ok { + sb.Encryption = expandBucketEncryption(v.([]interface{})) + } else { + sb.NullFields = append(sb.NullFields, "Encryption") + } + } + if d.HasChange("labels") { sb.Labels = expandLabels(d) if len(sb.Labels) == 0 { @@ -438,6 +464,7 @@ func resourceStorageBucketRead(d *schema.ResourceData, meta interface{}) error { d.Set("self_link", res.SelfLink) d.Set("url", fmt.Sprintf("gs://%s", bucket)) d.Set("storage_class", res.StorageClass) + d.Set("encryption", flattenBucketEncryption(res.Encryption)) d.Set("location", res.Location) d.Set("cors", flattenCors(res.Cors)) d.Set("logging", flattenBucketLogging(res.Logging)) @@ -542,6 +569,29 @@ func flattenCors(corsRules []*storage.BucketCors) []map[string]interface{} { return corsRulesSchema } +func expandBucketEncryption(configured interface{}) *storage.BucketEncryption { + encs := configured.([]interface{}) + enc := encs[0].(map[string]interface{}) + bucketenc := &storage.BucketEncryption{ + DefaultKmsKeyName: enc["default_kms_key_name"].(string), + } + return bucketenc +} + +func flattenBucketEncryption(enc *storage.BucketEncryption) []map[string]interface{} { + encryption := make([]map[string]interface{}, 0, 1) + + if enc == nil { + return encryption + } + + encryption = append(encryption, map[string]interface{}{ + "default_kms_key_name": enc.DefaultKmsKeyName, + }) + + return encryption +} + func expandBucketLogging(configured interface{}) *storage.BucketLogging { loggings := configured.([]interface{}) logging := loggings[0].(map[string]interface{}) diff --git a/google/resource_storage_bucket_test.go b/google/resource_storage_bucket_test.go index d8afed29..64c615a4 100644 --- a/google/resource_storage_bucket_test.go +++ b/google/resource_storage_bucket_test.go @@ -547,6 +547,33 @@ func TestAccStorageBucket_cors(t *testing.T) { } } +func TestAccStorageBucket_encryption(t *testing.T) { + t.Parallel() + + projectId := "terraform-" + acctest.RandString(10) + projectOrg := getTestOrgFromEnv(t) + projectBillingAccount := getTestBillingAccountFromEnv(t) + keyRingName := fmt.Sprintf("tf-test-%s", acctest.RandString(10)) + cryptoKeyName := fmt.Sprintf("tf-test-%s", acctest.RandString(10)) + bucketName := fmt.Sprintf("tf-test-crypto-bucket-%d", acctest.RandInt()) + var bucket storage.Bucket + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + Steps: []resource.TestStep{ + resource.TestStep{ + Config: testAccStorageBucket_encryption(projectId, projectOrg, projectBillingAccount, keyRingName, cryptoKeyName, bucketName), + Check: resource.ComposeTestCheckFunc( + testAccCheckGoogleKmsCryptoKeyExists("google_kms_crypto_key.crypto_key"), + testAccCheckStorageBucketExists( + "google_storage_bucket.bucket", bucketName, &bucket), + ), + }, + }, + }) +} + func TestAccStorageBucket_labels(t *testing.T) { t.Parallel() @@ -902,6 +929,44 @@ resource "google_storage_bucket" "bucket" { `, bucketName) } +func testAccStorageBucket_encryption(projectId, projectOrg, projectBillingAccount, keyRingName, cryptoKeyName, bucketName string) string { + return fmt.Sprintf(` +resource "google_project" "acceptance" { + name = "%s" + project_id = "%s" + org_id = "%s" + billing_account = "%s" +} + +resource "google_project_services" "acceptance" { + project = "${google_project.acceptance.project_id}" + + services = [ + "cloudkms.googleapis.com", + ] +} + +resource "google_kms_key_ring" "key_ring" { + project = "${google_project_services.acceptance.project}" + name = "%s" + location = "us" +} + +resource "google_kms_crypto_key" "crypto_key" { + name = "%s" + key_ring = "${google_kms_key_ring.key_ring.id}" + rotation_period = "1000000s" +} + +resource "google_storage_bucket" "bucket" { + name = "%s" + encryption { + default_kms_key_name = "${google_kms_crypto_key.crypto_key.self_link}" + } +} + `, projectId, projectId, projectOrg, projectBillingAccount, keyRingName, cryptoKeyName, bucketName) +} + func testAccStorageBucket_updateLabels(bucketName string) string { return fmt.Sprintf(` resource "google_storage_bucket" "bucket" { diff --git a/website/docs/r/google_kms_crypto_key.html.markdown b/website/docs/r/google_kms_crypto_key.html.markdown index 3ae2182f..498400d7 100644 --- a/website/docs/r/google_kms_crypto_key.html.markdown +++ b/website/docs/r/google_kms_crypto_key.html.markdown @@ -30,7 +30,7 @@ resource "google_kms_key_ring" "my_key_ring" { resource "google_kms_crypto_key" "my_crypto_key" { name = "my-crypto-key" - key_ring = "${google_kms_key_ring.my_key_ring.id}" + key_ring = "${google_kms_key_ring.my_key_ring.self_link}" rotation_period = "100000s" } ``` @@ -56,7 +56,7 @@ The following arguments are supported: In addition to the arguments listed above, the following computed attributes are exported: -* `id` - The ID of the created CryptoKey. Its format is `{projectId}/{location}/{keyRingName}/{cryptoKeyName}`. +* `self_link` - The self link of the created CryptoKey. Its format is `projects/{projectId}/locations/{location}/keyRings/{keyRingName}/cryptoKeys/{cryptoKeyName}`. ## Import diff --git a/website/docs/r/google_kms_key_ring.html.markdown b/website/docs/r/google_kms_key_ring.html.markdown index 323c689a..d5e8dacc 100644 --- a/website/docs/r/google_kms_key_ring.html.markdown +++ b/website/docs/r/google_kms_key_ring.html.markdown @@ -48,7 +48,7 @@ The following arguments are supported: In addition to the arguments listed above, the following computed attributes are exported: -* `id` - The ID of the created KeyRing. Its format is `{projectId}/{location}/{keyRingName}`. +* `self_link` - The self link of the created KeyRing. Its format is `projects/{projectId}/locations/{location}/keyRings/{keyRingName}`. ## Import diff --git a/website/docs/r/storage_bucket.html.markdown b/website/docs/r/storage_bucket.html.markdown index e42dc6b3..831adc8a 100644 --- a/website/docs/r/storage_bucket.html.markdown +++ b/website/docs/r/storage_bucket.html.markdown @@ -64,6 +64,8 @@ The following arguments are supported: * `logging` - (Optional) The bucket's [Access & Storage Logs](https://cloud.google.com/storage/docs/access-logs) configuration. +* `encryption` - (Optional) The bucket's encryption configuration. + The `lifecycle_rule` block supports: * `action` - (Required) The Lifecycle Rule's action configuration. A single block of this type is supported. Structure is documented below. @@ -117,6 +119,12 @@ The `logging` block supports: * `log_object_prefix` - (Optional, Computed) The object prefix for log objects. If it's not provided, by default GCS sets this to the log_bucket's name. +The `encryption` block supports: + +* `default_kms_key_name`: A Cloud KMS key that will be used to encrypt objects inserted into this bucket, if no encryption method is specified. + You must pay attention to whether the crypto key is available in the location that this bucket is created in. + See [the docs](https://cloud.google.com/storage/docs/encryption/using-customer-managed-keys) for more details. + ## Attributes Reference In addition to the arguments listed above, the following computed attributes are