From 46cc5b7d84ccea7ad25a9eb0d7f84e6f425986d7 Mon Sep 17 00:00:00 2001 From: Nick Jacques Date: Fri, 22 Dec 2017 15:24:05 -0600 Subject: [PATCH] WIP: Add "internal IP only" support for Dataproc clusters (#837) * Add internalIpOnly support for Dataproc clusters * Add internal_ip_only to dataproc cluster docs * Add default/basic dataproc internal ip test case * Add test for dataproc internal_ip_only=true * fixup cluster_config.gce_cluster_config to include .0. * Remove redundant depends_on * Add %s rnd to network and subnetwork * Use variable for subnet CIDR and reference via source_ranges * Add depends_on back to dataproc cluster test * Fix cluster attribute refs (.0. again) --- google/resource_dataproc_cluster.go | 17 +++- google/resource_dataproc_cluster_test.go | 92 +++++++++++++++++++ website/docs/r/dataproc_cluster.html.markdown | 5 + 3 files changed, 111 insertions(+), 3 deletions(-) diff --git a/google/resource_dataproc_cluster.go b/google/resource_dataproc_cluster.go index 5405dc04..625d9b45 100644 --- a/google/resource_dataproc_cluster.go +++ b/google/resource_dataproc_cluster.go @@ -174,6 +174,13 @@ func resourceDataprocCluster() *schema.Resource { }, Set: stringScopeHashcode, }, + + "internal_ip_only": { + Type: schema.TypeBool, + Optional: true, + ForceNew: true, + Default: false, + }, }, }, }, @@ -479,6 +486,9 @@ func expandGceClusterConfig(cfg map[string]interface{}) *dataproc.GceClusterConf } conf.ServiceAccountScopes = scopes } + if v, ok := cfg["internal_ip_only"]; ok { + conf.InternalIpOnly = v.(bool) + } return conf } @@ -720,9 +730,10 @@ func flattenInitializationActions(nia []*dataproc.NodeInitializationAction) ([]m func flattenGceClusterConfig(d *schema.ResourceData, gcc *dataproc.GceClusterConfig) []map[string]interface{} { gceConfig := map[string]interface{}{ - "tags": gcc.Tags, - "service_account": gcc.ServiceAccount, - "zone": extractLastResourceFromUri(gcc.ZoneUri), + "tags": gcc.Tags, + "service_account": gcc.ServiceAccount, + "zone": extractLastResourceFromUri(gcc.ZoneUri), + "internal_ip_only": gcc.InternalIpOnly, } if gcc.NetworkUri != "" { diff --git a/google/resource_dataproc_cluster_test.go b/google/resource_dataproc_cluster_test.go index b2b27880..6f5228ac 100644 --- a/google/resource_dataproc_cluster_test.go +++ b/google/resource_dataproc_cluster_test.go @@ -107,6 +107,9 @@ func TestAccDataprocCluster_basic(t *testing.T) { // Default behaviour is for Dataproc to autogen or autodiscover a config bucket resource.TestCheckResourceAttrSet("google_dataproc_cluster.basic", "cluster_config.0.bucket"), + // Default behavior is for Dataproc to not use only internal IP addresses + resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.gce_cluster_config.0.internal_ip_only", "false"), + // Expect 1 master instances with computed values resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.master_config.#", "1"), resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.master_config.0.num_instances", "1"), @@ -133,6 +136,29 @@ func TestAccDataprocCluster_basic(t *testing.T) { }) } +func TestAccDataprocCluster_basicWithInternalIpOnlyTrue(t *testing.T) { + t.Parallel() + + var cluster dataproc.Cluster + rnd := acctest.RandString(10) + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckDataprocClusterDestroy(false), + Steps: []resource.TestStep{ + { + Config: testAccDataprocCluster_basicWithInternalIpOnlyTrue(rnd), + Check: resource.ComposeTestCheckFunc( + testAccCheckDataprocClusterExists("google_dataproc_cluster.basic", &cluster), + + // Testing behavior for Dataproc to use only internal IP addresses + resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.gce_cluster_config.0.internal_ip_only", "true"), + ), + }, + }, + }) +} + func TestAccDataprocCluster_basicWithAutogenDeleteTrue(t *testing.T) { t.Parallel() @@ -648,6 +674,72 @@ resource "google_dataproc_cluster" "basic" { `, rnd) } +func testAccDataprocCluster_basicWithInternalIpOnlyTrue(rnd string) string { + return fmt.Sprintf(` +variable subnetwork_cidr { + default = "10.0.0.0/16" +} + +resource "google_compute_network" "dataproc_network" { + name = "dataproc-internalip-network-%s" + auto_create_subnetworks = false +} + +# +# Create a subnet with Private IP Access enabled to test +# deploying a Dataproc cluster with Internal IP Only enabled. +# +resource "google_compute_subnetwork" "dataproc_subnetwork" { + name = "dataproc-internalip-subnetwork-%s" + ip_cidr_range = "${var.subnetwork_cidr}" + network = "${google_compute_network.dataproc_network.self_link}" + region = "us-central1" + private_ip_google_access = true + } + +# +# The default network within GCP already comes pre configured with +# certain firewall rules open to allow internal communication. As we +# are creating a new one here for this test, we need to additionally +# open up similar rules to allow the nodes to talk to each other +# internally as part of their configuration or this will just hang. +# +resource "google_compute_firewall" "dataproc_network_firewall" { + name = "dproc-cluster-test-allow-internal" + description = "Firewall rules for dataproc Terraform acceptance testing" + network = "${google_compute_network.dataproc_network.name}" + + allow { + protocol = "icmp" + } + + allow { + protocol = "tcp" + ports = ["0-65535"] + } + + allow { + protocol = "udp" + ports = ["0-65535"] + } + + source_ranges = ["${var.subnetwork_cidr}"] +} +resource "google_dataproc_cluster" "basic" { + name = "dproc-cluster-test-%s" + region = "us-central1" + depends_on = ["google_compute_firewall.dataproc_network_firewall"] + + cluster_config { + gce_cluster_config { + subnetwork = "${google_compute_subnetwork.dataproc_subnetwork.name}" + internal_ip_only = true + } + } +} +`, rnd, rnd, rnd) +} + func testAccDataprocCluster_basicWithAutogenDeleteTrue(rnd string) string { return fmt.Sprintf(` resource "google_dataproc_cluster" "basic" { diff --git a/website/docs/r/dataproc_cluster.html.markdown b/website/docs/r/dataproc_cluster.html.markdown index 41324378..a6a73a2a 100644 --- a/website/docs/r/dataproc_cluster.html.markdown +++ b/website/docs/r/dataproc_cluster.html.markdown @@ -196,6 +196,11 @@ The **cluster_config.gce_cluster_config** block supports: * `tags` - (Optional) The list of instance tags applied to instances in the cluster. Tags are used to identify valid sources or targets for network firewalls. +* `internal_ip_only` - (Optional) By default, clusters are not restricted to internal IP addresses, + and will have ephemeral external IP addresses assigned to each instance. If set to true, all + instances in the cluster will only have internal IP addresses. Note: Private Google Access + (also known as `privateIpGoogleAccess`) must be enabled on the subnetwork that the cluster + will be launched in. - - - The **cluster_config.master_config** block supports: