2017-10-30 23:41:37 +00:00
|
|
|
package google
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
|
|
|
"net/http"
|
|
|
|
"reflect"
|
|
|
|
"regexp"
|
|
|
|
"strconv"
|
|
|
|
"testing"
|
|
|
|
|
|
|
|
"github.com/hashicorp/terraform/helper/acctest"
|
|
|
|
"github.com/hashicorp/terraform/helper/resource"
|
|
|
|
"github.com/hashicorp/terraform/terraform"
|
|
|
|
|
|
|
|
"google.golang.org/api/dataproc/v1"
|
|
|
|
"google.golang.org/api/googleapi"
|
|
|
|
)
|
|
|
|
|
|
|
|
const emptyTFDefinition = `
|
|
|
|
# empty def
|
|
|
|
`
|
|
|
|
|
|
|
|
func TestExtractInitTimeout(t *testing.T) {
|
2017-11-03 22:05:20 +00:00
|
|
|
t.Parallel()
|
|
|
|
|
2017-10-30 23:41:37 +00:00
|
|
|
actual, err := extractInitTimeout("500s")
|
|
|
|
expected := 500
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("Expected %d, but got error %v", expected, err)
|
|
|
|
}
|
|
|
|
if actual != expected {
|
|
|
|
t.Fatalf("Expected %d, but got %d", expected, actual)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestExtractInitTimeout_nonSeconds(t *testing.T) {
|
2017-11-03 22:05:20 +00:00
|
|
|
t.Parallel()
|
|
|
|
|
2017-10-30 23:41:37 +00:00
|
|
|
actual, err := extractInitTimeout("5m")
|
|
|
|
expected := 300
|
|
|
|
if err != nil {
|
|
|
|
t.Fatalf("Expected %d, but got error %v", expected, err)
|
|
|
|
}
|
|
|
|
if actual != expected {
|
|
|
|
t.Fatalf("Expected %d, but got %d", expected, actual)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestExtractInitTimeout_empty(t *testing.T) {
|
2017-11-03 22:05:20 +00:00
|
|
|
t.Parallel()
|
|
|
|
|
2017-10-30 23:41:37 +00:00
|
|
|
_, err := extractInitTimeout("")
|
|
|
|
expected := "time: invalid duration"
|
|
|
|
if err != nil && err.Error() != expected {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
t.Fatalf("Expected an error with message '%s', but got %v", expected, err.Error())
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAccDataprocCluster_missingZoneGlobalRegion1(t *testing.T) {
|
2017-11-03 22:05:20 +00:00
|
|
|
t.Parallel()
|
|
|
|
|
2017-10-30 23:41:37 +00:00
|
|
|
rnd := acctest.RandString(10)
|
|
|
|
resource.Test(t, resource.TestCase{
|
|
|
|
PreCheck: func() { testAccPreCheck(t) },
|
|
|
|
Providers: testAccProviders,
|
|
|
|
Steps: []resource.TestStep{
|
|
|
|
{
|
|
|
|
Config: testAccCheckDataproc_missingZoneGlobalRegion1(rnd),
|
|
|
|
ExpectError: regexp.MustCompile("zone is mandatory when region is set to 'global'"),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAccDataprocCluster_missingZoneGlobalRegion2(t *testing.T) {
|
2017-11-03 22:05:20 +00:00
|
|
|
t.Parallel()
|
|
|
|
|
2017-10-30 23:41:37 +00:00
|
|
|
rnd := acctest.RandString(10)
|
|
|
|
resource.Test(t, resource.TestCase{
|
|
|
|
PreCheck: func() { testAccPreCheck(t) },
|
|
|
|
Providers: testAccProviders,
|
|
|
|
Steps: []resource.TestStep{
|
|
|
|
{
|
|
|
|
Config: testAccCheckDataproc_missingZoneGlobalRegion2(rnd),
|
|
|
|
ExpectError: regexp.MustCompile("zone is mandatory when region is set to 'global'"),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAccDataprocCluster_basic(t *testing.T) {
|
2017-11-03 22:05:20 +00:00
|
|
|
t.Parallel()
|
|
|
|
|
2017-10-30 23:41:37 +00:00
|
|
|
var cluster dataproc.Cluster
|
|
|
|
rnd := acctest.RandString(10)
|
|
|
|
resource.Test(t, resource.TestCase{
|
|
|
|
PreCheck: func() { testAccPreCheck(t) },
|
|
|
|
Providers: testAccProviders,
|
2018-03-13 18:46:27 +00:00
|
|
|
CheckDestroy: testAccCheckDataprocClusterDestroy(),
|
2017-10-30 23:41:37 +00:00
|
|
|
Steps: []resource.TestStep{
|
|
|
|
{
|
|
|
|
Config: testAccDataprocCluster_basic(rnd),
|
|
|
|
Check: resource.ComposeTestCheckFunc(
|
|
|
|
testAccCheckDataprocClusterExists("google_dataproc_cluster.basic", &cluster),
|
|
|
|
|
|
|
|
// Default behaviour is for Dataproc to autogen or autodiscover a config bucket
|
|
|
|
resource.TestCheckResourceAttrSet("google_dataproc_cluster.basic", "cluster_config.0.bucket"),
|
|
|
|
|
2017-12-22 21:24:05 +00:00
|
|
|
// Default behavior is for Dataproc to not use only internal IP addresses
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.gce_cluster_config.0.internal_ip_only", "false"),
|
|
|
|
|
2017-10-30 23:41:37 +00:00
|
|
|
// Expect 1 master instances with computed values
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.master_config.#", "1"),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.master_config.0.num_instances", "1"),
|
|
|
|
resource.TestCheckResourceAttrSet("google_dataproc_cluster.basic", "cluster_config.0.master_config.0.disk_config.0.boot_disk_size_gb"),
|
|
|
|
resource.TestCheckResourceAttrSet("google_dataproc_cluster.basic", "cluster_config.0.master_config.0.disk_config.0.num_local_ssds"),
|
|
|
|
resource.TestCheckResourceAttrSet("google_dataproc_cluster.basic", "cluster_config.0.master_config.0.machine_type"),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.master_config.0.instance_names.#", "1"),
|
|
|
|
|
|
|
|
// Expect 2 worker instances with computed values
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.worker_config.#", "1"),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.worker_config.0.num_instances", "2"),
|
|
|
|
resource.TestCheckResourceAttrSet("google_dataproc_cluster.basic", "cluster_config.0.worker_config.0.disk_config.0.boot_disk_size_gb"),
|
|
|
|
resource.TestCheckResourceAttrSet("google_dataproc_cluster.basic", "cluster_config.0.worker_config.0.disk_config.0.num_local_ssds"),
|
|
|
|
resource.TestCheckResourceAttrSet("google_dataproc_cluster.basic", "cluster_config.0.worker_config.0.machine_type"),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.worker_config.0.instance_names.#", "2"),
|
|
|
|
|
|
|
|
// Expect 0 preemptible worker instances
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.preemptible_worker_config.#", "1"),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.preemptible_worker_config.0.num_instances", "0"),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.preemptible_worker_config.0.instance_names.#", "0"),
|
|
|
|
),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2017-12-22 21:24:05 +00:00
|
|
|
func TestAccDataprocCluster_basicWithInternalIpOnlyTrue(t *testing.T) {
|
|
|
|
t.Parallel()
|
|
|
|
|
|
|
|
var cluster dataproc.Cluster
|
|
|
|
rnd := acctest.RandString(10)
|
|
|
|
resource.Test(t, resource.TestCase{
|
|
|
|
PreCheck: func() { testAccPreCheck(t) },
|
|
|
|
Providers: testAccProviders,
|
2018-03-13 18:46:27 +00:00
|
|
|
CheckDestroy: testAccCheckDataprocClusterDestroy(),
|
2017-12-22 21:24:05 +00:00
|
|
|
Steps: []resource.TestStep{
|
|
|
|
{
|
|
|
|
Config: testAccDataprocCluster_basicWithInternalIpOnlyTrue(rnd),
|
|
|
|
Check: resource.ComposeTestCheckFunc(
|
|
|
|
testAccCheckDataprocClusterExists("google_dataproc_cluster.basic", &cluster),
|
|
|
|
|
|
|
|
// Testing behavior for Dataproc to use only internal IP addresses
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.gce_cluster_config.0.internal_ip_only", "true"),
|
|
|
|
),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2018-01-24 00:08:51 +00:00
|
|
|
func TestAccDataprocCluster_basicWithMetadata(t *testing.T) {
|
|
|
|
t.Parallel()
|
|
|
|
|
|
|
|
var cluster dataproc.Cluster
|
|
|
|
rnd := acctest.RandString(10)
|
|
|
|
resource.Test(t, resource.TestCase{
|
|
|
|
PreCheck: func() { testAccPreCheck(t) },
|
|
|
|
Providers: testAccProviders,
|
2018-03-13 18:46:27 +00:00
|
|
|
CheckDestroy: testAccCheckDataprocClusterDestroy(),
|
2018-01-24 00:08:51 +00:00
|
|
|
Steps: []resource.TestStep{
|
|
|
|
{
|
|
|
|
Config: testAccDataprocCluster_basicWithMetadata(rnd),
|
|
|
|
Check: resource.ComposeTestCheckFunc(
|
|
|
|
testAccCheckDataprocClusterExists("google_dataproc_cluster.basic", &cluster),
|
|
|
|
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.gce_cluster_config.0.metadata.foo", "bar"),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.gce_cluster_config.0.metadata.baz", "qux"),
|
|
|
|
),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2017-10-30 23:41:37 +00:00
|
|
|
func TestAccDataprocCluster_singleNodeCluster(t *testing.T) {
|
2017-11-03 22:05:20 +00:00
|
|
|
t.Parallel()
|
|
|
|
|
2017-10-30 23:41:37 +00:00
|
|
|
rnd := acctest.RandString(10)
|
|
|
|
var cluster dataproc.Cluster
|
|
|
|
resource.Test(t, resource.TestCase{
|
|
|
|
PreCheck: func() { testAccPreCheck(t) },
|
|
|
|
Providers: testAccProviders,
|
2018-03-13 18:46:27 +00:00
|
|
|
CheckDestroy: testAccCheckDataprocClusterDestroy(),
|
2017-10-30 23:41:37 +00:00
|
|
|
Steps: []resource.TestStep{
|
|
|
|
{
|
|
|
|
Config: testAccDataprocCluster_singleNodeCluster(rnd),
|
|
|
|
Check: resource.ComposeTestCheckFunc(
|
|
|
|
testAccCheckDataprocClusterExists("google_dataproc_cluster.single_node_cluster", &cluster),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.single_node_cluster", "cluster_config.0.master_config.0.num_instances", "1"),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.single_node_cluster", "cluster_config.0.worker_config.0.num_instances", "0"),
|
|
|
|
|
|
|
|
// We set the "dataproc:dataproc.allow.zero.workers" override property.
|
|
|
|
// GCP should populate the 'properties' value with this value, as well as many others
|
|
|
|
resource.TestCheckResourceAttrSet("google_dataproc_cluster.single_node_cluster", "cluster_config.0.software_config.0.properties.%"),
|
|
|
|
),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAccDataprocCluster_updatable(t *testing.T) {
|
2017-11-03 22:05:20 +00:00
|
|
|
t.Parallel()
|
|
|
|
|
2017-10-30 23:41:37 +00:00
|
|
|
rnd := acctest.RandString(10)
|
|
|
|
var cluster dataproc.Cluster
|
|
|
|
|
|
|
|
resource.Test(t, resource.TestCase{
|
|
|
|
PreCheck: func() { testAccPreCheck(t) },
|
|
|
|
Providers: testAccProviders,
|
2018-03-13 18:46:27 +00:00
|
|
|
CheckDestroy: testAccCheckDataprocClusterDestroy(),
|
2017-10-30 23:41:37 +00:00
|
|
|
Steps: []resource.TestStep{
|
|
|
|
{
|
|
|
|
Config: testAccDataprocCluster_updatable(rnd, 2, 1),
|
|
|
|
Check: resource.ComposeTestCheckFunc(
|
|
|
|
testAccCheckDataprocClusterExists("google_dataproc_cluster.updatable", &cluster),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.updatable", "cluster_config.0.master_config.0.num_instances", "1"),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.updatable", "cluster_config.0.worker_config.0.num_instances", "2"),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.updatable", "cluster_config.0.preemptible_worker_config.0.num_instances", "1")),
|
|
|
|
},
|
|
|
|
{
|
|
|
|
Config: testAccDataprocCluster_updatable(rnd, 3, 2),
|
|
|
|
Check: resource.ComposeTestCheckFunc(
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.updatable", "cluster_config.0.master_config.0.num_instances", "1"),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.updatable", "cluster_config.0.worker_config.0.num_instances", "3"),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.updatable", "cluster_config.0.preemptible_worker_config.0.num_instances", "2")),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAccDataprocCluster_withStagingBucket(t *testing.T) {
|
2017-11-03 22:05:20 +00:00
|
|
|
t.Parallel()
|
|
|
|
|
2017-10-30 23:41:37 +00:00
|
|
|
rnd := acctest.RandString(10)
|
|
|
|
var cluster dataproc.Cluster
|
|
|
|
clusterName := fmt.Sprintf("dproc-cluster-test-%s", rnd)
|
|
|
|
bucketName := fmt.Sprintf("%s-bucket", clusterName)
|
|
|
|
|
|
|
|
resource.Test(t, resource.TestCase{
|
|
|
|
PreCheck: func() { testAccPreCheck(t) },
|
|
|
|
Providers: testAccProviders,
|
2018-03-13 18:46:27 +00:00
|
|
|
CheckDestroy: testAccCheckDataprocClusterDestroy(),
|
2017-10-30 23:41:37 +00:00
|
|
|
Steps: []resource.TestStep{
|
|
|
|
{
|
|
|
|
Config: testAccDataprocCluster_withStagingBucketAndCluster(clusterName, bucketName),
|
|
|
|
Check: resource.ComposeTestCheckFunc(
|
|
|
|
testAccCheckDataprocClusterExists("google_dataproc_cluster.with_bucket", &cluster),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.with_bucket", "cluster_config.0.staging_bucket", bucketName),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.with_bucket", "cluster_config.0.bucket", bucketName)),
|
|
|
|
},
|
|
|
|
{
|
|
|
|
// Simulate destroy of cluster by removing it from definition,
|
|
|
|
// but leaving the storage bucket (should not be auto deleted)
|
|
|
|
Config: testAccDataprocCluster_withStagingBucketOnly(bucketName),
|
|
|
|
Check: resource.ComposeTestCheckFunc(
|
|
|
|
testAccCheckDataprocStagingBucketExists(bucketName),
|
|
|
|
),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAccDataprocCluster_withInitAction(t *testing.T) {
|
2017-11-03 22:05:20 +00:00
|
|
|
t.Parallel()
|
|
|
|
|
2017-10-30 23:41:37 +00:00
|
|
|
rnd := acctest.RandString(10)
|
|
|
|
var cluster dataproc.Cluster
|
|
|
|
bucketName := fmt.Sprintf("dproc-cluster-test-%s-init-bucket", rnd)
|
|
|
|
objectName := "msg.txt"
|
|
|
|
resource.Test(t, resource.TestCase{
|
|
|
|
PreCheck: func() { testAccPreCheck(t) },
|
|
|
|
Providers: testAccProviders,
|
2018-03-13 18:46:27 +00:00
|
|
|
CheckDestroy: testAccCheckDataprocClusterDestroy(),
|
2017-10-30 23:41:37 +00:00
|
|
|
Steps: []resource.TestStep{
|
|
|
|
{
|
|
|
|
Config: testAccDataprocCluster_withInitAction(rnd, bucketName, objectName),
|
|
|
|
Check: resource.ComposeTestCheckFunc(
|
|
|
|
testAccCheckDataprocClusterExists("google_dataproc_cluster.with_init_action", &cluster),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.with_init_action", "cluster_config.0.initialization_action.#", "2"),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.with_init_action", "cluster_config.0.initialization_action.0.timeout_sec", "500"),
|
|
|
|
testAccCheckDataprocClusterInitActionSucceeded(bucketName, objectName),
|
|
|
|
),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAccDataprocCluster_withConfigOverrides(t *testing.T) {
|
2017-11-03 22:05:20 +00:00
|
|
|
t.Parallel()
|
|
|
|
|
2017-10-30 23:41:37 +00:00
|
|
|
rnd := acctest.RandString(10)
|
|
|
|
var cluster dataproc.Cluster
|
|
|
|
resource.Test(t, resource.TestCase{
|
|
|
|
PreCheck: func() { testAccPreCheck(t) },
|
|
|
|
Providers: testAccProviders,
|
2018-03-13 18:46:27 +00:00
|
|
|
CheckDestroy: testAccCheckDataprocClusterDestroy(),
|
2017-10-30 23:41:37 +00:00
|
|
|
Steps: []resource.TestStep{
|
|
|
|
{
|
|
|
|
Config: testAccDataprocCluster_withConfigOverrides(rnd),
|
|
|
|
Check: resource.ComposeTestCheckFunc(
|
|
|
|
testAccCheckDataprocClusterExists("google_dataproc_cluster.with_config_overrides", &cluster),
|
|
|
|
validateDataprocCluster_withConfigOverrides("google_dataproc_cluster.with_config_overrides", &cluster),
|
|
|
|
),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAccDataprocCluster_withServiceAcc(t *testing.T) {
|
2017-11-03 22:05:20 +00:00
|
|
|
t.Parallel()
|
2017-10-30 23:41:37 +00:00
|
|
|
|
2017-11-03 22:05:20 +00:00
|
|
|
sa := "a" + acctest.RandString(10)
|
2017-11-21 17:34:32 +00:00
|
|
|
saEmail := fmt.Sprintf("%s@%s.iam.gserviceaccount.com", sa, getTestProjectFromEnv())
|
2017-10-30 23:41:37 +00:00
|
|
|
rnd := acctest.RandString(10)
|
2017-11-03 22:05:20 +00:00
|
|
|
|
|
|
|
var cluster dataproc.Cluster
|
|
|
|
|
2017-10-30 23:41:37 +00:00
|
|
|
resource.Test(t, resource.TestCase{
|
2017-11-03 22:05:20 +00:00
|
|
|
PreCheck: func() { testAccPreCheck(t) },
|
2017-10-30 23:41:37 +00:00
|
|
|
Providers: testAccProviders,
|
2018-03-13 18:46:27 +00:00
|
|
|
CheckDestroy: testAccCheckDataprocClusterDestroy(),
|
2017-10-30 23:41:37 +00:00
|
|
|
Steps: []resource.TestStep{
|
|
|
|
{
|
2017-11-03 22:05:20 +00:00
|
|
|
Config: testAccDataprocCluster_withServiceAcc(sa, rnd),
|
2017-10-30 23:41:37 +00:00
|
|
|
Check: resource.ComposeTestCheckFunc(
|
|
|
|
testAccCheckDataprocClusterExists(
|
|
|
|
"google_dataproc_cluster.with_service_account", &cluster),
|
|
|
|
testAccCheckDataprocClusterHasServiceScopes(t, &cluster,
|
|
|
|
"https://www.googleapis.com/auth/cloud.useraccounts.readonly",
|
|
|
|
"https://www.googleapis.com/auth/devstorage.read_write",
|
|
|
|
"https://www.googleapis.com/auth/logging.write",
|
|
|
|
"https://www.googleapis.com/auth/monitoring",
|
|
|
|
),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.with_service_account", "cluster_config.0.gce_cluster_config.0.service_account", saEmail),
|
|
|
|
),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAccDataprocCluster_withImageVersion(t *testing.T) {
|
2017-11-03 22:05:20 +00:00
|
|
|
t.Parallel()
|
|
|
|
|
2017-10-30 23:41:37 +00:00
|
|
|
rnd := acctest.RandString(10)
|
|
|
|
var cluster dataproc.Cluster
|
|
|
|
resource.Test(t, resource.TestCase{
|
|
|
|
PreCheck: func() { testAccPreCheck(t) },
|
|
|
|
Providers: testAccProviders,
|
2018-03-13 18:46:27 +00:00
|
|
|
CheckDestroy: testAccCheckDataprocClusterDestroy(),
|
2017-10-30 23:41:37 +00:00
|
|
|
Steps: []resource.TestStep{
|
|
|
|
{
|
|
|
|
Config: testAccDataprocCluster_withImageVersion(rnd),
|
|
|
|
Check: resource.ComposeTestCheckFunc(
|
|
|
|
testAccCheckDataprocClusterExists("google_dataproc_cluster.with_image_version", &cluster),
|
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.with_image_version", "cluster_config.0.software_config.0.image_version", "preview"),
|
|
|
|
),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAccDataprocCluster_withLabels(t *testing.T) {
|
2017-11-03 22:05:20 +00:00
|
|
|
t.Parallel()
|
|
|
|
|
2017-10-30 23:41:37 +00:00
|
|
|
rnd := acctest.RandString(10)
|
|
|
|
var cluster dataproc.Cluster
|
|
|
|
resource.Test(t, resource.TestCase{
|
|
|
|
PreCheck: func() { testAccPreCheck(t) },
|
|
|
|
Providers: testAccProviders,
|
2018-03-13 18:46:27 +00:00
|
|
|
CheckDestroy: testAccCheckDataprocClusterDestroy(),
|
2017-10-30 23:41:37 +00:00
|
|
|
Steps: []resource.TestStep{
|
|
|
|
{
|
|
|
|
Config: testAccDataprocCluster_withLabels(rnd),
|
|
|
|
Check: resource.ComposeTestCheckFunc(
|
|
|
|
testAccCheckDataprocClusterExists("google_dataproc_cluster.with_labels", &cluster),
|
|
|
|
|
2018-03-12 16:06:05 +00:00
|
|
|
// We only provide one, but GCP adds three, so expect 4. This means unfortunately a
|
2017-10-30 23:41:37 +00:00
|
|
|
// diff will exist unless the user adds these in. An alternative approach would
|
|
|
|
// be to follow the same approach as properties, i.e. split in into labels
|
|
|
|
// and override_labels
|
|
|
|
//
|
|
|
|
// The config is currently configured with ignore_changes = ["labels"] to handle this
|
|
|
|
//
|
2018-03-12 16:06:05 +00:00
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.with_labels", "labels.%", "4"),
|
2017-10-30 23:41:37 +00:00
|
|
|
resource.TestCheckResourceAttr("google_dataproc_cluster.with_labels", "labels.key1", "value1"),
|
|
|
|
),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
func TestAccDataprocCluster_withNetworkRefs(t *testing.T) {
|
2017-11-03 22:05:20 +00:00
|
|
|
t.Parallel()
|
|
|
|
|
2017-10-30 23:41:37 +00:00
|
|
|
var c1, c2 dataproc.Cluster
|
|
|
|
rnd := acctest.RandString(10)
|
|
|
|
netName := fmt.Sprintf(`dproc-cluster-test-%s-net`, rnd)
|
|
|
|
resource.Test(t, resource.TestCase{
|
|
|
|
PreCheck: func() { testAccPreCheck(t) },
|
|
|
|
Providers: testAccProviders,
|
2018-03-13 18:46:27 +00:00
|
|
|
CheckDestroy: testAccCheckDataprocClusterDestroy(),
|
2017-10-30 23:41:37 +00:00
|
|
|
Steps: []resource.TestStep{
|
|
|
|
{
|
|
|
|
Config: testAccDataprocCluster_withNetworkRefs(rnd, netName),
|
|
|
|
Check: resource.ComposeTestCheckFunc(
|
|
|
|
// successful creation of the clusters is good enough to assess it worked
|
|
|
|
testAccCheckDataprocClusterExists("google_dataproc_cluster.with_net_ref_by_url", &c1),
|
|
|
|
testAccCheckDataprocClusterExists("google_dataproc_cluster.with_net_ref_by_name", &c2),
|
|
|
|
),
|
|
|
|
},
|
|
|
|
},
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2018-03-13 18:46:27 +00:00
|
|
|
func testAccCheckDataprocClusterDestroy() resource.TestCheckFunc {
|
2017-10-30 23:41:37 +00:00
|
|
|
return func(s *terraform.State) error {
|
|
|
|
config := testAccProvider.Meta().(*Config)
|
|
|
|
|
|
|
|
for _, rs := range s.RootModule().Resources {
|
|
|
|
if rs.Type != "google_dataproc_cluster" {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
if rs.Primary.ID == "" {
|
|
|
|
return fmt.Errorf("Unable to verify delete of dataproc cluster, ID is empty")
|
|
|
|
}
|
|
|
|
|
|
|
|
attributes := rs.Primary.Attributes
|
|
|
|
project, err := getTestProject(rs.Primary, config)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2018-03-13 18:46:27 +00:00
|
|
|
_, err = config.clientDataproc.Projects.Regions.Clusters.Get(
|
|
|
|
project, attributes["region"], rs.Primary.ID).Do()
|
2017-10-30 23:41:37 +00:00
|
|
|
|
2018-03-13 18:46:27 +00:00
|
|
|
if err != nil {
|
|
|
|
if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == http.StatusNotFound {
|
|
|
|
return nil
|
|
|
|
} else if ok {
|
|
|
|
return fmt.Errorf("Error validating cluster deleted. Code: %d. Message: %s", gerr.Code, gerr.Message)
|
|
|
|
}
|
|
|
|
return fmt.Errorf("Error validating cluster deleted. %s", err.Error())
|
2017-10-30 23:41:37 +00:00
|
|
|
}
|
2018-03-13 18:46:27 +00:00
|
|
|
return fmt.Errorf("Dataproc cluster still exists")
|
2017-10-30 23:41:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func testAccCheckDataprocClusterHasServiceScopes(t *testing.T, cluster *dataproc.Cluster, scopes ...string) func(s *terraform.State) error {
|
|
|
|
return func(s *terraform.State) error {
|
|
|
|
|
|
|
|
if !reflect.DeepEqual(scopes, cluster.Config.GceClusterConfig.ServiceAccountScopes) {
|
|
|
|
return fmt.Errorf("Cluster does not contain expected set of service account scopes : %v : instead %v",
|
|
|
|
scopes, cluster.Config.GceClusterConfig.ServiceAccountScopes)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func validateBucketExists(bucket string, config *Config) (bool, error) {
|
|
|
|
_, err := config.clientStorage.Buckets.Get(bucket).Do()
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == http.StatusNotFound {
|
|
|
|
return false, nil
|
|
|
|
} else if ok {
|
|
|
|
return false, fmt.Errorf("Error validating bucket exists: http code error : %d, http message error: %s", gerr.Code, gerr.Message)
|
|
|
|
}
|
|
|
|
return false, fmt.Errorf("Error validating bucket exists: %s", err.Error())
|
|
|
|
}
|
|
|
|
return true, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func testAccCheckDataprocStagingBucketExists(bucketName string) resource.TestCheckFunc {
|
|
|
|
return func(s *terraform.State) error {
|
|
|
|
|
|
|
|
config := testAccProvider.Meta().(*Config)
|
|
|
|
|
|
|
|
exists, err := validateBucketExists(bucketName, config)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
if !exists {
|
|
|
|
return fmt.Errorf("Staging Bucket %s does not exist", bucketName)
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
func testAccCheckDataprocClusterInitActionSucceeded(bucket, object string) resource.TestCheckFunc {
|
|
|
|
|
|
|
|
// The init script will have created an object in the specified bucket.
|
|
|
|
// Ensure it exists
|
|
|
|
return func(s *terraform.State) error {
|
|
|
|
config := testAccProvider.Meta().(*Config)
|
|
|
|
_, err := config.clientStorage.Objects.Get(bucket, object).Do()
|
|
|
|
if err != nil {
|
|
|
|
return fmt.Errorf("Unable to verify init action success: Error reading object %s in bucket %s: %v", object, bucket, err)
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func validateDataprocCluster_withConfigOverrides(n string, cluster *dataproc.Cluster) resource.TestCheckFunc {
|
|
|
|
return func(s *terraform.State) error {
|
|
|
|
|
|
|
|
type tfAndGCPTestField struct {
|
|
|
|
tfAttr string
|
|
|
|
expectedVal string
|
|
|
|
actualGCPVal string
|
|
|
|
}
|
|
|
|
|
|
|
|
rs, ok := s.RootModule().Resources[n]
|
|
|
|
if !ok {
|
|
|
|
return fmt.Errorf("Terraform resource Not found: %s", n)
|
|
|
|
}
|
|
|
|
|
|
|
|
if cluster.Config.MasterConfig == nil || cluster.Config.WorkerConfig == nil || cluster.Config.SecondaryWorkerConfig == nil {
|
|
|
|
return fmt.Errorf("Master/Worker/SecondaryConfig values not set in GCP, expecting values")
|
|
|
|
}
|
|
|
|
|
|
|
|
clusterTests := []tfAndGCPTestField{
|
|
|
|
{"cluster_config.0.master_config.0.num_instances", "3", strconv.Itoa(int(cluster.Config.MasterConfig.NumInstances))},
|
|
|
|
{"cluster_config.0.master_config.0.disk_config.0.boot_disk_size_gb", "10", strconv.Itoa(int(cluster.Config.MasterConfig.DiskConfig.BootDiskSizeGb))},
|
|
|
|
{"cluster_config.0.master_config.0.disk_config.0.num_local_ssds", "0", strconv.Itoa(int(cluster.Config.MasterConfig.DiskConfig.NumLocalSsds))},
|
2018-01-17 18:45:28 +00:00
|
|
|
{"cluster_config.0.master_config.0.machine_type", "n1-standard-1", GetResourceNameFromSelfLink(cluster.Config.MasterConfig.MachineTypeUri)},
|
2017-10-30 23:41:37 +00:00
|
|
|
{"cluster_config.0.master_config.0.instance_names.#", "3", strconv.Itoa(len(cluster.Config.MasterConfig.InstanceNames))},
|
|
|
|
|
|
|
|
{"cluster_config.0.worker_config.0.num_instances", "3", strconv.Itoa(int(cluster.Config.WorkerConfig.NumInstances))},
|
|
|
|
{"cluster_config.0.worker_config.0.disk_config.0.boot_disk_size_gb", "11", strconv.Itoa(int(cluster.Config.WorkerConfig.DiskConfig.BootDiskSizeGb))},
|
|
|
|
{"cluster_config.0.worker_config.0.disk_config.0.num_local_ssds", "1", strconv.Itoa(int(cluster.Config.WorkerConfig.DiskConfig.NumLocalSsds))},
|
2018-01-17 18:45:28 +00:00
|
|
|
{"cluster_config.0.worker_config.0.machine_type", "n1-standard-1", GetResourceNameFromSelfLink(cluster.Config.WorkerConfig.MachineTypeUri)},
|
2017-10-30 23:41:37 +00:00
|
|
|
{"cluster_config.0.worker_config.0.instance_names.#", "3", strconv.Itoa(len(cluster.Config.WorkerConfig.InstanceNames))},
|
|
|
|
|
|
|
|
{"cluster_config.0.preemptible_worker_config.0.num_instances", "1", strconv.Itoa(int(cluster.Config.SecondaryWorkerConfig.NumInstances))},
|
|
|
|
{"cluster_config.0.preemptible_worker_config.0.disk_config.0.boot_disk_size_gb", "12", strconv.Itoa(int(cluster.Config.SecondaryWorkerConfig.DiskConfig.BootDiskSizeGb))},
|
|
|
|
{"cluster_config.0.preemptible_worker_config.0.instance_names.#", "1", strconv.Itoa(len(cluster.Config.SecondaryWorkerConfig.InstanceNames))},
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, attrs := range clusterTests {
|
|
|
|
tfVal := rs.Primary.Attributes[attrs.tfAttr]
|
|
|
|
if tfVal != attrs.expectedVal {
|
|
|
|
return fmt.Errorf("%s: Terraform Attribute value '%s' is not as expected '%s' ", attrs.tfAttr, tfVal, attrs.expectedVal)
|
|
|
|
}
|
|
|
|
if attrs.actualGCPVal != tfVal {
|
|
|
|
return fmt.Errorf("%s: Terraform Attribute value '%s' is not aligned with that in GCP '%s' ", attrs.tfAttr, tfVal, attrs.actualGCPVal)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func testAccCheckDataprocClusterExists(n string, cluster *dataproc.Cluster) resource.TestCheckFunc {
|
|
|
|
return func(s *terraform.State) error {
|
|
|
|
rs, ok := s.RootModule().Resources[n]
|
|
|
|
if !ok {
|
|
|
|
return fmt.Errorf("Terraform resource Not found: %s", n)
|
|
|
|
}
|
|
|
|
|
|
|
|
if rs.Primary.ID == "" {
|
|
|
|
return fmt.Errorf("No ID is set for Dataproc cluster")
|
|
|
|
}
|
|
|
|
|
|
|
|
config := testAccProvider.Meta().(*Config)
|
|
|
|
project, err := getTestProject(rs.Primary, config)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
found, err := config.clientDataproc.Projects.Regions.Clusters.Get(
|
|
|
|
project, rs.Primary.Attributes["region"], rs.Primary.ID).Do()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if found.ClusterName != rs.Primary.ID {
|
|
|
|
return fmt.Errorf("Dataproc cluster %s not found, found %s instead", rs.Primary.ID, cluster.ClusterName)
|
|
|
|
}
|
|
|
|
|
|
|
|
*cluster = *found
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func testAccCheckDataproc_missingZoneGlobalRegion1(rnd string) string {
|
|
|
|
return fmt.Sprintf(`
|
|
|
|
resource "google_dataproc_cluster" "basic" {
|
|
|
|
name = "dproc-cluster-test-%s"
|
|
|
|
region = "global"
|
|
|
|
}
|
|
|
|
`, rnd)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testAccCheckDataproc_missingZoneGlobalRegion2(rnd string) string {
|
|
|
|
return fmt.Sprintf(`
|
|
|
|
resource "google_dataproc_cluster" "basic" {
|
|
|
|
name = "dproc-cluster-test-%s"
|
|
|
|
region = "global"
|
|
|
|
|
|
|
|
cluster_config {
|
|
|
|
gce_cluster_config { }
|
|
|
|
}
|
|
|
|
}
|
|
|
|
`, rnd)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testAccDataprocCluster_basic(rnd string) string {
|
|
|
|
return fmt.Sprintf(`
|
|
|
|
resource "google_dataproc_cluster" "basic" {
|
|
|
|
name = "dproc-cluster-test-%s"
|
|
|
|
region = "us-central1"
|
|
|
|
}
|
|
|
|
`, rnd)
|
|
|
|
}
|
|
|
|
|
2017-12-22 21:24:05 +00:00
|
|
|
func testAccDataprocCluster_basicWithInternalIpOnlyTrue(rnd string) string {
|
|
|
|
return fmt.Sprintf(`
|
|
|
|
variable subnetwork_cidr {
|
|
|
|
default = "10.0.0.0/16"
|
|
|
|
}
|
|
|
|
|
|
|
|
resource "google_compute_network" "dataproc_network" {
|
|
|
|
name = "dataproc-internalip-network-%s"
|
|
|
|
auto_create_subnetworks = false
|
|
|
|
}
|
|
|
|
|
|
|
|
#
|
|
|
|
# Create a subnet with Private IP Access enabled to test
|
|
|
|
# deploying a Dataproc cluster with Internal IP Only enabled.
|
|
|
|
#
|
|
|
|
resource "google_compute_subnetwork" "dataproc_subnetwork" {
|
|
|
|
name = "dataproc-internalip-subnetwork-%s"
|
|
|
|
ip_cidr_range = "${var.subnetwork_cidr}"
|
|
|
|
network = "${google_compute_network.dataproc_network.self_link}"
|
|
|
|
region = "us-central1"
|
|
|
|
private_ip_google_access = true
|
|
|
|
}
|
|
|
|
|
|
|
|
#
|
|
|
|
# The default network within GCP already comes pre configured with
|
|
|
|
# certain firewall rules open to allow internal communication. As we
|
|
|
|
# are creating a new one here for this test, we need to additionally
|
|
|
|
# open up similar rules to allow the nodes to talk to each other
|
|
|
|
# internally as part of their configuration or this will just hang.
|
|
|
|
#
|
|
|
|
resource "google_compute_firewall" "dataproc_network_firewall" {
|
|
|
|
name = "dproc-cluster-test-allow-internal"
|
|
|
|
description = "Firewall rules for dataproc Terraform acceptance testing"
|
|
|
|
network = "${google_compute_network.dataproc_network.name}"
|
|
|
|
|
|
|
|
allow {
|
|
|
|
protocol = "icmp"
|
|
|
|
}
|
|
|
|
|
|
|
|
allow {
|
|
|
|
protocol = "tcp"
|
|
|
|
ports = ["0-65535"]
|
|
|
|
}
|
|
|
|
|
|
|
|
allow {
|
|
|
|
protocol = "udp"
|
|
|
|
ports = ["0-65535"]
|
|
|
|
}
|
|
|
|
|
|
|
|
source_ranges = ["${var.subnetwork_cidr}"]
|
|
|
|
}
|
|
|
|
resource "google_dataproc_cluster" "basic" {
|
|
|
|
name = "dproc-cluster-test-%s"
|
|
|
|
region = "us-central1"
|
|
|
|
depends_on = ["google_compute_firewall.dataproc_network_firewall"]
|
|
|
|
|
|
|
|
cluster_config {
|
|
|
|
gce_cluster_config {
|
|
|
|
subnetwork = "${google_compute_subnetwork.dataproc_subnetwork.name}"
|
|
|
|
internal_ip_only = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
`, rnd, rnd, rnd)
|
|
|
|
}
|
|
|
|
|
2018-01-24 00:08:51 +00:00
|
|
|
func testAccDataprocCluster_basicWithMetadata(rnd string) string {
|
|
|
|
return fmt.Sprintf(`
|
|
|
|
resource "google_dataproc_cluster" "basic" {
|
|
|
|
name = "dproc-cluster-test-%s"
|
|
|
|
region = "us-central1"
|
|
|
|
|
|
|
|
cluster_config {
|
|
|
|
gce_cluster_config {
|
|
|
|
metadata {
|
|
|
|
foo = "bar"
|
|
|
|
baz = "qux"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2017-10-30 23:41:37 +00:00
|
|
|
`, rnd)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testAccDataprocCluster_singleNodeCluster(rnd string) string {
|
|
|
|
return fmt.Sprintf(`
|
|
|
|
resource "google_dataproc_cluster" "single_node_cluster" {
|
|
|
|
name = "dproc-cluster-test-%s"
|
|
|
|
region = "us-central1"
|
|
|
|
|
|
|
|
cluster_config {
|
|
|
|
# Keep the costs down with smallest config we can get away with
|
|
|
|
software_config {
|
|
|
|
override_properties = {
|
|
|
|
"dataproc:dataproc.allow.zero.workers" = "true"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
`, rnd)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testAccDataprocCluster_withConfigOverrides(rnd string) string {
|
|
|
|
return fmt.Sprintf(`
|
|
|
|
resource "google_dataproc_cluster" "with_config_overrides" {
|
|
|
|
name = "dproc-cluster-test-%s"
|
|
|
|
region = "us-central1"
|
|
|
|
|
|
|
|
cluster_config {
|
|
|
|
|
|
|
|
master_config {
|
|
|
|
num_instances = 3
|
|
|
|
machine_type = "n1-standard-1"
|
|
|
|
disk_config {
|
|
|
|
boot_disk_size_gb = 10
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
worker_config {
|
|
|
|
num_instances = 3
|
|
|
|
machine_type = "n1-standard-1"
|
|
|
|
disk_config {
|
|
|
|
boot_disk_size_gb = 11
|
|
|
|
num_local_ssds = 1
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
preemptible_worker_config {
|
|
|
|
num_instances = 1
|
|
|
|
disk_config {
|
|
|
|
boot_disk_size_gb = 12
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}`, rnd)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testAccDataprocCluster_withInitAction(rnd, bucket, objName string) string {
|
|
|
|
return fmt.Sprintf(`
|
|
|
|
resource "google_storage_bucket" "init_bucket" {
|
|
|
|
name = "%s"
|
|
|
|
force_destroy = "true"
|
|
|
|
}
|
|
|
|
|
|
|
|
resource "google_storage_bucket_object" "init_script" {
|
|
|
|
name = "dproc-cluster-test-%s-init-script.sh"
|
|
|
|
bucket = "${google_storage_bucket.init_bucket.name}"
|
|
|
|
content = <<EOL
|
|
|
|
#!/bin/bash
|
|
|
|
echo "init action success" >> /tmp/%s
|
|
|
|
gsutil cp /tmp/%s ${google_storage_bucket.init_bucket.url}
|
|
|
|
EOL
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
resource "google_dataproc_cluster" "with_init_action" {
|
|
|
|
name = "dproc-cluster-test-%s"
|
|
|
|
region = "us-central1"
|
|
|
|
|
|
|
|
cluster_config {
|
|
|
|
# Keep the costs down with smallest config we can get away with
|
|
|
|
software_config {
|
|
|
|
override_properties = {
|
|
|
|
"dataproc:dataproc.allow.zero.workers" = "true"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
master_config {
|
|
|
|
machine_type = "n1-standard-1"
|
|
|
|
disk_config {
|
|
|
|
boot_disk_size_gb = 10
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
initialization_action {
|
|
|
|
script = "${google_storage_bucket.init_bucket.url}/${google_storage_bucket_object.init_script.name}"
|
|
|
|
timeout_sec = 500
|
|
|
|
}
|
|
|
|
initialization_action {
|
|
|
|
script = "${google_storage_bucket.init_bucket.url}/${google_storage_bucket_object.init_script.name}"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}`, bucket, rnd, objName, objName, rnd)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testAccDataprocCluster_updatable(rnd string, w, p int) string {
|
|
|
|
return fmt.Sprintf(`
|
|
|
|
resource "google_dataproc_cluster" "updatable" {
|
|
|
|
name = "dproc-cluster-test-%s"
|
|
|
|
region = "us-central1"
|
|
|
|
|
|
|
|
cluster_config {
|
|
|
|
|
|
|
|
master_config {
|
|
|
|
num_instances = "1"
|
|
|
|
machine_type = "n1-standard-1"
|
|
|
|
disk_config {
|
|
|
|
boot_disk_size_gb = 10
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
worker_config {
|
|
|
|
num_instances = "%d"
|
|
|
|
machine_type = "n1-standard-1"
|
|
|
|
disk_config {
|
|
|
|
boot_disk_size_gb = 10
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
preemptible_worker_config {
|
|
|
|
num_instances = "%d"
|
|
|
|
disk_config {
|
|
|
|
boot_disk_size_gb = 10
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
}`, rnd, w, p)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testAccDataprocCluster_withStagingBucketOnly(bucketName string) string {
|
|
|
|
return fmt.Sprintf(`
|
|
|
|
resource "google_storage_bucket" "bucket" {
|
|
|
|
name = "%s"
|
|
|
|
force_destroy = "true"
|
|
|
|
}`, bucketName)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testAccDataprocCluster_withStagingBucketAndCluster(clusterName, bucketName string) string {
|
|
|
|
return fmt.Sprintf(`
|
|
|
|
%s
|
|
|
|
|
|
|
|
resource "google_dataproc_cluster" "with_bucket" {
|
|
|
|
name = "%s"
|
|
|
|
region = "us-central1"
|
|
|
|
|
|
|
|
cluster_config {
|
|
|
|
staging_bucket = "${google_storage_bucket.bucket.name}"
|
|
|
|
|
|
|
|
# Keep the costs down with smallest config we can get away with
|
|
|
|
software_config {
|
|
|
|
override_properties = {
|
|
|
|
"dataproc:dataproc.allow.zero.workers" = "true"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
master_config {
|
|
|
|
machine_type = "n1-standard-1"
|
|
|
|
disk_config {
|
|
|
|
boot_disk_size_gb = 10
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}`, testAccDataprocCluster_withStagingBucketOnly(bucketName), clusterName)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testAccDataprocCluster_withLabels(rnd string) string {
|
|
|
|
return fmt.Sprintf(`
|
|
|
|
resource "google_dataproc_cluster" "with_labels" {
|
|
|
|
name = "dproc-cluster-test-%s"
|
|
|
|
region = "us-central1"
|
|
|
|
|
|
|
|
labels {
|
|
|
|
key1 = "value1"
|
|
|
|
}
|
|
|
|
|
|
|
|
# This is because GCP automatically adds its own labels as well.
|
|
|
|
# In this case we just want to test our newly added label is there
|
|
|
|
lifecycle {
|
|
|
|
ignore_changes = ["labels"]
|
|
|
|
}
|
|
|
|
|
|
|
|
}`, rnd)
|
|
|
|
}
|
|
|
|
|
|
|
|
func testAccDataprocCluster_withImageVersion(rnd string) string {
|
|
|
|
return fmt.Sprintf(`
|
|
|
|
resource "google_dataproc_cluster" "with_image_version" {
|
|
|
|
name = "dproc-cluster-test-%s"
|
|
|
|
region = "us-central1"
|
|
|
|
|
|
|
|
cluster_config {
|
|
|
|
software_config {
|
|
|
|
image_version = "preview"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}`, rnd)
|
|
|
|
}
|
|
|
|
|
2017-11-03 22:05:20 +00:00
|
|
|
func testAccDataprocCluster_withServiceAcc(sa string, rnd string) string {
|
2017-10-30 23:41:37 +00:00
|
|
|
return fmt.Sprintf(`
|
2017-11-03 22:05:20 +00:00
|
|
|
resource "google_service_account" "service_account" {
|
|
|
|
account_id = "%s"
|
|
|
|
}
|
|
|
|
|
|
|
|
resource "google_project_iam_member" "service_account" {
|
|
|
|
role = "roles/dataproc.worker"
|
|
|
|
member = "serviceAccount:${google_service_account.service_account.email}"
|
|
|
|
}
|
|
|
|
|
2017-10-30 23:41:37 +00:00
|
|
|
resource "google_dataproc_cluster" "with_service_account" {
|
|
|
|
name = "dproc-cluster-test-%s"
|
|
|
|
region = "us-central1"
|
|
|
|
|
|
|
|
cluster_config {
|
|
|
|
# Keep the costs down with smallest config we can get away with
|
|
|
|
software_config {
|
|
|
|
override_properties = {
|
|
|
|
"dataproc:dataproc.allow.zero.workers" = "true"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
master_config {
|
|
|
|
machine_type = "n1-standard-1"
|
|
|
|
disk_config {
|
|
|
|
boot_disk_size_gb = 10
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
gce_cluster_config {
|
2017-11-03 22:05:20 +00:00
|
|
|
service_account = "${google_service_account.service_account.email}"
|
2017-10-30 23:41:37 +00:00
|
|
|
service_account_scopes = [
|
|
|
|
# User supplied scopes
|
|
|
|
"https://www.googleapis.com/auth/monitoring",
|
|
|
|
|
|
|
|
# The following scopes necessary for the cluster to function properly are
|
|
|
|
# always added, even if not explicitly specified:
|
|
|
|
# useraccounts-ro: https://www.googleapis.com/auth/cloud.useraccounts.readonly
|
|
|
|
# storage-rw: https://www.googleapis.com/auth/devstorage.read_write
|
|
|
|
# logging-write: https://www.googleapis.com/auth/logging.write
|
|
|
|
"useraccounts-ro","storage-rw","logging-write"
|
|
|
|
]
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-11-03 22:05:20 +00:00
|
|
|
depends_on = ["google_project_iam_member.service_account"]
|
|
|
|
|
|
|
|
}`, sa, rnd)
|
2017-10-30 23:41:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func testAccDataprocCluster_withNetworkRefs(rnd, netName string) string {
|
|
|
|
return fmt.Sprintf(`
|
|
|
|
resource "google_compute_network" "dataproc_network" {
|
|
|
|
name = "%s"
|
|
|
|
auto_create_subnetworks = true
|
|
|
|
}
|
|
|
|
|
|
|
|
#
|
|
|
|
# The default network within GCP already comes pre configured with
|
|
|
|
# certain firewall rules open to allow internal communication. As we
|
|
|
|
# are creating a new one here for this test, we need to additionally
|
|
|
|
# open up similar rules to allow the nodes to talk to each other
|
|
|
|
# internally as part of their configuration or this will just hang.
|
|
|
|
#
|
|
|
|
resource "google_compute_firewall" "dataproc_network_firewall" {
|
|
|
|
name = "dproc-cluster-test-%s-allow-internal"
|
|
|
|
description = "Firewall rules for dataproc Terraform acceptance testing"
|
|
|
|
network = "${google_compute_network.dataproc_network.name}"
|
|
|
|
|
|
|
|
allow {
|
|
|
|
protocol = "icmp"
|
|
|
|
}
|
|
|
|
|
|
|
|
allow {
|
|
|
|
protocol = "tcp"
|
|
|
|
ports = ["0-65535"]
|
|
|
|
}
|
|
|
|
|
|
|
|
allow {
|
|
|
|
protocol = "udp"
|
|
|
|
ports = ["0-65535"]
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
resource "google_dataproc_cluster" "with_net_ref_by_name" {
|
|
|
|
name = "dproc-cluster-test-%s-name"
|
|
|
|
region = "us-central1"
|
|
|
|
depends_on = ["google_compute_firewall.dataproc_network_firewall"]
|
|
|
|
|
|
|
|
cluster_config {
|
|
|
|
# Keep the costs down with smallest config we can get away with
|
|
|
|
software_config {
|
|
|
|
override_properties = {
|
|
|
|
"dataproc:dataproc.allow.zero.workers" = "true"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
master_config {
|
|
|
|
machine_type = "n1-standard-1"
|
|
|
|
disk_config {
|
|
|
|
boot_disk_size_gb = 10
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
gce_cluster_config {
|
|
|
|
network = "${google_compute_network.dataproc_network.name}"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
resource "google_dataproc_cluster" "with_net_ref_by_url" {
|
|
|
|
name = "dproc-cluster-test-%s-url"
|
|
|
|
region = "us-central1"
|
|
|
|
depends_on = ["google_compute_firewall.dataproc_network_firewall"]
|
|
|
|
|
|
|
|
cluster_config {
|
|
|
|
# Keep the costs down with smallest config we can get away with
|
|
|
|
software_config {
|
|
|
|
override_properties = {
|
|
|
|
"dataproc:dataproc.allow.zero.workers" = "true"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
master_config {
|
|
|
|
machine_type = "n1-standard-1"
|
|
|
|
disk_config {
|
|
|
|
boot_disk_size_gb = 10
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
gce_cluster_config {
|
|
|
|
network = "${google_compute_network.dataproc_network.self_link}"
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
`, netName, rnd, rnd, rnd)
|
|
|
|
}
|