terraform-provider-google/google/node_config.go
Darren Haken 2b1b668953 #1300 Supporting regional clusters for node pools (#1320)
This PR also switched us to using the beta API in all cases, and that had a side effect which is worth noting, note included here for posterity.

=====
The problem is, we add a GPU, and as per the docs, GKE adds a taint to
the node pool saying "don't schedule here unless you tolerate GPUs",
which is pretty sensible.

Terraform doesn't know about that, because it didn't ask for the taint
to be added. So after apply, on refresh, it sees the state of the world
(1 taint) and the state of the config (0 taints) and wants to set the
world equal to the config. This introduces a diff, which makes the test
fail - tests fail if there's a diff after they run.

Taints are a beta feature, though. :) And since the config doesn't
contain any taints, terraform didn't see any beta features in that node
pool ... so it used to send the request to the v1 API. And since the v1
API didn't return anything about taints (since they're a beta feature),
terraform happily checked the state of the world (0 taints I know about)
vs the config (0 taints), and all was well.

This PR makes every node pool refresh request hit the beta API. So now
terraform finds out about the taints (which were always there) and the
test fails (which it always should have done).

The solution is probably to write a little bit of code which suppresses
the report of the diff of any taint with value 'nvidia.com/gpu', but
only if GPUs are enabled. I think that's something that can be done.
2018-04-24 17:55:21 -07:00

376 lines
9.4 KiB
Go

package google
import (
"github.com/hashicorp/terraform/helper/schema"
"github.com/hashicorp/terraform/helper/validation"
containerBeta "google.golang.org/api/container/v1beta1"
"strconv"
"strings"
)
// Matches gke-default scope from https://cloud.google.com/sdk/gcloud/reference/container/clusters/create
var defaultOauthScopes = []string{
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring",
"https://www.googleapis.com/auth/service.management.readonly",
"https://www.googleapis.com/auth/servicecontrol",
"https://www.googleapis.com/auth/trace.append",
}
var schemaNodeConfig = &schema.Schema{
Type: schema.TypeList,
Optional: true,
Computed: true,
ForceNew: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"disk_size_gb": {
Type: schema.TypeInt,
Optional: true,
Computed: true,
ForceNew: true,
ValidateFunc: validation.IntAtLeast(10),
},
"guest_accelerator": &schema.Schema{
Type: schema.TypeList,
Optional: true,
Computed: true,
ForceNew: true,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"count": &schema.Schema{
Type: schema.TypeInt,
Required: true,
ForceNew: true,
},
"type": &schema.Schema{
Type: schema.TypeString,
Required: true,
ForceNew: true,
DiffSuppressFunc: linkDiffSuppress,
},
},
},
},
"image_type": {
Type: schema.TypeString,
Optional: true,
Computed: true,
ForceNew: true,
},
"labels": {
Type: schema.TypeMap,
Optional: true,
ForceNew: true,
Elem: schema.TypeString,
},
"local_ssd_count": {
Type: schema.TypeInt,
Optional: true,
Computed: true,
ForceNew: true,
ValidateFunc: validation.IntAtLeast(0),
},
"machine_type": {
Type: schema.TypeString,
Optional: true,
Computed: true,
ForceNew: true,
},
"metadata": {
Type: schema.TypeMap,
Optional: true,
ForceNew: true,
Elem: schema.TypeString,
},
"min_cpu_platform": {
Type: schema.TypeString,
Optional: true,
ForceNew: true,
},
"oauth_scopes": {
Type: schema.TypeSet,
Optional: true,
Computed: true,
ForceNew: true,
Elem: &schema.Schema{
Type: schema.TypeString,
StateFunc: func(v interface{}) string {
return canonicalizeServiceScope(v.(string))
},
},
Set: stringScopeHashcode,
},
"preemptible": {
Type: schema.TypeBool,
Optional: true,
ForceNew: true,
Default: false,
},
"service_account": {
Type: schema.TypeString,
Optional: true,
Computed: true,
ForceNew: true,
},
"tags": {
Type: schema.TypeList,
Optional: true,
ForceNew: true,
Elem: &schema.Schema{Type: schema.TypeString},
},
"taint": {
Type: schema.TypeList,
Optional: true,
ForceNew: true,
DiffSuppressFunc: taintDiffSuppress,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"key": {
Type: schema.TypeString,
Required: true,
ForceNew: true,
},
"value": {
Type: schema.TypeString,
Required: true,
ForceNew: true,
},
"effect": {
Type: schema.TypeString,
Required: true,
ForceNew: true,
ValidateFunc: validation.StringInSlice([]string{"NO_SCHEDULE", "PREFER_NO_SCHEDULE", "NO_EXECUTE"}, false),
},
},
},
},
"workload_metadata_config": {
Type: schema.TypeList,
Optional: true,
ForceNew: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"node_metadata": {
Type: schema.TypeString,
Required: true,
ForceNew: true,
ValidateFunc: validation.StringInSlice([]string{"UNSPECIFIED", "SECURE", "EXPOSE"}, false),
},
},
},
},
},
},
}
func expandNodeConfig(v interface{}) *containerBeta.NodeConfig {
nodeConfigs := v.([]interface{})
nc := &containerBeta.NodeConfig{
// Defaults can't be set on a list/set in the schema, so set the default on create here.
OauthScopes: defaultOauthScopes,
}
if len(nodeConfigs) == 0 {
return nc
}
nodeConfig := nodeConfigs[0].(map[string]interface{})
if v, ok := nodeConfig["machine_type"]; ok {
nc.MachineType = v.(string)
}
if v, ok := nodeConfig["guest_accelerator"]; ok {
accels := v.([]interface{})
guestAccelerators := make([]*containerBeta.AcceleratorConfig, 0, len(accels))
for _, raw := range accels {
data := raw.(map[string]interface{})
if data["count"].(int) == 0 {
continue
}
guestAccelerators = append(guestAccelerators, &containerBeta.AcceleratorConfig{
AcceleratorCount: int64(data["count"].(int)),
AcceleratorType: data["type"].(string),
})
}
nc.Accelerators = guestAccelerators
}
if v, ok := nodeConfig["disk_size_gb"]; ok {
nc.DiskSizeGb = int64(v.(int))
}
if v, ok := nodeConfig["local_ssd_count"]; ok {
nc.LocalSsdCount = int64(v.(int))
}
if scopes, ok := nodeConfig["oauth_scopes"]; ok {
scopesSet := scopes.(*schema.Set)
scopes := make([]string, scopesSet.Len())
for i, scope := range scopesSet.List() {
scopes[i] = canonicalizeServiceScope(scope.(string))
}
nc.OauthScopes = scopes
}
if v, ok := nodeConfig["service_account"]; ok {
nc.ServiceAccount = v.(string)
}
if v, ok := nodeConfig["metadata"]; ok {
m := make(map[string]string)
for k, val := range v.(map[string]interface{}) {
m[k] = val.(string)
}
nc.Metadata = m
}
if v, ok := nodeConfig["image_type"]; ok {
nc.ImageType = v.(string)
}
if v, ok := nodeConfig["labels"]; ok {
m := make(map[string]string)
for k, val := range v.(map[string]interface{}) {
m[k] = val.(string)
}
nc.Labels = m
}
if v, ok := nodeConfig["tags"]; ok {
tagsList := v.([]interface{})
tags := []string{}
for _, v := range tagsList {
tags = append(tags, v.(string))
}
nc.Tags = tags
}
// Preemptible Is Optional+Default, so it always has a value
nc.Preemptible = nodeConfig["preemptible"].(bool)
if v, ok := nodeConfig["min_cpu_platform"]; ok {
nc.MinCpuPlatform = v.(string)
}
if v, ok := nodeConfig["taint"]; ok && len(v.([]interface{})) > 0 {
taints := v.([]interface{})
nodeTaints := make([]*containerBeta.NodeTaint, 0, len(taints))
for _, raw := range taints {
data := raw.(map[string]interface{})
taint := &containerBeta.NodeTaint{
Key: data["key"].(string),
Value: data["value"].(string),
Effect: data["effect"].(string),
}
nodeTaints = append(nodeTaints, taint)
}
nc.Taints = nodeTaints
}
if v, ok := nodeConfig["workload_metadata_config"]; ok && len(v.([]interface{})) > 0 {
conf := v.([]interface{})[0].(map[string]interface{})
nc.WorkloadMetadataConfig = &containerBeta.WorkloadMetadataConfig{
NodeMetadata: conf["node_metadata"].(string),
}
}
return nc
}
func flattenNodeConfig(c *containerBeta.NodeConfig) []map[string]interface{} {
config := make([]map[string]interface{}, 0, 1)
if c == nil {
return config
}
config = append(config, map[string]interface{}{
"machine_type": c.MachineType,
"disk_size_gb": c.DiskSizeGb,
"guest_accelerator": flattenContainerGuestAccelerators(c.Accelerators),
"local_ssd_count": c.LocalSsdCount,
"service_account": c.ServiceAccount,
"metadata": c.Metadata,
"image_type": c.ImageType,
"labels": c.Labels,
"tags": c.Tags,
"preemptible": c.Preemptible,
"min_cpu_platform": c.MinCpuPlatform,
"taint": flattenTaints(c.Taints),
"workload_metadata_config": flattenWorkloadMetadataConfig(c.WorkloadMetadataConfig),
})
if len(c.OauthScopes) > 0 {
config[0]["oauth_scopes"] = schema.NewSet(stringScopeHashcode, convertStringArrToInterface(c.OauthScopes))
}
return config
}
func flattenContainerGuestAccelerators(c []*containerBeta.AcceleratorConfig) []map[string]interface{} {
result := []map[string]interface{}{}
for _, accel := range c {
result = append(result, map[string]interface{}{
"count": accel.AcceleratorCount,
"type": accel.AcceleratorType,
})
}
return result
}
func flattenTaints(c []*containerBeta.NodeTaint) []map[string]interface{} {
result := []map[string]interface{}{}
for _, taint := range c {
result = append(result, map[string]interface{}{
"key": taint.Key,
"value": taint.Value,
"effect": taint.Effect,
})
}
return result
}
func flattenWorkloadMetadataConfig(c *containerBeta.WorkloadMetadataConfig) []map[string]interface{} {
result := []map[string]interface{}{}
if c != nil {
result = append(result, map[string]interface{}{
"node_metadata": c.NodeMetadata,
})
}
return result
}
func taintDiffSuppress(k, old, new string, d *schema.ResourceData) bool {
if strings.HasSuffix(k, "#") {
oldCount, oldErr := strconv.Atoi(old)
newCount, newErr := strconv.Atoi(new)
// If either of them isn't a number somehow, or if there's one that we didn't have before.
return oldErr != nil || newErr != nil || oldCount == newCount+1
} else {
lastDot := strings.LastIndex(k, ".")
taintKey := d.Get(k[:lastDot] + ".key").(string)
if taintKey == "nvidia.com/gpu" {
return true
} else {
return false
}
}
}