mirror of
https://github.com/letic/terraform-provider-google.git
synced 2024-09-06 09:06:44 +00:00
![Darren Haken](/assets/img/avatar_default.png)
This PR also switched us to using the beta API in all cases, and that had a side effect which is worth noting, note included here for posterity. ===== The problem is, we add a GPU, and as per the docs, GKE adds a taint to the node pool saying "don't schedule here unless you tolerate GPUs", which is pretty sensible. Terraform doesn't know about that, because it didn't ask for the taint to be added. So after apply, on refresh, it sees the state of the world (1 taint) and the state of the config (0 taints) and wants to set the world equal to the config. This introduces a diff, which makes the test fail - tests fail if there's a diff after they run. Taints are a beta feature, though. :) And since the config doesn't contain any taints, terraform didn't see any beta features in that node pool ... so it used to send the request to the v1 API. And since the v1 API didn't return anything about taints (since they're a beta feature), terraform happily checked the state of the world (0 taints I know about) vs the config (0 taints), and all was well. This PR makes every node pool refresh request hit the beta API. So now terraform finds out about the taints (which were always there) and the test fails (which it always should have done). The solution is probably to write a little bit of code which suppresses the report of the diff of any taint with value 'nvidia.com/gpu', but only if GPUs are enabled. I think that's something that can be done.
376 lines
9.4 KiB
Go
376 lines
9.4 KiB
Go
package google
|
|
|
|
import (
|
|
"github.com/hashicorp/terraform/helper/schema"
|
|
"github.com/hashicorp/terraform/helper/validation"
|
|
containerBeta "google.golang.org/api/container/v1beta1"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// Matches gke-default scope from https://cloud.google.com/sdk/gcloud/reference/container/clusters/create
|
|
var defaultOauthScopes = []string{
|
|
"https://www.googleapis.com/auth/devstorage.read_only",
|
|
"https://www.googleapis.com/auth/logging.write",
|
|
"https://www.googleapis.com/auth/monitoring",
|
|
"https://www.googleapis.com/auth/service.management.readonly",
|
|
"https://www.googleapis.com/auth/servicecontrol",
|
|
"https://www.googleapis.com/auth/trace.append",
|
|
}
|
|
|
|
var schemaNodeConfig = &schema.Schema{
|
|
Type: schema.TypeList,
|
|
Optional: true,
|
|
Computed: true,
|
|
ForceNew: true,
|
|
MaxItems: 1,
|
|
Elem: &schema.Resource{
|
|
Schema: map[string]*schema.Schema{
|
|
"disk_size_gb": {
|
|
Type: schema.TypeInt,
|
|
Optional: true,
|
|
Computed: true,
|
|
ForceNew: true,
|
|
ValidateFunc: validation.IntAtLeast(10),
|
|
},
|
|
|
|
"guest_accelerator": &schema.Schema{
|
|
Type: schema.TypeList,
|
|
Optional: true,
|
|
Computed: true,
|
|
ForceNew: true,
|
|
Elem: &schema.Resource{
|
|
Schema: map[string]*schema.Schema{
|
|
"count": &schema.Schema{
|
|
Type: schema.TypeInt,
|
|
Required: true,
|
|
ForceNew: true,
|
|
},
|
|
"type": &schema.Schema{
|
|
Type: schema.TypeString,
|
|
Required: true,
|
|
ForceNew: true,
|
|
DiffSuppressFunc: linkDiffSuppress,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
"image_type": {
|
|
Type: schema.TypeString,
|
|
Optional: true,
|
|
Computed: true,
|
|
ForceNew: true,
|
|
},
|
|
|
|
"labels": {
|
|
Type: schema.TypeMap,
|
|
Optional: true,
|
|
ForceNew: true,
|
|
Elem: schema.TypeString,
|
|
},
|
|
|
|
"local_ssd_count": {
|
|
Type: schema.TypeInt,
|
|
Optional: true,
|
|
Computed: true,
|
|
ForceNew: true,
|
|
ValidateFunc: validation.IntAtLeast(0),
|
|
},
|
|
|
|
"machine_type": {
|
|
Type: schema.TypeString,
|
|
Optional: true,
|
|
Computed: true,
|
|
ForceNew: true,
|
|
},
|
|
|
|
"metadata": {
|
|
Type: schema.TypeMap,
|
|
Optional: true,
|
|
ForceNew: true,
|
|
Elem: schema.TypeString,
|
|
},
|
|
|
|
"min_cpu_platform": {
|
|
Type: schema.TypeString,
|
|
Optional: true,
|
|
ForceNew: true,
|
|
},
|
|
|
|
"oauth_scopes": {
|
|
Type: schema.TypeSet,
|
|
Optional: true,
|
|
Computed: true,
|
|
ForceNew: true,
|
|
Elem: &schema.Schema{
|
|
Type: schema.TypeString,
|
|
StateFunc: func(v interface{}) string {
|
|
return canonicalizeServiceScope(v.(string))
|
|
},
|
|
},
|
|
Set: stringScopeHashcode,
|
|
},
|
|
|
|
"preemptible": {
|
|
Type: schema.TypeBool,
|
|
Optional: true,
|
|
ForceNew: true,
|
|
Default: false,
|
|
},
|
|
|
|
"service_account": {
|
|
Type: schema.TypeString,
|
|
Optional: true,
|
|
Computed: true,
|
|
ForceNew: true,
|
|
},
|
|
|
|
"tags": {
|
|
Type: schema.TypeList,
|
|
Optional: true,
|
|
ForceNew: true,
|
|
Elem: &schema.Schema{Type: schema.TypeString},
|
|
},
|
|
|
|
"taint": {
|
|
Type: schema.TypeList,
|
|
Optional: true,
|
|
ForceNew: true,
|
|
DiffSuppressFunc: taintDiffSuppress,
|
|
Elem: &schema.Resource{
|
|
Schema: map[string]*schema.Schema{
|
|
"key": {
|
|
Type: schema.TypeString,
|
|
Required: true,
|
|
ForceNew: true,
|
|
},
|
|
"value": {
|
|
Type: schema.TypeString,
|
|
Required: true,
|
|
ForceNew: true,
|
|
},
|
|
"effect": {
|
|
Type: schema.TypeString,
|
|
Required: true,
|
|
ForceNew: true,
|
|
ValidateFunc: validation.StringInSlice([]string{"NO_SCHEDULE", "PREFER_NO_SCHEDULE", "NO_EXECUTE"}, false),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
|
|
"workload_metadata_config": {
|
|
Type: schema.TypeList,
|
|
Optional: true,
|
|
ForceNew: true,
|
|
MaxItems: 1,
|
|
Elem: &schema.Resource{
|
|
Schema: map[string]*schema.Schema{
|
|
"node_metadata": {
|
|
Type: schema.TypeString,
|
|
Required: true,
|
|
ForceNew: true,
|
|
ValidateFunc: validation.StringInSlice([]string{"UNSPECIFIED", "SECURE", "EXPOSE"}, false),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
func expandNodeConfig(v interface{}) *containerBeta.NodeConfig {
|
|
nodeConfigs := v.([]interface{})
|
|
nc := &containerBeta.NodeConfig{
|
|
// Defaults can't be set on a list/set in the schema, so set the default on create here.
|
|
OauthScopes: defaultOauthScopes,
|
|
}
|
|
if len(nodeConfigs) == 0 {
|
|
return nc
|
|
}
|
|
|
|
nodeConfig := nodeConfigs[0].(map[string]interface{})
|
|
|
|
if v, ok := nodeConfig["machine_type"]; ok {
|
|
nc.MachineType = v.(string)
|
|
}
|
|
|
|
if v, ok := nodeConfig["guest_accelerator"]; ok {
|
|
accels := v.([]interface{})
|
|
guestAccelerators := make([]*containerBeta.AcceleratorConfig, 0, len(accels))
|
|
for _, raw := range accels {
|
|
data := raw.(map[string]interface{})
|
|
if data["count"].(int) == 0 {
|
|
continue
|
|
}
|
|
guestAccelerators = append(guestAccelerators, &containerBeta.AcceleratorConfig{
|
|
AcceleratorCount: int64(data["count"].(int)),
|
|
AcceleratorType: data["type"].(string),
|
|
})
|
|
}
|
|
nc.Accelerators = guestAccelerators
|
|
}
|
|
|
|
if v, ok := nodeConfig["disk_size_gb"]; ok {
|
|
nc.DiskSizeGb = int64(v.(int))
|
|
}
|
|
|
|
if v, ok := nodeConfig["local_ssd_count"]; ok {
|
|
nc.LocalSsdCount = int64(v.(int))
|
|
}
|
|
|
|
if scopes, ok := nodeConfig["oauth_scopes"]; ok {
|
|
scopesSet := scopes.(*schema.Set)
|
|
scopes := make([]string, scopesSet.Len())
|
|
for i, scope := range scopesSet.List() {
|
|
scopes[i] = canonicalizeServiceScope(scope.(string))
|
|
}
|
|
|
|
nc.OauthScopes = scopes
|
|
}
|
|
|
|
if v, ok := nodeConfig["service_account"]; ok {
|
|
nc.ServiceAccount = v.(string)
|
|
}
|
|
|
|
if v, ok := nodeConfig["metadata"]; ok {
|
|
m := make(map[string]string)
|
|
for k, val := range v.(map[string]interface{}) {
|
|
m[k] = val.(string)
|
|
}
|
|
nc.Metadata = m
|
|
}
|
|
|
|
if v, ok := nodeConfig["image_type"]; ok {
|
|
nc.ImageType = v.(string)
|
|
}
|
|
|
|
if v, ok := nodeConfig["labels"]; ok {
|
|
m := make(map[string]string)
|
|
for k, val := range v.(map[string]interface{}) {
|
|
m[k] = val.(string)
|
|
}
|
|
nc.Labels = m
|
|
}
|
|
|
|
if v, ok := nodeConfig["tags"]; ok {
|
|
tagsList := v.([]interface{})
|
|
tags := []string{}
|
|
for _, v := range tagsList {
|
|
tags = append(tags, v.(string))
|
|
}
|
|
nc.Tags = tags
|
|
}
|
|
// Preemptible Is Optional+Default, so it always has a value
|
|
nc.Preemptible = nodeConfig["preemptible"].(bool)
|
|
|
|
if v, ok := nodeConfig["min_cpu_platform"]; ok {
|
|
nc.MinCpuPlatform = v.(string)
|
|
}
|
|
|
|
if v, ok := nodeConfig["taint"]; ok && len(v.([]interface{})) > 0 {
|
|
taints := v.([]interface{})
|
|
nodeTaints := make([]*containerBeta.NodeTaint, 0, len(taints))
|
|
for _, raw := range taints {
|
|
data := raw.(map[string]interface{})
|
|
taint := &containerBeta.NodeTaint{
|
|
Key: data["key"].(string),
|
|
Value: data["value"].(string),
|
|
Effect: data["effect"].(string),
|
|
}
|
|
nodeTaints = append(nodeTaints, taint)
|
|
}
|
|
nc.Taints = nodeTaints
|
|
}
|
|
|
|
if v, ok := nodeConfig["workload_metadata_config"]; ok && len(v.([]interface{})) > 0 {
|
|
conf := v.([]interface{})[0].(map[string]interface{})
|
|
nc.WorkloadMetadataConfig = &containerBeta.WorkloadMetadataConfig{
|
|
NodeMetadata: conf["node_metadata"].(string),
|
|
}
|
|
}
|
|
|
|
return nc
|
|
}
|
|
|
|
func flattenNodeConfig(c *containerBeta.NodeConfig) []map[string]interface{} {
|
|
config := make([]map[string]interface{}, 0, 1)
|
|
|
|
if c == nil {
|
|
return config
|
|
}
|
|
|
|
config = append(config, map[string]interface{}{
|
|
"machine_type": c.MachineType,
|
|
"disk_size_gb": c.DiskSizeGb,
|
|
"guest_accelerator": flattenContainerGuestAccelerators(c.Accelerators),
|
|
"local_ssd_count": c.LocalSsdCount,
|
|
"service_account": c.ServiceAccount,
|
|
"metadata": c.Metadata,
|
|
"image_type": c.ImageType,
|
|
"labels": c.Labels,
|
|
"tags": c.Tags,
|
|
"preemptible": c.Preemptible,
|
|
"min_cpu_platform": c.MinCpuPlatform,
|
|
"taint": flattenTaints(c.Taints),
|
|
"workload_metadata_config": flattenWorkloadMetadataConfig(c.WorkloadMetadataConfig),
|
|
})
|
|
|
|
if len(c.OauthScopes) > 0 {
|
|
config[0]["oauth_scopes"] = schema.NewSet(stringScopeHashcode, convertStringArrToInterface(c.OauthScopes))
|
|
}
|
|
|
|
return config
|
|
}
|
|
|
|
func flattenContainerGuestAccelerators(c []*containerBeta.AcceleratorConfig) []map[string]interface{} {
|
|
result := []map[string]interface{}{}
|
|
for _, accel := range c {
|
|
result = append(result, map[string]interface{}{
|
|
"count": accel.AcceleratorCount,
|
|
"type": accel.AcceleratorType,
|
|
})
|
|
}
|
|
return result
|
|
}
|
|
|
|
func flattenTaints(c []*containerBeta.NodeTaint) []map[string]interface{} {
|
|
result := []map[string]interface{}{}
|
|
for _, taint := range c {
|
|
result = append(result, map[string]interface{}{
|
|
"key": taint.Key,
|
|
"value": taint.Value,
|
|
"effect": taint.Effect,
|
|
})
|
|
}
|
|
return result
|
|
}
|
|
|
|
func flattenWorkloadMetadataConfig(c *containerBeta.WorkloadMetadataConfig) []map[string]interface{} {
|
|
result := []map[string]interface{}{}
|
|
if c != nil {
|
|
result = append(result, map[string]interface{}{
|
|
"node_metadata": c.NodeMetadata,
|
|
})
|
|
}
|
|
return result
|
|
}
|
|
|
|
func taintDiffSuppress(k, old, new string, d *schema.ResourceData) bool {
|
|
if strings.HasSuffix(k, "#") {
|
|
oldCount, oldErr := strconv.Atoi(old)
|
|
newCount, newErr := strconv.Atoi(new)
|
|
// If either of them isn't a number somehow, or if there's one that we didn't have before.
|
|
return oldErr != nil || newErr != nil || oldCount == newCount+1
|
|
} else {
|
|
lastDot := strings.LastIndex(k, ".")
|
|
taintKey := d.Get(k[:lastDot] + ".key").(string)
|
|
if taintKey == "nvidia.com/gpu" {
|
|
return true
|
|
} else {
|
|
return false
|
|
}
|
|
}
|
|
}
|