2017-07-31 16:28:39 +00:00
package google
import (
2018-06-15 20:21:51 +00:00
"strconv"
"strings"
2017-07-31 16:28:39 +00:00
"github.com/hashicorp/terraform/helper/schema"
2017-08-10 20:01:45 +00:00
"github.com/hashicorp/terraform/helper/validation"
2018-03-07 01:44:05 +00:00
containerBeta "google.golang.org/api/container/v1beta1"
2017-07-31 16:28:39 +00:00
)
2018-01-08 23:54:45 +00:00
// Matches gke-default scope from https://cloud.google.com/sdk/gcloud/reference/container/clusters/create
var defaultOauthScopes = [ ] string {
"https://www.googleapis.com/auth/devstorage.read_only" ,
"https://www.googleapis.com/auth/logging.write" ,
"https://www.googleapis.com/auth/monitoring" ,
"https://www.googleapis.com/auth/service.management.readonly" ,
"https://www.googleapis.com/auth/servicecontrol" ,
"https://www.googleapis.com/auth/trace.append" ,
}
2017-07-31 16:28:39 +00:00
var schemaNodeConfig = & schema . Schema {
Type : schema . TypeList ,
Optional : true ,
Computed : true ,
ForceNew : true ,
MaxItems : 1 ,
Elem : & schema . Resource {
Schema : map [ string ] * schema . Schema {
"disk_size_gb" : {
2017-08-10 20:01:45 +00:00
Type : schema . TypeInt ,
Optional : true ,
Computed : true ,
ForceNew : true ,
ValidateFunc : validation . IntAtLeast ( 10 ) ,
2017-07-31 16:28:39 +00:00
} ,
2018-06-15 20:21:51 +00:00
"disk_type" : {
Type : schema . TypeString ,
Optional : true ,
Computed : true ,
ForceNew : true ,
ValidateFunc : validation . StringInSlice ( [ ] string { "pd-standard" , "pd-ssd" } , false ) ,
} ,
2018-02-24 00:55:07 +00:00
"guest_accelerator" : & schema . Schema {
Type : schema . TypeList ,
Optional : true ,
Computed : true ,
ForceNew : true ,
Elem : & schema . Resource {
Schema : map [ string ] * schema . Schema {
"count" : & schema . Schema {
Type : schema . TypeInt ,
Required : true ,
ForceNew : true ,
} ,
"type" : & schema . Schema {
Type : schema . TypeString ,
Required : true ,
ForceNew : true ,
DiffSuppressFunc : linkDiffSuppress ,
} ,
} ,
} ,
} ,
2017-10-27 22:18:34 +00:00
"image_type" : {
Type : schema . TypeString ,
Optional : true ,
Computed : true ,
} ,
"labels" : {
Type : schema . TypeMap ,
Optional : true ,
ForceNew : true ,
2018-08-14 23:53:39 +00:00
Elem : & schema . Schema { Type : schema . TypeString } ,
2017-10-27 22:18:34 +00:00
} ,
2017-07-31 16:28:39 +00:00
"local_ssd_count" : {
2017-08-10 20:01:45 +00:00
Type : schema . TypeInt ,
Optional : true ,
Computed : true ,
ForceNew : true ,
ValidateFunc : validation . IntAtLeast ( 0 ) ,
2017-07-31 16:28:39 +00:00
} ,
2017-10-27 22:18:34 +00:00
"machine_type" : {
2017-07-31 16:28:39 +00:00
Type : schema . TypeString ,
Optional : true ,
Computed : true ,
ForceNew : true ,
} ,
"metadata" : {
Type : schema . TypeMap ,
Optional : true ,
ForceNew : true ,
2018-08-14 23:53:39 +00:00
Elem : & schema . Schema { Type : schema . TypeString } ,
2017-07-31 16:28:39 +00:00
} ,
2017-10-27 22:18:34 +00:00
"min_cpu_platform" : {
2017-07-31 16:28:39 +00:00
Type : schema . TypeString ,
Optional : true ,
ForceNew : true ,
} ,
2017-10-27 22:18:34 +00:00
"oauth_scopes" : {
Type : schema . TypeSet ,
2017-07-31 16:28:39 +00:00
Optional : true ,
2017-10-27 22:18:34 +00:00
Computed : true ,
2017-07-31 16:28:39 +00:00
ForceNew : true ,
2017-10-27 22:18:34 +00:00
Elem : & schema . Schema {
Type : schema . TypeString ,
StateFunc : func ( v interface { } ) string {
return canonicalizeServiceScope ( v . ( string ) )
} ,
} ,
Set : stringScopeHashcode ,
2017-07-31 16:28:39 +00:00
} ,
2017-09-01 20:02:26 +00:00
"preemptible" : {
Type : schema . TypeBool ,
Optional : true ,
ForceNew : true ,
Default : false ,
} ,
2017-10-25 21:08:48 +00:00
2017-10-27 22:18:34 +00:00
"service_account" : {
2017-10-25 21:08:48 +00:00
Type : schema . TypeString ,
Optional : true ,
2017-10-27 22:18:34 +00:00
Computed : true ,
ForceNew : true ,
} ,
"tags" : {
Type : schema . TypeList ,
Optional : true ,
2017-10-25 21:08:48 +00:00
ForceNew : true ,
2017-10-27 22:18:34 +00:00
Elem : & schema . Schema { Type : schema . TypeString } ,
2017-10-25 21:08:48 +00:00
} ,
2018-03-15 20:28:30 +00:00
2018-03-27 21:55:42 +00:00
"taint" : {
2018-10-04 01:46:20 +00:00
Deprecated : "This field is in beta and will be removed from this provider. Use it in the the google-beta provider instead. See https://terraform.io/docs/providers/google/provider_versions.html for more details." ,
#1300 Supporting regional clusters for node pools (#1320)
This PR also switched us to using the beta API in all cases, and that had a side effect which is worth noting, note included here for posterity.
=====
The problem is, we add a GPU, and as per the docs, GKE adds a taint to
the node pool saying "don't schedule here unless you tolerate GPUs",
which is pretty sensible.
Terraform doesn't know about that, because it didn't ask for the taint
to be added. So after apply, on refresh, it sees the state of the world
(1 taint) and the state of the config (0 taints) and wants to set the
world equal to the config. This introduces a diff, which makes the test
fail - tests fail if there's a diff after they run.
Taints are a beta feature, though. :) And since the config doesn't
contain any taints, terraform didn't see any beta features in that node
pool ... so it used to send the request to the v1 API. And since the v1
API didn't return anything about taints (since they're a beta feature),
terraform happily checked the state of the world (0 taints I know about)
vs the config (0 taints), and all was well.
This PR makes every node pool refresh request hit the beta API. So now
terraform finds out about the taints (which were always there) and the
test fails (which it always should have done).
The solution is probably to write a little bit of code which suppresses
the report of the diff of any taint with value 'nvidia.com/gpu', but
only if GPUs are enabled. I think that's something that can be done.
2018-04-25 00:55:21 +00:00
Type : schema . TypeList ,
Optional : true ,
ForceNew : true ,
DiffSuppressFunc : taintDiffSuppress ,
2018-03-27 21:55:42 +00:00
Elem : & schema . Resource {
Schema : map [ string ] * schema . Schema {
"key" : {
Type : schema . TypeString ,
Required : true ,
ForceNew : true ,
} ,
"value" : {
Type : schema . TypeString ,
Required : true ,
ForceNew : true ,
} ,
"effect" : {
Type : schema . TypeString ,
Required : true ,
ForceNew : true ,
ValidateFunc : validation . StringInSlice ( [ ] string { "NO_SCHEDULE" , "PREFER_NO_SCHEDULE" , "NO_EXECUTE" } , false ) ,
} ,
} ,
} ,
} ,
2018-03-15 20:28:30 +00:00
"workload_metadata_config" : {
2018-10-04 01:46:20 +00:00
Deprecated : "This field is in beta and will be removed from this provider. Use it in the the google-beta provider instead. See https://terraform.io/docs/providers/google/provider_versions.html for more details." ,
2018-10-02 21:13:04 +00:00
Type : schema . TypeList ,
Optional : true ,
ForceNew : true ,
MaxItems : 1 ,
2018-03-15 20:28:30 +00:00
Elem : & schema . Resource {
Schema : map [ string ] * schema . Schema {
"node_metadata" : {
Type : schema . TypeString ,
Required : true ,
ForceNew : true ,
ValidateFunc : validation . StringInSlice ( [ ] string { "UNSPECIFIED" , "SECURE" , "EXPOSE" } , false ) ,
} ,
} ,
} ,
} ,
2017-07-31 16:28:39 +00:00
} ,
} ,
}
2017-08-11 19:23:17 +00:00
2018-03-07 01:44:05 +00:00
func expandNodeConfig ( v interface { } ) * containerBeta . NodeConfig {
2017-08-11 19:23:17 +00:00
nodeConfigs := v . ( [ ] interface { } )
2018-03-07 01:44:05 +00:00
nc := & containerBeta . NodeConfig {
2018-01-08 23:54:45 +00:00
// Defaults can't be set on a list/set in the schema, so set the default on create here.
OauthScopes : defaultOauthScopes ,
}
if len ( nodeConfigs ) == 0 {
return nc
}
2017-08-11 19:23:17 +00:00
2018-01-08 23:54:45 +00:00
nodeConfig := nodeConfigs [ 0 ] . ( map [ string ] interface { } )
2017-08-11 19:23:17 +00:00
if v , ok := nodeConfig [ "machine_type" ] ; ok {
nc . MachineType = v . ( string )
}
2018-02-24 00:55:07 +00:00
if v , ok := nodeConfig [ "guest_accelerator" ] ; ok {
accels := v . ( [ ] interface { } )
2018-03-07 01:44:05 +00:00
guestAccelerators := make ( [ ] * containerBeta . AcceleratorConfig , 0 , len ( accels ) )
2018-02-24 00:55:07 +00:00
for _ , raw := range accels {
data := raw . ( map [ string ] interface { } )
if data [ "count" ] . ( int ) == 0 {
continue
}
2018-03-07 01:44:05 +00:00
guestAccelerators = append ( guestAccelerators , & containerBeta . AcceleratorConfig {
2018-02-24 00:55:07 +00:00
AcceleratorCount : int64 ( data [ "count" ] . ( int ) ) ,
AcceleratorType : data [ "type" ] . ( string ) ,
} )
}
nc . Accelerators = guestAccelerators
}
2017-08-11 19:23:17 +00:00
if v , ok := nodeConfig [ "disk_size_gb" ] ; ok {
nc . DiskSizeGb = int64 ( v . ( int ) )
}
2018-06-15 20:21:51 +00:00
if v , ok := nodeConfig [ "disk_type" ] ; ok {
nc . DiskType = v . ( string )
}
2017-08-11 19:23:17 +00:00
if v , ok := nodeConfig [ "local_ssd_count" ] ; ok {
nc . LocalSsdCount = int64 ( v . ( int ) )
}
2017-10-25 20:41:42 +00:00
if scopes , ok := nodeConfig [ "oauth_scopes" ] ; ok {
scopesSet := scopes . ( * schema . Set )
scopes := make ( [ ] string , scopesSet . Len ( ) )
for i , scope := range scopesSet . List ( ) {
scopes [ i ] = canonicalizeServiceScope ( scope . ( string ) )
2017-08-11 19:23:17 +00:00
}
nc . OauthScopes = scopes
}
if v , ok := nodeConfig [ "service_account" ] ; ok {
nc . ServiceAccount = v . ( string )
}
if v , ok := nodeConfig [ "metadata" ] ; ok {
m := make ( map [ string ] string )
for k , val := range v . ( map [ string ] interface { } ) {
m [ k ] = val . ( string )
}
nc . Metadata = m
}
if v , ok := nodeConfig [ "image_type" ] ; ok {
nc . ImageType = v . ( string )
}
if v , ok := nodeConfig [ "labels" ] ; ok {
m := make ( map [ string ] string )
for k , val := range v . ( map [ string ] interface { } ) {
m [ k ] = val . ( string )
}
nc . Labels = m
}
if v , ok := nodeConfig [ "tags" ] ; ok {
tagsList := v . ( [ ] interface { } )
tags := [ ] string { }
for _ , v := range tagsList {
tags = append ( tags , v . ( string ) )
}
nc . Tags = tags
}
2017-09-01 20:02:26 +00:00
// Preemptible Is Optional+Default, so it always has a value
nc . Preemptible = nodeConfig [ "preemptible" ] . ( bool )
2017-08-11 19:23:17 +00:00
2017-10-25 21:08:48 +00:00
if v , ok := nodeConfig [ "min_cpu_platform" ] ; ok {
nc . MinCpuPlatform = v . ( string )
}
2018-03-27 21:55:42 +00:00
if v , ok := nodeConfig [ "taint" ] ; ok && len ( v . ( [ ] interface { } ) ) > 0 {
taints := v . ( [ ] interface { } )
nodeTaints := make ( [ ] * containerBeta . NodeTaint , 0 , len ( taints ) )
for _ , raw := range taints {
data := raw . ( map [ string ] interface { } )
taint := & containerBeta . NodeTaint {
Key : data [ "key" ] . ( string ) ,
Value : data [ "value" ] . ( string ) ,
Effect : data [ "effect" ] . ( string ) ,
}
nodeTaints = append ( nodeTaints , taint )
}
nc . Taints = nodeTaints
}
2018-03-16 00:08:39 +00:00
if v , ok := nodeConfig [ "workload_metadata_config" ] ; ok && len ( v . ( [ ] interface { } ) ) > 0 {
2018-03-15 20:28:30 +00:00
conf := v . ( [ ] interface { } ) [ 0 ] . ( map [ string ] interface { } )
nc . WorkloadMetadataConfig = & containerBeta . WorkloadMetadataConfig {
NodeMetadata : conf [ "node_metadata" ] . ( string ) ,
}
}
2017-08-11 19:23:17 +00:00
return nc
2017-09-01 20:02:26 +00:00
}
2018-03-07 01:44:05 +00:00
func flattenNodeConfig ( c * containerBeta . NodeConfig ) [ ] map [ string ] interface { } {
2017-09-26 22:32:12 +00:00
config := make ( [ ] map [ string ] interface { } , 0 , 1 )
if c == nil {
return config
2017-09-01 20:02:26 +00:00
}
2017-09-26 22:32:12 +00:00
config = append ( config , map [ string ] interface { } {
2018-03-15 20:28:30 +00:00
"machine_type" : c . MachineType ,
"disk_size_gb" : c . DiskSizeGb ,
2018-06-15 20:21:51 +00:00
"disk_type" : c . DiskType ,
2018-03-16 00:08:39 +00:00
"guest_accelerator" : flattenContainerGuestAccelerators ( c . Accelerators ) ,
2018-03-15 20:28:30 +00:00
"local_ssd_count" : c . LocalSsdCount ,
"service_account" : c . ServiceAccount ,
"metadata" : c . Metadata ,
"image_type" : c . ImageType ,
"labels" : c . Labels ,
"tags" : c . Tags ,
"preemptible" : c . Preemptible ,
"min_cpu_platform" : c . MinCpuPlatform ,
2018-03-27 21:55:42 +00:00
"taint" : flattenTaints ( c . Taints ) ,
2018-03-15 20:28:30 +00:00
"workload_metadata_config" : flattenWorkloadMetadataConfig ( c . WorkloadMetadataConfig ) ,
2017-09-26 22:32:12 +00:00
} )
2017-09-01 20:02:26 +00:00
if len ( c . OauthScopes ) > 0 {
2017-10-25 20:41:42 +00:00
config [ 0 ] [ "oauth_scopes" ] = schema . NewSet ( stringScopeHashcode , convertStringArrToInterface ( c . OauthScopes ) )
2017-09-01 20:02:26 +00:00
}
2017-08-11 19:23:17 +00:00
2017-09-01 20:02:26 +00:00
return config
2017-08-11 19:23:17 +00:00
}
2018-03-15 20:28:30 +00:00
2018-03-16 00:08:39 +00:00
func flattenContainerGuestAccelerators ( c [ ] * containerBeta . AcceleratorConfig ) [ ] map [ string ] interface { } {
result := [ ] map [ string ] interface { } { }
for _ , accel := range c {
result = append ( result , map [ string ] interface { } {
"count" : accel . AcceleratorCount ,
"type" : accel . AcceleratorType ,
} )
}
return result
}
2018-03-27 21:55:42 +00:00
func flattenTaints ( c [ ] * containerBeta . NodeTaint ) [ ] map [ string ] interface { } {
result := [ ] map [ string ] interface { } { }
for _ , taint := range c {
result = append ( result , map [ string ] interface { } {
"key" : taint . Key ,
"value" : taint . Value ,
"effect" : taint . Effect ,
} )
}
return result
}
2018-03-15 20:28:30 +00:00
func flattenWorkloadMetadataConfig ( c * containerBeta . WorkloadMetadataConfig ) [ ] map [ string ] interface { } {
result := [ ] map [ string ] interface { } { }
if c != nil {
result = append ( result , map [ string ] interface { } {
"node_metadata" : c . NodeMetadata ,
} )
}
return result
}
#1300 Supporting regional clusters for node pools (#1320)
This PR also switched us to using the beta API in all cases, and that had a side effect which is worth noting, note included here for posterity.
=====
The problem is, we add a GPU, and as per the docs, GKE adds a taint to
the node pool saying "don't schedule here unless you tolerate GPUs",
which is pretty sensible.
Terraform doesn't know about that, because it didn't ask for the taint
to be added. So after apply, on refresh, it sees the state of the world
(1 taint) and the state of the config (0 taints) and wants to set the
world equal to the config. This introduces a diff, which makes the test
fail - tests fail if there's a diff after they run.
Taints are a beta feature, though. :) And since the config doesn't
contain any taints, terraform didn't see any beta features in that node
pool ... so it used to send the request to the v1 API. And since the v1
API didn't return anything about taints (since they're a beta feature),
terraform happily checked the state of the world (0 taints I know about)
vs the config (0 taints), and all was well.
This PR makes every node pool refresh request hit the beta API. So now
terraform finds out about the taints (which were always there) and the
test fails (which it always should have done).
The solution is probably to write a little bit of code which suppresses
the report of the diff of any taint with value 'nvidia.com/gpu', but
only if GPUs are enabled. I think that's something that can be done.
2018-04-25 00:55:21 +00:00
func taintDiffSuppress ( k , old , new string , d * schema . ResourceData ) bool {
if strings . HasSuffix ( k , "#" ) {
oldCount , oldErr := strconv . Atoi ( old )
newCount , newErr := strconv . Atoi ( new )
// If either of them isn't a number somehow, or if there's one that we didn't have before.
return oldErr != nil || newErr != nil || oldCount == newCount + 1
} else {
lastDot := strings . LastIndex ( k , "." )
taintKey := d . Get ( k [ : lastDot ] + ".key" ) . ( string )
if taintKey == "nvidia.com/gpu" {
return true
} else {
return false
}
}
}