mirror of
https://github.com/letic/terraform-provider-google.git
synced 2024-07-09 03:28:29 +00:00
40094ba417
* Use errwrap to retain original error * Use built-in Page function, only return names when listing services This removes the custom logic on pagination and uses the built-in Page function in the SDK to make things a bit simpler. Additionally, I added a field filter to only return service names, which drastically reduces the size of the API call (important for slow connections, given how frequently this function is executed). Also added errwrap to better trace where errors originate. * Add helper function for diffing string slices This just looked really nasty inline * Batch 20 services at a time, handle precondition failed, better errwrap This commit does three things: 1. It batches services to be enabled 20 at a time. The API fails if you try to enable more than 20 services, and this is documented in the SDK and API. I learned this the hard way. I think Terraform should "do the right thing" here and batch them in series' of twenty, which is what this does. Each batch is tried in serial, but I think making it parallelized is not worth the complexity tradeoffs. 2. Handle the precondition failed error that occurs randomly. This just started happened, but it affects at least two APIs consistently, and a rudimentary test showed that it failed 78% of the time (78/100 times in an hour). We should fix this upstream, but that failure rate also necessitates (in my opinion) some mitigation on the Terraform side until a fix is in place at the API level. 3. Use errwrap on errors for better tracing. It was really difficult to trace exactly which error was being throw. That's fixed. * Updates from code review
362 lines
9.7 KiB
Go
362 lines
9.7 KiB
Go
package google
|
|
|
|
import (
|
|
"fmt"
|
|
"log"
|
|
"strings"
|
|
|
|
"github.com/hashicorp/errwrap"
|
|
"github.com/hashicorp/terraform/helper/schema"
|
|
"golang.org/x/net/context"
|
|
"google.golang.org/api/googleapi"
|
|
"google.golang.org/api/serviceusage/v1beta1"
|
|
)
|
|
|
|
func resourceGoogleProjectServices() *schema.Resource {
|
|
return &schema.Resource{
|
|
Create: resourceGoogleProjectServicesCreate,
|
|
Read: resourceGoogleProjectServicesRead,
|
|
Update: resourceGoogleProjectServicesUpdate,
|
|
Delete: resourceGoogleProjectServicesDelete,
|
|
Importer: &schema.ResourceImporter{
|
|
State: schema.ImportStatePassthrough,
|
|
},
|
|
|
|
Schema: map[string]*schema.Schema{
|
|
"project": &schema.Schema{
|
|
Type: schema.TypeString,
|
|
Optional: true,
|
|
ForceNew: true,
|
|
Computed: true,
|
|
},
|
|
"services": {
|
|
Type: schema.TypeSet,
|
|
Required: true,
|
|
Elem: &schema.Schema{Type: schema.TypeString},
|
|
Set: schema.HashString,
|
|
},
|
|
"disable_on_destroy": &schema.Schema{
|
|
Type: schema.TypeBool,
|
|
Optional: true,
|
|
Default: true,
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// These services can only be enabled as a side-effect of enabling other services,
|
|
// so don't bother storing them in the config or using them for diffing.
|
|
var ignoreProjectServices = map[string]struct{}{
|
|
"containeranalysis.googleapis.com": struct{}{},
|
|
"dataproc-control.googleapis.com": struct{}{},
|
|
"source.googleapis.com": struct{}{},
|
|
}
|
|
|
|
func resourceGoogleProjectServicesCreate(d *schema.ResourceData, meta interface{}) error {
|
|
config := meta.(*Config)
|
|
|
|
pid, err := getProject(d, config)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Get services from config
|
|
cfgServices := getConfigServices(d)
|
|
|
|
// Get services from API
|
|
apiServices, err := getApiServices(pid, config, ignoreProjectServices)
|
|
if err != nil {
|
|
return fmt.Errorf("Error creating services: %v", err)
|
|
}
|
|
|
|
// This call disables any APIs that aren't defined in cfgServices,
|
|
// and enables all of those that are
|
|
err = reconcileServices(cfgServices, apiServices, config, pid)
|
|
if err != nil {
|
|
return fmt.Errorf("Error creating services: %v", err)
|
|
}
|
|
|
|
d.SetId(pid)
|
|
return resourceGoogleProjectServicesRead(d, meta)
|
|
}
|
|
|
|
func resourceGoogleProjectServicesRead(d *schema.ResourceData, meta interface{}) error {
|
|
config := meta.(*Config)
|
|
|
|
services, err := getApiServices(d.Id(), config, ignoreProjectServices)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
d.Set("project", d.Id())
|
|
d.Set("services", services)
|
|
return nil
|
|
}
|
|
|
|
func resourceGoogleProjectServicesUpdate(d *schema.ResourceData, meta interface{}) error {
|
|
log.Printf("[DEBUG]: Updating google_project_services")
|
|
config := meta.(*Config)
|
|
|
|
// Get services from config
|
|
cfgServices := getConfigServices(d)
|
|
|
|
// Get services from API
|
|
apiServices, err := getApiServices(d.Id(), config, ignoreProjectServices)
|
|
if err != nil {
|
|
return fmt.Errorf("Error updating services: %v", err)
|
|
}
|
|
|
|
// This call disables any APIs that aren't defined in cfgServices,
|
|
// and enables all of those that are
|
|
err = reconcileServices(cfgServices, apiServices, config, d.Id())
|
|
if err != nil {
|
|
return fmt.Errorf("Error updating services: %v", err)
|
|
}
|
|
|
|
return resourceGoogleProjectServicesRead(d, meta)
|
|
}
|
|
|
|
func resourceGoogleProjectServicesDelete(d *schema.ResourceData, meta interface{}) error {
|
|
log.Printf("[DEBUG]: Deleting google_project_services")
|
|
|
|
if disable := d.Get("disable_on_destroy"); !(disable.(bool)) {
|
|
log.Printf("Not disabling service '%s', because disable_on_destroy is false.", d.Id())
|
|
d.SetId("")
|
|
return nil
|
|
}
|
|
|
|
config := meta.(*Config)
|
|
services := resourceServices(d)
|
|
for _, s := range services {
|
|
disableService(s, d.Id(), config)
|
|
}
|
|
d.SetId("")
|
|
return nil
|
|
}
|
|
|
|
// This function ensures that the services enabled for a project exactly match that
|
|
// in a config by disabling any services that are returned by the API but not present
|
|
// in the config
|
|
func reconcileServices(cfgServices, apiServices []string, config *Config, pid string) error {
|
|
// Helper to convert slice to map
|
|
m := func(vals []string) map[string]struct{} {
|
|
sm := make(map[string]struct{})
|
|
for _, s := range vals {
|
|
sm[s] = struct{}{}
|
|
}
|
|
return sm
|
|
}
|
|
|
|
cfgMap := m(cfgServices)
|
|
apiMap := m(apiServices)
|
|
|
|
for k, _ := range apiMap {
|
|
if _, ok := cfgMap[k]; !ok {
|
|
// The service in the API is not in the config; disable it.
|
|
err := disableService(k, pid, config)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
// The service exists in the config and the API, so we don't need
|
|
// to re-enable it
|
|
delete(cfgMap, k)
|
|
}
|
|
}
|
|
|
|
keys := make([]string, 0, len(cfgMap))
|
|
for k, _ := range cfgMap {
|
|
keys = append(keys, k)
|
|
}
|
|
err := enableServices(keys, pid, config)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Retrieve services defined in a config
|
|
func getConfigServices(d *schema.ResourceData) (services []string) {
|
|
if v, ok := d.GetOk("services"); ok {
|
|
for _, svc := range v.(*schema.Set).List() {
|
|
services = append(services, svc.(string))
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
// Retrieve a project's services from the API
|
|
func getApiServices(pid string, config *Config, ignore map[string]struct{}) ([]string, error) {
|
|
var apiServices []string
|
|
|
|
if ignore == nil {
|
|
ignore = make(map[string]struct{})
|
|
}
|
|
|
|
ctx := context.Background()
|
|
if err := config.clientServiceUsage.Services.
|
|
List("projects/"+pid).
|
|
Fields("services/name").
|
|
Filter("state:ENABLED").
|
|
Pages(ctx, func(r *serviceusage.ListServicesResponse) error {
|
|
for _, v := range r.Services {
|
|
// services are returned as "projects/PROJECT/services/NAME"
|
|
parts := strings.Split(v.Name, "/")
|
|
if len(parts) > 0 {
|
|
name := parts[len(parts)-1]
|
|
if _, ok := ignore[name]; !ok {
|
|
apiServices = append(apiServices, name)
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}); err != nil {
|
|
return nil, errwrap.Wrapf("failed to list services: {{err}}", err)
|
|
}
|
|
|
|
return apiServices, nil
|
|
}
|
|
|
|
func enableService(s, pid string, config *Config) error {
|
|
return enableServices([]string{s}, pid, config)
|
|
}
|
|
|
|
func enableServices(s []string, pid string, config *Config) error {
|
|
// It's not permitted to enable more than 20 services in one API call (even
|
|
// for batch).
|
|
//
|
|
// https://godoc.org/google.golang.org/api/serviceusage/v1beta1#BatchEnableServicesRequest
|
|
batchSize := 20
|
|
|
|
for i := 0; i < len(s); i += batchSize {
|
|
j := i + batchSize
|
|
if j > len(s) {
|
|
j = len(s)
|
|
}
|
|
|
|
services := s[i:j]
|
|
|
|
if err := retryTime(func() error {
|
|
var sop *serviceusage.Operation
|
|
var err error
|
|
|
|
if len(services) < 1 {
|
|
// No more services to enable
|
|
return nil
|
|
} else if len(services) == 1 {
|
|
// Use the singular enable - can't use batch for a single item
|
|
name := fmt.Sprintf("projects/%s/services/%s", pid, services[0])
|
|
req := &serviceusage.EnableServiceRequest{}
|
|
sop, err = config.clientServiceUsage.Services.Enable(name, req).Do()
|
|
} else {
|
|
// Batch enable 2+ services
|
|
name := fmt.Sprintf("projects/%s", pid)
|
|
req := &serviceusage.BatchEnableServicesRequest{ServiceIds: services}
|
|
sop, err = config.clientServiceUsage.Services.BatchEnable(name, req).Do()
|
|
}
|
|
if err != nil {
|
|
// Check for a "precondition failed" error. The API seems to randomly
|
|
// (although more than 50%) return this error when enabling certain
|
|
// APIs. It's transient, so we catch it and re-raise it as an error that
|
|
// is retryable instead.
|
|
if gerr, ok := err.(*googleapi.Error); ok {
|
|
if (gerr.Code == 400 || gerr.Code == 412) && gerr.Message == "Precondition check failed." {
|
|
return &googleapi.Error{
|
|
Code: 503,
|
|
Message: "api returned \"precondition failed\" while enabling service",
|
|
}
|
|
}
|
|
}
|
|
return errwrap.Wrapf("failed to issue request: {{err}}", err)
|
|
}
|
|
|
|
// Poll for the API to return
|
|
activity := fmt.Sprintf("apis %q to be enabled for %s", services, pid)
|
|
_, waitErr := serviceUsageOperationWait(config, sop, activity)
|
|
if waitErr != nil {
|
|
return waitErr
|
|
}
|
|
|
|
// Accumulate the list of services that are enabled on the project
|
|
enabledServices, err := getApiServices(pid, config, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Diff the list of requested services to enable against the list of
|
|
// services on the project.
|
|
missing := diffStringSlice(services, enabledServices)
|
|
|
|
// If there are any missing, force a retry
|
|
if len(missing) > 0 {
|
|
// Spoof a googleapi Error so retryTime will try again
|
|
return &googleapi.Error{
|
|
Code: 503,
|
|
Message: fmt.Sprintf("The service(s) %q are still being enabled for project %s. This isn't a real API error, this is just eventual consistency.", missing, pid),
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}, 10); err != nil {
|
|
return errwrap.Wrap(err, fmt.Errorf("failed to enable service(s) %q for project %s", services, pid))
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func diffStringSlice(wanted, actual []string) []string {
|
|
var missing []string
|
|
|
|
for _, want := range wanted {
|
|
found := false
|
|
|
|
for _, act := range actual {
|
|
if want == act {
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
|
|
if !found {
|
|
missing = append(missing, want)
|
|
}
|
|
}
|
|
|
|
return missing
|
|
}
|
|
|
|
func disableService(s, pid string, config *Config) error {
|
|
err := retryTime(func() error {
|
|
name := fmt.Sprintf("projects/%s/services/%s", pid, s)
|
|
sop, err := config.clientServiceUsage.Services.Disable(name, &serviceusage.DisableServiceRequest{}).Do()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// Wait for the operation to complete
|
|
_, waitErr := serviceUsageOperationWait(config, sop, "api to disable")
|
|
if waitErr != nil {
|
|
return waitErr
|
|
}
|
|
return nil
|
|
}, 10)
|
|
if err != nil {
|
|
return fmt.Errorf("Error disabling service %q for project %q: %v", s, pid, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func resourceServices(d *schema.ResourceData) []string {
|
|
// Calculate the tags
|
|
var services []string
|
|
if s := d.Get("services"); s != nil {
|
|
ss := s.(*schema.Set)
|
|
services = make([]string, ss.Len())
|
|
for i, v := range ss.List() {
|
|
services[i] = v.(string)
|
|
}
|
|
}
|
|
return services
|
|
}
|