diff --git a/backend/init/init.go b/backend/init/init.go index fdc0ad263d..5684182394 100644 --- a/backend/init/init.go +++ b/backend/init/init.go @@ -14,6 +14,7 @@ import ( backendAzure "github.com/hashicorp/terraform/backend/remote-state/azure" backendconsul "github.com/hashicorp/terraform/backend/remote-state/consul" backendetcdv3 "github.com/hashicorp/terraform/backend/remote-state/etcdv3" + backendGCloud "github.com/hashicorp/terraform/backend/remote-state/gcloud" backendinmem "github.com/hashicorp/terraform/backend/remote-state/inmem" backendS3 "github.com/hashicorp/terraform/backend/remote-state/s3" backendSwift "github.com/hashicorp/terraform/backend/remote-state/swift" @@ -47,6 +48,7 @@ func init() { `Warning: "azure" name is deprecated, please use "azurerm"`), "azurerm": func() backend.Backend { return backendAzure.New() }, "etcdv3": func() backend.Backend { return backendetcdv3.New() }, + "gcloud": func() backend.Backend { return backendGCloud.New() }, } // Add the legacy remote backends that haven't yet been convertd to diff --git a/backend/remote-state/gcloud/backend.go b/backend/remote-state/gcloud/backend.go new file mode 100644 index 0000000000..ac23ed0408 --- /dev/null +++ b/backend/remote-state/gcloud/backend.go @@ -0,0 +1,101 @@ +package gcloud + +import ( + "cloud.google.com/go/storage" + "context" + "fmt" + "github.com/hashicorp/terraform/backend" + "github.com/hashicorp/terraform/helper/pathorcontents" + "github.com/hashicorp/terraform/helper/schema" + googleContext "golang.org/x/net/context" + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "google.golang.org/api/option" +) + +func New() backend.Backend { + s := &schema.Backend{ + Schema: map[string]*schema.Schema{ + "bucket": { + Type: schema.TypeString, + Required: true, + Description: "The name of the Google Cloud Storage bucket", + }, + + "state_dir": { + Type: schema.TypeString, + Optional: true, + Description: "The directory where state files will be saved inside the bucket", + }, + + "credentials": { + Type: schema.TypeString, + Optional: true, + Description: "Google Cloud JSON Account Key", + Default: "", + }, + }, + } + + result := &Backend{Backend: s} + result.Backend.ConfigureFunc = result.configure + return result +} + +type Backend struct { + *schema.Backend + + storageClient *storage.Client + storageContext googleContext.Context + + bucketName string + stateDir string +} + +func (b *Backend) configure(ctx context.Context) error { + if b.storageClient != nil { + return nil + } + + storageOAuth2Scope := "https://www.googleapis.com/auth/devstorage.read_write" + + data := schema.FromContextBackendConfig(ctx) + + b.bucketName = data.Get("bucket").(string) + b.stateDir = data.Get("state_dir").(string) + b.storageContext = googleContext.Background() + + credentials := data.Get("credentials").(string) + + var tokenSource oauth2.TokenSource + + if credentials != "" { + credentialsJson, _, err := pathorcontents.Read(data.Get("credentials").(string)) + if err != nil { + return fmt.Errorf("Error loading credentials: %v", err) + } + + jwtConfig, err := google.JWTConfigFromJSON([]byte(credentialsJson), storageOAuth2Scope) + if err != nil { + return fmt.Errorf("Failed to get Google OAuth2 token: %v", err) + } + + tokenSource = jwtConfig.TokenSource(b.storageContext) + } else { + defaultTokenSource, err := google.DefaultTokenSource(b.storageContext, storageOAuth2Scope) + if err != nil { + return fmt.Errorf("Failed to get Google Application Default Credentials: %v", err) + } + + tokenSource = defaultTokenSource + } + + client, err := storage.NewClient(b.storageContext, option.WithTokenSource(tokenSource)) + if err != nil { + return fmt.Errorf("Failed to create Google Storage client: %v", err) + } + + b.storageClient = client + + return nil +} diff --git a/backend/remote-state/gcloud/backend_state.go b/backend/remote-state/gcloud/backend_state.go new file mode 100644 index 0000000000..5f2b79a0bb --- /dev/null +++ b/backend/remote-state/gcloud/backend_state.go @@ -0,0 +1,165 @@ +package gcloud + +import ( + "cloud.google.com/go/storage" + "errors" + "fmt" + "github.com/hashicorp/terraform/backend" + "github.com/hashicorp/terraform/state" + "github.com/hashicorp/terraform/state/remote" + "github.com/hashicorp/terraform/terraform" + "google.golang.org/api/iterator" + "regexp" + "sort" + "strings" +) + +func (b *Backend) States() ([]string, error) { + workspaces := []string{backend.DefaultStateName} + var stateRegex *regexp.Regexp + var err error + if b.stateDir == "" { + stateRegex = regexp.MustCompile(`^(.+)\.tfstate$`) + } else { + stateRegex, err = regexp.Compile(fmt.Sprintf("^%v/(.+)\\.tfstate$", regexp.QuoteMeta(b.stateDir))) + if err != nil { + return []string{}, fmt.Errorf("Failed to compile regex for querying states: %v", err) + } + } + + bucket := b.storageClient.Bucket(b.bucketName) + query := &storage.Query{ + Prefix: b.stateDir, + } + + files := bucket.Objects(b.storageContext, query) + for { + attrs, err := files.Next() + if err == iterator.Done { + break + } + if err != nil { + return []string{}, fmt.Errorf("Failed to query remote states: %v", err) + } + + matches := stateRegex.FindStringSubmatch(attrs.Name) + if len(matches) == 2 && matches[1] != backend.DefaultStateName { + workspaces = append(workspaces, matches[1]) + } + } + + sort.Strings(workspaces[1:]) + return workspaces, nil +} + +func (b *Backend) DeleteState(name string) error { + if name == backend.DefaultStateName || name == "" { + return fmt.Errorf("Can't delete default state") + } + + client, err := b.remoteClient(name) + if err != nil { + return fmt.Errorf("Failed to create Google Storage client: %v", err) + } + + err = client.Delete() + if err != nil { + return fmt.Errorf("Failed to delete state file %v: %v", client.stateFileUrl(), err) + } + + return nil +} + +// get a remote client configured for this state +func (b *Backend) remoteClient(name string) (*RemoteClient, error) { + if name == "" { + return nil, errors.New("Missing state name") + } + + client := &RemoteClient{ + storageContext: b.storageContext, + storageClient: b.storageClient, + bucketName: b.bucketName, + stateFilePath: b.stateFileName(name), + lockFilePath: b.lockFileName(name), + } + + return client, nil +} + +func (b *Backend) State(name string) (state.State, error) { + client, err := b.remoteClient(name) + if err != nil { + return nil, fmt.Errorf("Failed to create Google Storage client: %v", err) + } + + stateMgr := &remote.State{Client: client} + lockInfo := state.NewLockInfo() + lockInfo.Operation = "init" + lockId, err := stateMgr.Lock(lockInfo) + if err != nil { + return nil, fmt.Errorf("Failed to lock state in Google Cloud Storage: %v", err) + } + + // Local helper function so we can call it multiple places + lockUnlock := func(parent error) error { + if err := stateMgr.Unlock(lockId); err != nil { + return fmt.Errorf(strings.TrimSpace(errStateUnlock), lockId, client.lockFileUrl(), err) + } + + return parent + } + + // Grab the value + if err := stateMgr.RefreshState(); err != nil { + err = lockUnlock(err) + return nil, err + } + + // If we have no state, we have to create an empty state + if v := stateMgr.State(); v == nil { + if err := stateMgr.WriteState(terraform.NewState()); err != nil { + err = lockUnlock(err) + return nil, err + } + if err := stateMgr.PersistState(); err != nil { + err = lockUnlock(err) + return nil, err + } + } + + // Unlock, the state should now be initialized + if err := lockUnlock(nil); err != nil { + return nil, err + } + + return stateMgr, nil +} + +func (b *Backend) stateFileName(stateName string) string { + if b.stateDir == "" { + return fmt.Sprintf("%v.tfstate", stateName) + } else { + return fmt.Sprintf("%v/%v.tfstate", b.stateDir, stateName) + } +} + +func (b *Backend) lockFileName(stateName string) string { + if b.stateDir == "" { + return fmt.Sprintf("%v.tflock", stateName) + } else { + return fmt.Sprintf("%v/%v.tflock", b.stateDir, stateName) + } +} + +const errStateUnlock = ` +Error unlocking Google Cloud Storage state. + +Lock ID: %v +Lock file URL: %v +Error: %v + +You may have to force-unlock this state in order to use it again. +The GCloud backend acquires a lock during initialization to ensure +the initial state file is created. +` diff --git a/backend/remote-state/gcloud/client.go b/backend/remote-state/gcloud/client.go new file mode 100644 index 0000000000..41ec5257ed --- /dev/null +++ b/backend/remote-state/gcloud/client.go @@ -0,0 +1,168 @@ +package gcloud + +import ( + "cloud.google.com/go/storage" + "encoding/json" + "fmt" + uuid "github.com/hashicorp/go-uuid" + "github.com/hashicorp/terraform/state" + "github.com/hashicorp/terraform/state/remote" + "golang.org/x/net/context" + "io/ioutil" +) + +type RemoteClient struct { + storageContext context.Context + storageClient *storage.Client + bucketName string + stateFilePath string + lockFilePath string +} + +func (c *RemoteClient) Get() (payload *remote.Payload, err error) { + bucket := c.storageClient.Bucket(c.bucketName) + stateFile := bucket.Object(c.stateFilePath) + stateFileUrl := c.stateFileUrl() + + stateFileReader, err := stateFile.NewReader(c.storageContext) + if err != nil { + if err == storage.ErrObjectNotExist { + return nil, nil + } else { + return nil, fmt.Errorf("Failed to open state file at %v: %v", stateFileUrl, err) + } + } + defer stateFileReader.Close() + + stateFileContents, err := ioutil.ReadAll(stateFileReader) + if err != nil { + return nil, fmt.Errorf("Failed to read state file from %v: %v", stateFileUrl, err) + } + + stateFileAttrs, err := stateFile.Attrs(c.storageContext) + if err != nil { + return nil, fmt.Errorf("Failed to read state file attrs from %v: %v", stateFileUrl, err) + } + + result := &remote.Payload{ + Data: stateFileContents, + MD5: stateFileAttrs.MD5, + } + + return result, nil +} + +func (c *RemoteClient) Put(data []byte) error { + bucket := c.storageClient.Bucket(c.bucketName) + stateFile := bucket.Object(c.stateFilePath) + + stateFileWriter := stateFile.NewWriter(c.storageContext) + + stateFileWriter.Write(data) + err := stateFileWriter.Close() + + if err != nil { + return fmt.Errorf("Failed to upload state to %v: %v", c.stateFileUrl(), err) + } + + return nil +} + +func (c *RemoteClient) Delete() error { + bucket := c.storageClient.Bucket(c.bucketName) + stateFile := bucket.Object(c.stateFilePath) + + err := stateFile.Delete(c.storageContext) + + if err != nil { + return fmt.Errorf("Failed to delete state file %v: %v", c.stateFileUrl(), err) + } + + return nil +} + +func (c *RemoteClient) Lock(info *state.LockInfo) (string, error) { + if info.ID == "" { + lockID, err := uuid.GenerateUUID() + if err != nil { + return "", err + } + + info.ID = lockID + } + + info.Path = c.lockFileUrl() + + infoJson, err := json.Marshal(info) + if err != nil { + return "", err + } + + bucket := c.storageClient.Bucket(c.bucketName) + lockFile := bucket.Object(c.lockFilePath) + + writer := lockFile.If(storage.Conditions{DoesNotExist: true}).NewWriter(c.storageContext) + writer.Write(infoJson) + if err := writer.Close(); err != nil { + return "", fmt.Errorf("Error while saving lock file (%v): %v", info.Path, err) + } + + return info.ID, nil +} + +func (c *RemoteClient) Unlock(id string) error { + lockErr := &state.LockError{} + + bucket := c.storageClient.Bucket(c.bucketName) + lockFile := bucket.Object(c.lockFilePath) + lockFileUrl := c.lockFileUrl() + + lockFileReader, err := lockFile.NewReader(c.storageContext) + if err != nil { + lockErr.Err = fmt.Errorf("Failed to retrieve lock info (%v): %v", lockFileUrl, err) + return lockErr + } + defer lockFileReader.Close() + + lockFileContents, err := ioutil.ReadAll(lockFileReader) + if err != nil { + lockErr.Err = fmt.Errorf("Failed to retrieve lock info (%v): %v", lockFileUrl, err) + return lockErr + } + + lockInfo := &state.LockInfo{} + err = json.Unmarshal(lockFileContents, lockInfo) + if err != nil { + lockErr.Err = fmt.Errorf("Failed to unmarshal lock info (%v): %v", lockFileUrl, err) + return lockErr + } + + lockErr.Info = lockInfo + + if lockInfo.ID != id { + lockErr.Err = fmt.Errorf("Lock id %q does not match existing lock", id) + return lockErr + } + + lockFileAttrs, err := lockFile.Attrs(c.storageContext) + if err != nil { + lockErr.Err = fmt.Errorf("Failed to fetch lock file attrs (%v): %v", lockFileUrl, err) + return lockErr + } + + err = lockFile.If(storage.Conditions{GenerationMatch: lockFileAttrs.Generation}).Delete(c.storageContext) + if err != nil { + lockErr.Err = fmt.Errorf("Failed to delete lock file (%v): %v", lockFileUrl, err) + return lockErr + } + + return nil +} + +func (c *RemoteClient) stateFileUrl() string { + return fmt.Sprintf("gs://%v/%v", c.bucketName, c.stateFilePath) +} + +func (c *RemoteClient) lockFileUrl() string { + return fmt.Sprintf("gs://%v/%v", c.bucketName, c.lockFilePath) +}