diff --git a/state/remote/remote.go b/state/remote/remote.go index 19632a9fdf..cb525e6a32 100644 --- a/state/remote/remote.go +++ b/state/remote/remote.go @@ -39,6 +39,7 @@ var BuiltinClients = map[string]Factory{ "atlas": atlasFactory, "consul": consulFactory, "http": httpFactory, + "s3": s3Factory, // This is used for development purposes only. "_local": fileFactory, diff --git a/state/remote/s3.go b/state/remote/s3.go new file mode 100644 index 0000000000..cd72a941af --- /dev/null +++ b/state/remote/s3.go @@ -0,0 +1,125 @@ +package remote + +import ( + "bytes" + "fmt" + "io" + "os" + + "github.com/awslabs/aws-sdk-go/aws" + "github.com/awslabs/aws-sdk-go/service/s3" +) + +func s3Factory(conf map[string]string) (Client, error) { + bucketName, ok := conf["bucket"] + if !ok { + return nil, fmt.Errorf("missing 'bucket' configuration") + } + + keyName, ok := conf["key"] + if !ok { + return nil, fmt.Errorf("missing 'key' configuration") + } + + regionName, ok := conf["region"] + if !ok { + regionName = os.Getenv("AWS_DEFAULT_REGION") + if regionName == "" { + return nil, fmt.Errorf("missing 'region' configuration or AWS_DEFAULT_REGION environment variable") + } + } + + accessKeyId := conf["access_key"] + secretAccessKey := conf["secret_key"] + + credentialsProvider := aws.DetectCreds(accessKeyId, secretAccessKey, "") + + // Make sure we got some sort of working credentials. + _, err := credentialsProvider.Credentials() + if err != nil { + return nil, fmt.Errorf("Unable to determine AWS credentials. Set the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables.\n(error was: %s)", err) + } + + awsConfig := &aws.Config{ + Credentials: credentialsProvider, + Region: regionName, + } + nativeClient := s3.New(awsConfig) + + return &S3Client{ + nativeClient: nativeClient, + bucketName: bucketName, + keyName: keyName, + }, nil +} + +type S3Client struct { + nativeClient *s3.S3 + bucketName string + keyName string +} + +func (c *S3Client) Get() (*Payload, error) { + output, err := c.nativeClient.GetObject(&s3.GetObjectInput{ + Bucket: &c.bucketName, + Key: &c.keyName, + }) + + if err != nil { + if awserr := aws.Error(err); awserr != nil { + if awserr.Code == "NoSuchKey" { + return nil, nil + } else { + return nil, err + } + } else { + return nil, err + } + } + + defer output.Body.Close() + + buf := bytes.NewBuffer(nil) + if _, err := io.Copy(buf, output.Body); err != nil { + return nil, fmt.Errorf("Failed to read remote state: %s", err) + } + + payload := &Payload{ + Data: buf.Bytes(), + } + + // If there was no data, then return nil + if len(payload.Data) == 0 { + return nil, nil + } + + return payload, nil +} + +func (c *S3Client) Put(data []byte) error { + contentType := "application/octet-stream" + contentLength := int64(len(data)) + + _, err := c.nativeClient.PutObject(&s3.PutObjectInput{ + ContentType: &contentType, + ContentLength: &contentLength, + Body: bytes.NewReader(data), + Bucket: &c.bucketName, + Key: &c.keyName, + }) + + if err == nil { + return nil + } else { + return fmt.Errorf("Failed to upload state: %v", err) + } +} + +func (c *S3Client) Delete() error { + _, err := c.nativeClient.DeleteObject(&s3.DeleteObjectInput{ + Bucket: &c.bucketName, + Key: &c.keyName, + }) + + return err +} diff --git a/state/remote/s3_test.go b/state/remote/s3_test.go new file mode 100644 index 0000000000..35d06ec4f3 --- /dev/null +++ b/state/remote/s3_test.go @@ -0,0 +1,120 @@ +package remote + +import ( + "fmt" + "os" + "testing" + "time" + + "github.com/awslabs/aws-sdk-go/service/s3" +) + +func TestS3Client_impl(t *testing.T) { + var _ Client = new(S3Client) +} + +func TestS3Factory(t *testing.T) { + // This test just instantiates the client. Shouldn't make any actual + // requests nor incur any costs. + + config := make(map[string]string) + + // Empty config is an error + _, err := s3Factory(config) + if err == nil { + t.Fatalf("Empty config should be error") + } + + config["region"] = "us-west-1" + config["bucket"] = "foo" + config["key"] = "bar" + // For this test we'll provide the credentials as config. The + // acceptance tests implicitly test passing credentials as + // environment variables. + config["access_key"] = "bazkey" + config["secret_key"] = "bazsecret" + + client, err := s3Factory(config) + if err != nil { + t.Fatalf("Error for valid config") + } + + s3Client := client.(*S3Client) + + if s3Client.nativeClient.Config.Region != "us-west-1" { + t.Fatalf("Incorrect region was populated") + } + if s3Client.bucketName != "foo" { + t.Fatalf("Incorrect bucketName was populated") + } + if s3Client.keyName != "bar" { + t.Fatalf("Incorrect keyName was populated") + } + + credentials, err := s3Client.nativeClient.Config.Credentials.Credentials() + if err != nil { + t.Fatalf("Error when requesting credentials") + } + if credentials.AccessKeyID != "bazkey" { + t.Fatalf("Incorrect Access Key Id was populated") + } + if credentials.SecretAccessKey != "bazsecret" { + t.Fatalf("Incorrect Secret Access Key was populated") + } +} + +func TestS3Client(t *testing.T) { + // This test creates a bucket in S3 and populates it. + // It may incur costs, so it will only run if AWS credential environment + // variables are present. + + accessKeyId := os.Getenv("AWS_ACCESS_KEY_ID") + if accessKeyId == "" { + t.Skipf("skipping; AWS_ACCESS_KEY_ID must be set") + } + + regionName := os.Getenv("AWS_DEFAULT_REGION") + if regionName == "" { + regionName = "us-west-2" + } + + bucketName := fmt.Sprintf("terraform-remote-s3-test-%x", time.Now().Unix()) + keyName := "testState" + + config := make(map[string]string) + config["region"] = regionName + config["bucket"] = bucketName + config["key"] = keyName + + client, err := s3Factory(config) + if err != nil { + t.Fatalf("Error for valid config") + } + + s3Client := client.(*S3Client) + nativeClient := s3Client.nativeClient + + createBucketReq := &s3.CreateBucketInput{ + Bucket: &bucketName, + } + + // Be clear about what we're doing in case the user needs to clean + // this up later. + t.Logf("Creating S3 bucket %s in %s", bucketName, regionName) + _, err = nativeClient.CreateBucket(createBucketReq) + if err != nil { + t.Skipf("Failed to create test S3 bucket, so skipping") + } + defer func () { + deleteBucketReq := &s3.DeleteBucketInput{ + Bucket: &bucketName, + } + + _, err := nativeClient.DeleteBucket(deleteBucketReq) + if err != nil { + t.Logf("WARNING: Failed to delete the test S3 bucket. It has been left in your AWS account and may incur storage charges. (error was %s)", err) + } + }() + + testClient(t, client) +} diff --git a/website/source/docs/commands/remote-config.html.markdown b/website/source/docs/commands/remote-config.html.markdown index 3ced2a4341..481fa8f758 100644 --- a/website/source/docs/commands/remote-config.html.markdown +++ b/website/source/docs/commands/remote-config.html.markdown @@ -50,6 +50,14 @@ The following backends are supported: variables can optionally be provided. Address is assumed to be the local agent if not provided. +* S3 - Stores the state as a given key in a given bucket on Amazon S3. + Requires the `bucket` and `key` variables. Supports and honors the standard + AWS environment variables `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` + and `AWS_DEFAULT_REGION`. These can optionally be provided as parameters + in the `aws_access_key`, `aws_secret_key` and `region` variables + respectively, but passing credentials this way is not recommended since they + will be included in cleartext inside the persisted state. + * HTTP - Stores the state using a simple REST client. State will be fetched via GET, updated via POST, and purged with DELETE. Requires the `address` variable.