From 48a92cfb1f6f0aea100c7f55608653cb25093bbb Mon Sep 17 00:00:00 2001 From: Jakub Janczak Date: Thu, 5 Feb 2015 09:33:56 +0100 Subject: [PATCH 1/4] remote/s3: s3 remote state storage support --- command/remote.go | 27 +++++-- remote/README.md | 11 +++ remote/client.go | 2 + remote/s3.go | 192 ++++++++++++++++++++++++++++++++++++++++++++++ remote/s3_test.go | 134 ++++++++++++++++++++++++++++++++ 5 files changed, 359 insertions(+), 7 deletions(-) create mode 100644 remote/README.md create mode 100644 remote/s3.go create mode 100644 remote/s3_test.go diff --git a/command/remote.go b/command/remote.go index d9a773704a..3b9e8c57b1 100644 --- a/command/remote.go +++ b/command/remote.go @@ -32,7 +32,7 @@ type RemoteCommand struct { func (c *RemoteCommand) Run(args []string) int { args = c.Meta.process(args, false) - var address, accessToken, name, path string + var address, accessToken, name, path, region, securityToken, bucket string cmdFlags := flag.NewFlagSet("remote", flag.ContinueOnError) cmdFlags.BoolVar(&c.conf.disableRemote, "disable", false, "") cmdFlags.BoolVar(&c.conf.pullOnDisable, "pull", true, "") @@ -41,6 +41,9 @@ func (c *RemoteCommand) Run(args []string) int { cmdFlags.StringVar(&c.remoteConf.Type, "backend", "atlas", "") cmdFlags.StringVar(&address, "address", "", "") cmdFlags.StringVar(&accessToken, "access-token", "", "") + cmdFlags.StringVar(&securityToken, "security-token", "", "") + cmdFlags.StringVar(&bucket, "bucket", "", "") + cmdFlags.StringVar(®ion, "region", "", "") cmdFlags.StringVar(&name, "name", "", "") cmdFlags.StringVar(&path, "path", "", "") cmdFlags.Usage = func() { c.Ui.Error(c.Help()) } @@ -57,10 +60,13 @@ func (c *RemoteCommand) Run(args []string) int { // Populate the various configurations c.remoteConf.Config = map[string]string{ - "address": address, - "access_token": accessToken, - "name": name, - "path": path, + "address": address, + "access_token": accessToken, + "security_token": securityToken, + "name": name, + "path": path, + "bucket": bucket, + "region": region, } // Check if have an existing local state file @@ -329,13 +335,17 @@ Options: -access-token=token Authentication token for state storage server. Required for Atlas backend, optional for Consul. + -security-token=token Security token. Specific to S3 (required). + -backend=Atlas Specifies the type of remote backend. Must be one - of Atlas, Consul, or HTTP. Defaults to Atlas. + of Atlas, Consul,HTTP or S3. Defaults to Atlas. -backup=path Path to backup the existing state file before modifying. Defaults to the "-state" path with ".backup" extension. Set to "-" to disable backup. + -bucket=bucket S3 bucket name. Specific to S3 (required). + -disable Disables remote state management and migrates the state to the -state path. @@ -343,12 +353,15 @@ Options: Required for Atlas backend. -path=path Path of the remote state in Consul. Required for the - Consul backend. + Consul and S3 backend. -pull=true Controls if the remote state is pulled before disabling. This defaults to true to ensure the latest state is cached before disabling. + -region=region AWS region to use. Specific for S3 (not required if AWS_DEFAULT_REGION + env variable is set). + -state=path Path to read state. Defaults to "terraform.tfstate" unless remote state is enabled. diff --git a/remote/README.md b/remote/README.md new file mode 100644 index 0000000000..60d53a1e26 --- /dev/null +++ b/remote/README.md @@ -0,0 +1,11 @@ +## How to test + +### S3 remote state storage +To run S3 integration tests you need following env variables to be set: + * AWS_ACCESS_KEY + * AWS_SECRET_KEY + * AWS_DEFAULT_REGION + * TERRAFORM_STATE_BUCKET + +Additionally specified bucket should exist in the defined region and should be accessible +using specified credentials. diff --git a/remote/client.go b/remote/client.go index c57e40c780..56849553e9 100644 --- a/remote/client.go +++ b/remote/client.go @@ -59,6 +59,8 @@ func NewClientByType(ctype string, conf map[string]string) (RemoteClient, error) return NewConsulRemoteClient(conf) case "http": return NewHTTPRemoteClient(conf) + case "s3": + return NewS3RemoteClient(conf) default: return nil, fmt.Errorf("Unknown remote client type '%s'", ctype) } diff --git a/remote/s3.go b/remote/s3.go new file mode 100644 index 0000000000..04b202094a --- /dev/null +++ b/remote/s3.go @@ -0,0 +1,192 @@ +package remote + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/goamz/goamz/aws" + "github.com/goamz/goamz/s3" +) + +type S3RemoteClient struct { + Bucket *s3.Bucket + Path string +} + +func GetRegion(conf map[string]string) (aws.Region, error) { + regionName, ok := conf["region"] + if !ok || regionName == "" { + regionName = os.Getenv("AWS_DEFAULT_REGION") + if regionName == "" { + return aws.Region{}, fmt.Errorf("AWS region not set") + } + } + + region, ok := aws.Regions[regionName] + if !ok { + return aws.Region{}, fmt.Errorf("AWS region set in configuration '%v' doesn't exist", regionName) + } + return region, nil +} + +func NewS3RemoteClient(conf map[string]string) (*S3RemoteClient, error) { + client := &S3RemoteClient{} + + auth, err := aws.GetAuth(conf["access_token"], conf["secret_token"], "", time.Now()) + if err != nil { + return nil, err + } + + region, err := GetRegion(conf) + if err != nil { + return nil, err + } + + bucketName, ok := conf["bucket"] + if !ok { + return nil, fmt.Errorf("Missing 'bucket_name' configuration") + } + + client.Bucket = s3.New(auth, region).Bucket(bucketName) + + path, ok := conf["path"] + if !ok { + return nil, fmt.Errorf("Missing 'path' configuration") + } + client.Path = path + + return client, nil +} + +func (c *S3RemoteClient) GetState() (*RemoteStatePayload, error) { + resp, err := c.Bucket.GetResponse(c.Path) + defer func() { + if resp != nil && resp.Body != nil { + resp.Body.Close() + } + }() + + if err != nil { + switch err.(type) { + case *s3.Error: + s3Err := err.(*s3.Error) + + // FIXME copied from Atlas + // Handle the common status codes + switch s3Err.StatusCode { + case http.StatusOK: + // Handled after + case http.StatusNoContent: + return nil, nil + case http.StatusNotFound: + return nil, nil + case http.StatusUnauthorized: + return nil, ErrRequireAuth + case http.StatusForbidden: + return nil, ErrInvalidAuth + case http.StatusInternalServerError: + return nil, ErrRemoteInternal + default: + return nil, fmt.Errorf("Unexpected HTTP response code %d", s3Err.StatusCode) + } + default: + return nil, err + } + } + + // Read in the body + buf := bytes.NewBuffer(nil) + if _, err := io.Copy(buf, resp.Body); err != nil { + return nil, fmt.Errorf("Failed to read remote state: %v", err) + } + + // Create the payload + payload := &RemoteStatePayload{ + State: buf.Bytes(), + } + + // Check for the MD5 + if raw := resp.Header.Get("Content-MD5"); raw != "" { + md5, err := base64.StdEncoding.DecodeString(raw) + if err != nil { + return nil, fmt.Errorf("Failed to decode Content-MD5 '%s': %v", raw, err) + } + payload.MD5 = md5 + + } else { + // Generate the MD5 + hash := md5.Sum(payload.State) + payload.MD5 = hash[:md5.Size] + } + + return payload, nil +} + +func (c *S3RemoteClient) PutState(state []byte, force bool) error { + // Generate the MD5 + hash := md5.Sum(state) + b64 := base64.StdEncoding.EncodeToString(hash[:md5.Size]) + + options := s3.Options{ + ContentMD5: b64, + } + + err := c.Bucket.Put(c.Path, state, "application/json", s3.Private, options) + switch err.(type) { + case *s3.Error: + s3Err := err.(*s3.Error) + + // Handle the error codes + switch s3Err.StatusCode { + case http.StatusOK: + return nil + case http.StatusConflict: + return ErrConflict + case http.StatusPreconditionFailed: + return ErrServerNewer + case http.StatusUnauthorized: + return ErrRequireAuth + case http.StatusForbidden: + return ErrInvalidAuth + case http.StatusInternalServerError: + return ErrRemoteInternal + default: + return fmt.Errorf("Unexpected HTTP response code %d", s3Err.StatusCode) + } + default: + return err + } +} + +func (c *S3RemoteClient) DeleteState() error { + err := c.Bucket.Del(c.Path) + switch err.(type) { + case *s3.Error: + s3Err := err.(*s3.Error) + // Handle the error codes + switch s3Err.StatusCode { + case http.StatusOK: + return nil + case http.StatusNoContent: + return nil + case http.StatusNotFound: + return nil + case http.StatusUnauthorized: + return ErrRequireAuth + case http.StatusForbidden: + return ErrInvalidAuth + case http.StatusInternalServerError: + return ErrRemoteInternal + default: + return fmt.Errorf("Unexpected HTTP response code %d", s3Err.StatusCode) + } + default: + return err + } +} diff --git a/remote/s3_test.go b/remote/s3_test.go new file mode 100644 index 0000000000..41309ae3f1 --- /dev/null +++ b/remote/s3_test.go @@ -0,0 +1,134 @@ +package remote + +import ( + "bytes" + "crypto/md5" + "os" + "testing" + + "github.com/hashicorp/terraform/terraform" +) + +func TestS3Remote_NewClient(t *testing.T) { + conf := map[string]string{} + if _, err := NewS3RemoteClient(conf); err == nil { + t.Fatalf("expect error") + } + + conf["access_token"] = "test" + conf["secret_token"] = "test" + conf["path"] = "hashicorp/test-state" + conf["bucket"] = "plan3-test" + conf["region"] = "eu-west-1" + if _, err := NewS3RemoteClient(conf); err != nil { + t.Fatalf("err: %v", err) + } +} + +func TestS3Remote_Validate_envVar(t *testing.T) { + conf := map[string]string{} + if _, err := NewS3RemoteClient(conf); err == nil { + t.Fatalf("expect error") + } + + defer os.Setenv("AWS_ACCESS_KEY", os.Getenv("AWS_ACCESS_KEY")) + os.Setenv("AWS_ACCESS_KEY", "foo") + + defer os.Setenv("AWS_SECRET_KEY", os.Getenv("AWS_SECRET_KEY")) + os.Setenv("AWS_SECRET_KEY", "foo") + + defer os.Setenv("AWS_DEFAULT_REGION", os.Getenv("AWS_DEFAULT_REGION")) + os.Setenv("AWS_DEFAULT_REGION", "eu-west-1") + + conf["path"] = "hashicorp/test-state" + conf["bucket"] = "plan3-test" + if _, err := NewS3RemoteClient(conf); err != nil { + t.Fatalf("err: %v", err) + } +} + +func checkS3(t *testing.T) { + if os.Getenv("AWS_ACCESS_KEY") == "" || os.Getenv("AWS_SECRET_KEY") == "" || os.Getenv("AWS_DEFAULT_REGION") == "" || os.Getenv("TERRAFORM_STATE_BUCKET") == "" { + t.SkipNow() + } +} + +func TestS3Remote(t *testing.T) { + checkS3(t) + remote := &terraform.RemoteState{ + Type: "atlas", + Config: map[string]string{ + "access_token": "some-access-token", + "name": "hashicorp/test-remote-state", + }, + } + r, err := NewClientByType("s3", map[string]string{ + "bucket": os.Getenv("TERRAFORM_STATE_BUCKET"), + "path": "test-remote-state", + }) + if err != nil { + t.Fatalf("Err: %v", err) + } + + // Get a valid input + inp, err := blankState(remote) + if err != nil { + t.Fatalf("Err: %v", err) + } + inpMD5 := md5.Sum(inp) + hash := inpMD5[:16] + + // Delete the state, should be none + err = r.DeleteState() + if err != nil { + t.Fatalf("err: %v", err) + } + + // Ensure no state + payload, err := r.GetState() + if err != nil { + t.Fatalf("Err: %v", err) + } + if payload != nil { + t.Fatalf("unexpected payload") + } + + // Put the state + err = r.PutState(inp, false) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Get it back + payload, err = r.GetState() + if err != nil { + t.Fatalf("Err: %v", err) + } + if payload == nil { + t.Fatalf("unexpected payload") + } + + // Check the payload + if !bytes.Equal(payload.MD5, hash) { + t.Fatalf("bad hash: %x %x", payload.MD5, hash) + } + if !bytes.Equal(payload.State, inp) { + t.Errorf("inp: %s", inp) + t.Fatalf("bad response: %s", payload.State) + } + + // Delete the state + err = r.DeleteState() + if err != nil { + t.Fatalf("err: %v", err) + } + + // Should be gone + payload, err = r.GetState() + if err != nil { + t.Fatalf("Err: %v", err) + } + if payload != nil { + t.Fatalf("unexpected payload") + } +} From bcfb8df1164a4dab4e1fa31388798cd7e5005ca1 Mon Sep 17 00:00:00 2001 From: Jakub Janczak Date: Fri, 6 Feb 2015 00:02:04 +0100 Subject: [PATCH 2/4] remote/s3: removing code that is serving versioning puproses which is not the case for s3 for the moment --- remote/s3.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/remote/s3.go b/remote/s3.go index 04b202094a..a4c34e53a7 100644 --- a/remote/s3.go +++ b/remote/s3.go @@ -146,10 +146,6 @@ func (c *S3RemoteClient) PutState(state []byte, force bool) error { switch s3Err.StatusCode { case http.StatusOK: return nil - case http.StatusConflict: - return ErrConflict - case http.StatusPreconditionFailed: - return ErrServerNewer case http.StatusUnauthorized: return ErrRequireAuth case http.StatusForbidden: From 98d7bcefef6391ee421fd173ce9a60c0fb38f915 Mon Sep 17 00:00:00 2001 From: Jakub Janczak Date: Fri, 6 Feb 2015 10:41:50 +0100 Subject: [PATCH 3/4] remote/s3: there is no ok answer on error --- command/remote.go | 10 +++------- remote/s3.go | 6 ------ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/command/remote.go b/command/remote.go index 3b9e8c57b1..74ddddbdac 100644 --- a/command/remote.go +++ b/command/remote.go @@ -32,7 +32,7 @@ type RemoteCommand struct { func (c *RemoteCommand) Run(args []string) int { args = c.Meta.process(args, false) - var address, accessToken, name, path, region, securityToken, bucket string + var address, accessToken, name, path, region, securityToken string cmdFlags := flag.NewFlagSet("remote", flag.ContinueOnError) cmdFlags.BoolVar(&c.conf.disableRemote, "disable", false, "") cmdFlags.BoolVar(&c.conf.pullOnDisable, "pull", true, "") @@ -42,7 +42,6 @@ func (c *RemoteCommand) Run(args []string) int { cmdFlags.StringVar(&address, "address", "", "") cmdFlags.StringVar(&accessToken, "access-token", "", "") cmdFlags.StringVar(&securityToken, "security-token", "", "") - cmdFlags.StringVar(&bucket, "bucket", "", "") cmdFlags.StringVar(®ion, "region", "", "") cmdFlags.StringVar(&name, "name", "", "") cmdFlags.StringVar(&path, "path", "", "") @@ -65,7 +64,6 @@ func (c *RemoteCommand) Run(args []string) int { "security_token": securityToken, "name": name, "path": path, - "bucket": bucket, "region": region, } @@ -330,7 +328,7 @@ Usage: terraform remote [options] Options: -address=url URL of the remote storage server. - Required for HTTP backend, optional for Atlas and Consul. + Required for HTTP and S3 backend, optional for Atlas and Consul. -access-token=token Authentication token for state storage server. Required for Atlas backend, optional for Consul. @@ -344,8 +342,6 @@ Options: modifying. Defaults to the "-state" path with ".backup" extension. Set to "-" to disable backup. - -bucket=bucket S3 bucket name. Specific to S3 (required). - -disable Disables remote state management and migrates the state to the -state path. @@ -353,7 +349,7 @@ Options: Required for Atlas backend. -path=path Path of the remote state in Consul. Required for the - Consul and S3 backend. + Consul. -pull=true Controls if the remote state is pulled before disabling. This defaults to true to ensure the latest state is cached diff --git a/remote/s3.go b/remote/s3.go index a4c34e53a7..6c4d1a41f7 100644 --- a/remote/s3.go +++ b/remote/s3.go @@ -80,8 +80,6 @@ func (c *S3RemoteClient) GetState() (*RemoteStatePayload, error) { // FIXME copied from Atlas // Handle the common status codes switch s3Err.StatusCode { - case http.StatusOK: - // Handled after case http.StatusNoContent: return nil, nil case http.StatusNotFound: @@ -144,8 +142,6 @@ func (c *S3RemoteClient) PutState(state []byte, force bool) error { // Handle the error codes switch s3Err.StatusCode { - case http.StatusOK: - return nil case http.StatusUnauthorized: return ErrRequireAuth case http.StatusForbidden: @@ -167,8 +163,6 @@ func (c *S3RemoteClient) DeleteState() error { s3Err := err.(*s3.Error) // Handle the error codes switch s3Err.StatusCode { - case http.StatusOK: - return nil case http.StatusNoContent: return nil case http.StatusNotFound: From 84e6364bff670f9230a5a90ccf0af0bf53aeb7fe Mon Sep 17 00:00:00 2001 From: Jakub Janczak Date: Fri, 6 Feb 2015 12:26:11 +0100 Subject: [PATCH 4/4] remote/s3: using address for bucket and path --- remote/s3.go | 28 ++++++++++++++++++---------- remote/s3_test.go | 6 ++---- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/remote/s3.go b/remote/s3.go index 6c4d1a41f7..bc43e9d17f 100644 --- a/remote/s3.go +++ b/remote/s3.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "os" + "regexp" "time" "github.com/goamz/goamz/aws" @@ -19,7 +20,7 @@ type S3RemoteClient struct { Path string } -func GetRegion(conf map[string]string) (aws.Region, error) { +func getRegion(conf map[string]string) (aws.Region, error) { regionName, ok := conf["region"] if !ok || regionName == "" { regionName = os.Getenv("AWS_DEFAULT_REGION") @@ -35,6 +36,15 @@ func GetRegion(conf map[string]string) (aws.Region, error) { return region, nil } +func getBucketAndPath(address string) (string, string, error) { + re := regexp.MustCompile("^s3://([^/]+)/(.+)$") + matches := re.FindStringSubmatch(address) + if len(matches) < 3 { + return "", "", fmt.Errorf("Address for s3 should be of form: s3:///") + } + return matches[1], matches[2], nil +} + func NewS3RemoteClient(conf map[string]string) (*S3RemoteClient, error) { client := &S3RemoteClient{} @@ -43,22 +53,20 @@ func NewS3RemoteClient(conf map[string]string) (*S3RemoteClient, error) { return nil, err } - region, err := GetRegion(conf) + region, err := getRegion(conf) if err != nil { return nil, err } - bucketName, ok := conf["bucket"] + address, ok := conf["address"] if !ok { - return nil, fmt.Errorf("Missing 'bucket_name' configuration") + return nil, fmt.Errorf("'address' configuration not set for S3 remote state storage backend") } - - client.Bucket = s3.New(auth, region).Bucket(bucketName) - - path, ok := conf["path"] - if !ok { - return nil, fmt.Errorf("Missing 'path' configuration") + bucketName, path, err := getBucketAndPath(address) + if err != nil { + return nil, err } + client.Bucket = s3.New(auth, region).Bucket(bucketName) client.Path = path return client, nil diff --git a/remote/s3_test.go b/remote/s3_test.go index 41309ae3f1..07e1c2fd59 100644 --- a/remote/s3_test.go +++ b/remote/s3_test.go @@ -17,8 +17,7 @@ func TestS3Remote_NewClient(t *testing.T) { conf["access_token"] = "test" conf["secret_token"] = "test" - conf["path"] = "hashicorp/test-state" - conf["bucket"] = "plan3-test" + conf["address"] = "s3://plan3-test/hashicorp/test-state" conf["region"] = "eu-west-1" if _, err := NewS3RemoteClient(conf); err != nil { t.Fatalf("err: %v", err) @@ -40,8 +39,7 @@ func TestS3Remote_Validate_envVar(t *testing.T) { defer os.Setenv("AWS_DEFAULT_REGION", os.Getenv("AWS_DEFAULT_REGION")) os.Setenv("AWS_DEFAULT_REGION", "eu-west-1") - conf["path"] = "hashicorp/test-state" - conf["bucket"] = "plan3-test" + conf["address"] = "s3://terraform-state/hashicorp/test-state" if _, err := NewS3RemoteClient(conf); err != nil { t.Fatalf("err: %v", err) }