diff --git a/internal/backend/remote-state/s3/backend.go b/internal/backend/remote-state/s3/backend.go index 86690a1cd0..0b645bec42 100644 --- a/internal/backend/remote-state/s3/backend.go +++ b/internal/backend/remote-state/s3/backend.go @@ -45,6 +45,7 @@ type Backend struct { kmsKeyID string ddbTable string workspaceKeyPrefix string + skipS3Checksum bool } // ConfigSchema returns a description of the expected configuration @@ -183,7 +184,7 @@ func (b *Backend) ConfigSchema() *configschema.Block { "skip_credentials_validation": { Type: cty.Bool, Optional: true, - Description: "Skip the credentials validation via STS API.", + Description: "Skip the credentials validation via STS API. Useful for testing and for AWS API implementations that do not have STS available.", }, "skip_requesting_account_id": { Type: cty.Bool, @@ -200,6 +201,11 @@ func (b *Backend) ConfigSchema() *configschema.Block { Optional: true, Description: "Skip static validation of region name.", }, + "skip_s3_checksum": { + Type: cty.Bool, + Optional: true, + Description: "Do not include checksum when uploading S3 Objects. Useful for some S3-Compatible APIs.", + }, "sse_customer_key": { Type: cty.String, Optional: true, @@ -903,6 +909,7 @@ func (b *Backend) Configure(obj cty.Value) tfdiags.Diagnostics { b.serverSideEncryption = boolAttr(obj, "encrypt") b.kmsKeyID = stringAttr(obj, "kms_key_id") b.ddbTable = stringAttr(obj, "dynamodb_table") + b.skipS3Checksum = boolAttr(obj, "skip_s3_checksum") if _, ok := stringAttrOk(obj, "kms_key_id"); ok { if customerKey := os.Getenv("AWS_SSE_CUSTOMER_KEY"); customerKey != "" { diff --git a/internal/backend/remote-state/s3/backend_state.go b/internal/backend/remote-state/s3/backend_state.go index 331dfbfb7d..12d4511b69 100644 --- a/internal/backend/remote-state/s3/backend_state.go +++ b/internal/backend/remote-state/s3/backend_state.go @@ -151,6 +151,7 @@ func (b *Backend) remoteClient(name string) (*RemoteClient, error) { acl: b.acl, kmsKeyID: b.kmsKeyID, ddbTable: b.ddbTable, + skipS3Checksum: b.skipS3Checksum, } return client, nil diff --git a/internal/backend/remote-state/s3/backend_test.go b/internal/backend/remote-state/s3/backend_test.go index e680b10d42..f9f28f2fbe 100644 --- a/internal/backend/remote-state/s3/backend_test.go +++ b/internal/backend/remote-state/s3/backend_test.go @@ -61,6 +61,15 @@ func TestBackend_impl(t *testing.T) { var _ backend.Backend = new(Backend) } +func TestBackend_InternalValidate(t *testing.T) { + b := New() + + schema := b.ConfigSchema() + if err := schema.InternalValidate(); err != nil { + t.Fatalf("failed InternalValidate: %s", err) + } +} + func TestBackendConfig_original(t *testing.T) { testACC(t) diff --git a/internal/backend/remote-state/s3/client.go b/internal/backend/remote-state/s3/client.go index c4528e861e..204ddde314 100644 --- a/internal/backend/remote-state/s3/client.go +++ b/internal/backend/remote-state/s3/client.go @@ -47,6 +47,7 @@ type RemoteClient struct { acl string kmsKeyID string ddbTable string + skipS3Checksum bool } var ( @@ -182,6 +183,10 @@ func (c *RemoteClient) get(ctx context.Context) (*remote.Payload, error) { } func (c *RemoteClient) Put(data []byte) error { + return c.put(data) +} + +func (c *RemoteClient) put(data []byte, optFns ...func(*s3.Options)) error { ctx := context.TODO() log := c.logger(operationClientPut) @@ -193,11 +198,13 @@ func (c *RemoteClient) Put(data []byte) error { sum := md5.Sum(data) input := &s3.PutObjectInput{ - ContentType: aws.String(contentType), - Body: bytes.NewReader(data), - Bucket: aws.String(c.bucketName), - Key: aws.String(c.path), - ChecksumAlgorithm: s3types.ChecksumAlgorithmSha256, + ContentType: aws.String(contentType), + Body: bytes.NewReader(data), + Bucket: aws.String(c.bucketName), + Key: aws.String(c.path), + } + if !c.skipS3Checksum { + input.ChecksumAlgorithm = s3types.ChecksumAlgorithmSha256 } if c.serverSideEncryption { @@ -219,7 +226,9 @@ func (c *RemoteClient) Put(data []byte) error { log.Info("Uploading remote state") - uploader := manager.NewUploader(c.s3Client) + uploader := manager.NewUploader(c.s3Client, func(u *manager.Uploader) { + u.ClientOptions = optFns + }) _, err := uploader.Upload(ctx, input) if err != nil { return fmt.Errorf("failed to upload state: %s", err) diff --git a/internal/backend/remote-state/s3/client_test.go b/internal/backend/remote-state/s3/client_test.go index 2197e766f9..49133f3d7d 100644 --- a/internal/backend/remote-state/s3/client_test.go +++ b/internal/backend/remote-state/s3/client_test.go @@ -13,11 +13,15 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go/middleware" + smithyhttp "github.com/aws/smithy-go/transport/http" "github.com/hashicorp/terraform/internal/backend" "github.com/hashicorp/terraform/internal/states/remote" "github.com/hashicorp/terraform/internal/states/statefile" "github.com/hashicorp/terraform/internal/states/statemgr" + "golang.org/x/exp/maps" ) func TestRemoteClient_impl(t *testing.T) { @@ -383,3 +387,104 @@ func (b neverEnding) Read(p []byte) (n int, err error) { } return len(p), nil } + +func TestRemoteClientSkipS3Checksum(t *testing.T) { + testACC(t) + + ctx := context.TODO() + + bucketName := fmt.Sprintf("terraform-remote-s3-test-%x", time.Now().Unix()) + keyName := "testState" + + testcases := map[string]struct { + config map[string]any + expected string + }{ + "default": { + config: map[string]any{}, + expected: string(s3types.ChecksumAlgorithmSha256), + }, + "true": { + config: map[string]any{ + "skip_s3_checksum": true, + }, + expected: "", + }, + "false": { + config: map[string]any{ + "skip_s3_checksum": false, + }, + expected: string(s3types.ChecksumAlgorithmSha256), + }, + } + + for name, testcase := range testcases { + t.Run(name, func(t *testing.T) { + config := map[string]interface{}{ + "bucket": bucketName, + "key": keyName, + } + maps.Copy(config, testcase.config) + b := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(config)).(*Backend) + + createS3Bucket(ctx, t, b.s3Client, bucketName, b.awsConfig.Region) + defer deleteS3Bucket(ctx, t, b.s3Client, bucketName, b.awsConfig.Region) + + state, err := b.StateMgr(backend.DefaultStateName) + if err != nil { + t.Fatal(err) + } + + c := state.(*remote.State).Client + client := c.(*RemoteClient) + + s := statemgr.TestFullInitialState() + sf := &statefile.File{State: s} + var stateBuf bytes.Buffer + if err := statefile.Write(sf, &stateBuf); err != nil { + t.Fatal(err) + } + + var header string + err = client.put(stateBuf.Bytes(), func(opts *s3.Options) { + opts.APIOptions = append(opts.APIOptions, + addRetrieveChecksumHeaderMiddleware(t, &header), + addCancelRequestMiddleware(), + ) + }) + if err != nil { + t.Fatal(err) + } + + if a, e := header, testcase.expected; a != e { + t.Fatalf("expected %q, got %q", e, a) + } + }) + } +} + +func addRetrieveChecksumHeaderMiddleware(t *testing.T, stuff *string) func(*middleware.Stack) error { + return func(stack *middleware.Stack) error { + return stack.Finalize.Add( + retrieveChecksumHeaderMiddleware(t, stuff), + middleware.After, + ) + } +} + +func retrieveChecksumHeaderMiddleware(t *testing.T, stuff *string) middleware.FinalizeMiddleware { + return middleware.FinalizeMiddlewareFunc( + "Test: Retrieve Stuff", + func(ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler) (middleware.FinalizeOutput, middleware.Metadata, error) { + t.Helper() + + request, ok := in.Request.(*smithyhttp.Request) + if !ok { + t.Fatalf("Expected *github.com/aws/smithy-go/transport/http.Request, got %s", fullTypeName(in.Request)) + } + + *stuff = request.Header.Get("x-amz-sdk-checksum-algorithm") + + return next.HandleFinalize(ctx, in) + }) +} diff --git a/website/docs/language/settings/backends/s3.mdx b/website/docs/language/settings/backends/s3.mdx index 07d3352ca0..b46939bb47 100644 --- a/website/docs/language/settings/backends/s3.mdx +++ b/website/docs/language/settings/backends/s3.mdx @@ -173,9 +173,13 @@ The following configuration is optional: * `shared_credentials_file` - (Optional, **Deprecated**, use `shared_credentials_files` instead) Path to the AWS shared credentials file. Defaults to `~/.aws/credentials`. * `shared_credentials_files` - (Optional) List of paths to AWS shared credentials files. Defaults to `~/.aws/credentials`. * `skip_credentials_validation` - (Optional) Skip credentials validation via the STS API. + Useful for testing and for AWS API implementations that do not have STS available. * `skip_region_validation` - (Optional) Skip validation of provided region name. -* `skip_requesting_account_id` - (Optional) Whether to skip requesting the account ID. Useful for AWS API implementations that do not have the IAM, STS API, or metadata API. +* `skip_requesting_account_id` - (Optional) Whether to skip requesting the account ID. + Useful for AWS API implementations that do not have the IAM, STS API, or metadata API. * `skip_metadata_api_check` - (Optional) Skip usage of EC2 Metadata API. +* `skip_s3_checksum` - (Optional) Do not include checksum when uploading S3 Objects. + Useful for some S3-Compatible APIs. * `sts_endpoint` - (Optional, **Deprecated**) Custom endpoint URL for the AWS Security Token Service (STS) API. Use `endpoints.sts` instead. * `sts_region` - (Optional) AWS region for STS. If unset, AWS will use the same region for STS as other non-STS operations.