diff --git a/CHANGELOG.md b/CHANGELOG.md index c1340d3f19..82b49e2e10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,10 +20,12 @@ BUG FIXES: S3 BACKEND -Enhancements +Update Notes * Moves arguments associated with assuming an IAM role into nested block `assume_role`. This deprecates the arguments `role_arn`, `session_name`, `external_id`, `assume_role_duration_seconds`, `assume_role_policy`, `assume_role_policy_arns`, `assume_role_tags`, and `assume_role_transitive_tag_keys`. [GH-33630] +* Supports the default AWS environment variables for overrding API endpoints: `AWS_ENDPOINT_URL_DYNAMODB`, `AWS_ENDPOINT_URL_IAM`, `AWS_ENDPOINT_URL_S3`, and `AWS_ENDPOINT_URL_STS`. + This deprecates the environment variables `AWS_DYNAMODB_ENDPOINT`, `AWS_IAM_ENDPOINT`, `AWS_S3_ENDPOINT`, and `AWS_STS_ENDPOINT`. [GH-33715] ## Previous Releases diff --git a/internal/backend/remote-state/s3/backend.go b/internal/backend/remote-state/s3/backend.go index f43f1e758d..37ec9bb97a 100644 --- a/internal/backend/remote-state/s3/backend.go +++ b/internal/backend/remote-state/s3/backend.go @@ -329,8 +329,7 @@ func (b *Backend) PrepareConfig(obj cty.Value) (cty.Value, tfdiags.Diagnostics) } } else { if defined := findDeprecatedFields(obj, assumeRoleDeprecatedFields); len(defined) != 0 { - diags = diags.Append(tfdiags.WholeContainingBody( - tfdiags.Warning, + diags = diags.Append(wholeBodyWarningDiag( "Deprecated Parameters", `The following parameters have been deprecated. Replace them as follows:`+"\n"+ formatDeprecations(defined), @@ -455,20 +454,31 @@ func (b *Backend) Configure(obj cty.Value) tfdiags.Diagnostics { } } + for envvar, replacement := range map[string]string{ + "AWS_DYNAMODB_ENDPOINT": "AWS_ENDPOINT_URL_DYNAMODB", + "AWS_IAM_ENDPOINT": "AWS_ENDPOINT_URL_IAM", + "AWS_S3_ENDPOINT": "AWS_ENDPOINT_URL_S3", + "AWS_STS_ENDPOINT": "AWS_ENDPOINT_URL_STS", + } { + if val := os.Getenv(envvar); val != "" { + diags = diags.Append(deprecatedEnvVarDiag(envvar, replacement)) + } + } + cfg := &awsbase.Config{ AccessKey: stringAttr(obj, "access_key"), CallerDocumentationURL: "https://www.terraform.io/docs/language/settings/backends/s3.html", CallerName: "S3 Backend", CredsFilename: stringAttr(obj, "shared_credentials_file"), DebugLogging: logging.IsDebugOrHigher(), - IamEndpoint: stringAttrDefaultEnvVar(obj, "iam_endpoint", "AWS_IAM_ENDPOINT"), + IamEndpoint: stringAttrDefaultEnvVar(obj, "iam_endpoint", "AWS_ENDPOINT_URL_IAM", "AWS_IAM_ENDPOINT"), MaxRetries: intAttrDefault(obj, "max_retries", 5), Profile: stringAttr(obj, "profile"), Region: stringAttr(obj, "region"), SecretKey: stringAttr(obj, "secret_key"), SkipCredsValidation: boolAttr(obj, "skip_credentials_validation"), SkipMetadataApiCheck: boolAttr(obj, "skip_metadata_api_check"), - StsEndpoint: stringAttrDefaultEnvVar(obj, "sts_endpoint", "AWS_STS_ENDPOINT"), + StsEndpoint: stringAttrDefaultEnvVar(obj, "sts_endpoint", "AWS_ENDPOINT_URL_STS", "AWS_STS_ENDPOINT"), Token: stringAttr(obj, "token"), UserAgentProducts: []*awsbase.UserAgentProduct{ {Name: "APN", Version: "1.0"}, @@ -535,13 +545,13 @@ func (b *Backend) Configure(obj cty.Value) tfdiags.Diagnostics { } var dynamoConfig aws.Config - if v, ok := stringAttrDefaultEnvVarOk(obj, "dynamodb_endpoint", "AWS_DYNAMODB_ENDPOINT"); ok { + if v, ok := stringAttrDefaultEnvVarOk(obj, "dynamodb_endpoint", "AWS_ENDPOINT_URL_DYNAMODB", "AWS_DYNAMODB_ENDPOINT"); ok { dynamoConfig.Endpoint = aws.String(v) } b.dynClient = dynamodb.New(sess.Copy(&dynamoConfig)) var s3Config aws.Config - if v, ok := stringAttrDefaultEnvVarOk(obj, "endpoint", "AWS_S3_ENDPOINT"); ok { + if v, ok := stringAttrDefaultEnvVarOk(obj, "endpoint", "AWS_ENDPOINT_URL_S3", "AWS_S3_ENDPOINT"); ok { s3Config.Endpoint = aws.String(v) } if v, ok := boolAttrOk(obj, "force_path_style"); ok { @@ -966,3 +976,10 @@ func assumeRoleFullSchema() objectSchema { }, } } + +func deprecatedEnvVarDiag(envvar, replacement string) tfdiags.Diagnostic { + return wholeBodyWarningDiag( + "Deprecated Environment Variable", + fmt.Sprintf(`The environment variable "%s" is deprecated. Use environment variable "%s" instead.`, envvar, replacement), + ) +} diff --git a/internal/backend/remote-state/s3/backend_test.go b/internal/backend/remote-state/s3/backend_test.go index 97ca43fc7f..bee5383a17 100644 --- a/internal/backend/remote-state/s3/backend_test.go +++ b/internal/backend/remote-state/s3/backend_test.go @@ -18,6 +18,8 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/google/go-cmp/cmp" awsbase "github.com/hashicorp/aws-sdk-go-base" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hcldec" "github.com/hashicorp/terraform/internal/backend" "github.com/hashicorp/terraform/internal/configs/configschema" "github.com/hashicorp/terraform/internal/configs/hcl2shim" @@ -200,24 +202,34 @@ func TestBackendConfig_DynamoDBEndpoint(t *testing.T) { testACC(t) cases := map[string]struct { - config map[string]any - vars map[string]string - expected string + config map[string]any + vars map[string]string + expectedEndpoint string + expectedDiags tfdiags.Diagnostics }{ "none": { - expected: "", + expectedEndpoint: "", }, "config": { config: map[string]any{ "dynamodb_endpoint": "dynamo.test", }, - expected: "dynamo.test", + expectedEndpoint: "dynamo.test", }, "envvar": { + vars: map[string]string{ + "AWS_ENDPOINT_URL_DYNAMODB": "dynamo.test", + }, + expectedEndpoint: "dynamo.test", + }, + "deprecated envvar": { vars: map[string]string{ "AWS_DYNAMODB_ENDPOINT": "dynamo.test", }, - expected: "dynamo.test", + expectedEndpoint: "dynamo.test", + expectedDiags: tfdiags.Diagnostics{ + deprecatedEnvVarDiag("AWS_DYNAMODB_ENDPOINT", "AWS_ENDPOINT_URL_DYNAMODB"), + }, }, } @@ -246,9 +258,14 @@ func TestBackendConfig_DynamoDBEndpoint(t *testing.T) { } } - b := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(config)).(*Backend) + raw, diags := testBackendConfigDiags(t, New(), backend.TestWrapConfig(config)) + b := raw.(*Backend) + + checkClientEndpoint(t, b.dynClient.Config, tc.expectedEndpoint) - checkClientEndpoint(t, b.dynClient.Config, tc.expected) + if diff := cmp.Diff(diags, tc.expectedDiags, cmp.Comparer(diagnosticComparer)); diff != "" { + t.Errorf("unexpected diagnostics difference: %s", diff) + } }) } } @@ -257,24 +274,34 @@ func TestBackendConfig_S3Endpoint(t *testing.T) { testACC(t) cases := map[string]struct { - config map[string]any - vars map[string]string - expected string + config map[string]any + vars map[string]string + expectedEndpoint string + expectedDiags tfdiags.Diagnostics }{ "none": { - expected: "", + expectedEndpoint: "", }, "config": { config: map[string]any{ "endpoint": "s3.test", }, - expected: "s3.test", + expectedEndpoint: "s3.test", }, "envvar": { + vars: map[string]string{ + "AWS_ENDPOINT_URL_S3": "s3.test", + }, + expectedEndpoint: "s3.test", + }, + "deprecated envvar": { vars: map[string]string{ "AWS_S3_ENDPOINT": "s3.test", }, - expected: "s3.test", + expectedEndpoint: "s3.test", + expectedDiags: tfdiags.Diagnostics{ + deprecatedEnvVarDiag("AWS_S3_ENDPOINT", "AWS_ENDPOINT_URL_S3"), + }, }, } @@ -303,9 +330,14 @@ func TestBackendConfig_S3Endpoint(t *testing.T) { } } - b := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(config)).(*Backend) + raw, diags := testBackendConfigDiags(t, New(), backend.TestWrapConfig(config)) + b := raw.(*Backend) + + checkClientEndpoint(t, b.s3Client.Config, tc.expectedEndpoint) - checkClientEndpoint(t, b.s3Client.Config, tc.expected) + if diff := cmp.Diff(diags, tc.expectedDiags, cmp.Comparer(diagnosticComparer)); diff != "" { + t.Errorf("unexpected diagnostics difference: %s", diff) + } }) } } @@ -1653,3 +1685,39 @@ func must[T any](v T, err error) T { return v } } + +// testBackendConfigDiags is an equivalent to `backend.TestBackendConfig` which returns the diags to the caller +// instead of failing the test +func testBackendConfigDiags(t *testing.T, b backend.Backend, c hcl.Body) (backend.Backend, tfdiags.Diagnostics) { + t.Helper() + + t.Logf("TestBackendConfig on %T with %#v", b, c) + + var diags tfdiags.Diagnostics + + // To make things easier for test authors, we'll allow a nil body here + // (even though that's not normally valid) and just treat it as an empty + // body. + if c == nil { + c = hcl.EmptyBody() + } + + schema := b.ConfigSchema() + spec := schema.DecoderSpec() + obj, decDiags := hcldec.Decode(c, spec, nil) + diags = diags.Append(decDiags) + + newObj, valDiags := b.PrepareConfig(obj) + diags = diags.Append(valDiags.InConfigBody(c, "")) + + // it's valid for a Backend to have warnings (e.g. a Deprecation) as such we should only raise on errors + if diags.HasErrors() { + return b, diags + } + + obj = newObj + + confDiags := b.Configure(obj) + + return b, diags.Append(confDiags) +} diff --git a/internal/backend/remote-state/s3/validate.go b/internal/backend/remote-state/s3/validate.go index 610b6c2e7d..03bac21225 100644 --- a/internal/backend/remote-state/s3/validate.go +++ b/internal/backend/remote-state/s3/validate.go @@ -352,3 +352,7 @@ func attributeErrDiag(summary, detail string, attrPath cty.Path) tfdiags.Diagnos func wholeBodyErrDiag(summary, detail string) tfdiags.Diagnostic { return tfdiags.WholeContainingBody(tfdiags.Error, summary, detail) } + +func wholeBodyWarningDiag(summary, detail string) tfdiags.Diagnostic { + return tfdiags.WholeContainingBody(tfdiags.Warning, summary, detail) +} diff --git a/website/docs/language/settings/backends/s3.mdx b/website/docs/language/settings/backends/s3.mdx index ef85f68c0a..5a6e1b7190 100644 --- a/website/docs/language/settings/backends/s3.mdx +++ b/website/docs/language/settings/backends/s3.mdx @@ -154,14 +154,14 @@ The following configuration is optional: * `access_key` - (Optional) AWS access key. If configured, must also configure `secret_key`. This can also be sourced from the `AWS_ACCESS_KEY_ID` environment variable, AWS shared credentials file (e.g. `~/.aws/credentials`), or AWS shared configuration file (e.g. `~/.aws/config`). * `secret_key` - (Optional) AWS access key. If configured, must also configure `access_key`. This can also be sourced from the `AWS_SECRET_ACCESS_KEY` environment variable, AWS shared credentials file (e.g. `~/.aws/credentials`), or AWS shared configuration file (e.g. `~/.aws/config`). -* `iam_endpoint` - (Optional) Custom endpoint for the AWS Identity and Access Management (IAM) API. This can also be sourced from the `AWS_IAM_ENDPOINT` environment variable. +* `iam_endpoint` - (Optional) Custom endpoint for the AWS Identity and Access Management (IAM) API. This can also be sourced from the environment variable `AWS_ENDPOINT_URL_IAM` or the deprecated environment variable `AWS_IAM_ENDPOINT`. * `max_retries` - (Optional) The maximum number of times an AWS API request is retried on retryable failure. Defaults to 5. * `profile` - (Optional) Name of AWS profile in AWS shared credentials file (e.g. `~/.aws/credentials`) or AWS shared configuration file (e.g. `~/.aws/config`) to use for credentials and/or configuration. This can also be sourced from the `AWS_PROFILE` environment variable. * `shared_credentials_file` - (Optional) Path to the AWS shared credentials file. Defaults to `~/.aws/credentials`. * `skip_credentials_validation` - (Optional) Skip credentials validation via the STS API. * `skip_region_validation` - (Optional) Skip validation of provided region name. * `skip_metadata_api_check` - (Optional) Skip usage of EC2 Metadata API. -* `sts_endpoint` - (Optional) Custom endpoint for the AWS Security Token Service (STS) API. This can also be sourced from the `AWS_STS_ENDPOINT` environment variable. +* `sts_endpoint` - (Optional) Custom endpoint for the AWS Security Token Service (STS) API. This can also be sourced from the environment variable `AWS_ENDPOINT_URL_STS` or the deprecated environment variable `AWS_STS_ENDPOINT`. * `token` - (Optional) Multi-Factor Authentication (MFA) token. This can also be sourced from the `AWS_SESSION_TOKEN` environment variable. #### Assume Role Configuration @@ -214,7 +214,7 @@ The following configuration is optional: * `acl` - (Optional) [Canned ACL](https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl) to be applied to the state file. * `encrypt` - (Optional) Enable [server side encryption](https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingServerSideEncryption.html) of the state file. -* `endpoint` - (Optional) Custom endpoint for the AWS S3 API. This can also be sourced from the `AWS_S3_ENDPOINT` environment variable. +* `endpoint` - (Optional) Custom endpoint for the AWS S3 API. This can also be sourced from the environment variable `AWS_ENDPOINT_URL_S3` or the deprecated environment variable `AWS_S3_ENDPOINT`. * `force_path_style` - (Optional) Enable path-style S3 URLs (`https:///` instead of `https://.`). * `kms_key_id` - (Optional) Amazon Resource Name (ARN) of a Key Management Service (KMS) Key to use for encrypting the state. Note that if this value is specified, Terraform will need `kms:Encrypt`, `kms:Decrypt` and `kms:GenerateDataKey` permissions on this KMS key. * `sse_customer_key` - (Optional) The key to use for encrypting state with [Server-Side Encryption with Customer-Provided Keys (SSE-C)](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html). This is the base64-encoded value of the key, which must decode to 256 bits. This can also be sourced from the `AWS_SSE_CUSTOMER_KEY` environment variable, which is recommended due to the sensitivity of the value. Setting it inside a terraform file will cause it to be persisted to disk in `terraform.tfstate`. @@ -224,7 +224,7 @@ The following configuration is optional: The following configuration is optional: -* `dynamodb_endpoint` - (Optional) Custom endpoint for the AWS DynamoDB API. This can also be sourced from the `AWS_DYNAMODB_ENDPOINT` environment variable. +* `dynamodb_endpoint` - (Optional) Custom endpoint for the AWS DynamoDB API. This can also be sourced from the environment variable `AWS_ENDPOINT_URL_DYNAMODB` or the deprecated environment variable `AWS_DYNAMODB_ENDPOINT`. * `dynamodb_table` - (Optional) Name of DynamoDB Table to use for state locking and consistency. The table must have a partition key named `LockID` with type of `String`. If not configured, state locking will be disabled. ## Multi-account AWS Architecture