From d1ada744e17e6d1e223568653031e21385063c83 Mon Sep 17 00:00:00 2001 From: Sylvia Moss Date: Fri, 22 Jan 2021 14:49:45 +0100 Subject: [PATCH] Aws Secrets Manager data sources (#10505) --- cmd/packer-plugin-amazon/main.go | 2 + command/plugin.go | 4 +- datasource/amazon/ami/data.go | 19 +- datasource/amazon/secretsmanager/data.go | 168 ++++++++++++++++ .../amazon/secretsmanager/data.hcl2spec.go | 99 ++++++++++ .../amazon/secretsmanager/data_acc_test.go | 179 ++++++++++++++++++ datasource/amazon/secretsmanager/data_test.go | 39 ++++ go.mod | 2 +- go.sum | 8 + hcl2template/types.datasource.go | 7 + hcl2template/types.hcl_post-processor.go | 8 + hcl2template/types.hcl_provisioner.go | 8 + .../packer-plugin-sdk/acctest/datasources.go | 7 + .../aws/secretsmanager/secretsmanager.go | 18 +- .../packer-plugin-sdk/version/version.go | 4 +- vendor/modules.txt | 2 +- .../{amazon-ami.mdx => amazon/ami.mdx} | 7 +- .../content/docs/datasources/amazon/index.mdx | 42 ++++ .../datasources/amazon/secretsmanager.mdx | 51 +++++ .../ami/DatasourceOutput-not-required.mdx | 13 ++ .../secretsmanager/Config-not-required.mdx | 10 + .../amazon/secretsmanager/Config-required.mdx | 4 + .../DatasourceOutput-not-required.mdx | 12 ++ website/data/docs-navigation.js | 49 +++-- 24 files changed, 723 insertions(+), 39 deletions(-) create mode 100644 datasource/amazon/secretsmanager/data.go create mode 100644 datasource/amazon/secretsmanager/data.hcl2spec.go create mode 100644 datasource/amazon/secretsmanager/data_acc_test.go create mode 100644 datasource/amazon/secretsmanager/data_test.go rename website/content/docs/datasources/{amazon-ami.mdx => amazon/ami.mdx} (81%) create mode 100644 website/content/docs/datasources/amazon/index.mdx create mode 100644 website/content/docs/datasources/amazon/secretsmanager.mdx create mode 100644 website/content/partials/datasource/amazon/ami/DatasourceOutput-not-required.mdx create mode 100644 website/content/partials/datasource/amazon/secretsmanager/Config-not-required.mdx create mode 100644 website/content/partials/datasource/amazon/secretsmanager/Config-required.mdx create mode 100644 website/content/partials/datasource/amazon/secretsmanager/DatasourceOutput-not-required.mdx diff --git a/cmd/packer-plugin-amazon/main.go b/cmd/packer-plugin-amazon/main.go index 01d77d70d..e05a799de 100644 --- a/cmd/packer-plugin-amazon/main.go +++ b/cmd/packer-plugin-amazon/main.go @@ -10,6 +10,7 @@ import ( "github.com/hashicorp/packer/builder/amazon/ebsvolume" "github.com/hashicorp/packer/builder/osc/chroot" amazonami "github.com/hashicorp/packer/datasource/amazon/ami" + "github.com/hashicorp/packer/datasource/amazon/secretsmanager" amazonimport "github.com/hashicorp/packer/post-processor/amazon-import" ) @@ -21,6 +22,7 @@ func main() { pps.RegisterBuilder("ebsvolume", new(ebsvolume.Builder)) pps.RegisterPostProcessor("import", new(amazonimport.PostProcessor)) pps.RegisterDatasource("ami", new(amazonami.Datasource)) + pps.RegisterDatasource("secretsmanager", new(secretsmanager.Datasource)) err := pps.Run() if err != nil { fmt.Fprintln(os.Stderr, err.Error()) diff --git a/command/plugin.go b/command/plugin.go index cfc505700..b5be6a4fa 100644 --- a/command/plugin.go +++ b/command/plugin.go @@ -66,6 +66,7 @@ import ( vsphereisobuilder "github.com/hashicorp/packer/builder/vsphere/iso" yandexbuilder "github.com/hashicorp/packer/builder/yandex" amazonamidatasource "github.com/hashicorp/packer/datasource/amazon/ami" + amazonsecretsmanagerdatasource "github.com/hashicorp/packer/datasource/amazon/secretsmanager" alicloudimportpostprocessor "github.com/hashicorp/packer/post-processor/alicloud-import" amazonimportpostprocessor "github.com/hashicorp/packer/post-processor/amazon-import" artificepostprocessor "github.com/hashicorp/packer/post-processor/artifice" @@ -214,7 +215,8 @@ var PostProcessors = map[string]packersdk.PostProcessor{ } var Datasources = map[string]packersdk.Datasource{ - "amazon-ami": new(amazonamidatasource.Datasource), + "amazon-ami": new(amazonamidatasource.Datasource), + "amazon-secretsmanager": new(amazonsecretsmanagerdatasource.Datasource), } var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner|datasource)-(.+)") diff --git a/datasource/amazon/ami/data.go b/datasource/amazon/ami/data.go index 77a10e1f5..11f7563b4 100644 --- a/datasource/amazon/ami/data.go +++ b/datasource/amazon/ami/data.go @@ -1,3 +1,4 @@ +//go:generate struct-markdown //go:generate mapstructure-to-hcl2 -type DatasourceOutput,Config package ami @@ -50,12 +51,18 @@ func (d *Datasource) Configure(raws ...interface{}) error { } type DatasourceOutput struct { - ID string `mapstructure:"id"` - Name string `mapstructure:"name"` - CreationDate string `mapstructure:"creation_date"` - Owner string `mapstructure:"owner"` - OwnerName string `mapstructure:"owner_name"` - Tags map[string]string `mapstructure:"tags"` + // The ID of the AMI. + ID string `mapstructure:"id"` + // The name of the AMI. + Name string `mapstructure:"name"` + // The date of creation of the AMI. + CreationDate string `mapstructure:"creation_date"` + // The AWS account ID of the owner. + Owner string `mapstructure:"owner"` + // The owner alias. + OwnerName string `mapstructure:"owner_name"` + // The key/value combination of the tags assigned to the AMI. + Tags map[string]string `mapstructure:"tags"` } func (d *Datasource) OutputSpec() hcldec.ObjectSpec { diff --git a/datasource/amazon/secretsmanager/data.go b/datasource/amazon/secretsmanager/data.go new file mode 100644 index 000000000..e28ff4b8f --- /dev/null +++ b/datasource/amazon/secretsmanager/data.go @@ -0,0 +1,168 @@ +//go:generate struct-markdown +//go:generate mapstructure-to-hcl2 -type DatasourceOutput,Config +package secretsmanager + +import ( + "encoding/json" + "fmt" + "strconv" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/secretsmanager" + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/hashicorp/packer-plugin-sdk/hcl2helper" + packersdk "github.com/hashicorp/packer-plugin-sdk/packer" + "github.com/hashicorp/packer-plugin-sdk/template/config" + awscommon "github.com/hashicorp/packer/builder/amazon/common" + "github.com/hashicorp/packer/builder/amazon/common/awserrors" + "github.com/zclconf/go-cty/cty" +) + +type Datasource struct { + config Config +} + +type Config struct { + // Specifies the secret containing the version that you want to retrieve. + // You can specify either the Amazon Resource Name (ARN) or the friendly name of the secret. + Name string `mapstructure:"name" required:"true"` + // Optional key for JSON secrets that contain more than one value. When set, the `value` output will + // contain the value for the provided key. + Key string `mapstructure:"key"` + // Specifies the unique identifier of the version of the secret that you want to retrieve. + // Overrides version_stage. + VersionId string `mapstructure:"version_id"` + // Specifies the secret version that you want to retrieve by the staging label attached to the version. + // Defaults to AWSCURRENT. + VersionStage string `mapstructure:"version_stage"` + awscommon.AccessConfig `mapstructure:",squash"` +} + +func (d *Datasource) ConfigSpec() hcldec.ObjectSpec { + return d.config.FlatMapstructure().HCL2Spec() +} + +func (d *Datasource) Configure(raws ...interface{}) error { + err := config.Decode(&d.config, nil, raws...) + if err != nil { + return err + } + + var errs *packersdk.MultiError + errs = packersdk.MultiErrorAppend(errs, d.config.AccessConfig.Prepare()...) + + if d.config.Name == "" { + errs = packersdk.MultiErrorAppend(errs, fmt.Errorf("a 'name' must be provided")) + } + + if d.config.VersionStage == "" { + d.config.VersionStage = "AWSCURRENT" + } + + if errs != nil && len(errs.Errors) > 0 { + return errs + } + return nil +} + +type DatasourceOutput struct { + // When a [key](#key) is provided, this will be the value for that key. If a key is not provided, + // `value` will contain the first value found in the secret string. + Value string `mapstructure:"value"` + // The decrypted part of the protected secret information that + // was originally provided as a string. + SecretString string `mapstructure:"secret_string"` + // The decrypted part of the protected secret information that + // was originally provided as a binary. Base64 encoded. + SecretBinary string `mapstructure:"secret_binary"` + // The unique identifier of this version of the secret. + VersionId string `mapstructure:"version_id"` +} + +func (d *Datasource) OutputSpec() hcldec.ObjectSpec { + return (&DatasourceOutput{}).FlatMapstructure().HCL2Spec() +} + +func (d *Datasource) Execute() (cty.Value, error) { + session, err := d.config.Session() + if err != nil { + return cty.NullVal(cty.EmptyObject), err + } + + input := &secretsmanager.GetSecretValueInput{ + SecretId: aws.String(d.config.Name), + } + + version := "" + if d.config.VersionId != "" { + input.VersionId = aws.String(d.config.VersionId) + version = d.config.VersionId + } else { + input.VersionStage = aws.String(d.config.VersionStage) + version = d.config.VersionStage + } + + secretsApi := secretsmanager.New(session) + secret, err := secretsApi.GetSecretValue(input) + if err != nil { + if awserrors.Matches(err, secretsmanager.ErrCodeResourceNotFoundException, "") { + return cty.NullVal(cty.EmptyObject), fmt.Errorf("Secrets Manager Secret %q Version %q not found", d.config.Name, version) + } + if awserrors.Matches(err, secretsmanager.ErrCodeInvalidRequestException, "You can’t perform this operation on the secret because it was deleted") { + return cty.NullVal(cty.EmptyObject), fmt.Errorf("Secrets Manager Secret %q Version %q not found", d.config.Name, version) + } + return cty.NullVal(cty.EmptyObject), fmt.Errorf("error reading Secrets Manager Secret Version: %s", err) + } + + value, err := getSecretValue(aws.StringValue(secret.SecretString), d.config.Key) + if err != nil { + return cty.NullVal(cty.EmptyObject), fmt.Errorf("error to get secret value: %q", err.Error()) + } + + versionId := aws.StringValue(secret.VersionId) + output := DatasourceOutput{ + Value: value, + SecretString: aws.StringValue(secret.SecretString), + SecretBinary: fmt.Sprintf("%s", secret.SecretBinary), + VersionId: versionId, + } + return hcl2helper.HCL2ValueFromConfig(output, d.OutputSpec()), nil +} + +func getSecretValue(secretString string, key string) (string, error) { + var secretValue map[string]interface{} + blob := []byte(secretString) + + //For those plaintext secrets just return the value + if json.Valid(blob) != true { + return secretString, nil + } + + err := json.Unmarshal(blob, &secretValue) + if err != nil { + return "", err + } + + if key == "" { + for _, v := range secretValue { + return getStringSecretValue(v) + } + } + + if v, ok := secretValue[key]; ok { + return getStringSecretValue(v) + } + + return "", nil +} + +func getStringSecretValue(v interface{}) (string, error) { + switch valueType := v.(type) { + case string: + return valueType, nil + case float64: + return strconv.FormatFloat(valueType, 'f', 0, 64), nil + default: + return "", fmt.Errorf("Unsupported secret value type: %T", valueType) + } +} diff --git a/datasource/amazon/secretsmanager/data.hcl2spec.go b/datasource/amazon/secretsmanager/data.hcl2spec.go new file mode 100644 index 000000000..6100a97bb --- /dev/null +++ b/datasource/amazon/secretsmanager/data.hcl2spec.go @@ -0,0 +1,99 @@ +// Code generated by "mapstructure-to-hcl2 -type DatasourceOutput,Config"; DO NOT EDIT. + +package secretsmanager + +import ( + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/hashicorp/packer/builder/amazon/common" + "github.com/zclconf/go-cty/cty" +) + +// FlatConfig is an auto-generated flat version of Config. +// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up. +type FlatConfig struct { + Name *string `mapstructure:"name" required:"true" cty:"name" hcl:"name"` + Key *string `mapstructure:"key" cty:"key" hcl:"key"` + VersionId *string `mapstructure:"version_id" cty:"version_id" hcl:"version_id"` + VersionStage *string `mapstructure:"version_stage" cty:"version_stage" hcl:"version_stage"` + AccessKey *string `mapstructure:"access_key" required:"true" cty:"access_key" hcl:"access_key"` + AssumeRole *common.FlatAssumeRoleConfig `mapstructure:"assume_role" required:"false" cty:"assume_role" hcl:"assume_role"` + CustomEndpointEc2 *string `mapstructure:"custom_endpoint_ec2" required:"false" cty:"custom_endpoint_ec2" hcl:"custom_endpoint_ec2"` + CredsFilename *string `mapstructure:"shared_credentials_file" required:"false" cty:"shared_credentials_file" hcl:"shared_credentials_file"` + DecodeAuthZMessages *bool `mapstructure:"decode_authorization_messages" required:"false" cty:"decode_authorization_messages" hcl:"decode_authorization_messages"` + InsecureSkipTLSVerify *bool `mapstructure:"insecure_skip_tls_verify" required:"false" cty:"insecure_skip_tls_verify" hcl:"insecure_skip_tls_verify"` + MaxRetries *int `mapstructure:"max_retries" required:"false" cty:"max_retries" hcl:"max_retries"` + MFACode *string `mapstructure:"mfa_code" required:"false" cty:"mfa_code" hcl:"mfa_code"` + ProfileName *string `mapstructure:"profile" required:"false" cty:"profile" hcl:"profile"` + RawRegion *string `mapstructure:"region" required:"true" cty:"region" hcl:"region"` + SecretKey *string `mapstructure:"secret_key" required:"true" cty:"secret_key" hcl:"secret_key"` + SkipMetadataApiCheck *bool `mapstructure:"skip_metadata_api_check" cty:"skip_metadata_api_check" hcl:"skip_metadata_api_check"` + SkipCredsValidation *bool `mapstructure:"skip_credential_validation" cty:"skip_credential_validation" hcl:"skip_credential_validation"` + Token *string `mapstructure:"token" required:"false" cty:"token" hcl:"token"` + VaultAWSEngine *common.FlatVaultAWSEngineOptions `mapstructure:"vault_aws_engine" required:"false" cty:"vault_aws_engine" hcl:"vault_aws_engine"` + PollingConfig *common.FlatAWSPollingConfig `mapstructure:"aws_polling" required:"false" cty:"aws_polling" hcl:"aws_polling"` +} + +// FlatMapstructure returns a new FlatConfig. +// FlatConfig is an auto-generated flat version of Config. +// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up. +func (*Config) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } { + return new(FlatConfig) +} + +// HCL2Spec returns the hcl spec of a Config. +// This spec is used by HCL to read the fields of Config. +// The decoded values from this spec will then be applied to a FlatConfig. +func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec { + s := map[string]hcldec.Spec{ + "name": &hcldec.AttrSpec{Name: "name", Type: cty.String, Required: false}, + "key": &hcldec.AttrSpec{Name: "key", Type: cty.String, Required: false}, + "version_id": &hcldec.AttrSpec{Name: "version_id", Type: cty.String, Required: false}, + "version_stage": &hcldec.AttrSpec{Name: "version_stage", Type: cty.String, Required: false}, + "access_key": &hcldec.AttrSpec{Name: "access_key", Type: cty.String, Required: false}, + "assume_role": &hcldec.BlockSpec{TypeName: "assume_role", Nested: hcldec.ObjectSpec((*common.FlatAssumeRoleConfig)(nil).HCL2Spec())}, + "custom_endpoint_ec2": &hcldec.AttrSpec{Name: "custom_endpoint_ec2", Type: cty.String, Required: false}, + "shared_credentials_file": &hcldec.AttrSpec{Name: "shared_credentials_file", Type: cty.String, Required: false}, + "decode_authorization_messages": &hcldec.AttrSpec{Name: "decode_authorization_messages", Type: cty.Bool, Required: false}, + "insecure_skip_tls_verify": &hcldec.AttrSpec{Name: "insecure_skip_tls_verify", Type: cty.Bool, Required: false}, + "max_retries": &hcldec.AttrSpec{Name: "max_retries", Type: cty.Number, Required: false}, + "mfa_code": &hcldec.AttrSpec{Name: "mfa_code", Type: cty.String, Required: false}, + "profile": &hcldec.AttrSpec{Name: "profile", Type: cty.String, Required: false}, + "region": &hcldec.AttrSpec{Name: "region", Type: cty.String, Required: false}, + "secret_key": &hcldec.AttrSpec{Name: "secret_key", Type: cty.String, Required: false}, + "skip_metadata_api_check": &hcldec.AttrSpec{Name: "skip_metadata_api_check", Type: cty.Bool, Required: false}, + "skip_credential_validation": &hcldec.AttrSpec{Name: "skip_credential_validation", Type: cty.Bool, Required: false}, + "token": &hcldec.AttrSpec{Name: "token", Type: cty.String, Required: false}, + "vault_aws_engine": &hcldec.BlockSpec{TypeName: "vault_aws_engine", Nested: hcldec.ObjectSpec((*common.FlatVaultAWSEngineOptions)(nil).HCL2Spec())}, + "aws_polling": &hcldec.BlockSpec{TypeName: "aws_polling", Nested: hcldec.ObjectSpec((*common.FlatAWSPollingConfig)(nil).HCL2Spec())}, + } + return s +} + +// FlatDatasourceOutput is an auto-generated flat version of DatasourceOutput. +// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up. +type FlatDatasourceOutput struct { + Value *string `mapstructure:"value" cty:"value" hcl:"value"` + SecretString *string `mapstructure:"secret_string" cty:"secret_string" hcl:"secret_string"` + SecretBinary *string `mapstructure:"secret_binary" cty:"secret_binary" hcl:"secret_binary"` + VersionId *string `mapstructure:"version_id" cty:"version_id" hcl:"version_id"` +} + +// FlatMapstructure returns a new FlatDatasourceOutput. +// FlatDatasourceOutput is an auto-generated flat version of DatasourceOutput. +// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up. +func (*DatasourceOutput) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } { + return new(FlatDatasourceOutput) +} + +// HCL2Spec returns the hcl spec of a DatasourceOutput. +// This spec is used by HCL to read the fields of DatasourceOutput. +// The decoded values from this spec will then be applied to a FlatDatasourceOutput. +func (*FlatDatasourceOutput) HCL2Spec() map[string]hcldec.Spec { + s := map[string]hcldec.Spec{ + "value": &hcldec.AttrSpec{Name: "value", Type: cty.String, Required: false}, + "secret_string": &hcldec.AttrSpec{Name: "secret_string", Type: cty.String, Required: false}, + "secret_binary": &hcldec.AttrSpec{Name: "secret_binary", Type: cty.String, Required: false}, + "version_id": &hcldec.AttrSpec{Name: "version_id", Type: cty.String, Required: false}, + } + return s +} diff --git a/datasource/amazon/secretsmanager/data_acc_test.go b/datasource/amazon/secretsmanager/data_acc_test.go new file mode 100644 index 000000000..2aeec663b --- /dev/null +++ b/datasource/amazon/secretsmanager/data_acc_test.go @@ -0,0 +1,179 @@ +package secretsmanager + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "os/exec" + "regexp" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/secretsmanager" + "github.com/hashicorp/packer-plugin-sdk/acctest" + "github.com/hashicorp/packer-plugin-sdk/retry" + awscommon "github.com/hashicorp/packer/builder/amazon/common" + "github.com/hashicorp/packer/builder/amazon/common/awserrors" +) + +func TestAmazonSecretsManager(t *testing.T) { + secret := &AmazonSecret{ + Name: "packer_datasource_secretsmanager_test_secret", + Key: "packer_test_key", + Value: "this_is_the_packer_test_secret_value", + Description: "this is a secret used in a packer acc test", + } + + testCase := &acctest.DatasourceTestCase{ + Name: "amazon_secretsmanager_datasource_basic_test", + Setup: func() error { + return secret.Create() + }, + Teardown: func() error { + return secret.Delete() + }, + Template: testDatasourceBasic, + Type: "amazon-secrestmanager", + Check: func(buildCommand *exec.Cmd, logfile string) error { + if buildCommand.ProcessState != nil { + if buildCommand.ProcessState.ExitCode() != 0 { + return fmt.Errorf("Bad exit code. Logfile: %s", logfile) + } + } + + logs, err := os.Open(logfile) + if err != nil { + return fmt.Errorf("Unable find %s", logfile) + } + defer logs.Close() + + logsBytes, err := ioutil.ReadAll(logs) + if err != nil { + return fmt.Errorf("Unable to read %s", logfile) + } + logsString := string(logsBytes) + + valueLog := fmt.Sprintf("null.basic-example: secret value: %s", secret.Value) + secretStringLog := fmt.Sprintf("null.basic-example: secret secret_string: %s", fmt.Sprintf("{%s:%s}", secret.Key, secret.Value)) + versionIdLog := fmt.Sprintf("null.basic-example: secret version_id: %s", aws.StringValue(secret.Info.VersionId)) + secretValueLog := fmt.Sprintf("null.basic-example: secret value: %s", secret.Value) + + if matched, _ := regexp.MatchString(valueLog+".*", logsString); !matched { + t.Fatalf("logs doesn't contain expected arn %q", logsString) + } + if matched, _ := regexp.MatchString(secretStringLog+".*", logsString); !matched { + t.Fatalf("logs doesn't contain expected secret_string %q", logsString) + } + if matched, _ := regexp.MatchString(versionIdLog+".*", logsString); !matched { + t.Fatalf("logs doesn't contain expected version_id %q", logsString) + } + if matched, _ := regexp.MatchString(secretValueLog+".*", logsString); !matched { + t.Fatalf("logs doesn't contain expected value %q", logsString) + } + return nil + }, + } + acctest.TestDatasource(t, testCase) +} + +const testDatasourceBasic = ` +data "amazon-secretsmanager" "test" { + name = "packer_datasource_secretsmanager_test_secret" + key = "packer_test_key" +} + +locals { + value = data.amazon-secretsmanager.test.value + secret_string = data.amazon-secretsmanager.test.secret_string + version_id = data.amazon-secretsmanager.test.version_id + secret_value = jsondecode(data.amazon-secretsmanager.test.secret_string)["packer_test_key"] +} + +source "null" "basic-example" { + communicator = "none" +} + +build { + sources = [ + "source.null.basic-example" + ] + + provisioner "shell-local" { + inline = [ + "echo secret value: ${local.value}", + "echo secret secret_string: ${local.secret_string}", + "echo secret version_id: ${local.version_id}", + "echo secret value: ${local.secret_value}" + ] + } +} +` + +type AmazonSecret struct { + Name string + Key string + Value string + Description string + + Info *secretsmanager.CreateSecretOutput + manager *secretsmanager.SecretsManager +} + +func (as *AmazonSecret) Create() error { + if as.manager == nil { + accessConfig := &awscommon.AccessConfig{} + session, err := accessConfig.Session() + if err != nil { + return fmt.Errorf("Unable to create aws session %s", err.Error()) + } + as.manager = secretsmanager.New(session) + } + + newSecret := &secretsmanager.CreateSecretInput{ + Description: aws.String(as.Description), + Name: aws.String(as.Name), + SecretString: aws.String(fmt.Sprintf(`{%q:%q}`, as.Key, as.Value)), + } + + secret := new(secretsmanager.CreateSecretOutput) + var err error + err = retry.Config{ + Tries: 11, + ShouldRetry: func(err error) bool { + if awserrors.Matches(err, "ResourceExistsException", "") { + _ = as.Delete() + return true + } + if awserrors.Matches(err, "InvalidRequestException", "already scheduled for deletion") { + return true + } + return false + }, + RetryDelay: (&retry.Backoff{InitialBackoff: 200 * time.Millisecond, MaxBackoff: 30 * time.Second, Multiplier: 2}).Linear, + }.Run(context.TODO(), func(_ context.Context) error { + secret, err = as.manager.CreateSecret(newSecret) + return err + }) + as.Info = secret + return err +} + +func (as *AmazonSecret) Delete() error { + if as.manager == nil { + accessConfig := &awscommon.AccessConfig{} + session, err := accessConfig.Session() + if err != nil { + return fmt.Errorf("Unable to create aws session %s", err.Error()) + } + as.manager = secretsmanager.New(session) + } + + secret := &secretsmanager.DeleteSecretInput{ + ForceDeleteWithoutRecovery: aws.Bool(true), + SecretId: aws.String(as.Name), + } + _, err := as.manager.DeleteSecret(secret) + return err +} diff --git a/datasource/amazon/secretsmanager/data_test.go b/datasource/amazon/secretsmanager/data_test.go new file mode 100644 index 000000000..5802b3b29 --- /dev/null +++ b/datasource/amazon/secretsmanager/data_test.go @@ -0,0 +1,39 @@ +package secretsmanager + +import ( + "testing" +) + +func TestDatasourceConfigure_EmptySecretId(t *testing.T) { + datasource := Datasource{ + config: Config{}, + } + if err := datasource.Configure(nil); err == nil { + t.Fatalf("Should error if secret id is not specified") + } +} + +func TestDatasourceConfigure_Dafaults(t *testing.T) { + datasource := Datasource{ + config: Config{ + Name: "arn:1223", + }, + } + if err := datasource.Configure(nil); err != nil { + t.Fatalf("err: %s", err) + } + if datasource.config.VersionStage != "AWSCURRENT" { + t.Fatalf("VersionStage not set correctly") + } +} + +func TestDatasourceConfigure(t *testing.T) { + datasource := Datasource{ + config: Config{ + Name: "arn:1223", + }, + } + if err := datasource.Configure(nil); err != nil { + t.Fatalf("err: %s", err) + } +} diff --git a/go.mod b/go.mod index 875879d92..b8dee3302 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/hashicorp/go-uuid v1.0.2 github.com/hashicorp/go-version v1.2.0 github.com/hashicorp/hcl/v2 v2.8.0 - github.com/hashicorp/packer-plugin-sdk v0.0.7 + github.com/hashicorp/packer-plugin-sdk v0.0.9 github.com/hashicorp/vault/api v1.0.4 github.com/hetznercloud/hcloud-go v1.15.1 github.com/hyperonecom/h1-client-go v0.0.0-20191203060043-b46280e4c4a4 diff --git a/go.sum b/go.sum index 579ae2ce6..1d6317ba2 100644 --- a/go.sum +++ b/go.sum @@ -359,8 +359,16 @@ github.com/hashicorp/packer v1.6.7-0.20210107234516-6564ee76e807/go.mod h1:fBz28 github.com/hashicorp/packer-plugin-sdk v0.0.6/go.mod h1:Nvh28f+Jmpp2rcaN79bULTouNkGNDRfHckhHKTAXtyU= github.com/hashicorp/packer-plugin-sdk v0.0.7-0.20210113192617-8a28198491f7 h1:2N1NAfBCmG1vIkbdlIOb/YbaYXCW40YOllWqMZDjnHM= github.com/hashicorp/packer-plugin-sdk v0.0.7-0.20210113192617-8a28198491f7/go.mod h1:YdWTt5w6cYfaQG7IOi5iorL+3SXnz8hI0gJCi8Db/LI= +github.com/hashicorp/packer-plugin-sdk v0.0.7-0.20210120130732-6167b5e5b2e8 h1:50/m5nP40RaXnXyd0GHHUd+CfkmcYeTNGAY5eXQlBeY= +github.com/hashicorp/packer-plugin-sdk v0.0.7-0.20210120130732-6167b5e5b2e8/go.mod h1:YdWTt5w6cYfaQG7IOi5iorL+3SXnz8hI0gJCi8Db/LI= +github.com/hashicorp/packer-plugin-sdk v0.0.7-0.20210121103409-4b079ce99178 h1:AVT2ugu3+UzTDEViAxMFbUzzxgUpSVMMpbuaOEd97HY= +github.com/hashicorp/packer-plugin-sdk v0.0.7-0.20210121103409-4b079ce99178/go.mod h1:YdWTt5w6cYfaQG7IOi5iorL+3SXnz8hI0gJCi8Db/LI= github.com/hashicorp/packer-plugin-sdk v0.0.7 h1:adELlId/KOGWXmQ79L+NwYSgKES6811RVXiRCj4FE0s= github.com/hashicorp/packer-plugin-sdk v0.0.7/go.mod h1:YdWTt5w6cYfaQG7IOi5iorL+3SXnz8hI0gJCi8Db/LI= +github.com/hashicorp/packer-plugin-sdk v0.0.8 h1:/qyCO9YqALnaHSE++y+//tNy68++4SThZctqTwqikrU= +github.com/hashicorp/packer-plugin-sdk v0.0.8/go.mod h1:YdWTt5w6cYfaQG7IOi5iorL+3SXnz8hI0gJCi8Db/LI= +github.com/hashicorp/packer-plugin-sdk v0.0.9 h1:PWX6g0TeAbev5zhiRR91k3Z0wVCqsivs6xyBTRmPMkQ= +github.com/hashicorp/packer-plugin-sdk v0.0.9/go.mod h1:YdWTt5w6cYfaQG7IOi5iorL+3SXnz8hI0gJCi8Db/LI= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hashicorp/serf v0.9.2 h1:yJoyfZXo4Pk2p/M/viW+YLibBFiIbKoP79gu7kDAFP0= github.com/hashicorp/serf v0.9.2/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= diff --git a/hcl2template/types.datasource.go b/hcl2template/types.datasource.go index 9298452dc..0484927b5 100644 --- a/hcl2template/types.datasource.go +++ b/hcl2template/types.datasource.go @@ -6,6 +6,7 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" + hcl2shim "github.com/hashicorp/packer/hcl2template/shim" "github.com/hashicorp/packer/packer" "github.com/zclconf/go-cty/cty" ) @@ -107,6 +108,12 @@ func (cfg *PackerConfig) startDatasource(dataSourceStore packer.DatasourceStore, return nil, diags } + // In case of cty.Unknown values, this will write a equivalent placeholder of the same type + // Unknown types are not recognized by the json marshal during the RPC call and we have to do this here + // to avoid json parsing failures when running the validate command. + // We don't do this before so we can validate if variable types matches correctly on decodeHCL2Spec. + decoded = hcl2shim.WriteUnknownPlaceholderValues(decoded) + if err := datasource.Configure(decoded); err != nil { diags = append(diags, &hcl.Diagnostic{ Summary: err.Error(), diff --git a/hcl2template/types.hcl_post-processor.go b/hcl2template/types.hcl_post-processor.go index 0a703c594..b09bcd73c 100644 --- a/hcl2template/types.hcl_post-processor.go +++ b/hcl2template/types.hcl_post-processor.go @@ -7,6 +7,7 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hcldec" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" + hcl2shim "github.com/hashicorp/packer/hcl2template/shim" "github.com/zclconf/go-cty/cty" ) @@ -55,6 +56,13 @@ func (p *HCL2PostProcessor) HCL2Prepare(buildVars map[string]interface{}) error if diags.HasErrors() { return diags } + + // In case of cty.Unknown values, this will write a equivalent placeholder of the same type + // Unknown types are not recognized by the json marshal during the RPC call and we have to do this here + // to avoid json parsing failures when running the validate command. + // We don't do this before so we can validate if variable types matches correctly on decodeHCL2Spec. + flatPostProcessorCfg = hcl2shim.WriteUnknownPlaceholderValues(flatPostProcessorCfg) + return p.PostProcessor.Configure(p.builderVariables, flatPostProcessorCfg) } diff --git a/hcl2template/types.hcl_provisioner.go b/hcl2template/types.hcl_provisioner.go index daf06c65a..df962f3a0 100644 --- a/hcl2template/types.hcl_provisioner.go +++ b/hcl2template/types.hcl_provisioner.go @@ -7,6 +7,7 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hcldec" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" + hcl2shim "github.com/hashicorp/packer/hcl2template/shim" "github.com/zclconf/go-cty/cty" ) @@ -59,6 +60,13 @@ func (p *HCL2Provisioner) HCL2Prepare(buildVars map[string]interface{}) error { if diags.HasErrors() { return diags } + + // In case of cty.Unknown values, this will write a equivalent placeholder of the same type + // Unknown types are not recognized by the json marshal during the RPC call and we have to do this here + // to avoid json parsing failures when running the validate command. + // We don't do this before so we can validate if variable types matches correctly on decodeHCL2Spec. + flatProvisionerCfg = hcl2shim.WriteUnknownPlaceholderValues(flatProvisionerCfg) + return p.Provisioner.Prepare(p.builderVariables, flatProvisionerCfg, p.override) } diff --git a/vendor/github.com/hashicorp/packer-plugin-sdk/acctest/datasources.go b/vendor/github.com/hashicorp/packer-plugin-sdk/acctest/datasources.go index a0f23bd65..c41dde742 100644 --- a/vendor/github.com/hashicorp/packer-plugin-sdk/acctest/datasources.go +++ b/vendor/github.com/hashicorp/packer-plugin-sdk/acctest/datasources.go @@ -46,6 +46,13 @@ func TestDatasource(t *testing.T, testCase *DatasourceTestCase) { return } + if testCase.Setup != nil { + err := testCase.Setup() + if err != nil { + t.Fatalf("test %s setup failed: %s", testCase.Name, err) + } + } + logfile := fmt.Sprintf("packer_log_%s.txt", testCase.Name) templatePath := fmt.Sprintf("./%s.pkr.hcl", testCase.Name) diff --git a/vendor/github.com/hashicorp/packer-plugin-sdk/template/interpolate/aws/secretsmanager/secretsmanager.go b/vendor/github.com/hashicorp/packer-plugin-sdk/template/interpolate/aws/secretsmanager/secretsmanager.go index 67ec3b90b..b7e77ab5e 100644 --- a/vendor/github.com/hashicorp/packer-plugin-sdk/template/interpolate/aws/secretsmanager/secretsmanager.go +++ b/vendor/github.com/hashicorp/packer-plugin-sdk/template/interpolate/aws/secretsmanager/secretsmanager.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "strconv" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" @@ -76,7 +77,7 @@ func (c *Client) GetSecret(spec *SecretSpec) (string, error) { } func getSecretValue(s *SecretString, spec *SecretSpec) (string, error) { - var secretValue map[string]string + var secretValue map[string]interface{} blob := []byte(s.SecretString) //For those plaintext secrets just return the value @@ -96,13 +97,24 @@ func getSecretValue(s *SecretString, spec *SecretSpec) (string, error) { if spec.Key == "" { for _, v := range secretValue { - return v, nil + return getStringSecretValue(v) } } if v, ok := secretValue[spec.Key]; ok { - return v, nil + return getStringSecretValue(v) } return "", fmt.Errorf("No secret found for key %q", spec.Key) } + +func getStringSecretValue(v interface{}) (string, error) { + switch valueType := v.(type) { + case string: + return valueType, nil + case float64: + return strconv.FormatFloat(valueType, 'f', 0, 64), nil + default: + return "", fmt.Errorf("Unsupported secret value type: %T", valueType) + } +} diff --git a/vendor/github.com/hashicorp/packer-plugin-sdk/version/version.go b/vendor/github.com/hashicorp/packer-plugin-sdk/version/version.go index a815dabfb..fd262a0a8 100644 --- a/vendor/github.com/hashicorp/packer-plugin-sdk/version/version.go +++ b/vendor/github.com/hashicorp/packer-plugin-sdk/version/version.go @@ -13,12 +13,12 @@ import ( var GitCommit string // Package version helps plugin creators set and track the sdk version using -const Version = "0.0.7" +const Version = "0.0.9" // A pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. -const VersionPrerelease = "" +const VersionPrerelease = "dev" // InitializePluginVersion initializes the SemVer and returns a version var. // If the provided "version" string is not valid, the call to version.Must diff --git a/vendor/modules.txt b/vendor/modules.txt index db5f768a4..b47029723 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -355,7 +355,7 @@ github.com/hashicorp/hcl/v2/hclparse github.com/hashicorp/hcl/v2/hclsyntax github.com/hashicorp/hcl/v2/hclwrite github.com/hashicorp/hcl/v2/json -# github.com/hashicorp/packer-plugin-sdk v0.0.7 +# github.com/hashicorp/packer-plugin-sdk v0.0.9 github.com/hashicorp/packer-plugin-sdk/acctest github.com/hashicorp/packer-plugin-sdk/acctest/provisioneracc github.com/hashicorp/packer-plugin-sdk/acctest/testutils diff --git a/website/content/docs/datasources/amazon-ami.mdx b/website/content/docs/datasources/amazon/ami.mdx similarity index 81% rename from website/content/docs/datasources/amazon-ami.mdx rename to website/content/docs/datasources/amazon/ami.mdx index bf2ebe217..0053dda5e 100644 --- a/website/content/docs/datasources/amazon-ami.mdx +++ b/website/content/docs/datasources/amazon/ami.mdx @@ -38,9 +38,4 @@ This selects the most recent Ubuntu 16.04 HVM EBS AMI from Canonical. Note that ## Output Data -- `id` - The ID of the AMI. -- `name` - The name of the AMI. -- `creation_date` - The date of creation of the AMI. -- `owner` - The AWS account ID of the owner. -- `owner_name` - The owner alias. -- `tags` - The key/value combination of the tags assigned to the AMI. +@include 'datasource/amazon/ami/DatasourceOutput-not-required.mdx' diff --git a/website/content/docs/datasources/amazon/index.mdx b/website/content/docs/datasources/amazon/index.mdx new file mode 100644 index 000000000..da4300a02 --- /dev/null +++ b/website/content/docs/datasources/amazon/index.mdx @@ -0,0 +1,42 @@ +--- +description: | + Packer is able to fetch data from AWS. To achieve this, Packer comes with + data sources to retrieve AMI and secrets information. +page_title: Amazon - Data Sources +sidebar_title: Amazon +--- + +# Amazon Data Sources + +Packer is able to fetch data from AWS. To achieve this, Packer comes with data sources to retrieve AMI and secrets information. +Packer supports the following data sources at the moment: + +- [amazon-ami](/docs/datasources/amazon/ami) - Filter and fetch an Amazon AMI to output all the AMI information. + +- [amazon-secretsmanager](/docs/datasources/amazon/secretsmanager) - Retrieve information +about a Secrets Manager secret version, including its secret value. + + +## Authentication + +The Amazon Data Sources authentication works just like for the [Amazon Builders](/docs/builders/amazon). Both +have the same authentication options and you can refer to the [Amazon Builders authentication](/docs/builders/amazon#authentication) +to learn the options to authenticate for data sources. + +-> **Note:** A data source will start and execute in your own authentication session. The authentication in the data source +doesn't relate with the authentication on Amazon Builders. + +Basic example of an Amazon data source authentication using `assume_role`: + +```hcl +data "amazon-secretsmanager" "basic-example" { + name = "packer_test_secret" + key = "packer_test_key" + + assume_role { + role_arn = "arn:aws:iam::ACCOUNT_ID:role/ROLE_NAME" + session_name = "SESSION_NAME" + external_id = "EXTERNAL_ID" + } +} +``` diff --git a/website/content/docs/datasources/amazon/secretsmanager.mdx b/website/content/docs/datasources/amazon/secretsmanager.mdx new file mode 100644 index 000000000..625493bcd --- /dev/null +++ b/website/content/docs/datasources/amazon/secretsmanager.mdx @@ -0,0 +1,51 @@ +--- +description: | + The Amazon Secrets Manager data source provides information about a Secrets Manager secret version, + including its secret value. + +page_title: Secrets Manager - Data Source +sidebar_title: Secrets Manager +--- + +# Amazon Secrets Manager Data Source + +The Secrets Manager data source provides information about a Secrets Manager secret version, +including its secret value. + +-> **Note:** Data sources is a feature exclusively to HCL2 templates. + +Basic examples of usage: + +```hcl +data "amazon-secretsmanager" "basic-example" { + name = "packer_test_secret" + key = "packer_test_key" + version_stage = "example" +} + +# usage example of the data source output +locals { + value = data.amazon-secretsmanager.basic-example.value + secret_string = data.amazon-secretsmanager.basic-example.secret_string + version_id = data.amazon-secretsmanager.basic-example.version_id + secret_value = jsondecode(data.amazon-secretsmanager.basic-example.secret_string)["packer_test_key"] +} +``` + +Reading key-value pairs from JSON back into a native Packer map can be accomplished +with the [jsondecode() function](/docs/templates/hcl_templates/functions/encoding/jsondecode). + + +## Configuration Reference + +### Required + +@include 'datasource/amazon/secretsmanager/Config-required.mdx' + +### Optional + +@include 'datasource/amazon/secretsmanager/Config-not-required.mdx' + +## Output Data + +@include 'datasource/amazon/secretsmanager/DatasourceOutput-not-required.mdx' diff --git a/website/content/partials/datasource/amazon/ami/DatasourceOutput-not-required.mdx b/website/content/partials/datasource/amazon/ami/DatasourceOutput-not-required.mdx new file mode 100644 index 000000000..19ada68ff --- /dev/null +++ b/website/content/partials/datasource/amazon/ami/DatasourceOutput-not-required.mdx @@ -0,0 +1,13 @@ + + +- `id` (string) - The ID of the AMI. + +- `name` (string) - The name of the AMI. + +- `creation_date` (string) - The date of creation of the AMI. + +- `owner` (string) - The AWS account ID of the owner. + +- `owner_name` (string) - The owner alias. + +- `tags` (map[string]string) - The key/value combination of the tags assigned to the AMI. diff --git a/website/content/partials/datasource/amazon/secretsmanager/Config-not-required.mdx b/website/content/partials/datasource/amazon/secretsmanager/Config-not-required.mdx new file mode 100644 index 000000000..b4a94b5cd --- /dev/null +++ b/website/content/partials/datasource/amazon/secretsmanager/Config-not-required.mdx @@ -0,0 +1,10 @@ + + +- `key` (string) - Optional key for JSON secrets that contain more than one value. When set, the `value` output will + contain the value for the provided key. + +- `version_id` (string) - Specifies the unique identifier of the version of the secret that you want to retrieve. + Overrides version_stage. + +- `version_stage` (string) - Specifies the secret version that you want to retrieve by the staging label attached to the version. + Defaults to AWSCURRENT. diff --git a/website/content/partials/datasource/amazon/secretsmanager/Config-required.mdx b/website/content/partials/datasource/amazon/secretsmanager/Config-required.mdx new file mode 100644 index 000000000..c63df6888 --- /dev/null +++ b/website/content/partials/datasource/amazon/secretsmanager/Config-required.mdx @@ -0,0 +1,4 @@ + + +- `name` (string) - Specifies the secret containing the version that you want to retrieve. + You can specify either the Amazon Resource Name (ARN) or the friendly name of the secret. diff --git a/website/content/partials/datasource/amazon/secretsmanager/DatasourceOutput-not-required.mdx b/website/content/partials/datasource/amazon/secretsmanager/DatasourceOutput-not-required.mdx new file mode 100644 index 000000000..6c34f99a9 --- /dev/null +++ b/website/content/partials/datasource/amazon/secretsmanager/DatasourceOutput-not-required.mdx @@ -0,0 +1,12 @@ + + +- `value` (string) - When a [key](#key) is provided, this will be the value for that key. If a key is not provided, + `value` will contain the first value found in the secret string. + +- `secret_string` (string) - The decrypted part of the protected secret information that + was originally provided as a string. + +- `secret_binary` (string) - The decrypted part of the protected secret information that + was originally provided as a binary. Base64 encoded. + +- `version_id` (string) - The unique identifier of this version of the secret. diff --git a/website/data/docs-navigation.js b/website/data/docs-navigation.js index dc5018331..66adf5edd 100644 --- a/website/data/docs-navigation.js +++ b/website/data/docs-navigation.js @@ -15,19 +15,19 @@ export default [ { category: 'templates', content: [ - { - category: "legacy_json_templates", - content: [ - 'builders', - 'communicator', - 'engine', - 'post-processors', - 'provisioners', - 'user-variables', - ] - }, - { - category: 'hcl_templates', + { + category: "legacy_json_templates", + content: [ + 'builders', + 'communicator', + 'engine', + 'post-processors', + 'provisioners', + 'user-variables', + ] + }, + { + category: 'hcl_templates', content: [ { category: 'blocks', @@ -191,7 +191,7 @@ export default [ ], }, '----------', - { category: 'communicators', content: ['ssh', 'winrm'] }, + {category: 'communicators', content: ['ssh', 'winrm']}, { category: 'builders', content: [ @@ -211,7 +211,7 @@ export default [ 'googlecompute', 'hetzner-cloud', 'hyperone', - { category: 'hyperv', content: ['iso', 'vmcx'] }, + {category: 'hyperv', content: ['iso', 'vmcx']}, 'linode', 'lxc', 'lxd', @@ -219,14 +219,14 @@ export default [ 'null', 'oneandone', 'openstack', - { category: 'oracle', content: ['classic', 'oci'] }, + {category: 'oracle', content: ['classic', 'oci']}, { category: 'outscale', content: ['chroot', 'bsu', 'bsusurrogate', 'bsuvolume'], }, - { category: 'parallels', content: ['iso', 'pvm'] }, + {category: 'parallels', content: ['iso', 'pvm']}, 'profitbricks', - { category: 'proxmox', content: ['iso', 'clone'] }, + {category: 'proxmox', content: ['iso', 'clone']}, 'qemu', 'scaleway', 'tencentcloud-cvm', @@ -247,7 +247,18 @@ export default [ 'community-supported', ], }, - { category: 'datasources', content: ['amazon-ami'] }, + { + category: 'datasources', + content: [ + { + category: 'amazon', + content: [ + 'ami', + 'secretsmanager' + ], + }, + ] + }, { category: 'provisioners', content: [