From 0b7251a4bb5f78a103aa6acb7ec04f3a3d17471e Mon Sep 17 00:00:00 2001 From: Sylvia Moss Date: Mon, 20 Jan 2020 16:29:38 +0100 Subject: [PATCH] Share .Device and .MountPath between builders, provisioners and post-processors (#8621) --- builder/amazon/chroot/builder.go | 12 ++++++-- builder/amazon/chroot/builder_test.go | 25 ++++++++++++++++ builder/amazon/chroot/step_mount_device.go | 5 +++- builder/amazon/chroot/step_prepare_device.go | 3 ++ .../amazon/common/interpolate_build_info.go | 6 ++-- .../common/interpolate_build_info_test.go | 21 +++++++++---- .../common/step_modify_ami_attributes.go | 6 ++-- builder/amazon/common/tags.go | 4 ++- builder/amazon/ebs/builder.go | 4 ++- builder/amazon/ebssurrogate/builder.go | 4 ++- builder/amazon/instance/builder.go | 3 ++ builder/generated_data.go | 20 +++++++++++++ builder/generated_data_test.go | 30 +++++++++++++++++++ common/chroot/step_chroot_provision.go | 5 +++- common/step_provision.go | 2 +- 15 files changed, 131 insertions(+), 19 deletions(-) create mode 100644 builder/generated_data.go create mode 100644 builder/generated_data_test.go diff --git a/builder/amazon/chroot/builder.go b/builder/amazon/chroot/builder.go index ddadd76c4..7b1fd70b9 100644 --- a/builder/amazon/chroot/builder.go +++ b/builder/amazon/chroot/builder.go @@ -10,6 +10,7 @@ package chroot import ( "context" "errors" + "github.com/hashicorp/packer/builder" "runtime" "github.com/aws/aws-sdk-go/service/ec2" @@ -326,7 +327,8 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, []string, error) { } packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token) - return nil, warns, nil + generatedData := []string{"SourceAMIName", "Device", "MountPath"} + return generatedData, warns, nil } func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (packer.Artifact, error) { @@ -356,6 +358,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack state.Put("hook", hook) state.Put("ui", ui) state.Put("wrappedCommand", common.CommandWrapper(wrappedCommand)) + generatedData := &builder.GeneratedData{State: state} // Build the steps steps := []multistep.Step{ @@ -381,7 +384,9 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack steps = append(steps, &StepFlock{}, - &StepPrepareDevice{}, + &StepPrepareDevice{ + GeneratedData: generatedData, + }, &StepCreateVolume{ RootVolumeType: b.config.RootVolumeType, RootVolumeSize: b.config.RootVolumeSize, @@ -396,6 +401,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack &StepMountDevice{ MountOptions: b.config.MountOptions, MountPartition: b.config.MountPartition, + GeneratedData: generatedData, }, &chroot.StepPostMountCommands{ Commands: b.config.PostMountCommands, @@ -439,6 +445,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack SnapshotUsers: b.config.SnapshotUsers, SnapshotGroups: b.config.SnapshotGroups, Ctx: b.config.ctx, + GeneratedData: generatedData, }, &awscommon.StepCreateTags{ Tags: b.config.AMITags, @@ -466,6 +473,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack Amis: state.Get("amis").(map[string]string), BuilderIdValue: BuilderId, Session: session, + StateData: map[string]interface{}{"generated_data": state.Get("generated_data")}, } return artifact, nil diff --git a/builder/amazon/chroot/builder_test.go b/builder/amazon/chroot/builder_test.go index b181b878f..dcab9e92b 100644 --- a/builder/amazon/chroot/builder_test.go +++ b/builder/amazon/chroot/builder_test.go @@ -209,3 +209,28 @@ func TestBuilderPrepare_RootDeviceNameNoAMIMappings(t *testing.T) { t.Fatalf("should have error") } } + +func TestBuilderPrepare_ReturnGeneratedData(t *testing.T) { + var b Builder + config := testConfig() + + generatedData, warnings, err := b.Prepare(config) + if len(warnings) > 0 { + t.Fatalf("bad: %#v", warnings) + } + if err != nil { + t.Fatalf("should not have error: %s", err) + } + if len(generatedData) == 0 { + t.Fatalf("Generated data should not be empty") + } + if generatedData[0] != "SourceAMIName" { + t.Fatalf("Generated data should contain SourceAMIName") + } + if generatedData[1] != "Device" { + t.Fatalf("Generated data should contain Device") + } + if generatedData[2] != "MountPath" { + t.Fatalf("Generated data should contain MountPath") + } +} diff --git a/builder/amazon/chroot/step_mount_device.go b/builder/amazon/chroot/step_mount_device.go index 052a2d90a..262789620 100644 --- a/builder/amazon/chroot/step_mount_device.go +++ b/builder/amazon/chroot/step_mount_device.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "github.com/hashicorp/packer/builder" "log" "os" "path/filepath" @@ -29,7 +30,8 @@ type StepMountDevice struct { MountOptions []string MountPartition string - mountPath string + mountPath string + GeneratedData *builder.GeneratedData } func (s *StepMountDevice) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { @@ -118,6 +120,7 @@ func (s *StepMountDevice) Run(ctx context.Context, state multistep.StateBag) mul // Set the mount path so we remember to unmount it later s.mountPath = mountPath state.Put("mount_path", s.mountPath) + s.GeneratedData.Put("MountPath", s.mountPath) state.Put("mount_device_cleanup", s) return multistep.ActionContinue diff --git a/builder/amazon/chroot/step_prepare_device.go b/builder/amazon/chroot/step_prepare_device.go index 7006d7f49..cbc40d0f6 100644 --- a/builder/amazon/chroot/step_prepare_device.go +++ b/builder/amazon/chroot/step_prepare_device.go @@ -3,6 +3,7 @@ package chroot import ( "context" "fmt" + "github.com/hashicorp/packer/builder" "log" "os" @@ -12,6 +13,7 @@ import ( // StepPrepareDevice finds an available device and sets it. type StepPrepareDevice struct { + GeneratedData *builder.GeneratedData } func (s *StepPrepareDevice) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { @@ -40,6 +42,7 @@ func (s *StepPrepareDevice) Run(ctx context.Context, state multistep.StateBag) m log.Printf("Device: %s", device) state.Put("device", device) + s.GeneratedData.Put("Device", device) return multistep.ActionContinue } diff --git a/builder/amazon/common/interpolate_build_info.go b/builder/amazon/common/interpolate_build_info.go index cba66fcfe..3a2ab4611 100644 --- a/builder/amazon/common/interpolate_build_info.go +++ b/builder/amazon/common/interpolate_build_info.go @@ -3,6 +3,7 @@ package common import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/hashicorp/packer/builder" "github.com/hashicorp/packer/helper/multistep" ) @@ -15,7 +16,7 @@ type BuildInfoTemplate struct { SourceAMITags map[string]string } -func extractBuildInfo(region string, state multistep.StateBag) *BuildInfoTemplate { +func extractBuildInfo(region string, state multistep.StateBag, generatedData *builder.GeneratedData) *BuildInfoTemplate { rawSourceAMI, hasSourceAMI := state.GetOk("source_image") if !hasSourceAMI { return &BuildInfoTemplate{ @@ -37,7 +38,6 @@ func extractBuildInfo(region string, state multistep.StateBag) *BuildInfoTemplat SourceAMIOwnerName: aws.StringValue(sourceAMI.ImageOwnerAlias), SourceAMITags: sourceAMITags, } - state.Put("generated_data", map[string]interface{}{"SourceAMIName": buildInfoTemplate.SourceAMIName}) - + generatedData.Put("SourceAMIName", buildInfoTemplate.SourceAMIName) return buildInfoTemplate } diff --git a/builder/amazon/common/interpolate_build_info_test.go b/builder/amazon/common/interpolate_build_info_test.go index 5a725fd58..7f72f82ea 100644 --- a/builder/amazon/common/interpolate_build_info_test.go +++ b/builder/amazon/common/interpolate_build_info_test.go @@ -1,6 +1,7 @@ package common import ( + "github.com/hashicorp/packer/builder" "reflect" "testing" @@ -33,9 +34,15 @@ func testState() multistep.StateBag { return state } +func testGeneratedData(state multistep.StateBag) builder.GeneratedData { + generatedData := builder.GeneratedData{State: state} + return generatedData +} + func TestInterpolateBuildInfo_extractBuildInfo_noSourceImage(t *testing.T) { state := testState() - buildInfo := extractBuildInfo("foo", state) + generatedData := testGeneratedData(state) + buildInfo := extractBuildInfo("foo", state, &generatedData) expected := BuildInfoTemplate{ BuildRegion: "foo", @@ -48,7 +55,8 @@ func TestInterpolateBuildInfo_extractBuildInfo_noSourceImage(t *testing.T) { func TestInterpolateBuildInfo_extractBuildInfo_withSourceImage(t *testing.T) { state := testState() state.Put("source_image", testImage()) - buildInfo := extractBuildInfo("foo", state) + generatedData := testGeneratedData(state) + buildInfo := extractBuildInfo("foo", state, &generatedData) expected := BuildInfoTemplate{ BuildRegion: "foo", @@ -69,11 +77,12 @@ func TestInterpolateBuildInfo_extractBuildInfo_withSourceImage(t *testing.T) { func TestInterpolateBuildInfo_extractBuildInfo_GeneratedDataWithSourceImageName(t *testing.T) { state := testState() state.Put("source_image", testImage()) - extractBuildInfo("foo", state) + generatedData := testGeneratedData(state) + extractBuildInfo("foo", state, &generatedData) - generatedData := state.Get("generated_data").(map[string]interface{}) + generatedDataState := state.Get("generated_data").(map[string]interface{}) - if generatedData["SourceAMIName"] != "ami_test_name" { - t.Fatalf("Unexpected state SourceAMIName: expected %#v got %#v\n", "ami_test_name", generatedData["SourceAMIName"]) + if generatedDataState["SourceAMIName"] != "ami_test_name" { + t.Fatalf("Unexpected state SourceAMIName: expected %#v got %#v\n", "ami_test_name", generatedDataState["SourceAMIName"]) } } diff --git a/builder/amazon/common/step_modify_ami_attributes.go b/builder/amazon/common/step_modify_ami_attributes.go index f6e969578..ba34037af 100644 --- a/builder/amazon/common/step_modify_ami_attributes.go +++ b/builder/amazon/common/step_modify_ami_attributes.go @@ -3,10 +3,10 @@ package common import ( "context" "fmt" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/hashicorp/packer/builder" "github.com/hashicorp/packer/helper/multistep" "github.com/hashicorp/packer/packer" "github.com/hashicorp/packer/template/interpolate" @@ -20,6 +20,8 @@ type StepModifyAMIAttributes struct { ProductCodes []string Description string Ctx interpolate.Context + + GeneratedData *builder.GeneratedData } func (s *StepModifyAMIAttributes) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { @@ -43,7 +45,7 @@ func (s *StepModifyAMIAttributes) Run(ctx context.Context, state multistep.State } var err error - s.Ctx.Data = extractBuildInfo(*ec2conn.Config.Region, state) + s.Ctx.Data = extractBuildInfo(*ec2conn.Config.Region, state, s.GeneratedData) s.Description, err = interpolate.Render(s.Description, &s.Ctx) if err != nil { err = fmt.Errorf("Error interpolating AMI description: %s", err) diff --git a/builder/amazon/common/tags.go b/builder/amazon/common/tags.go index 750ad1184..6a171d759 100644 --- a/builder/amazon/common/tags.go +++ b/builder/amazon/common/tags.go @@ -5,6 +5,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/hashicorp/packer/builder" "github.com/hashicorp/packer/helper/multistep" "github.com/hashicorp/packer/packer" "github.com/hashicorp/packer/template/interpolate" @@ -26,7 +27,8 @@ func (t TagMap) IsSet() bool { func (t TagMap) EC2Tags(ictx interpolate.Context, region string, state multistep.StateBag) (EC2Tags, error) { var ec2Tags []*ec2.Tag - ictx.Data = extractBuildInfo(region, state) + generatedData := builder.GeneratedData{State: state} + ictx.Data = extractBuildInfo(region, state, &generatedData) for key, value := range t { interpolatedKey, err := interpolate.Render(key, &ictx) diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index f02fa6a68..895e0d953 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -11,10 +11,10 @@ package ebs import ( "context" "fmt" - "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/iam" "github.com/hashicorp/hcl/v2/hcldec" + "github.com/hashicorp/packer/builder" awscommon "github.com/hashicorp/packer/builder/amazon/common" "github.com/hashicorp/packer/common" "github.com/hashicorp/packer/helper/communicator" @@ -154,6 +154,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack state.Put("awsSession", session) state.Put("hook", hook) state.Put("ui", ui) + generatedData := &builder.GeneratedData{State: state} var instanceStep multistep.Step @@ -301,6 +302,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack SnapshotUsers: b.config.SnapshotUsers, SnapshotGroups: b.config.SnapshotGroups, Ctx: b.config.ctx, + GeneratedData: generatedData, }, &awscommon.StepCreateTags{ Tags: b.config.AMITags, diff --git a/builder/amazon/ebssurrogate/builder.go b/builder/amazon/ebssurrogate/builder.go index 3a7d762cd..2de9fd943 100644 --- a/builder/amazon/ebssurrogate/builder.go +++ b/builder/amazon/ebssurrogate/builder.go @@ -9,10 +9,10 @@ import ( "context" "errors" "fmt" - "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/iam" "github.com/hashicorp/hcl/v2/hcldec" + "github.com/hashicorp/packer/builder" awscommon "github.com/hashicorp/packer/builder/amazon/common" "github.com/hashicorp/packer/common" "github.com/hashicorp/packer/helper/communicator" @@ -177,6 +177,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack state.Put("awsSession", session) state.Put("hook", hook) state.Put("ui", ui) + generatedData := &builder.GeneratedData{State: state} var instanceStep multistep.Step @@ -335,6 +336,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack SnapshotUsers: b.config.SnapshotUsers, SnapshotGroups: b.config.SnapshotGroups, Ctx: b.config.ctx, + GeneratedData: generatedData, }, &awscommon.StepCreateTags{ Tags: b.config.AMITags, diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index f06f22d77..ae098926c 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -9,6 +9,7 @@ import ( "context" "errors" "fmt" + "github.com/hashicorp/packer/builder" "os" "strings" @@ -248,6 +249,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack state.Put("awsSession", session) state.Put("hook", hook) state.Put("ui", ui) + generatedData := &builder.GeneratedData{State: state} var instanceStep multistep.Step @@ -383,6 +385,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack SnapshotUsers: b.config.SnapshotUsers, SnapshotGroups: b.config.SnapshotGroups, Ctx: b.config.ctx, + GeneratedData: generatedData, }, &awscommon.StepCreateTags{ Tags: b.config.AMITags, diff --git a/builder/generated_data.go b/builder/generated_data.go new file mode 100644 index 000000000..9b84c8d2a --- /dev/null +++ b/builder/generated_data.go @@ -0,0 +1,20 @@ +package builder + +import "github.com/hashicorp/packer/helper/multistep" + +// GeneratedData manages variables exported by a builder after +// it started. It uses the builder's multistep.StateBag internally, make sure it +// is not nil before calling any functions. +type GeneratedData struct { + // The builder's StateBag + State multistep.StateBag +} + +func (gd *GeneratedData) Put(key string, data interface{}) { + genData := make(map[string]interface{}) + if _, ok := gd.State.GetOk("generated_data"); ok { + genData = gd.State.Get("generated_data").(map[string]interface{}) + } + genData[key] = data + gd.State.Put("generated_data", genData) +} diff --git a/builder/generated_data_test.go b/builder/generated_data_test.go new file mode 100644 index 000000000..998569237 --- /dev/null +++ b/builder/generated_data_test.go @@ -0,0 +1,30 @@ +package builder + +import ( + "github.com/hashicorp/packer/helper/multistep" + "testing" +) + +func TestGeneratedData_Put(t *testing.T) { + state := new(multistep.BasicStateBag) + generatedData := GeneratedData{ + State: state, + } + expectedValue := "data value" + secondExpectedValue := "another data value" + + generatedData.Put("data_key", expectedValue) + generatedData.Put("another_data_key", secondExpectedValue) + + if _, ok := generatedData.State.GetOk("generated_data"); !ok { + t.Fatalf("BAD: StateBag should contain generated_data") + } + + generatedDataState := generatedData.State.Get("generated_data").(map[string]interface{}) + if generatedDataState["data_key"] != expectedValue { + t.Fatalf("Unexpected state for data_key: expected %#v got %#v\n", expectedValue, generatedDataState["data_key"]) + } + if generatedDataState["another_data_key"] != secondExpectedValue { + t.Fatalf("Unexpected state for another_data_key: expected %#v got %#v\n", secondExpectedValue, generatedDataState["another_data_key"]) + } +} diff --git a/common/chroot/step_chroot_provision.go b/common/chroot/step_chroot_provision.go index a90c79767..19a6ec8aa 100644 --- a/common/chroot/step_chroot_provision.go +++ b/common/chroot/step_chroot_provision.go @@ -25,9 +25,12 @@ func (s *StepChrootProvision) Run(ctx context.Context, state multistep.StateBag) CmdWrapper: wrappedCommand, } + // Loads hook data from builder's state, if it has been set. + hookData := common.PopulateProvisionHookData(state) + // Provision log.Println("Running the provision hook") - if err := hook.Run(ctx, packer.HookProvision, ui, comm, nil); err != nil { + if err := hook.Run(ctx, packer.HookProvision, ui, comm, hookData); err != nil { state.Put("error", err) return multistep.ActionHalt } diff --git a/common/step_provision.go b/common/step_provision.go index af15b3dad..5a45fb720 100644 --- a/common/step_provision.go +++ b/common/step_provision.go @@ -52,7 +52,7 @@ func PopulateProvisionHookData(state multistep.StateBag) map[string]interface{} // Read communicator data into hook data comm, ok := state.GetOk("communicator_config") if !ok { - log.Printf("Unable to load config from state to populate provisionHookData") + log.Printf("Unable to load communicator config from state to populate provisionHookData") return hookData } commConf := comm.(*communicator.Config)