From f7e68f1d711ca4fcc2f1e0562a63e41566fa500b Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Sat, 9 Jan 2021 23:58:23 +1100 Subject: [PATCH 1/4] Add spot tags as resource tags of instances and volumes in launch template According to the APIReference: , the resource types `instance` and `volume` support tagging on creation. It is useful to add the spot tags here as it should be more reliable than tagging them after the spot request is fulfilled as we currently do. --- builder/amazon/common/step_run_spot_instance.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/builder/amazon/common/step_run_spot_instance.go b/builder/amazon/common/step_run_spot_instance.go index 210498eef..58ebec885 100644 --- a/builder/amazon/common/step_run_spot_instance.go +++ b/builder/amazon/common/step_run_spot_instance.go @@ -260,6 +260,16 @@ func (s *StepRunSpotInstance) Run(ctx context.Context, state multistep.StateBag) Tags: spotTags, }, } + launchTemplate.LaunchTemplateData.TagSpecifications = []*ec2.LaunchTemplateTagSpecificationRequest{ + { + ResourceType: aws.String("instance"), + Tags: spotTags, + }, + { + ResourceType: aws.String("volume"), + Tags: spotTags, + }, + } } // Tell EC2 to create the template From f7588a3737936ccd4aa4d460531be428018deda5 Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Sat, 9 Jan 2021 23:59:31 +1100 Subject: [PATCH 2/4] Add printing of launch template id --- builder/amazon/common/step_run_spot_instance.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/builder/amazon/common/step_run_spot_instance.go b/builder/amazon/common/step_run_spot_instance.go index 58ebec885..c67027ffe 100644 --- a/builder/amazon/common/step_run_spot_instance.go +++ b/builder/amazon/common/step_run_spot_instance.go @@ -273,7 +273,7 @@ func (s *StepRunSpotInstance) Run(ctx context.Context, state multistep.StateBag) } // Tell EC2 to create the template - _, err = ec2conn.CreateLaunchTemplate(launchTemplate) + createLaunchTemplateOutput, err := ec2conn.CreateLaunchTemplate(launchTemplate) if err != nil { err := fmt.Errorf("Error creating launch template for spot instance: %s", err) state.Put("error", err) @@ -281,6 +281,9 @@ func (s *StepRunSpotInstance) Run(ctx context.Context, state multistep.StateBag) return multistep.ActionHalt } + launchTemplateId := createLaunchTemplateOutput.LaunchTemplate.LaunchTemplateId + ui.Message(fmt.Sprintf("Created Spot Fleet launch template: %s", *launchTemplateId)) + // Add overrides for each user-provided instance type var overrides []*ec2.FleetLaunchTemplateOverridesRequest for _, instanceType := range s.SpotInstanceTypes { From bb94df2d02e172d4806a0f5245213bd89ed760b4 Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Sun, 10 Jan 2021 00:00:16 +1100 Subject: [PATCH 3/4] Add mocking of launch template id It is being printed after it is created so we need to mock it to prevent a nil pointer dereference when the tests are run with the launch template create request is mocked. --- .../amazon/common/step_run_spot_instance_test.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/builder/amazon/common/step_run_spot_instance_test.go b/builder/amazon/common/step_run_spot_instance_test.go index a30bc4702..71caa4bd3 100644 --- a/builder/amazon/common/step_run_spot_instance_test.go +++ b/builder/amazon/common/step_run_spot_instance_test.go @@ -190,7 +190,7 @@ func (m *runSpotEC2ConnMock) CreateTags(req *ec2.CreateTagsInput) (*ec2.CreateTa } } -func defaultEc2Mock(instanceId, spotRequestId, volumeId *string) *runSpotEC2ConnMock { +func defaultEc2Mock(instanceId, spotRequestId, volumeId, launchTemplateId *string) *runSpotEC2ConnMock { instance := &ec2.Instance{ InstanceId: instanceId, SpotInstanceRequestId: spotRequestId, @@ -205,8 +205,10 @@ func defaultEc2Mock(instanceId, spotRequestId, volumeId *string) *runSpotEC2Conn return &runSpotEC2ConnMock{ CreateLaunchTemplateFn: func(in *ec2.CreateLaunchTemplateInput) (*ec2.CreateLaunchTemplateOutput, error) { return &ec2.CreateLaunchTemplateOutput{ - LaunchTemplate: nil, - Warning: nil, + LaunchTemplate: &ec2.LaunchTemplate{ + LaunchTemplateId: launchTemplateId, + }, + Warning: nil, }, nil }, CreateFleetFn: func(*ec2.CreateFleetInput) (*ec2.CreateFleetOutput, error) { @@ -237,7 +239,8 @@ func TestRun(t *testing.T) { instanceId := aws.String("test-instance-id") spotRequestId := aws.String("spot-id") volumeId := aws.String("volume-id") - ec2Mock := defaultEc2Mock(instanceId, spotRequestId, volumeId) + launchTemplateId := aws.String("launchTemplateId") + ec2Mock := defaultEc2Mock(instanceId, spotRequestId, volumeId, launchTemplateId) uiMock := packersdk.TestUi(t) @@ -329,7 +332,8 @@ func TestRun_NoSpotTags(t *testing.T) { instanceId := aws.String("test-instance-id") spotRequestId := aws.String("spot-id") volumeId := aws.String("volume-id") - ec2Mock := defaultEc2Mock(instanceId, spotRequestId, volumeId) + launchTemplateId := aws.String("lt-id") + ec2Mock := defaultEc2Mock(instanceId, spotRequestId, volumeId, launchTemplateId) uiMock := packersdk.TestUi(t) From cf30b49e6c49e5b29dd34a9d84835ba2e323aecc Mon Sep 17 00:00:00 2001 From: Narthana Epa Date: Tue, 12 Jan 2021 20:46:41 +1100 Subject: [PATCH 4/4] Use run_tags (resp run_volume_tags) to tags intances (resp volumes) created from the spot instance launch template --- .../amazon/common/step_run_spot_instance.go | 41 +++++++++++-------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/builder/amazon/common/step_run_spot_instance.go b/builder/amazon/common/step_run_spot_instance.go index c67027ffe..2f454b7c0 100644 --- a/builder/amazon/common/step_run_spot_instance.go +++ b/builder/amazon/common/step_run_spot_instance.go @@ -210,6 +210,15 @@ func (s *StepRunSpotInstance) Run(ctx context.Context, state multistep.StateBag) // instance yet ec2Tags.Report(ui) + volumeTags, err := TagMap(s.VolumeTags).EC2Tags(s.Ctx, s.Region, state) + if err != nil { + err := fmt.Errorf("Error generating volume tags: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + volumeTags.Report(ui) + spotOptions := ec2.LaunchTemplateSpotMarketOptionsRequest{} // The default is to set the maximum price to the OnDemand price. if s.SpotPrice != "auto" { @@ -260,16 +269,26 @@ func (s *StepRunSpotInstance) Run(ctx context.Context, state multistep.StateBag) Tags: spotTags, }, } - launchTemplate.LaunchTemplateData.TagSpecifications = []*ec2.LaunchTemplateTagSpecificationRequest{ - { + } + + if len(ec2Tags) > 0 { + launchTemplate.LaunchTemplateData.TagSpecifications = append( + launchTemplate.LaunchTemplateData.TagSpecifications, + &ec2.LaunchTemplateTagSpecificationRequest{ ResourceType: aws.String("instance"), - Tags: spotTags, + Tags: ec2Tags, }, - { + ) + } + + if len(volumeTags) > 0 { + launchTemplate.LaunchTemplateData.TagSpecifications = append( + launchTemplate.LaunchTemplateData.TagSpecifications, + &ec2.LaunchTemplateTagSpecificationRequest{ ResourceType: aws.String("volume"), - Tags: spotTags, + Tags: volumeTags, }, - } + ) } // Tell EC2 to create the template @@ -450,16 +469,6 @@ func (s *StepRunSpotInstance) Run(ctx context.Context, state multistep.StateBag) if len(volumeIds) > 0 && len(s.VolumeTags) > 0 { ui.Say("Adding tags to source EBS Volumes") - - volumeTags, err := TagMap(s.VolumeTags).EC2Tags(s.Ctx, s.Region, state) - if err != nil { - err := fmt.Errorf("Error tagging source EBS Volumes on %s: %s", *instance.InstanceId, err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } - volumeTags.Report(ui) - _, err = ec2conn.CreateTags(&ec2.CreateTagsInput{ Resources: volumeIds, Tags: volumeTags,