From c33e7cc8674e4208853ef7ee40b40251b4897cf1 Mon Sep 17 00:00:00 2001 From: Henry Huang Date: Thu, 8 May 2014 01:13:27 +0800 Subject: [PATCH 1/8] Add the support of launching spot instances in "amazon-ebs" AMI --- builder/amazon/common/run_config.go | 1 + builder/amazon/common/state.go | 26 +++++ .../amazon/common/step_run_source_instance.go | 94 +++++++++++++++---- builder/amazon/ebs/builder.go | 3 +- builder/amazon/ebs/step_stop_instance.go | 10 +- 5 files changed, 113 insertions(+), 21 deletions(-) diff --git a/builder/amazon/common/run_config.go b/builder/amazon/common/run_config.go index c50c22f7e..95fc7b3be 100644 --- a/builder/amazon/common/run_config.go +++ b/builder/amazon/common/run_config.go @@ -17,6 +17,7 @@ type RunConfig struct { InstanceType string `mapstructure:"instance_type"` RunTags map[string]string `mapstructure:"run_tags"` SourceAmi string `mapstructure:"source_ami"` + SpotPrice string `mapstructure:"spot_price"` RawSSHTimeout string `mapstructure:"ssh_timeout"` SSHUsername string `mapstructure:"ssh_username"` SSHPrivateKeyFile string `mapstructure:"ssh_private_key_file"` diff --git a/builder/amazon/common/state.go b/builder/amazon/common/state.go index 688a918a5..bab45d5db 100644 --- a/builder/amazon/common/state.go +++ b/builder/amazon/common/state.go @@ -81,6 +81,32 @@ func InstanceStateRefreshFunc(conn *ec2.EC2, i *ec2.Instance) StateRefreshFunc { } } +// SpotRequestStateRefreshFunc returns a StateRefreshFunc that is used to watch +// a spot request for state changes. +func SpotRequestStateRefreshFunc(conn *ec2.EC2, spotRequestId string) StateRefreshFunc { + return func() (interface{}, string, error) { + resp, err := conn.DescribeSpotRequests([]string{spotRequestId}, ec2.NewFilter()) + if err != nil { + if ec2err, ok := err.(*ec2.Error); ok && ec2err.Code == "InvalidSpotInstanceRequestID.NotFound" { + // Set this to nil as if we didn't find anything. + resp = nil + } else { + log.Printf("Error on SpotRequestStateRefresh: %s", err) + return nil, "", err + } + } + + if resp == nil || len(resp.SpotRequestResults) == 0 { + // Sometimes AWS has consistency issues and doesn't see the + // SpotRequest. Return an empty state. + return nil, "", nil + } + + i := resp.SpotRequestResults[0] + return i, i.State, nil + } +} + // WaitForState watches an object and waits for it to achieve a certain // state. func WaitForState(conf *StateChangeConf) (i interface{}, err error) { diff --git a/builder/amazon/common/step_run_source_instance.go b/builder/amazon/common/step_run_source_instance.go index c9827e589..60289af4c 100644 --- a/builder/amazon/common/step_run_source_instance.go +++ b/builder/amazon/common/step_run_source_instance.go @@ -10,6 +10,7 @@ import ( type StepRunSourceInstance struct { AssociatePublicIpAddress bool + SpotPrice string AvailabilityZone string BlockDevices BlockDevices Debug bool @@ -47,21 +48,6 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi securityGroups[n] = ec2.SecurityGroup{Id: securityGroupId} } - runOpts := &ec2.RunInstances{ - KeyName: keyName, - ImageId: s.SourceAMI, - InstanceType: s.InstanceType, - UserData: []byte(userData), - MinCount: 0, - MaxCount: 0, - SecurityGroups: securityGroups, - IamInstanceProfile: s.IamInstanceProfile, - SubnetId: s.SubnetId, - AssociatePublicIpAddress: s.AssociatePublicIpAddress, - BlockDevices: s.BlockDevices.BuildLaunchDevices(), - AvailZone: s.AvailabilityZone, - } - ui.Say("Launching a source AWS instance...") imageResp, err := ec2conn.Images([]string{s.SourceAMI}, ec2.NewFilter()) if err != nil { @@ -82,15 +68,85 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi return multistep.ActionHalt } - runResp, err := ec2conn.RunInstances(runOpts) + var instanceId []string + if s.SpotPrice == "" { + runOpts := &ec2.RunInstances{ + KeyName: keyName, + ImageId: s.SourceAMI, + InstanceType: s.InstanceType, + UserData: []byte(userData), + MinCount: 0, + MaxCount: 0, + SecurityGroups: securityGroups, + IamInstanceProfile: s.IamInstanceProfile, + SubnetId: s.SubnetId, + AssociatePublicIpAddress: s.AssociatePublicIpAddress, + BlockDevices: s.BlockDevices.BuildLaunchDevices(), + AvailZone: s.AvailabilityZone, + } + runResp, err := ec2conn.RunInstances(runOpts) + if err != nil { + err := fmt.Errorf("Error launching source instance: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + instanceId = []string{runResp.Instances[0].InstanceId} + + } else { + runOpts := &ec2.RequestSpotInstances{ + SpotPrice: s.SpotPrice, + KeyName: keyName, + ImageId: s.SourceAMI, + InstanceType: s.InstanceType, + UserData: []byte(userData), + SecurityGroups: securityGroups, + IamInstanceProfile: s.IamInstanceProfile, + SubnetId: s.SubnetId, + AssociatePublicIpAddress: s.AssociatePublicIpAddress, + BlockDevices: s.BlockDevices.BuildLaunchDevices(), + AvailZone: s.AvailabilityZone, + } + runSpotResp, err := ec2conn.RequestSpotInstances(runOpts) + if err != nil { + err := fmt.Errorf("Error launching source spot instance: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + spotRequestId := runSpotResp.SpotRequestResults[0].SpotRequestId + ui.Say(fmt.Sprintf("Waiting for spot request (%s) to become ready...", spotRequestId)) + stateChange := StateChangeConf{ + Pending: []string{"open"}, + Target: "active", + Refresh: SpotRequestStateRefreshFunc(ec2conn, spotRequestId), + StepState: state, + } + _, err = WaitForState(&stateChange) + if err != nil { + err := fmt.Errorf("Error waiting for spot request (%s) to become ready: %s", spotRequestId, err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + spotResp, err := ec2conn.DescribeSpotRequests([]string{spotRequestId}, nil) + if err != nil { + err := fmt.Errorf("Error finding spot request (%s): %s", spotRequestId, err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + instanceId = []string{spotResp.SpotRequestResults[0].InstanceId} + } + + instanceResp, err := ec2conn.Instances(instanceId, nil) if err != nil { - err := fmt.Errorf("Error launching source instance: %s", err) + err := fmt.Errorf("Error finding source instance (%s): %s", instanceId, err) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } - - s.instance = &runResp.Instances[0] + s.instance = &instanceResp.Reservations[0].Instances[0] ui.Message(fmt.Sprintf("Instance ID: %s", s.instance.InstanceId)) ec2Tags := make([]ec2.Tag, 1, len(s.Tags)+1) diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index 2e898671e..a5240a61b 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -96,6 +96,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &awscommon.StepRunSourceInstance{ Debug: b.config.PackerDebug, ExpectedRootDevice: "ebs", + SpotPrice: b.config.SpotPrice, InstanceType: b.config.InstanceType, UserData: b.config.UserData, UserDataFile: b.config.UserDataFile, @@ -113,7 +114,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe SSHWaitTimeout: b.config.SSHTimeout(), }, &common.StepProvision{}, - &stepStopInstance{}, + &stepStopInstance{SpotPrice: b.config.SpotPrice}, &stepCreateAMI{}, &awscommon.StepAMIRegionCopy{ Regions: b.config.AMIRegions, diff --git a/builder/amazon/ebs/step_stop_instance.go b/builder/amazon/ebs/step_stop_instance.go index 8603fb807..c64e23c7d 100644 --- a/builder/amazon/ebs/step_stop_instance.go +++ b/builder/amazon/ebs/step_stop_instance.go @@ -8,13 +8,21 @@ import ( "github.com/mitchellh/packer/packer" ) -type stepStopInstance struct{} +type stepStopInstance struct{ + SpotPrice string +} func (s *stepStopInstance) Run(state multistep.StateBag) multistep.StepAction { ec2conn := state.Get("ec2").(*ec2.EC2) instance := state.Get("instance").(*ec2.Instance) ui := state.Get("ui").(packer.Ui) + // Skip when it is a spot instance + if s.SpotPrice != "" { + ui.Say(fmt.Sprintf("This is a spot instance, no need to stop for the AMI")) + return multistep.ActionContinue + } + // Stop the instance so we can create an AMI from it ui.Say("Stopping the source instance...") _, err := ec2conn.StopInstances(instance.InstanceId) From a411405628705808068629ee3d6f55d2837e88fd Mon Sep 17 00:00:00 2001 From: Henry Huang Date: Thu, 8 May 2014 13:58:05 +0800 Subject: [PATCH 2/8] Clean up code via "go fmt" --- builder/amazon/ebs/step_stop_instance.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/builder/amazon/ebs/step_stop_instance.go b/builder/amazon/ebs/step_stop_instance.go index c64e23c7d..c266b46d4 100644 --- a/builder/amazon/ebs/step_stop_instance.go +++ b/builder/amazon/ebs/step_stop_instance.go @@ -8,8 +8,8 @@ import ( "github.com/mitchellh/packer/packer" ) -type stepStopInstance struct{ - SpotPrice string +type stepStopInstance struct { + SpotPrice string } func (s *stepStopInstance) Run(state multistep.StateBag) multistep.StepAction { @@ -17,11 +17,11 @@ func (s *stepStopInstance) Run(state multistep.StateBag) multistep.StepAction { instance := state.Get("instance").(*ec2.Instance) ui := state.Get("ui").(packer.Ui) - // Skip when it is a spot instance - if s.SpotPrice != "" { + // Skip when it is a spot instance + if s.SpotPrice != "" { ui.Say(fmt.Sprintf("This is a spot instance, no need to stop for the AMI")) - return multistep.ActionContinue - } + return multistep.ActionContinue + } // Stop the instance so we can create an AMI from it ui.Say("Stopping the source instance...") From 3a74c469e5fb53aee66caa1aa3638dbe278a21f1 Mon Sep 17 00:00:00 2001 From: Henry Huang Date: Sat, 10 May 2014 18:08:46 +0800 Subject: [PATCH 3/8] Add "spot_price" param into template processing --- builder/amazon/common/run_config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/builder/amazon/common/run_config.go b/builder/amazon/common/run_config.go index 95fc7b3be..42be279a9 100644 --- a/builder/amazon/common/run_config.go +++ b/builder/amazon/common/run_config.go @@ -91,6 +91,7 @@ func (c *RunConfig) Prepare(t *packer.ConfigTemplate) []error { templates := map[string]*string{ "iam_instance_profile": &c.IamInstanceProfile, "instance_type": &c.InstanceType, + "spot_price": &c.SpotPrice, "ssh_timeout": &c.RawSSHTimeout, "ssh_username": &c.SSHUsername, "ssh_private_key_file": &c.SSHPrivateKeyFile, From 5d410bddc9b7d353827bc04657a9edaa60ff4c60 Mon Sep 17 00:00:00 2001 From: Henry Huang Date: Sun, 11 May 2014 00:40:50 +0800 Subject: [PATCH 4/8] When unexpected state found in waiting, notify the caller func with a correct error message --- builder/amazon/common/state.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/builder/amazon/common/state.go b/builder/amazon/common/state.go index bab45d5db..4f7d4b11d 100644 --- a/builder/amazon/common/state.go +++ b/builder/amazon/common/state.go @@ -151,8 +151,8 @@ func WaitForState(conf *StateChangeConf) (i interface{}, err error) { } if !found { - fmt.Errorf("unexpected state '%s', wanted target '%s'", currentState, conf.Target) - return + err := fmt.Errorf("unexpected state '%s', wanted target '%s'", currentState, conf.Target) + return nil, err } } From 1e1fa3de26c0d7ce18f2a27563761f983b15fabb Mon Sep 17 00:00:00 2001 From: Henry Huang Date: Sun, 11 May 2014 00:51:35 +0800 Subject: [PATCH 5/8] Add the cleanup for existing spot request --- .../amazon/common/step_run_source_instance.go | 51 ++++++++++++------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/builder/amazon/common/step_run_source_instance.go b/builder/amazon/common/step_run_source_instance.go index 60289af4c..802371ee3 100644 --- a/builder/amazon/common/step_run_source_instance.go +++ b/builder/amazon/common/step_run_source_instance.go @@ -22,8 +22,8 @@ type StepRunSourceInstance struct { Tags map[string]string UserData string UserDataFile string - - instance *ec2.Instance + spotRequest *ec2.SpotRequestResult + instance *ec2.Instance } func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepAction { @@ -92,7 +92,6 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi return multistep.ActionHalt } instanceId = []string{runResp.Instances[0].InstanceId} - } else { runOpts := &ec2.RequestSpotInstances{ SpotPrice: s.SpotPrice, @@ -136,7 +135,8 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi ui.Error(err.Error()) return multistep.ActionHalt } - instanceId = []string{spotResp.SpotRequestResults[0].InstanceId} + s.spotRequest = &spotResp.SpotRequestResults[0] + instanceId = []string{s.spotRequest.InstanceId} } instanceResp, err := ec2conn.Instances(instanceId, nil) @@ -198,24 +198,41 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi } func (s *StepRunSourceInstance) Cleanup(state multistep.StateBag) { - if s.instance == nil { - return - } ec2conn := state.Get("ec2").(*ec2.EC2) ui := state.Get("ui").(packer.Ui) - ui.Say("Terminating the source AWS instance...") - if _, err := ec2conn.TerminateInstances([]string{s.instance.InstanceId}); err != nil { - ui.Error(fmt.Sprintf("Error terminating instance, may still be around: %s", err)) - return - } + // Cancel the spot request if it exists + if s.spotRequest != nil { + ui.Say("Cancelling the spot request...") + if _, err := ec2conn.CancelSpotRequests([]string{s.spotRequest.SpotRequestId}); err != nil { + ui.Error(fmt.Sprintf("Error cancelling the spot request, may still be around: %s", err)) + return + } + stateChange := StateChangeConf{ + Pending: []string{"active", "open"}, + Refresh: SpotRequestStateRefreshFunc(ec2conn, s.spotRequest.SpotRequestId), + Target: "cancelled", + } + + WaitForState(&stateChange) - stateChange := StateChangeConf{ - Pending: []string{"pending", "running", "shutting-down", "stopped", "stopping"}, - Refresh: InstanceStateRefreshFunc(ec2conn, s.instance), - Target: "terminated", } - WaitForState(&stateChange) + // Terminate the source instance if it exists + if s.instance != nil { + + ui.Say("Terminating the source AWS instance...") + if _, err := ec2conn.TerminateInstances([]string{s.instance.InstanceId}); err != nil { + ui.Error(fmt.Sprintf("Error terminating instance, may still be around: %s", err)) + return + } + stateChange := StateChangeConf{ + Pending: []string{"pending", "running", "shutting-down", "stopped", "stopping"}, + Refresh: InstanceStateRefreshFunc(ec2conn, s.instance), + Target: "terminated", + } + + WaitForState(&stateChange) + } } From 3980c7dcfddc346a0760c08abfddbf6ce2eaff7a Mon Sep 17 00:00:00 2001 From: Henry Huang Date: Mon, 12 May 2014 12:51:13 +0800 Subject: [PATCH 6/8] Save the spot request before waiting for its active state --- builder/amazon/common/step_run_source_instance.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/builder/amazon/common/step_run_source_instance.go b/builder/amazon/common/step_run_source_instance.go index 802371ee3..b3c7e9ad9 100644 --- a/builder/amazon/common/step_run_source_instance.go +++ b/builder/amazon/common/step_run_source_instance.go @@ -113,7 +113,8 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi ui.Error(err.Error()) return multistep.ActionHalt } - spotRequestId := runSpotResp.SpotRequestResults[0].SpotRequestId + s.spotRequest = &runSpotResp.SpotRequestResults[0] + spotRequestId := s.spotRequest.SpotRequestId ui.Say(fmt.Sprintf("Waiting for spot request (%s) to become ready...", spotRequestId)) stateChange := StateChangeConf{ Pending: []string{"open"}, @@ -135,8 +136,7 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi ui.Error(err.Error()) return multistep.ActionHalt } - s.spotRequest = &spotResp.SpotRequestResults[0] - instanceId = []string{s.spotRequest.InstanceId} + instanceId = []string{spotResp.SpotRequestResults[0].InstanceId} } instanceResp, err := ec2conn.Instances(instanceId, nil) From da29d684a8cb9e271ba830b124bd17d731cabf94 Mon Sep 17 00:00:00 2001 From: Henry Huang Date: Mon, 12 May 2014 12:54:06 +0800 Subject: [PATCH 7/8] Enable the spot_price in "amazon-instance-store" AMI --- builder/amazon/instance/builder.go | 1 + 1 file changed, 1 insertion(+) diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index 81be72f72..47c5826c0 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -200,6 +200,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &awscommon.StepRunSourceInstance{ Debug: b.config.PackerDebug, ExpectedRootDevice: "instance-store", + SpotPrice: b.config.SpotPrice, InstanceType: b.config.InstanceType, IamInstanceProfile: b.config.IamInstanceProfile, UserData: b.config.UserData, From bf7b8199daac8940a920f800e959ba15ebe51bbc Mon Sep 17 00:00:00 2001 From: Henry Huang Date: Mon, 26 May 2014 23:03:49 +0800 Subject: [PATCH 8/8] Add the document for the new spot_price parameter --- website/source/docs/builders/amazon-ebs.html.markdown | 7 +++++++ website/source/docs/builders/amazon-instance.html.markdown | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/website/source/docs/builders/amazon-ebs.html.markdown b/website/source/docs/builders/amazon-ebs.html.markdown index f46e2d0da..012e0fb9a 100644 --- a/website/source/docs/builders/amazon-ebs.html.markdown +++ b/website/source/docs/builders/amazon-ebs.html.markdown @@ -117,6 +117,13 @@ each category, the available configuration keys are alphabetized. described above. Note that if this is specified, you must omit the security_group_id. +* `spot_price` (string) - The maximum hourly price to launch a spot instance + to create the AMI. It is a type of instances that EC2 starts when the maximum + price that you specify exceeds the current spot price. Spot price will be + updated based on available spot instance capacity and current spot Instance + requests. It may save you some costs. For example, it takes only "0.001" to + launch a spot "m3.medium" instance while "0.07" needed for on-demand. + * `ssh_port` (integer) - The port that SSH will be available on. This defaults to port 22. diff --git a/website/source/docs/builders/amazon-instance.html.markdown b/website/source/docs/builders/amazon-instance.html.markdown index 238eaead2..8f9b6f407 100644 --- a/website/source/docs/builders/amazon-instance.html.markdown +++ b/website/source/docs/builders/amazon-instance.html.markdown @@ -155,6 +155,13 @@ each category, the available configuration keys are alphabetized. described above. Note that if this is specified, you must omit the security_group_id. +* `spot_price` (string) - The maximum hourly price to launch a spot instance + to create the AMI. It is a type of instances that EC2 starts when the maximum + price that you specify exceeds the current spot price. Spot price will be + updated based on available spot instance capacity and current spot Instance + requests. It may save you some costs. For example, it takes only "0.001" to + launch a spot "m3.medium" instance while "0.07" needed for on-demand. + * `ssh_port` (integer) - The port that SSH will be available on. This defaults to port 22.