From db90c16118201cf44d1030fbd71326b5dfae2e2b Mon Sep 17 00:00:00 2001 From: Mitchell Hashimoto Date: Sat, 6 Sep 2014 10:44:12 -0700 Subject: [PATCH] builder/amazon: support auto spot price discovery [GH-1465] --- builder/amazon/common/run_config.go | 9 +++ builder/amazon/common/run_config_test.go | 13 ++++ .../amazon/common/step_run_source_instance.go | 72 ++++++++++++++++--- builder/amazon/ebs/builder.go | 1 + builder/amazon/instance/builder.go | 1 + .../docs/builders/amazon-ebs.html.markdown | 9 ++- .../builders/amazon-instance.html.markdown | 9 ++- 7 files changed, 99 insertions(+), 15 deletions(-) diff --git a/builder/amazon/common/run_config.go b/builder/amazon/common/run_config.go index 6cd2f3e73..a71387623 100644 --- a/builder/amazon/common/run_config.go +++ b/builder/amazon/common/run_config.go @@ -20,6 +20,7 @@ type RunConfig struct { RunTags map[string]string `mapstructure:"run_tags"` SourceAmi string `mapstructure:"source_ami"` SpotPrice string `mapstructure:"spot_price"` + SpotPriceAutoProduct string `mapstructure:"spot_price_auto_product"` RawSSHTimeout string `mapstructure:"ssh_timeout"` SSHUsername string `mapstructure:"ssh_username"` SSHPrivateKeyFile string `mapstructure:"ssh_private_key_file"` @@ -50,6 +51,7 @@ func (c *RunConfig) Prepare(t *packer.ConfigTemplate) []error { "iam_instance_profile": &c.IamInstanceProfile, "instance_type": &c.InstanceType, "spot_price": &c.SpotPrice, + "spot_price_auto_product": &c.SpotPriceAutoProduct, "ssh_timeout": &c.RawSSHTimeout, "ssh_username": &c.SSHUsername, "ssh_private_key_file": &c.SSHPrivateKeyFile, @@ -97,6 +99,13 @@ func (c *RunConfig) Prepare(t *packer.ConfigTemplate) []error { errs = append(errs, errors.New("An instance_type must be specified")) } + if c.SpotPrice == "auto" { + if c.SpotPriceAutoProduct == "" { + errs = append(errs, errors.New( + "spot_price_auto_product must be specified when spot_price is auto")) + } + } + if c.SSHUsername == "" { errs = append(errs, errors.New("An ssh_username must be specified")) } diff --git a/builder/amazon/common/run_config_test.go b/builder/amazon/common/run_config_test.go index 1d376b1dd..8e9c4b6b9 100644 --- a/builder/amazon/common/run_config_test.go +++ b/builder/amazon/common/run_config_test.go @@ -47,6 +47,19 @@ func TestRunConfigPrepare_SourceAmi(t *testing.T) { } } +func TestRunConfigPrepare_SpotAuto(t *testing.T) { + c := testConfig() + c.SpotPrice = "auto" + if err := c.Prepare(nil); len(err) != 1 { + t.Fatalf("err: %s", err) + } + + c.SpotPriceAutoProduct = "foo" + if err := c.Prepare(nil); len(err) != 0 { + t.Fatalf("err: %s", err) + } +} + func TestRunConfigPrepare_SSHPort(t *testing.T) { c := testConfig() c.SSHPort = 0 diff --git a/builder/amazon/common/step_run_source_instance.go b/builder/amazon/common/step_run_source_instance.go index 7e8627095..50cedf6ea 100644 --- a/builder/amazon/common/step_run_source_instance.go +++ b/builder/amazon/common/step_run_source_instance.go @@ -2,15 +2,18 @@ package common import ( "fmt" + "io/ioutil" + "log" + "strconv" + "time" + "github.com/mitchellh/goamz/ec2" "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" - "io/ioutil" ) type StepRunSourceInstance struct { AssociatePublicIpAddress bool - SpotPrice string AvailabilityZone string BlockDevices BlockDevices Debug bool @@ -18,12 +21,15 @@ type StepRunSourceInstance struct { InstanceType string IamInstanceProfile string SourceAMI string + SpotPrice string + SpotPriceProduct string SubnetId string Tags map[string]string UserData string UserDataFile string - spotRequest *ec2.SpotRequestResult - instance *ec2.Instance + + instance *ec2.Instance + spotRequest *ec2.SpotRequestResult } func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepAction { @@ -68,8 +74,52 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi return multistep.ActionHalt } - var instanceId []string - if s.SpotPrice == "" { + spotPrice := s.SpotPrice + if spotPrice == "auto" { + ui.Message(fmt.Sprintf( + "Finding spot price for %s %s...", + s.SpotPriceProduct, s.InstanceType)) + + // Detect the spot price + startTime := time.Now().Add(-1 * time.Hour) + resp, err := ec2conn.DescribeSpotPriceHistory(&ec2.DescribeSpotPriceHistory{ + InstanceType: []string{s.InstanceType}, + ProductDescription: []string{s.SpotPriceProduct}, + AvailabilityZone: s.AvailabilityZone, + StartTime: startTime, + }) + if err != nil { + err := fmt.Errorf("Error finding spot price: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + var price float64 + for _, history := range resp.History { + log.Printf("[INFO] Candidate spot price: %s", history.SpotPrice) + current, err := strconv.ParseFloat(history.SpotPrice, 64) + if err != nil { + log.Printf("[ERR] Error parsing spot price: %s", err) + continue + } + if price == 0 || current < price { + price = current + } + } + if price == 0 { + err := fmt.Errorf("No candidate spot prices found!") + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + spotPrice = strconv.FormatFloat(price, 'f', -1, 64) + } + + var instanceId string + + if spotPrice == "" { runOpts := &ec2.RunInstances{ KeyName: keyName, ImageId: s.SourceAMI, @@ -91,14 +141,14 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi ui.Error(err.Error()) return multistep.ActionHalt } - instanceId = []string{runResp.Instances[0].InstanceId} + instanceId = runResp.Instances[0].InstanceId } else { ui.Message(fmt.Sprintf( "Requesting spot instance '%s' for: %s", - s.InstanceType, s.SpotPrice)) + s.InstanceType, spotPrice)) runOpts := &ec2.RequestSpotInstances{ - SpotPrice: s.SpotPrice, + SpotPrice: spotPrice, KeyName: keyName, ImageId: s.SourceAMI, InstanceType: s.InstanceType, @@ -142,10 +192,10 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi ui.Error(err.Error()) return multistep.ActionHalt } - instanceId = []string{spotResp.SpotRequestResults[0].InstanceId} + instanceId = spotResp.SpotRequestResults[0].InstanceId } - instanceResp, err := ec2conn.Instances(instanceId, nil) + instanceResp, err := ec2conn.Instances([]string{instanceId}, nil) if err != nil { err := fmt.Errorf("Error finding source instance (%s): %s", instanceId, err) state.Put("error", err) diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index 9d66ba459..889cc7b60 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -103,6 +103,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe Debug: b.config.PackerDebug, ExpectedRootDevice: "ebs", SpotPrice: b.config.SpotPrice, + SpotPriceProduct: b.config.SpotPriceAutoProduct, InstanceType: b.config.InstanceType, UserData: b.config.UserData, UserDataFile: b.config.UserDataFile, diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index fcc92273a..1f5c1d9c8 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -207,6 +207,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &awscommon.StepRunSourceInstance{ Debug: b.config.PackerDebug, SpotPrice: b.config.SpotPrice, + SpotPriceProduct: b.config.SpotPriceAutoProduct, InstanceType: b.config.InstanceType, IamInstanceProfile: b.config.IamInstanceProfile, UserData: b.config.UserData, diff --git a/website/source/docs/builders/amazon-ebs.html.markdown b/website/source/docs/builders/amazon-ebs.html.markdown index 1c502942a..5cbd4003a 100644 --- a/website/source/docs/builders/amazon-ebs.html.markdown +++ b/website/source/docs/builders/amazon-ebs.html.markdown @@ -124,8 +124,13 @@ each category, the available configuration keys are alphabetized. to create the AMI. It is a type of instances that EC2 starts when the maximum price that you specify exceeds the current spot price. Spot price will be updated based on available spot instance capacity and current spot Instance - requests. It may save you some costs. For example, it takes only "0.001" to - launch a spot "m3.medium" instance while "0.07" needed for on-demand. + requests. It may save you some costs. You can set this to "auto" for + Packer to automatically discover the best spot price. + +* `spot_price_auto_product` (string) - Required if `spot_price` is set to + "auto". This tells Packer what sort of AMI you're launching to find the best + spot price. This must be one of: `Linux/UNIX`, `SUSE Linux`, `Windows`, + `Linux/UNIX (Amazon VPC)`, `SUSE Linux (Amazon VPC)`, `Windows (Amazon VPC)` * `ssh_port` (integer) - The port that SSH will be available on. This defaults to port 22. diff --git a/website/source/docs/builders/amazon-instance.html.markdown b/website/source/docs/builders/amazon-instance.html.markdown index f0b044650..db1cb132b 100644 --- a/website/source/docs/builders/amazon-instance.html.markdown +++ b/website/source/docs/builders/amazon-instance.html.markdown @@ -162,8 +162,13 @@ each category, the available configuration keys are alphabetized. to create the AMI. It is a type of instances that EC2 starts when the maximum price that you specify exceeds the current spot price. Spot price will be updated based on available spot instance capacity and current spot Instance - requests. It may save you some costs. For example, it takes only "0.001" to - launch a spot "m3.medium" instance while "0.07" needed for on-demand. + requests. It may save you some costs. You can set this to "auto" for + Packer to automatically discover the best spot price. + +* `spot_price_auto_product` (string) - Required if `spot_price` is set to + "auto". This tells Packer what sort of AMI you're launching to find the best + spot price. This must be one of: `Linux/UNIX`, `SUSE Linux`, `Windows`, + `Linux/UNIX (Amazon VPC)`, `SUSE Linux (Amazon VPC)`, `Windows (Amazon VPC)` * `ssh_port` (integer) - The port that SSH will be available on. This defaults to port 22.