Merge pull request #3817 from ChrisLundquist/dynamic-source-ami

Dynamic source ami

builder/amazon/chroot/builder.go

@@ -30,19 +30,20 @@ type Config struct {
 	awscommon.AMIConfig       `mapstructure:",squash"`
 	awscommon.AccessConfig    `mapstructure:",squash"`
 
-	ChrootMounts      [][]string `mapstructure:"chroot_mounts"`
-	CommandWrapper    string     `mapstructure:"command_wrapper"`
-	CopyFiles         []string   `mapstructure:"copy_files"`
-	DevicePath        string     `mapstructure:"device_path"`
-	FromScratch       bool       `mapstructure:"from_scratch"`
-	MountOptions      []string   `mapstructure:"mount_options"`
-	MountPartition    int        `mapstructure:"mount_partition"`
-	MountPath         string     `mapstructure:"mount_path"`
-	PostMountCommands []string   `mapstructure:"post_mount_commands"`
-	PreMountCommands  []string   `mapstructure:"pre_mount_commands"`
-	RootDeviceName    string     `mapstructure:"root_device_name"`
-	RootVolumeSize    int64      `mapstructure:"root_volume_size"`
-	SourceAmi         string     `mapstructure:"source_ami"`
+	ChrootMounts      [][]string                 `mapstructure:"chroot_mounts"`
+	CommandWrapper    string                     `mapstructure:"command_wrapper"`
+	CopyFiles         []string                   `mapstructure:"copy_files"`
+	DevicePath        string                     `mapstructure:"device_path"`
+	FromScratch       bool                       `mapstructure:"from_scratch"`
+	MountOptions      []string                   `mapstructure:"mount_options"`
+	MountPartition    int                        `mapstructure:"mount_partition"`
+	MountPath         string                     `mapstructure:"mount_path"`
+	PostMountCommands []string                   `mapstructure:"post_mount_commands"`
+	PreMountCommands  []string                   `mapstructure:"pre_mount_commands"`
+	RootDeviceName    string                     `mapstructure:"root_device_name"`
+	RootVolumeSize    int64                      `mapstructure:"root_volume_size"`
+	SourceAmi         string                     `mapstructure:"source_ami"`
+	SourceAmiFilter   awscommon.AmiFilterOptions `mapstructure:"source_ami_filter"`
 
 	ctx interpolate.Context
 }
@@ -125,8 +126,8 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
 	}
 
 	if b.config.FromScratch {
-		if b.config.SourceAmi != "" {
-			warns = append(warns, "source_ami is unused when from_scratch is true")
+		if b.config.SourceAmi != "" || !b.config.SourceAmiFilter.Empty() {
+			warns = append(warns, "source_ami and source_ami_filter are unused when from_scratch is true")
 		}
 		if b.config.RootVolumeSize == 0 {
 			errs = packer.MultiErrorAppend(
@@ -149,9 +150,9 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
 				errs, errors.New("ami_block_device_mappings is required with from_scratch."))
 		}
 	} else {
-		if b.config.SourceAmi == "" {
+		if b.config.SourceAmi == "" && b.config.SourceAmiFilter.Empty() {
 			errs = packer.MultiErrorAppend(
-				errs, errors.New("source_ami is required."))
+				errs, errors.New("source_ami or source_ami_filter is required."))
 		}
 		if len(b.config.AMIMappings) != 0 {
 			warns = append(warns, "ami_block_device_mappings are unused when from_scratch is false")
@@ -210,6 +211,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 			&awscommon.StepSourceAMIInfo{
 				SourceAmi:          b.config.SourceAmi,
 				EnhancedNetworking: b.config.AMIEnhancedNetworking,
+				AmiFilters:         b.config.SourceAmiFilter,
 			},
 			&StepCheckRootDevice{},
 		)

builder/amazon/common/run_config.go

@@ -14,6 +14,16 @@ import (
 
 var reShutdownBehavior = regexp.MustCompile("^(stop|terminate)$")
 
+type AmiFilterOptions struct {
+	Filters    map[*string]*string
+	Owners     []*string
+	MostRecent bool `mapstructure:"most_recent"`
+}
+
+func (d *AmiFilterOptions) Empty() bool {
+	return len(d.Owners) == 0 && len(d.Filters) == 0
+}
+
 // RunConfig contains configuration for running an instance from a source
 // AMI and details on how to access that launched image.
 type RunConfig struct {
@@ -24,6 +34,7 @@ type RunConfig struct {
 	InstanceType                      string            `mapstructure:"instance_type"`
 	RunTags                           map[string]string `mapstructure:"run_tags"`
 	SourceAmi                         string            `mapstructure:"source_ami"`
+	SourceAmiFilter                   AmiFilterOptions  `mapstructure:"source_ami_filter"`
 	SpotPrice                         string            `mapstructure:"spot_price"`
 	SpotPriceAutoProduct              string            `mapstructure:"spot_price_auto_product"`
 	DisableStopInstance               bool              `mapstructure:"disable_stop_instance"`
@@ -60,8 +71,8 @@ func (c *RunConfig) Prepare(ctx *interpolate.Context) []error {
 
 	// Validation
 	errs := c.Comm.Prepare(ctx)
-	if c.SourceAmi == "" {
-		errs = append(errs, errors.New("A source_ami must be specified"))
+	if c.SourceAmi == "" && c.SourceAmiFilter.Empty() {
+		errs = append(errs, errors.New("A source_ami or source_ami_filter must be specified"))
 	}
 
 	if c.InstanceType == "" {

builder/amazon/common/run_config_test.go

@@ -29,6 +29,13 @@ func testConfig() *RunConfig {
 	}
 }
 
+func testConfigFilter() *RunConfig {
+	config := testConfig()
+	config.SourceAmi = ""
+	config.SourceAmiFilter = AmiFilterOptions{}
+	return config
+}
+
 func TestRunConfigPrepare(t *testing.T) {
 	c := testConfig()
 	err := c.Prepare(nil)
@@ -53,6 +60,25 @@ func TestRunConfigPrepare_SourceAmi(t *testing.T) {
 	}
 }
 
+func TestRunConfigPrepare_SourceAmiFilterBlank(t *testing.T) {
+	c := testConfigFilter()
+	if err := c.Prepare(nil); len(err) != 1 {
+		t.Fatalf("err: %s", err)
+	}
+}
+
+func TestRunConfigPrepare_SourceAmiFilterGood(t *testing.T) {
+	c := testConfigFilter()
+	owner := "123"
+	filter_key := "name"
+	filter_value := "foo"
+	goodFilter := AmiFilterOptions{Owners: []*string{&owner}, Filters: map[*string]*string{&filter_key: &filter_value}}
+	c.SourceAmiFilter = goodFilter
+	if err := c.Prepare(nil); len(err) != 0 {
+		t.Fatalf("err: %s", err)
+	}
+}
+
 func TestRunConfigPrepare_SpotAuto(t *testing.T) {
 	c := testConfig()
 	c.SpotPrice = "auto"

builder/amazon/common/step_run_source_instance.go

@@ -84,24 +84,18 @@ func (s *StepRunSourceInstance) Run(state multistep.StateBag) multistep.StepActi
 	}
 
 	ui.Say("Launching a source AWS instance...")
-	imageResp, err := ec2conn.DescribeImages(&ec2.DescribeImagesInput{
-		ImageIds: []*string{&s.SourceAMI},
-	})
-	if err != nil {
-		state.Put("error", fmt.Errorf("There was a problem with the source AMI: %s", err))
-		return multistep.ActionHalt
-	}
-
-	if len(imageResp.Images) != 1 {
-		state.Put("error", fmt.Errorf("The source AMI '%s' could not be found.", s.SourceAMI))
+	image, ok := state.Get("source_image").(*ec2.Image)
+	if !ok {
+		state.Put("error", fmt.Errorf("source_image type assertion failed"))
 		return multistep.ActionHalt
 	}
+	s.SourceAMI = *image.ImageId
 
-	if s.ExpectedRootDevice != "" && *imageResp.Images[0].RootDeviceType != s.ExpectedRootDevice {
+	if s.ExpectedRootDevice != "" && *image.RootDeviceType != s.ExpectedRootDevice {
 		state.Put("error", fmt.Errorf(
 			"The provided source AMI has an invalid root device type.\n"+
 				"Expected '%s', got '%s'.",
-			s.ExpectedRootDevice, *imageResp.Images[0].RootDeviceType))
+			s.ExpectedRootDevice, *image.RootDeviceType))
 		return multistep.ActionHalt
 	}
 

builder/amazon/common/step_source_ami_info.go

@@ -2,6 +2,9 @@ package common
 
 import (
 	"fmt"
+	"log"
+	"sort"
+	"time"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/mitchellh/multistep"
@@ -16,14 +19,55 @@ import (
 type StepSourceAMIInfo struct {
 	SourceAmi          string
 	EnhancedNetworking bool
+	AmiFilters         AmiFilterOptions
+}
+
+// Build a slice of AMI filter options from the filters provided.
+func buildAmiFilters(input map[*string]*string) []*ec2.Filter {
+	var filters []*ec2.Filter
+	for k, v := range input {
+		filters = append(filters, &ec2.Filter{
+			Name:   k,
+			Values: []*string{v},
+		})
+	}
+	return filters
+}
+
+type imageSort []*ec2.Image
+
+func (a imageSort) Len() int      { return len(a) }
+func (a imageSort) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a imageSort) Less(i, j int) bool {
+	itime, _ := time.Parse(time.RFC3339, *a[i].CreationDate)
+	jtime, _ := time.Parse(time.RFC3339, *a[j].CreationDate)
+	return itime.Unix() < jtime.Unix()
+}
+
+// Returns the most recent AMI out of a slice of images.
+func mostRecentAmi(images []*ec2.Image) *ec2.Image {
+	sortedImages := images
+	sort.Sort(imageSort(sortedImages))
+	return sortedImages[len(sortedImages)-1]
 }
 
 func (s *StepSourceAMIInfo) Run(state multistep.StateBag) multistep.StepAction {
 	ec2conn := state.Get("ec2").(*ec2.EC2)
 	ui := state.Get("ui").(packer.Ui)
 
-	ui.Say(fmt.Sprintf("Inspecting the source AMI (%s)...", s.SourceAmi))
-	imageResp, err := ec2conn.DescribeImages(&ec2.DescribeImagesInput{ImageIds: []*string{&s.SourceAmi}})
+	params := &ec2.DescribeImagesInput{}
+
+	if s.SourceAmi != "" {
+		params.ImageIds = []*string{&s.SourceAmi}
+	}
+
+	// We have filters to apply
+	if len(s.AmiFilters.Filters) > 0 {
+		params.Filters = buildAmiFilters(s.AmiFilters.Filters)
+	}
+
+	log.Printf("Using AMI Filters %v", params)
+	imageResp, err := ec2conn.DescribeImages(params)
 	if err != nil {
 		err := fmt.Errorf("Error querying AMI: %s", err)
 		state.Put("error", err)
@@ -32,13 +76,27 @@ func (s *StepSourceAMIInfo) Run(state multistep.StateBag) multistep.StepAction {
 	}
 
 	if len(imageResp.Images) == 0 {
-		err := fmt.Errorf("Source AMI '%s' was not found!", s.SourceAmi)
+		err := fmt.Errorf("No AMI was found matching filters: %v", params)
+		state.Put("error", err)
+		ui.Error(err.Error())
+		return multistep.ActionHalt
+	}
+
+	if len(imageResp.Images) > 1 && s.AmiFilters.MostRecent == false {
+		err := fmt.Errorf("Your query returned more than one result. Please try a more specific search, or set most_recent to true.")
 		state.Put("error", err)
 		ui.Error(err.Error())
 		return multistep.ActionHalt
 	}
 
-	image := imageResp.Images[0]
+	var image *ec2.Image
+	if s.AmiFilters.MostRecent {
+		image = mostRecentAmi(imageResp.Images)
+	} else {
+		image = imageResp.Images[0]
+	}
+
+	log.Printf(fmt.Sprintf("Got Image %v", image))
 
 	// Enhanced Networking (SriovNetSupport) can only be enabled on HVM AMIs.
 	// See http://goo.gl/icuXh5

builder/amazon/ebs/builder.go

@@ -100,6 +100,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 		&awscommon.StepSourceAMIInfo{
 			SourceAmi:          b.config.SourceAmi,
 			EnhancedNetworking: b.config.AMIEnhancedNetworking,
+			AmiFilters:         b.config.SourceAmiFilter,
 		},
 		&awscommon.StepKeyPair{
 			Debug:                b.config.PackerDebug,

builder/amazon/instance/builder.go

@@ -190,6 +190,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 		&awscommon.StepSourceAMIInfo{
 			SourceAmi:          b.config.SourceAmi,
 			EnhancedNetworking: b.config.AMIEnhancedNetworking,
+			AmiFilters:         b.config.SourceAmiFilter,
 		},
 		&awscommon.StepKeyPair{
 			Debug:                b.config.PackerDebug,

website/source/docs/builders/amazon-chroot.html.md

@@ -170,6 +170,34 @@ each category, the available configuration keys are alphabetized.
 -   `skip_region_validation` (boolean) - Set to true if you want to skip
     validation of the `ami_regions` configuration option. Defaults to false.
 
+-   `source_ami_filter` (object) - Filters used to populate the `source_ami` field.
+    Example:
+    ``` {.javascript}
+    "source_ami_filter": {
+        "filters": {
+          "virtualization-type": "hvm",
+          "name": "*ubuntu-xenial-16.04-amd64-server-*",
+          "root-device-type": "ebs"
+        },
+        "owners": ["099720109477"],
+        "most_recent": true
+    }
+    ```
+    This selects the most recent Ubuntu 16.04 HVM EBS AMI from Canonical.
+    NOTE: This will fail unless *exactly* one AMI is returned. In the above
+    example, `most_recent` will cause this to succeed by selecting the newest image.
+
+    -   `filters` (map of strings) - filters used to select a `source_ami`.
+         NOTE: This will fail unless *exactly* one AMI is returned.
+         Any filter described in the docs for [DescribeImages](http://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeImages.html)
+         is valid.
+
+    -   `owners` (array of strings) - This scopes the AMIs to certain Amazon account IDs.
+         This is helpful to limit the AMIs to a trusted third party, or to your own account.
+
+    -   `most_recent` (bool) - Selects the newest created image when true.
+         This is most useful for selecting a daily distro build.
+
 -   `tags` (object of key/value strings) - Tags applied to the AMI.
 
 ## Basic Example

website/source/docs/builders/amazon-ebs.html.md

@@ -58,7 +58,8 @@ builder.
     how to set this.](/docs/builders/amazon.html#specifying-amazon-credentials)
 
 -   `source_ami` (string) - The initial AMI used as a base for the newly
-    created machine.
+    created machine. `source_ami_filter` may be used instead to populate this
+    automatically.
 
 ### Optional:
 
@@ -174,6 +175,34 @@ builder.
 -   `skip_region_validation` (boolean) - Set to true if you want to skip 
     validation of the region configuration option.  Defaults to false.
 
+-   `source_ami_filter` (object) - Filters used to populate the `source_ami` field.
+    Example:
+    ``` {.javascript}
+    "source_ami_filter": {
+        "filters": {
+          "virtualization-type": "hvm",
+          "name": "*ubuntu-xenial-16.04-amd64-server-*",
+          "root-device-type": "ebs"
+        },
+        "owners": ["099720109477"],
+        "most_recent": true
+    }
+    ```
+    This selects the most recent Ubuntu 16.04 HVM EBS AMI from Canonical.
+    NOTE: This will fail unless *exactly* one AMI is returned. In the above
+    example, `most_recent` will cause this to succeed by selecting the newest image.
+
+    -   `filters` (map of strings) - filters used to select a `source_ami`.
+         NOTE: This will fail unless *exactly* one AMI is returned.
+         Any filter described in the docs for [DescribeImages](http://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeImages.html)
+         is valid.
+
+    -   `owners` (array of strings) - This scopes the AMIs to certain Amazon account IDs.
+         This is helpful to limit the AMIs to a trusted third party, or to your own account.
+
+    -   `most_recent` (bool) - Selects the newest created image when true.
+         This is most useful for selecting a daily distro build.
+
 -   `spot_price` (string) - The maximum hourly price to pay for a spot instance
     to create the AMI. Spot instances are a type of instance that EC2 starts
     when the current spot price is less than the maximum price you specify. Spot

website/source/docs/builders/amazon-instance.html.md

@@ -191,6 +191,34 @@ builder.
 -   `skip_region_validation` (boolean) - Set to true if you want to skip 
     validation of the region configuration option.  Defaults to false.
 
+-   `source_ami_filter` (object) - Filters used to populate the `source_ami` field.
+    Example:
+    ``` {.javascript}
+    "source_ami_filter": {
+        "filters": {
+          "virtualization-type": "hvm",
+          "name": "*ubuntu-xenial-16.04-amd64-server-*",
+          "root-device-type": "ebs"
+        },
+        "owners": ["099720109477"],
+        "most_recent": true
+    }
+    ```
+    This selects the most recent Ubuntu 16.04 HVM EBS AMI from Canonical.
+    NOTE: This will fail unless *exactly* one AMI is returned. In the above
+    example, `most_recent` will cause this to succeed by selecting the newest image.
+
+    -   `filters` (map of strings) - filters used to select a `source_ami`.
+         NOTE: This will fail unless *exactly* one AMI is returned.
+         Any filter described in the docs for [DescribeImages](http://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeImages.html)
+         is valid.
+
+    -   `owners` (array of strings) - This scopes the AMIs to certain Amazon account IDs.
+         This is helpful to limit the AMIs to a trusted third party, or to your own account.
+
+    -   `most_recent` (bool) - Selects the newest created image when true.
+         This is most useful for selecting a daily distro build.
+
 -   `spot_price` (string) - The maximum hourly price to launch a spot instance
     to create the AMI. It is a type of instances that EC2 starts when the
     maximum price that you specify exceeds the current spot price. Spot price