From 984086275719fa638e22a2ca8ce50fdfc77d0229 Mon Sep 17 00:00:00 2001 From: Rickard von Essen Date: Tue, 12 Jun 2018 12:05:16 +0200 Subject: [PATCH 1/4] builder/amazon: Add suppport for vpc_filter and subnet_filter First step of adding support for discovering VPC's and Subnets using filters. --- builder/amazon/common/build_filter.go | 17 +++ builder/amazon/common/run_config.go | 66 ++++++--- builder/amazon/common/step_network_info.go | 131 ++++++++++++++++++ .../amazon/common/step_run_source_instance.go | 7 + builder/amazon/common/step_security_group.go | 7 + builder/amazon/common/step_source_ami_info.go | 14 +- builder/amazon/ebs/builder.go | 16 ++- 7 files changed, 221 insertions(+), 37 deletions(-) create mode 100644 builder/amazon/common/build_filter.go create mode 100644 builder/amazon/common/step_network_info.go diff --git a/builder/amazon/common/build_filter.go b/builder/amazon/common/build_filter.go new file mode 100644 index 000000000..8f621efc8 --- /dev/null +++ b/builder/amazon/common/build_filter.go @@ -0,0 +1,17 @@ +package common + +import ( + "github.com/aws/aws-sdk-go/service/ec2" +) + +// Build a slice of EC2 (AMI/Subnet/VPC) filter options from the filters provided. +func buildEc2Filters(input map[*string]*string) []*ec2.Filter { + var filters []*ec2.Filter + for k, v := range input { + filters = append(filters, &ec2.Filter{ + Name: k, + Values: []*string{v}, + }) + } + return filters +} diff --git a/builder/amazon/common/run_config.go b/builder/amazon/common/run_config.go index dbafd0f38..5eb2cfa42 100644 --- a/builder/amazon/common/run_config.go +++ b/builder/amazon/common/run_config.go @@ -29,32 +29,52 @@ func (d *AmiFilterOptions) NoOwner() bool { return len(d.Owners) == 0 } +type SubnetFilterOptions struct { + Filters map[*string]*string + MostFree bool `mapstructure:"most_free"` + Random bool `mapstructure:"random"` +} + +func (d *SubnetFilterOptions) Empty() bool { + return len(d.Filters) == 0 +} + +type VpcFilterOptions struct { + Filters map[*string]*string +} + +func (d *VpcFilterOptions) Empty() bool { + return len(d.Filters) == 0 +} + // RunConfig contains configuration for running an instance from a source // AMI and details on how to access that launched image. type RunConfig struct { - AssociatePublicIpAddress bool `mapstructure:"associate_public_ip_address"` - AvailabilityZone string `mapstructure:"availability_zone"` - DisableStopInstance bool `mapstructure:"disable_stop_instance"` - EbsOptimized bool `mapstructure:"ebs_optimized"` - EnableT2Unlimited bool `mapstructure:"enable_t2_unlimited"` - IamInstanceProfile string `mapstructure:"iam_instance_profile"` - InstanceInitiatedShutdownBehavior string `mapstructure:"shutdown_behavior"` - InstanceType string `mapstructure:"instance_type"` - RunTags map[string]string `mapstructure:"run_tags"` - SecurityGroupId string `mapstructure:"security_group_id"` - SecurityGroupIds []string `mapstructure:"security_group_ids"` - SourceAmi string `mapstructure:"source_ami"` - SourceAmiFilter AmiFilterOptions `mapstructure:"source_ami_filter"` - SpotPrice string `mapstructure:"spot_price"` - SpotPriceAutoProduct string `mapstructure:"spot_price_auto_product"` - SpotTags map[string]string `mapstructure:"spot_tags"` - SubnetId string `mapstructure:"subnet_id"` - TemporaryKeyPairName string `mapstructure:"temporary_key_pair_name"` - TemporarySGSourceCidr string `mapstructure:"temporary_security_group_source_cidr"` - UserData string `mapstructure:"user_data"` - UserDataFile string `mapstructure:"user_data_file"` - VpcId string `mapstructure:"vpc_id"` - WindowsPasswordTimeout time.Duration `mapstructure:"windows_password_timeout"` + AssociatePublicIpAddress bool `mapstructure:"associate_public_ip_address"` + AvailabilityZone string `mapstructure:"availability_zone"` + DisableStopInstance bool `mapstructure:"disable_stop_instance"` + EbsOptimized bool `mapstructure:"ebs_optimized"` + EnableT2Unlimited bool `mapstructure:"enable_t2_unlimited"` + IamInstanceProfile string `mapstructure:"iam_instance_profile"` + InstanceInitiatedShutdownBehavior string `mapstructure:"shutdown_behavior"` + InstanceType string `mapstructure:"instance_type"` + RunTags map[string]string `mapstructure:"run_tags"` + SecurityGroupId string `mapstructure:"security_group_id"` + SecurityGroupIds []string `mapstructure:"security_group_ids"` + SourceAmi string `mapstructure:"source_ami"` + SourceAmiFilter AmiFilterOptions `mapstructure:"source_ami_filter"` + SpotPrice string `mapstructure:"spot_price"` + SpotPriceAutoProduct string `mapstructure:"spot_price_auto_product"` + SpotTags map[string]string `mapstructure:"spot_tags"` + SubnetFilter SubnetFilterOptions `mapstructure:"subnet_filter"` + SubnetId string `mapstructure:"subnet_id"` + TemporaryKeyPairName string `mapstructure:"temporary_key_pair_name"` + TemporarySGSourceCidr string `mapstructure:"temporary_security_group_source_cidr"` + UserData string `mapstructure:"user_data"` + UserDataFile string `mapstructure:"user_data_file"` + VpcFilter VpcFilterOptions `mapstructure:"vpc_filter"` + VpcId string `mapstructure:"vpc_id"` + WindowsPasswordTimeout time.Duration `mapstructure:"windows_password_timeout"` // Communicator settings Comm communicator.Config `mapstructure:",squash"` diff --git a/builder/amazon/common/step_network_info.go b/builder/amazon/common/step_network_info.go new file mode 100644 index 000000000..3f75c62b2 --- /dev/null +++ b/builder/amazon/common/step_network_info.go @@ -0,0 +1,131 @@ +package common + +import ( + "context" + "fmt" + "log" + "math/rand" + "sort" + + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/hashicorp/packer/helper/multistep" + "github.com/hashicorp/packer/packer" +) + +// StepNetworkInfo queries AWS for information about +// VPC's, Subnets, and Security Groups that is used +// throughout the AMI creation process. +// +// Produces: +// vpc_id string - the VPC ID +// subnet_id string - the Subnet ID +// az string - the AZ name +// sg_ids []string - the SG IDs +type StepNetworkInfo struct { + VpcId string + VpcFilter VpcFilterOptions + SubnetId string + SubnetFilter SubnetFilterOptions + AvailabilityZone string + // TODO Security groups + filter +} + +type subnetsSort []*ec2.Subnet + +func (a subnetsSort) Len() int { return len(a) } +func (a subnetsSort) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a subnetsSort) Less(i, j int) bool { + return *a[i].AvailableIpAddressCount < *a[j].AvailableIpAddressCount +} + +// Returns the most recent AMI out of a slice of images. +func mostFreeSubnet(subnets []*ec2.Subnet) *ec2.Subnet { + sortedSubnets := subnets + sort.Sort(subnetsSort(sortedSubnets)) + return sortedSubnets[len(sortedSubnets)-1] +} + +func (s *StepNetworkInfo) Run(_ context.Context, state multistep.StateBag) multistep.StepAction { + ec2conn := state.Get("ec2").(*ec2.EC2) + ui := state.Get("ui").(packer.Ui) + + // VPC + if s.VpcId == "" && !s.VpcFilter.Empty() { + params := &ec2.DescribeVpcsInput{} + + params.Filters = buildEc2Filters(s.VpcFilter.Filters) + log.Printf("Using VPC Filters %v", params) + + vpcResp, err := ec2conn.DescribeVpcs(params) + if err != nil { + err := fmt.Errorf("Error querying VPCs: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + if len(vpcResp.Vpcs) != 1 { + err := fmt.Errorf("No or more than one VPC was found matching filters: %v", params) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + s.VpcId = *vpcResp.Vpcs[0].VpcId + ui.Message(fmt.Sprintf("Found VPC ID: %s", s.VpcId)) + } + + // Subnet + if s.SubnetId == "" && !s.SubnetFilter.Empty() { + params := &ec2.DescribeSubnetsInput{} + + vpcId := "vpc-id" + s.SubnetFilter.Filters[&vpcId] = &s.VpcId + if s.AvailabilityZone != "" { + az := "availability-zone" + s.SubnetFilter.Filters[&az] = &s.AvailabilityZone + } + params.Filters = buildEc2Filters(s.SubnetFilter.Filters) + log.Printf("Using Subnet Filters %v", params) + + subnetsResp, err := ec2conn.DescribeSubnets(params) + if err != nil { + err := fmt.Errorf("Error querying Subnets: %s", err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + if len(subnetsResp.Subnets) == 0 { + err := fmt.Errorf("No Subnets was found matching filters: %v", params) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + if len(subnetsResp.Subnets) > 1 && !s.SubnetFilter.Random && !s.SubnetFilter.MostFree { + err := fmt.Errorf("Your query returned more than one result. Please try a more specific search, or set random or most_free to true.") + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + + var subnet *ec2.Subnet + switch { + case s.SubnetFilter.MostFree: + subnet = mostFreeSubnet(subnetsResp.Subnets) + case s.SubnetFilter.Random: + subnet = subnetsResp.Subnets[rand.Intn(len(subnetsResp.Subnets))] + default: + subnet = subnetsResp.Subnets[0] + } + s.SubnetId = *subnet.SubnetId + ui.Message(fmt.Sprintf("Found Subnet ID: %s", s.SubnetId)) + } + + state.Put("vpc_id", s.VpcId) + state.Put("subnet_id", s.SubnetId) + return multistep.ActionContinue +} + +func (s *StepNetworkInfo) Cleanup(multistep.StateBag) {} diff --git a/builder/amazon/common/step_run_source_instance.go b/builder/amazon/common/step_run_source_instance.go index 7c31fa1f8..e4ad43157 100644 --- a/builder/amazon/common/step_run_source_instance.go +++ b/builder/amazon/common/step_run_source_instance.go @@ -154,6 +154,13 @@ func (s *StepRunSourceInstance) Run(ctx context.Context, state multistep.StateBa runOpts.KeyName = &keyName } + // TODO always get subnet_id from state. + if s.SubnetId == "" { + if subnetId, ok := state.GetOk("subnet_id"); ok { + s.SubnetId = subnetId.(string) + } + } + if s.SubnetId != "" && s.AssociatePublicIpAddress { runOpts.NetworkInterfaces = []*ec2.InstanceNetworkInterfaceSpecification{ { diff --git a/builder/amazon/common/step_security_group.go b/builder/amazon/common/step_security_group.go index 031842cd3..6ef3f4dbb 100644 --- a/builder/amazon/common/step_security_group.go +++ b/builder/amazon/common/step_security_group.go @@ -60,6 +60,13 @@ func (s *StepSecurityGroup) Run(_ context.Context, state multistep.StateBag) mul Description: aws.String("Temporary group for Packer"), } + // TODO always get vpc_id from state. + if s.VpcId == "" { + if vpcId, ok := state.GetOk("vpc_id"); ok { + s.VpcId = vpcId.(string) + } + } + if s.VpcId != "" { group.VpcId = &s.VpcId } diff --git a/builder/amazon/common/step_source_ami_info.go b/builder/amazon/common/step_source_ami_info.go index 5e57c2fbf..ce322130e 100644 --- a/builder/amazon/common/step_source_ami_info.go +++ b/builder/amazon/common/step_source_ami_info.go @@ -24,18 +24,6 @@ type StepSourceAMIInfo struct { AmiFilters AmiFilterOptions } -// Build a slice of AMI filter options from the filters provided. -func buildAmiFilters(input map[*string]*string) []*ec2.Filter { - var filters []*ec2.Filter - for k, v := range input { - filters = append(filters, &ec2.Filter{ - Name: k, - Values: []*string{v}, - }) - } - return filters -} - type imageSort []*ec2.Image func (a imageSort) Len() int { return len(a) } @@ -65,7 +53,7 @@ func (s *StepSourceAMIInfo) Run(_ context.Context, state multistep.StateBag) mul // We have filters to apply if len(s.AmiFilters.Filters) > 0 { - params.Filters = buildAmiFilters(s.AmiFilters.Filters) + params.Filters = buildEc2Filters(s.AmiFilters.Filters) } if len(s.AmiFilters.Owners) > 0 { params.Owners = s.AmiFilters.Owners diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index f2c0bcbec..d846f617d 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -93,7 +93,12 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe } ec2conn := ec2.New(session) + // TODO Translate this into VpcFilter/SubnetFilter and move the describe into apropriate step. // If the subnet is specified but not the VpcId or AZ, try to determine them automatically + + /* + * If subnet => vpc, az + */ if b.config.SubnetId != "" && (b.config.AvailabilityZone == "" || b.config.VpcId == "") { log.Printf("[INFO] Finding AZ and VpcId for the given subnet '%s'", b.config.SubnetId) resp, err := ec2conn.DescribeSubnets(&ec2.DescribeSubnetsInput{SubnetIds: []*string{&b.config.SubnetId}}) @@ -177,6 +182,13 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe EnableAMIENASupport: b.config.AMIENASupport, AmiFilters: b.config.SourceAmiFilter, }, + &awscommon.StepNetworkInfo{ + VpcId: b.config.VpcId, + VpcFilter: b.config.VpcFilter, + SubnetId: b.config.SubnetId, + SubnetFilter: b.config.SubnetFilter, + AvailabilityZone: b.config.AvailabilityZone, + }, &awscommon.StepKeyPair{ Debug: b.config.PackerDebug, SSHAgentAuth: b.config.Comm.SSHAgentAuth, @@ -186,9 +198,11 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe PrivateKeyFile: b.config.RunConfig.Comm.SSHPrivateKey, }, &awscommon.StepSecurityGroup{ + // TODO remove SecurityGroupIds: b.config.SecurityGroupIds, CommConfig: &b.config.RunConfig.Comm, - VpcId: b.config.VpcId, + // TODO remove + VpcId: b.config.VpcId, TemporarySGSourceCidr: b.config.TemporarySGSourceCidr, }, &awscommon.StepCleanupVolumes{ From 533967cb66a60063ecc2e6106a050e3701c8771a Mon Sep 17 00:00:00 2001 From: Rickard von Essen Date: Tue, 14 Aug 2018 12:04:13 +0200 Subject: [PATCH 2/4] builder/amazon: Add suppport for security_group_filter Allow selecting SG's using a filter. Also some cleanup of vpc_filter and subnet_filter. --- builder/amazon/common/run_config.go | 59 +++++++++++-------- builder/amazon/common/step_network_info.go | 57 ++++++++++++------ .../amazon/common/step_run_source_instance.go | 18 ++---- .../amazon/common/step_run_spot_instance.go | 25 ++++---- builder/amazon/common/step_security_group.go | 48 +++++++++++---- builder/amazon/ebs/builder.go | 46 ++++----------- builder/amazon/ebssurrogate/builder.go | 36 ++++------- builder/amazon/ebsvolume/builder.go | 36 ++++------- builder/amazon/instance/builder.go | 36 ++++------- 9 files changed, 177 insertions(+), 184 deletions(-) diff --git a/builder/amazon/common/run_config.go b/builder/amazon/common/run_config.go index 5eb2cfa42..f1b4fa960 100644 --- a/builder/amazon/common/run_config.go +++ b/builder/amazon/common/run_config.go @@ -47,34 +47,43 @@ func (d *VpcFilterOptions) Empty() bool { return len(d.Filters) == 0 } +type SecurityGroupFilterOptions struct { + Filters map[*string]*string +} + +func (d *SecurityGroupFilterOptions) Empty() bool { + return len(d.Filters) == 0 +} + // RunConfig contains configuration for running an instance from a source // AMI and details on how to access that launched image. type RunConfig struct { - AssociatePublicIpAddress bool `mapstructure:"associate_public_ip_address"` - AvailabilityZone string `mapstructure:"availability_zone"` - DisableStopInstance bool `mapstructure:"disable_stop_instance"` - EbsOptimized bool `mapstructure:"ebs_optimized"` - EnableT2Unlimited bool `mapstructure:"enable_t2_unlimited"` - IamInstanceProfile string `mapstructure:"iam_instance_profile"` - InstanceInitiatedShutdownBehavior string `mapstructure:"shutdown_behavior"` - InstanceType string `mapstructure:"instance_type"` - RunTags map[string]string `mapstructure:"run_tags"` - SecurityGroupId string `mapstructure:"security_group_id"` - SecurityGroupIds []string `mapstructure:"security_group_ids"` - SourceAmi string `mapstructure:"source_ami"` - SourceAmiFilter AmiFilterOptions `mapstructure:"source_ami_filter"` - SpotPrice string `mapstructure:"spot_price"` - SpotPriceAutoProduct string `mapstructure:"spot_price_auto_product"` - SpotTags map[string]string `mapstructure:"spot_tags"` - SubnetFilter SubnetFilterOptions `mapstructure:"subnet_filter"` - SubnetId string `mapstructure:"subnet_id"` - TemporaryKeyPairName string `mapstructure:"temporary_key_pair_name"` - TemporarySGSourceCidr string `mapstructure:"temporary_security_group_source_cidr"` - UserData string `mapstructure:"user_data"` - UserDataFile string `mapstructure:"user_data_file"` - VpcFilter VpcFilterOptions `mapstructure:"vpc_filter"` - VpcId string `mapstructure:"vpc_id"` - WindowsPasswordTimeout time.Duration `mapstructure:"windows_password_timeout"` + AssociatePublicIpAddress bool `mapstructure:"associate_public_ip_address"` + AvailabilityZone string `mapstructure:"availability_zone"` + DisableStopInstance bool `mapstructure:"disable_stop_instance"` + EbsOptimized bool `mapstructure:"ebs_optimized"` + EnableT2Unlimited bool `mapstructure:"enable_t2_unlimited"` + IamInstanceProfile string `mapstructure:"iam_instance_profile"` + InstanceInitiatedShutdownBehavior string `mapstructure:"shutdown_behavior"` + InstanceType string `mapstructure:"instance_type"` + RunTags map[string]string `mapstructure:"run_tags"` + SecurityGroupFilter SecurityGroupFilterOptions `mapstructure:"security_group_filter"` + SecurityGroupId string `mapstructure:"security_group_id"` + SecurityGroupIds []string `mapstructure:"security_group_ids"` + SourceAmi string `mapstructure:"source_ami"` + SourceAmiFilter AmiFilterOptions `mapstructure:"source_ami_filter"` + SpotPrice string `mapstructure:"spot_price"` + SpotPriceAutoProduct string `mapstructure:"spot_price_auto_product"` + SpotTags map[string]string `mapstructure:"spot_tags"` + SubnetFilter SubnetFilterOptions `mapstructure:"subnet_filter"` + SubnetId string `mapstructure:"subnet_id"` + TemporaryKeyPairName string `mapstructure:"temporary_key_pair_name"` + TemporarySGSourceCidr string `mapstructure:"temporary_security_group_source_cidr"` + UserData string `mapstructure:"user_data"` + UserDataFile string `mapstructure:"user_data_file"` + VpcFilter VpcFilterOptions `mapstructure:"vpc_filter"` + VpcId string `mapstructure:"vpc_id"` + WindowsPasswordTimeout time.Duration `mapstructure:"windows_password_timeout"` // Communicator settings Comm communicator.Config `mapstructure:",squash"` diff --git a/builder/amazon/common/step_network_info.go b/builder/amazon/common/step_network_info.go index 3f75c62b2..7fc026335 100644 --- a/builder/amazon/common/step_network_info.go +++ b/builder/amazon/common/step_network_info.go @@ -7,27 +7,27 @@ import ( "math/rand" "sort" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/hashicorp/packer/helper/multistep" "github.com/hashicorp/packer/packer" ) // StepNetworkInfo queries AWS for information about -// VPC's, Subnets, and Security Groups that is used -// throughout the AMI creation process. +// VPC's and Subnets that is used throughout the AMI creation process. // -// Produces: +// Produces (adding them to the state bag): // vpc_id string - the VPC ID // subnet_id string - the Subnet ID -// az string - the AZ name -// sg_ids []string - the SG IDs +// availability_zone string - the AZ name type StepNetworkInfo struct { - VpcId string - VpcFilter VpcFilterOptions - SubnetId string - SubnetFilter SubnetFilterOptions - AvailabilityZone string - // TODO Security groups + filter + VpcId string + VpcFilter VpcFilterOptions + SubnetId string + SubnetFilter SubnetFilterOptions + AvailabilityZone string + SecurityGroupIds []string + SecurityGroupFilter SecurityGroupFilterOptions } type subnetsSort []*ec2.Subnet @@ -52,8 +52,8 @@ func (s *StepNetworkInfo) Run(_ context.Context, state multistep.StateBag) multi // VPC if s.VpcId == "" && !s.VpcFilter.Empty() { params := &ec2.DescribeVpcsInput{} - params.Filters = buildEc2Filters(s.VpcFilter.Filters) + log.Printf("Using VPC Filters %v", params) vpcResp, err := ec2conn.DescribeVpcs(params) @@ -65,7 +65,7 @@ func (s *StepNetworkInfo) Run(_ context.Context, state multistep.StateBag) multi } if len(vpcResp.Vpcs) != 1 { - err := fmt.Errorf("No or more than one VPC was found matching filters: %v", params) + err := fmt.Errorf("Exactly one VPC should match the filter, but %d VPC's was found matching filters: %v", len(vpcResp.Vpcs), params) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt @@ -79,11 +79,11 @@ func (s *StepNetworkInfo) Run(_ context.Context, state multistep.StateBag) multi if s.SubnetId == "" && !s.SubnetFilter.Empty() { params := &ec2.DescribeSubnetsInput{} - vpcId := "vpc-id" - s.SubnetFilter.Filters[&vpcId] = &s.VpcId + if s.VpcId != "" { + s.SubnetFilter.Filters[aws.String("vpc-id")] = &s.VpcId + } if s.AvailabilityZone != "" { - az := "availability-zone" - s.SubnetFilter.Filters[&az] = &s.AvailabilityZone + s.SubnetFilter.Filters[aws.String("availability-zone")] = &s.AvailabilityZone } params.Filters = buildEc2Filters(s.SubnetFilter.Filters) log.Printf("Using Subnet Filters %v", params) @@ -104,7 +104,7 @@ func (s *StepNetworkInfo) Run(_ context.Context, state multistep.StateBag) multi } if len(subnetsResp.Subnets) > 1 && !s.SubnetFilter.Random && !s.SubnetFilter.MostFree { - err := fmt.Errorf("Your query returned more than one result. Please try a more specific search, or set random or most_free to true.") + err := fmt.Errorf("Your filter matched %d Subnets. Please try a more specific search, or set random or most_free to true.", len(subnetsResp.Subnets)) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt @@ -123,7 +123,28 @@ func (s *StepNetworkInfo) Run(_ context.Context, state multistep.StateBag) multi ui.Message(fmt.Sprintf("Found Subnet ID: %s", s.SubnetId)) } + // Try to find AZ and VPC Id from Subnet if they are not yet found/given + if s.SubnetId != "" && (s.AvailabilityZone == "" || s.VpcId == "") { + log.Printf("[INFO] Finding AZ and VpcId for the given subnet '%s'", s.SubnetId) + resp, err := ec2conn.DescribeSubnets(&ec2.DescribeSubnetsInput{SubnetIds: []*string{&s.SubnetId}}) + if err != nil { + err := fmt.Errorf("Describing the subnet: %s returned error: %s.", s.SubnetId, err) + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + if s.AvailabilityZone == "" { + s.AvailabilityZone = *resp.Subnets[0].AvailabilityZone + log.Printf("[INFO] AvailabilityZone found: '%s'", s.AvailabilityZone) + } + if s.VpcId == "" { + s.VpcId = *resp.Subnets[0].VpcId + log.Printf("[INFO] VpcId found: '%s'", s.VpcId) + } + } + state.Put("vpc_id", s.VpcId) + state.Put("availability_zone", s.AvailabilityZone) state.Put("subnet_id", s.SubnetId) return multistep.ActionContinue } diff --git a/builder/amazon/common/step_run_source_instance.go b/builder/amazon/common/step_run_source_instance.go index e4ad43157..55db40f64 100644 --- a/builder/amazon/common/step_run_source_instance.go +++ b/builder/amazon/common/step_run_source_instance.go @@ -19,7 +19,6 @@ import ( type StepRunSourceInstance struct { AssociatePublicIpAddress bool - AvailabilityZone string BlockDevices BlockDevices Ctx interpolate.Context Debug bool @@ -31,7 +30,6 @@ type StepRunSourceInstance struct { InstanceType string IsRestricted bool SourceAMI string - SubnetId string Tags TagMap UserData string UserDataFile string @@ -105,6 +103,7 @@ func (s *StepRunSourceInstance) Run(ctx context.Context, state multistep.StateBa return multistep.ActionHalt } + az := state.Get("availability_zone").(string) runOpts := &ec2.RunInstancesInput{ ImageId: &s.SourceAMI, InstanceType: &s.InstanceType, @@ -113,7 +112,7 @@ func (s *StepRunSourceInstance) Run(ctx context.Context, state multistep.StateBa MinCount: aws.Int64(1), IamInstanceProfile: &ec2.IamInstanceProfileSpecification{Name: &s.IamInstanceProfile}, BlockDeviceMappings: s.BlockDevices.BuildLaunchDevices(), - Placement: &ec2.Placement{AvailabilityZone: &s.AvailabilityZone}, + Placement: &ec2.Placement{AvailabilityZone: &az}, EbsOptimized: &s.EbsOptimized, } @@ -154,25 +153,20 @@ func (s *StepRunSourceInstance) Run(ctx context.Context, state multistep.StateBa runOpts.KeyName = &keyName } - // TODO always get subnet_id from state. - if s.SubnetId == "" { - if subnetId, ok := state.GetOk("subnet_id"); ok { - s.SubnetId = subnetId.(string) - } - } + subnetId := state.Get("subnet_id").(string) - if s.SubnetId != "" && s.AssociatePublicIpAddress { + if subnetId != "" && s.AssociatePublicIpAddress { runOpts.NetworkInterfaces = []*ec2.InstanceNetworkInterfaceSpecification{ { DeviceIndex: aws.Int64(0), AssociatePublicIpAddress: &s.AssociatePublicIpAddress, - SubnetId: &s.SubnetId, + SubnetId: aws.String(subnetId), Groups: securityGroupIds, DeleteOnTermination: aws.Bool(true), }, } } else { - runOpts.SubnetId = &s.SubnetId + runOpts.SubnetId = aws.String(subnetId) runOpts.SecurityGroupIds = securityGroupIds } diff --git a/builder/amazon/common/step_run_spot_instance.go b/builder/amazon/common/step_run_spot_instance.go index 09d456496..279e3aab5 100644 --- a/builder/amazon/common/step_run_spot_instance.go +++ b/builder/amazon/common/step_run_spot_instance.go @@ -21,7 +21,6 @@ import ( type StepRunSpotInstance struct { AssociatePublicIpAddress bool - AvailabilityZone string BlockDevices BlockDevices Debug bool EbsOptimized bool @@ -33,7 +32,6 @@ type StepRunSpotInstance struct { SpotPrice string SpotPriceProduct string SpotTags TagMap - SubnetId string Tags TagMap VolumeTags TagMap UserData string @@ -87,7 +85,12 @@ func (s *StepRunSpotInstance) Run(ctx context.Context, state multistep.StateBag) } spotPrice := s.SpotPrice - availabilityZone := s.AvailabilityZone + azConfig := "" + if azRaw, ok := state.GetOk("availability_zone"); ok { + azConfig = azRaw.(string) + } + az := azConfig + if spotPrice == "auto" { ui.Message(fmt.Sprintf( "Finding spot price for %s %s...", @@ -98,7 +101,7 @@ func (s *StepRunSpotInstance) Run(ctx context.Context, state multistep.StateBag) resp, err := ec2conn.DescribeSpotPriceHistory(&ec2.DescribeSpotPriceHistoryInput{ InstanceTypes: []*string{&s.InstanceType}, ProductDescriptions: []*string{&s.SpotPriceProduct}, - AvailabilityZone: &s.AvailabilityZone, + AvailabilityZone: &az, StartTime: &startTime, }) if err != nil { @@ -118,8 +121,8 @@ func (s *StepRunSpotInstance) Run(ctx context.Context, state multistep.StateBag) } if price == 0 || current < price { price = current - if s.AvailabilityZone == "" { - availabilityZone = *history.AvailabilityZone + if azConfig == "" { + az = *history.AvailabilityZone } } } @@ -163,24 +166,26 @@ func (s *StepRunSpotInstance) Run(ctx context.Context, state multistep.StateBag) UserData: &userData, IamInstanceProfile: &ec2.IamInstanceProfileSpecification{Name: &s.IamInstanceProfile}, Placement: &ec2.SpotPlacement{ - AvailabilityZone: &availabilityZone, + AvailabilityZone: &az, }, BlockDeviceMappings: s.BlockDevices.BuildLaunchDevices(), EbsOptimized: &s.EbsOptimized, } - if s.SubnetId != "" && s.AssociatePublicIpAddress { + subnetId := state.Get("subnet_id").(string) + + if subnetId != "" && s.AssociatePublicIpAddress { runOpts.NetworkInterfaces = []*ec2.InstanceNetworkInterfaceSpecification{ { DeviceIndex: aws.Int64(0), AssociatePublicIpAddress: &s.AssociatePublicIpAddress, - SubnetId: &s.SubnetId, + SubnetId: &subnetId, Groups: securityGroupIds, DeleteOnTermination: aws.Bool(true), }, } } else { - runOpts.SubnetId = &s.SubnetId + runOpts.SubnetId = &subnetId runOpts.SecurityGroupIds = securityGroupIds } diff --git a/builder/amazon/common/step_security_group.go b/builder/amazon/common/step_security_group.go index 6ef3f4dbb..4f8788222 100644 --- a/builder/amazon/common/step_security_group.go +++ b/builder/amazon/common/step_security_group.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "strings" "time" "github.com/aws/aws-sdk-go/aws" @@ -17,8 +18,8 @@ import ( type StepSecurityGroup struct { CommConfig *communicator.Config + SecurityGroupFilter SecurityGroupFilterOptions SecurityGroupIds []string - VpcId string TemporarySGSourceCidr string createdGroupId string @@ -27,6 +28,7 @@ type StepSecurityGroup struct { func (s *StepSecurityGroup) Run(_ context.Context, state multistep.StateBag) multistep.StepAction { ec2conn := state.Get("ec2").(*ec2.EC2) ui := state.Get("ui").(packer.Ui) + vpcId := state.Get("vpc_id").(string) if len(s.SecurityGroupIds) > 0 { _, err := ec2conn.DescribeSecurityGroups( @@ -45,6 +47,39 @@ func (s *StepSecurityGroup) Run(_ context.Context, state multistep.StateBag) mul return multistep.ActionContinue } + if !s.SecurityGroupFilter.Empty() { + + params := &ec2.DescribeSecurityGroupsInput{} + params.Filters = buildEc2Filters(s.SecurityGroupFilter.Filters) + vpcFilter := ec2.Filter{ + Name: aws.String("vpc-id"), + Values: []*string{ + aws.String(vpcId), + }, + } + params.Filters = append(params.Filters, &vpcFilter) + + log.Printf("Using SecurityGroup Filters %v", params) + + sgResp, err := ec2conn.DescribeSecurityGroups(params) + if err != nil { + err := fmt.Errorf("Couldn't find security groups for filter: %s", err) + log.Printf("[DEBUG] %s", err.Error()) + state.Put("error", err) + return multistep.ActionHalt + } + + securityGroupIds := []string{} + for _, sg := range sgResp.SecurityGroups { + securityGroupIds = append(securityGroupIds, *sg.GroupId) + } + + ui.Message(fmt.Sprintf("Found Security Group(s): %s", strings.Join(securityGroupIds, ", "))) + state.Put("securityGroupIds", securityGroupIds) + + return multistep.ActionContinue + } + port := s.CommConfig.Port() if port == 0 { if s.CommConfig.Type != "none" { @@ -60,16 +95,7 @@ func (s *StepSecurityGroup) Run(_ context.Context, state multistep.StateBag) mul Description: aws.String("Temporary group for Packer"), } - // TODO always get vpc_id from state. - if s.VpcId == "" { - if vpcId, ok := state.GetOk("vpc_id"); ok { - s.VpcId = vpcId.(string) - } - } - - if s.VpcId != "" { - group.VpcId = &s.VpcId - } + group.VpcId = &vpcId groupResp, err := ec2conn.CreateSecurityGroup(group) if err != nil { diff --git a/builder/amazon/ebs/builder.go b/builder/amazon/ebs/builder.go index d846f617d..61acd2c2a 100644 --- a/builder/amazon/ebs/builder.go +++ b/builder/amazon/ebs/builder.go @@ -93,28 +93,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe } ec2conn := ec2.New(session) - // TODO Translate this into VpcFilter/SubnetFilter and move the describe into apropriate step. - // If the subnet is specified but not the VpcId or AZ, try to determine them automatically - - /* - * If subnet => vpc, az - */ - if b.config.SubnetId != "" && (b.config.AvailabilityZone == "" || b.config.VpcId == "") { - log.Printf("[INFO] Finding AZ and VpcId for the given subnet '%s'", b.config.SubnetId) - resp, err := ec2conn.DescribeSubnets(&ec2.DescribeSubnetsInput{SubnetIds: []*string{&b.config.SubnetId}}) - if err != nil { - return nil, err - } - if b.config.AvailabilityZone == "" { - b.config.AvailabilityZone = *resp.Subnets[0].AvailabilityZone - log.Printf("[INFO] AvailabilityZone found: '%s'", b.config.AvailabilityZone) - } - if b.config.VpcId == "" { - b.config.VpcId = *resp.Subnets[0].VpcId - log.Printf("[INFO] VpcId found: '%s'", b.config.VpcId) - } - } - // Setup the state bag and initial state for the steps state := new(multistep.BasicStateBag) state.Put("config", b.config) @@ -128,7 +106,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe if b.config.IsSpotInstance() { instanceStep = &awscommon.StepRunSpotInstance{ AssociatePublicIpAddress: b.config.AssociatePublicIpAddress, - AvailabilityZone: b.config.AvailabilityZone, BlockDevices: b.config.BlockDevices, Ctx: b.config.ctx, Debug: b.config.PackerDebug, @@ -141,7 +118,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe SpotPrice: b.config.SpotPrice, SpotPriceProduct: b.config.SpotPriceAutoProduct, SpotTags: b.config.SpotTags, - SubnetId: b.config.SubnetId, Tags: b.config.RunTags, UserData: b.config.UserData, UserDataFile: b.config.UserDataFile, @@ -150,7 +126,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe } else { instanceStep = &awscommon.StepRunSourceInstance{ AssociatePublicIpAddress: b.config.AssociatePublicIpAddress, - AvailabilityZone: b.config.AvailabilityZone, BlockDevices: b.config.BlockDevices, Ctx: b.config.ctx, Debug: b.config.PackerDebug, @@ -162,7 +137,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe InstanceType: b.config.InstanceType, IsRestricted: b.config.IsChinaCloud() || b.config.IsGovCloud(), SourceAMI: b.config.SourceAmi, - SubnetId: b.config.SubnetId, Tags: b.config.RunTags, UserData: b.config.UserData, UserDataFile: b.config.UserDataFile, @@ -183,11 +157,13 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe AmiFilters: b.config.SourceAmiFilter, }, &awscommon.StepNetworkInfo{ - VpcId: b.config.VpcId, - VpcFilter: b.config.VpcFilter, - SubnetId: b.config.SubnetId, - SubnetFilter: b.config.SubnetFilter, - AvailabilityZone: b.config.AvailabilityZone, + VpcId: b.config.VpcId, + VpcFilter: b.config.VpcFilter, + SecurityGroupIds: b.config.SecurityGroupIds, + SecurityGroupFilter: b.config.SecurityGroupFilter, + SubnetId: b.config.SubnetId, + SubnetFilter: b.config.SubnetFilter, + AvailabilityZone: b.config.AvailabilityZone, }, &awscommon.StepKeyPair{ Debug: b.config.PackerDebug, @@ -198,11 +174,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe PrivateKeyFile: b.config.RunConfig.Comm.SSHPrivateKey, }, &awscommon.StepSecurityGroup{ - // TODO remove - SecurityGroupIds: b.config.SecurityGroupIds, - CommConfig: &b.config.RunConfig.Comm, - // TODO remove - VpcId: b.config.VpcId, + SecurityGroupFilter: b.config.SecurityGroupFilter, + SecurityGroupIds: b.config.SecurityGroupIds, + CommConfig: &b.config.RunConfig.Comm, TemporarySGSourceCidr: b.config.TemporarySGSourceCidr, }, &awscommon.StepCleanupVolumes{ diff --git a/builder/amazon/ebssurrogate/builder.go b/builder/amazon/ebssurrogate/builder.go index e9cc8ad7d..7abd1b6dc 100644 --- a/builder/amazon/ebssurrogate/builder.go +++ b/builder/amazon/ebssurrogate/builder.go @@ -107,23 +107,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe } ec2conn := ec2.New(session) - // If the subnet is specified but not the VpcId or AZ, try to determine them automatically - if b.config.SubnetId != "" && (b.config.AvailabilityZone == "" || b.config.VpcId == "") { - log.Printf("[INFO] Finding AZ and VpcId for the given subnet '%s'", b.config.SubnetId) - resp, err := ec2conn.DescribeSubnets(&ec2.DescribeSubnetsInput{SubnetIds: []*string{&b.config.SubnetId}}) - if err != nil { - return nil, err - } - if b.config.AvailabilityZone == "" { - b.config.AvailabilityZone = *resp.Subnets[0].AvailabilityZone - log.Printf("[INFO] AvailabilityZone found: '%s'", b.config.AvailabilityZone) - } - if b.config.VpcId == "" { - b.config.VpcId = *resp.Subnets[0].VpcId - log.Printf("[INFO] VpcId found: '%s'", b.config.VpcId) - } - } - // Setup the state bag and initial state for the steps state := new(multistep.BasicStateBag) state.Put("config", &b.config) @@ -137,7 +120,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe if b.config.IsSpotInstance() { instanceStep = &awscommon.StepRunSpotInstance{ AssociatePublicIpAddress: b.config.AssociatePublicIpAddress, - AvailabilityZone: b.config.AvailabilityZone, BlockDevices: b.config.BlockDevices, Ctx: b.config.ctx, Debug: b.config.PackerDebug, @@ -150,7 +132,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe SpotPrice: b.config.SpotPrice, SpotPriceProduct: b.config.SpotPriceAutoProduct, SpotTags: b.config.SpotTags, - SubnetId: b.config.SubnetId, Tags: b.config.RunTags, UserData: b.config.UserData, UserDataFile: b.config.UserDataFile, @@ -159,7 +140,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe } else { instanceStep = &awscommon.StepRunSourceInstance{ AssociatePublicIpAddress: b.config.AssociatePublicIpAddress, - AvailabilityZone: b.config.AvailabilityZone, BlockDevices: b.config.BlockDevices, Ctx: b.config.ctx, Debug: b.config.PackerDebug, @@ -171,7 +151,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe InstanceType: b.config.InstanceType, IsRestricted: b.config.IsChinaCloud() || b.config.IsGovCloud(), SourceAMI: b.config.SourceAmi, - SubnetId: b.config.SubnetId, Tags: b.config.RunTags, UserData: b.config.UserData, UserDataFile: b.config.UserDataFile, @@ -194,6 +173,15 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe EnableAMIENASupport: b.config.AMIENASupport, AmiFilters: b.config.SourceAmiFilter, }, + &awscommon.StepNetworkInfo{ + VpcId: b.config.VpcId, + VpcFilter: b.config.VpcFilter, + SecurityGroupIds: b.config.SecurityGroupIds, + SecurityGroupFilter: b.config.SecurityGroupFilter, + SubnetId: b.config.SubnetId, + SubnetFilter: b.config.SubnetFilter, + AvailabilityZone: b.config.AvailabilityZone, + }, &awscommon.StepKeyPair{ Debug: b.config.PackerDebug, SSHAgentAuth: b.config.Comm.SSHAgentAuth, @@ -203,9 +191,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe PrivateKeyFile: b.config.RunConfig.Comm.SSHPrivateKey, }, &awscommon.StepSecurityGroup{ - SecurityGroupIds: b.config.SecurityGroupIds, - CommConfig: &b.config.RunConfig.Comm, - VpcId: b.config.VpcId, + SecurityGroupFilter: b.config.SecurityGroupFilter, + SecurityGroupIds: b.config.SecurityGroupIds, + CommConfig: &b.config.RunConfig.Comm, TemporarySGSourceCidr: b.config.TemporarySGSourceCidr, }, &awscommon.StepCleanupVolumes{ diff --git a/builder/amazon/ebsvolume/builder.go b/builder/amazon/ebsvolume/builder.go index 222febc56..acdc882bf 100644 --- a/builder/amazon/ebsvolume/builder.go +++ b/builder/amazon/ebsvolume/builder.go @@ -92,23 +92,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe } ec2conn := ec2.New(session) - // If the subnet is specified but not the VpcId or AZ, try to determine them automatically - if b.config.SubnetId != "" && (b.config.AvailabilityZone == "" || b.config.VpcId == "") { - log.Printf("[INFO] Finding AZ and VpcId for the given subnet '%s'", b.config.SubnetId) - resp, err := ec2conn.DescribeSubnets(&ec2.DescribeSubnetsInput{SubnetIds: []*string{&b.config.SubnetId}}) - if err != nil { - return nil, err - } - if b.config.AvailabilityZone == "" { - b.config.AvailabilityZone = *resp.Subnets[0].AvailabilityZone - log.Printf("[INFO] AvailabilityZone found: '%s'", b.config.AvailabilityZone) - } - if b.config.VpcId == "" { - b.config.VpcId = *resp.Subnets[0].VpcId - log.Printf("[INFO] VpcId found: '%s'", b.config.VpcId) - } - } - // Setup the state bag and initial state for the steps state := new(multistep.BasicStateBag) state.Put("config", b.config) @@ -121,7 +104,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe if b.config.IsSpotInstance() { instanceStep = &awscommon.StepRunSpotInstance{ AssociatePublicIpAddress: b.config.AssociatePublicIpAddress, - AvailabilityZone: b.config.AvailabilityZone, BlockDevices: b.config.launchBlockDevices, Ctx: b.config.ctx, Debug: b.config.PackerDebug, @@ -134,7 +116,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe SpotPrice: b.config.SpotPrice, SpotPriceProduct: b.config.SpotPriceAutoProduct, SpotTags: b.config.SpotTags, - SubnetId: b.config.SubnetId, Tags: b.config.RunTags, UserData: b.config.UserData, UserDataFile: b.config.UserDataFile, @@ -142,7 +123,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe } else { instanceStep = &awscommon.StepRunSourceInstance{ AssociatePublicIpAddress: b.config.AssociatePublicIpAddress, - AvailabilityZone: b.config.AvailabilityZone, BlockDevices: b.config.launchBlockDevices, Ctx: b.config.ctx, Debug: b.config.PackerDebug, @@ -154,7 +134,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe InstanceType: b.config.InstanceType, IsRestricted: b.config.IsChinaCloud() || b.config.IsGovCloud(), SourceAMI: b.config.SourceAmi, - SubnetId: b.config.SubnetId, Tags: b.config.RunTags, UserData: b.config.UserData, UserDataFile: b.config.UserDataFile, @@ -169,6 +148,15 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe EnableAMIENASupport: b.config.AMIENASupport, AmiFilters: b.config.SourceAmiFilter, }, + &awscommon.StepNetworkInfo{ + VpcId: b.config.VpcId, + VpcFilter: b.config.VpcFilter, + SecurityGroupIds: b.config.SecurityGroupIds, + SecurityGroupFilter: b.config.SecurityGroupFilter, + SubnetId: b.config.SubnetId, + SubnetFilter: b.config.SubnetFilter, + AvailabilityZone: b.config.AvailabilityZone, + }, &awscommon.StepKeyPair{ Debug: b.config.PackerDebug, SSHAgentAuth: b.config.Comm.SSHAgentAuth, @@ -178,9 +166,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe PrivateKeyFile: b.config.RunConfig.Comm.SSHPrivateKey, }, &awscommon.StepSecurityGroup{ - SecurityGroupIds: b.config.SecurityGroupIds, - CommConfig: &b.config.RunConfig.Comm, - VpcId: b.config.VpcId, + SecurityGroupFilter: b.config.SecurityGroupFilter, + SecurityGroupIds: b.config.SecurityGroupIds, + CommConfig: &b.config.RunConfig.Comm, TemporarySGSourceCidr: b.config.TemporarySGSourceCidr, }, instanceStep, diff --git a/builder/amazon/instance/builder.go b/builder/amazon/instance/builder.go index fb59fff95..d26fc1c10 100644 --- a/builder/amazon/instance/builder.go +++ b/builder/amazon/instance/builder.go @@ -178,23 +178,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe } ec2conn := ec2.New(session) - // If the subnet is specified but not the VpcId or AZ, try to determine them automatically - if b.config.SubnetId != "" && (b.config.AvailabilityZone == "" || b.config.VpcId == "") { - log.Printf("[INFO] Finding AZ and VpcId for the given subnet '%s'", b.config.SubnetId) - resp, err := ec2conn.DescribeSubnets(&ec2.DescribeSubnetsInput{SubnetIds: []*string{&b.config.SubnetId}}) - if err != nil { - return nil, err - } - if b.config.AvailabilityZone == "" { - b.config.AvailabilityZone = *resp.Subnets[0].AvailabilityZone - log.Printf("[INFO] AvailabilityZone found: '%s'", b.config.AvailabilityZone) - } - if b.config.VpcId == "" { - b.config.VpcId = *resp.Subnets[0].VpcId - log.Printf("[INFO] VpcId found: '%s'", b.config.VpcId) - } - } - // Setup the state bag and initial state for the steps state := new(multistep.BasicStateBag) state.Put("config", &b.config) @@ -208,7 +191,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe if b.config.IsSpotInstance() { instanceStep = &awscommon.StepRunSpotInstance{ AssociatePublicIpAddress: b.config.AssociatePublicIpAddress, - AvailabilityZone: b.config.AvailabilityZone, BlockDevices: b.config.BlockDevices, Ctx: b.config.ctx, Debug: b.config.PackerDebug, @@ -218,7 +200,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe SourceAMI: b.config.SourceAmi, SpotPrice: b.config.SpotPrice, SpotPriceProduct: b.config.SpotPriceAutoProduct, - SubnetId: b.config.SubnetId, Tags: b.config.RunTags, SpotTags: b.config.SpotTags, UserData: b.config.UserData, @@ -227,7 +208,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe } else { instanceStep = &awscommon.StepRunSourceInstance{ AssociatePublicIpAddress: b.config.AssociatePublicIpAddress, - AvailabilityZone: b.config.AvailabilityZone, BlockDevices: b.config.BlockDevices, Ctx: b.config.ctx, Debug: b.config.PackerDebug, @@ -237,7 +217,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe InstanceType: b.config.InstanceType, IsRestricted: b.config.IsChinaCloud() || b.config.IsGovCloud(), SourceAMI: b.config.SourceAmi, - SubnetId: b.config.SubnetId, Tags: b.config.RunTags, UserData: b.config.UserData, UserDataFile: b.config.UserDataFile, @@ -256,6 +235,15 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe EnableAMIENASupport: b.config.AMIENASupport, AmiFilters: b.config.SourceAmiFilter, }, + &awscommon.StepNetworkInfo{ + VpcId: b.config.VpcId, + VpcFilter: b.config.VpcFilter, + SecurityGroupIds: b.config.SecurityGroupIds, + SecurityGroupFilter: b.config.SecurityGroupFilter, + SubnetId: b.config.SubnetId, + SubnetFilter: b.config.SubnetFilter, + AvailabilityZone: b.config.AvailabilityZone, + }, &awscommon.StepKeyPair{ Debug: b.config.PackerDebug, SSHAgentAuth: b.config.Comm.SSHAgentAuth, @@ -265,9 +253,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe TemporaryKeyPairName: b.config.TemporaryKeyPairName, }, &awscommon.StepSecurityGroup{ - CommConfig: &b.config.RunConfig.Comm, - SecurityGroupIds: b.config.SecurityGroupIds, - VpcId: b.config.VpcId, + CommConfig: &b.config.RunConfig.Comm, + SecurityGroupFilter: b.config.SecurityGroupFilter, + SecurityGroupIds: b.config.SecurityGroupIds, TemporarySGSourceCidr: b.config.TemporarySGSourceCidr, }, instanceStep, From 23f62f221e6ed929f5b640d5ab83cb0e7045095b Mon Sep 17 00:00:00 2001 From: Rickard von Essen Date: Fri, 24 Aug 2018 19:47:11 +0200 Subject: [PATCH 3/4] Only filter SG's on VPC if it's set. Only use VPC's and Subnets that are available --- builder/amazon/common/step_network_info.go | 2 ++ builder/amazon/common/step_security_group.go | 10 +++------- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/builder/amazon/common/step_network_info.go b/builder/amazon/common/step_network_info.go index 7fc026335..27cd99d73 100644 --- a/builder/amazon/common/step_network_info.go +++ b/builder/amazon/common/step_network_info.go @@ -53,6 +53,7 @@ func (s *StepNetworkInfo) Run(_ context.Context, state multistep.StateBag) multi if s.VpcId == "" && !s.VpcFilter.Empty() { params := &ec2.DescribeVpcsInput{} params.Filters = buildEc2Filters(s.VpcFilter.Filters) + s.VpcFilter.Filters[aws.String("state")] = aws.String("available") log.Printf("Using VPC Filters %v", params) @@ -78,6 +79,7 @@ func (s *StepNetworkInfo) Run(_ context.Context, state multistep.StateBag) multi // Subnet if s.SubnetId == "" && !s.SubnetFilter.Empty() { params := &ec2.DescribeSubnetsInput{} + s.SubnetFilter.Filters[aws.String("state")] = aws.String("available") if s.VpcId != "" { s.SubnetFilter.Filters[aws.String("vpc-id")] = &s.VpcId diff --git a/builder/amazon/common/step_security_group.go b/builder/amazon/common/step_security_group.go index 4f8788222..6c358b9a9 100644 --- a/builder/amazon/common/step_security_group.go +++ b/builder/amazon/common/step_security_group.go @@ -50,14 +50,10 @@ func (s *StepSecurityGroup) Run(_ context.Context, state multistep.StateBag) mul if !s.SecurityGroupFilter.Empty() { params := &ec2.DescribeSecurityGroupsInput{} - params.Filters = buildEc2Filters(s.SecurityGroupFilter.Filters) - vpcFilter := ec2.Filter{ - Name: aws.String("vpc-id"), - Values: []*string{ - aws.String(vpcId), - }, + if vpcId != "" { + s.SecurityGroupFilter.Filters[aws.String("vpc-id")] = &vpcId } - params.Filters = append(params.Filters, &vpcFilter) + params.Filters = buildEc2Filters(s.SecurityGroupFilter.Filters) log.Printf("Using SecurityGroup Filters %v", params) From 414e10d43de88940e06168443848b174768bbaf0 Mon Sep 17 00:00:00 2001 From: Rickard von Essen Date: Fri, 24 Aug 2018 20:11:45 +0200 Subject: [PATCH 4/4] Added docs for subnet_filter, vpc_filter, and security_group_filter --- .../source/docs/builders/amazon-ebs.html.md | 80 +++++++++++++++++++ .../docs/builders/amazon-ebssurrogate.html.md | 80 +++++++++++++++++++ .../docs/builders/amazon-ebsvolume.html.md | 80 +++++++++++++++++++ .../docs/builders/amazon-instance.html.md | 80 +++++++++++++++++++ 4 files changed, 320 insertions(+) diff --git a/website/source/docs/builders/amazon-ebs.html.md b/website/source/docs/builders/amazon-ebs.html.md index f238d5190..fe3282fb7 100644 --- a/website/source/docs/builders/amazon-ebs.html.md +++ b/website/source/docs/builders/amazon-ebs.html.md @@ -267,6 +267,27 @@ builder. described above. Note that if this is specified, you must omit the `security_group_id`. +- `security_group_filter` (object) - Filters used to populate the `security_group_ids` field. + Example: + + ``` json + { + "security_group_filter": { + "filters": { + "tag:Class": "packer" + } + } + } + ``` + + This selects the SG's with tag `Class` with the value `packer`. + + - `filters` (map of strings) - filters used to select a `security_group_ids`. + Any filter described in the docs for [DescribeSecurityGroups](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSecurityGroups.html) + is valid. + + `security_group_ids` take precendense over this. + - `temporary_security_group_source_cidr` (string) - An IPv4 CIDR block to be authorized access to the instance, when packer is creating a temporary security group. The default is `0.0.0.0/0` (ie, allow any IPv4 source). This is only used @@ -387,6 +408,39 @@ builder. `subnet-12345def`, where Packer will launch the EC2 instance. This field is required if you are using an non-default VPC. +- `subnet_filter` (object) - Filters used to populate the `subnet_id` field. + Example: + + ``` json + { + "subnet_filter": { + "filters": { + "tag:Class": "build" + }, + "most_free": true, + "random": false + } + } + ``` + + This selects the Subnet with tag `Class` with the value `build`, which has + the most free IP addresses. + NOTE: This will fail unless *exactly* one Subnet is returned. By using + `most_free` or `random` one will be selected from those matching the filter. + + - `filters` (map of strings) - filters used to select a `subnet_id`. + NOTE: This will fail unless *exactly* one Subnet is returned. + Any filter described in the docs for [DescribeSubnets](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSubnets.html) + is valid. + + - `most_free` (boolean) - The Subnet with the most free IPv4 addresses + will be used if multiple Subnets matches the filter. + + - `random` (boolean) - A random Subnet will be used if multiple Subnets + matches the filter. `most_free` have precendence over this. + + `subnet_id` take precedence over this. + - `tags` (object of key/value strings) - Tags applied to the AMI and relevant snapshots. This is a [template engine](/docs/templates/engine.html), @@ -413,6 +467,32 @@ builder. to be set. If this field is left blank, Packer will try to get the VPC ID from the `subnet_id`. +- `vpc_filter` (object) - Filters used to populate the `vpc_id` field. + Example: + + ``` json + { + "vpc_filter": { + "filters": { + "tag:Class": "build", + "isDefault": "false", + "cidr": "/24" + } + } + } + ``` + + This selects the VPC with tag `Class` with the value `build`, which is not the + default VPC, and have a IPv4 CIDR block of `/24`. + NOTE: This will fail unless *exactly* one VPC is returned. + + - `filters` (map of strings) - filters used to select a `vpc_id`. + NOTE: This will fail unless *exactly* one VPC is returned. + Any filter described in the docs for [DescribeVpcs](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeVpcs.html) + is valid. + + `vpc_id` take precedence over this. + - `windows_password_timeout` (string) - The timeout for waiting for a Windows password for Windows instances. Defaults to 20 minutes. Example value: `10m` diff --git a/website/source/docs/builders/amazon-ebssurrogate.html.md b/website/source/docs/builders/amazon-ebssurrogate.html.md index d2672be0d..bd699ca0b 100644 --- a/website/source/docs/builders/amazon-ebssurrogate.html.md +++ b/website/source/docs/builders/amazon-ebssurrogate.html.md @@ -260,6 +260,27 @@ builder. described above. Note that if this is specified, you must omit the `security_group_id`. +- `security_group_filter` (object) - Filters used to populate the `security_group_ids` field. + Example: + + ``` json + { + "security_group_filter": { + "filters": { + "tag:Class": "packer" + } + } + } + ``` + + This selects the SG's with tag `Class` with the value `packer`. + + - `filters` (map of strings) - filters used to select a `security_group_ids`. + Any filter described in the docs for [DescribeSecurityGroups](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSecurityGroups.html) + is valid. + + `security_group_ids` take precendense over this. + - `temporary_security_group_source_cidr` (string) - An IPv4 CIDR block to be authorized access to the instance, when packer is creating a temporary security group. The default is `0.0.0.0/0` (ie, allow any IPv4 source). This is only used @@ -380,6 +401,39 @@ builder. `subnet-12345def`, where Packer will launch the EC2 instance. This field is required if you are using an non-default VPC. +- `subnet_filter` (object) - Filters used to populate the `subnet_id` field. + Example: + + ``` json + { + "subnet_filter": { + "filters": { + "tag:Class": "build" + }, + "most_free": true, + "random": false + } + } + ``` + + This selects the Subnet with tag `Class` with the value `build`, which has + the most free IP addresses. + NOTE: This will fail unless *exactly* one Subnet is returned. By using + `most_free` or `random` one will be selected from those matching the filter. + + - `filters` (map of strings) - filters used to select a `subnet_id`. + NOTE: This will fail unless *exactly* one Subnet is returned. + Any filter described in the docs for [DescribeSubnets](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSubnets.html) + is valid. + + - `most_free` (boolean) - The Subnet with the most free IPv4 addresses + will be used if multiple Subnets matches the filter. + + - `random` (boolean) - A random Subnet will be used if multiple Subnets + matches the filter. `most_free` have precendence over this. + + `subnet_id` take precedence over this. + - `tags` (object of key/value strings) - Tags applied to the AMI and relevant snapshots. This is a [template engine](/docs/templates/engine.html), @@ -405,6 +459,32 @@ builder. to be set. If this field is left blank, Packer will try to get the VPC ID from the `subnet_id`. +- `vpc_filter` (object) - Filters used to populate the `vpc_id` field. + Example: + + ``` json + { + "vpc_filter": { + "filters": { + "tag:Class": "build", + "isDefault": "false", + "cidr": "/24" + } + } + } + ``` + + This selects the VPC with tag `Class` with the value `build`, which is not the + default VPC, and have a IPv4 CIDR block of `/24`. + NOTE: This will fail unless *exactly* one VPC is returned. + + - `filters` (map of strings) - filters used to select a `vpc_id`. + NOTE: This will fail unless *exactly* one VPC is returned. + Any filter described in the docs for [DescribeVpcs](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeVpcs.html) + is valid. + + `vpc_id` take precedence over this. + - `windows_password_timeout` (string) - The timeout for waiting for a Windows password for Windows instances. Defaults to 20 minutes. Example value: `10m` diff --git a/website/source/docs/builders/amazon-ebsvolume.html.md b/website/source/docs/builders/amazon-ebsvolume.html.md index 26306a1ef..63cea5bcf 100644 --- a/website/source/docs/builders/amazon-ebsvolume.html.md +++ b/website/source/docs/builders/amazon-ebsvolume.html.md @@ -193,6 +193,27 @@ builder. described above. Note that if this is specified, you must omit the `security_group_id`. +- `security_group_filter` (object) - Filters used to populate the `security_group_ids` field. + Example: + + ``` json + { + "security_group_filter": { + "filters": { + "tag:Class": "packer" + } + } + } + ``` + + This selects the SG's with tag `Class` with the value `packer`. + + - `filters` (map of strings) - filters used to select a `security_group_ids`. + Any filter described in the docs for [DescribeSecurityGroups](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSecurityGroups.html) + is valid. + + `security_group_ids` take precendense over this. + - `temporary_security_group_source_cidr` (string) - An IPv4 CIDR block to be authorized access to the instance, when packer is creating a temporary security group. The default is `0.0.0.0/0` (ie, allow any IPv4 source). This is only used @@ -299,6 +320,39 @@ builder. `subnet-12345def`, where Packer will launch the EC2 instance. This field is required if you are using an non-default VPC. +- `subnet_filter` (object) - Filters used to populate the `subnet_id` field. + Example: + + ``` json + { + "subnet_filter": { + "filters": { + "tag:Class": "build" + }, + "most_free": true, + "random": false + } + } + ``` + + This selects the Subnet with tag `Class` with the value `build`, which has + the most free IP addresses. + NOTE: This will fail unless *exactly* one Subnet is returned. By using + `most_free` or `random` one will be selected from those matching the filter. + + - `filters` (map of strings) - filters used to select a `subnet_id`. + NOTE: This will fail unless *exactly* one Subnet is returned. + Any filter described in the docs for [DescribeSubnets](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSubnets.html) + is valid. + + - `most_free` (boolean) - The Subnet with the most free IPv4 addresses + will be used if multiple Subnets matches the filter. + + - `random` (boolean) - A random Subnet will be used if multiple Subnets + matches the filter. `most_free` have precendence over this. + + `subnet_id` take precedence over this. + - `temporary_key_pair_name` (string) - The name of the temporary key pair to generate. By default, Packer generates a name that looks like `packer_`, where <UUID> is a 36 character unique identifier. @@ -320,6 +374,32 @@ builder. to be set. If this field is left blank, Packer will try to get the VPC ID from the `subnet_id`. +- `vpc_filter` (object) - Filters used to populate the `vpc_id` field. + Example: + + ``` json + { + "vpc_filter": { + "filters": { + "tag:Class": "build", + "isDefault": "false", + "cidr": "/24" + } + } + } + ``` + + This selects the VPC with tag `Class` with the value `build`, which is not the + default VPC, and have a IPv4 CIDR block of `/24`. + NOTE: This will fail unless *exactly* one VPC is returned. + + - `filters` (map of strings) - filters used to select a `vpc_id`. + NOTE: This will fail unless *exactly* one VPC is returned. + Any filter described in the docs for [DescribeVpcs](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeVpcs.html) + is valid. + + `vpc_id` take precedence over this. + - `windows_password_timeout` (string) - The timeout for waiting for a Windows password for Windows instances. Defaults to 20 minutes. Example value: `10m` diff --git a/website/source/docs/builders/amazon-instance.html.md b/website/source/docs/builders/amazon-instance.html.md index fa0a8d3e0..f4d136843 100644 --- a/website/source/docs/builders/amazon-instance.html.md +++ b/website/source/docs/builders/amazon-instance.html.md @@ -269,6 +269,27 @@ builder. described above. Note that if this is specified, you must omit the `security_group_id`. +- `security_group_filter` (object) - Filters used to populate the `security_group_ids` field. + Example: + + ``` json + { + "security_group_filter": { + "filters": { + "tag:Class": "packer" + } + } + } + ``` + + This selects the SG's with tag `Class` with the value `packer`. + + - `filters` (map of strings) - filters used to select a `security_group_ids`. + Any filter described in the docs for [DescribeSecurityGroups](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSecurityGroups.html) + is valid. + + `security_group_ids` take precendense over this. + - `temporary_security_group_source_cidr` (string) - An IPv4 CIDR block to be authorized access to the instance, when packer is creating a temporary security group. The default is `0.0.0.0/0` (ie, allow any IPv4 source). This is only used @@ -383,6 +404,39 @@ builder. `subnet-12345def`, where Packer will launch the EC2 instance. This field is required if you are using an non-default VPC. +- `subnet_filter` (object) - Filters used to populate the `subnet_id` field. + Example: + + ``` json + { + "subnet_filter": { + "filters": { + "tag:Class": "build" + }, + "most_free": true, + "random": false + } + } + ``` + + This selects the Subnet with tag `Class` with the value `build`, which has + the most free IP addresses. + NOTE: This will fail unless *exactly* one Subnet is returned. By using + `most_free` or `random` one will be selected from those matching the filter. + + - `filters` (map of strings) - filters used to select a `subnet_id`. + NOTE: This will fail unless *exactly* one Subnet is returned. + Any filter described in the docs for [DescribeSubnets](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSubnets.html) + is valid. + + - `most_free` (boolean) - The Subnet with the most free IPv4 addresses + will be used if multiple Subnets matches the filter. + + - `random` (boolean) - A random Subnet will be used if multiple Subnets + matches the filter. `most_free` have precendence over this. + + `subnet_id` take precedence over this. + - `tags` (object of key/value strings) - Tags applied to the AMI. This is a [template engine](/docs/templates/engine.html), see [Build template data](#build-template-data) for more information. @@ -403,6 +457,32 @@ builder. to be set. If this field is left blank, Packer will try to get the VPC ID from the `subnet_id`. +- `vpc_filter` (object) - Filters used to populate the `vpc_id` field. + Example: + + ``` json + { + "vpc_filter": { + "filters": { + "tag:Class": "build", + "isDefault": "false", + "cidr": "/24" + } + } + } + ``` + + This selects the VPC with tag `Class` with the value `build`, which is not the + default VPC, and have a IPv4 CIDR block of `/24`. + NOTE: This will fail unless *exactly* one VPC is returned. + + - `filters` (map of strings) - filters used to select a `vpc_id`. + NOTE: This will fail unless *exactly* one VPC is returned. + Any filter described in the docs for [DescribeVpcs](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeVpcs.html) + is valid. + + `vpc_id` take precedence over this. + - `x509_upload_path` (string) - The path on the remote machine where the X509 certificate will be uploaded. This path must already exist and be writable. X509 certificates are uploaded after provisioning is run, so it is perfectly