Merge pull request #5841 from hashicorp/fix5447

Add volume and run tags if in us-gov/china

builder/amazon/chroot/device_test.go

@@ -0,0 +1,10 @@
+package chroot
+
+import "testing"
+
+func TestDevicePrefixMatch(t *testing.T) {
+	/*
+		if devicePrefixMatch("nvme0n1") != "" {
+		}
+	*/
+}

builder/amazon/common/access_config.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -80,6 +81,21 @@ func (c *AccessConfig) Session() (*session.Session, error) {
 	return c.session, nil
 }
 
+func (c *AccessConfig) SessionRegion() string {
+	if c.session == nil {
+		panic("access config session should be set.")
+	}
+	return aws.StringValue(c.session.Config.Region)
+}
+
+func (c *AccessConfig) IsGovCloud() bool {
+	return strings.HasPrefix(c.SessionRegion(), "us-gov-")
+}
+
+func (c *AccessConfig) IsChinaCloud() bool {
+	return strings.HasPrefix(c.SessionRegion(), "cn-")
+}
+
 // metadataRegion returns the region from the metadata service
 func (c *AccessConfig) metadataRegion() string {
 

builder/amazon/common/access_config_test.go

@@ -2,6 +2,9 @@ package common
 
 import (
 	"testing"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
 )
 
 func testAccessConfig() *AccessConfig {
@@ -38,3 +41,20 @@ func TestAccessConfigPrepare_Region(t *testing.T) {
 	c.SkipValidation = false
 
 }
+
+func TestAccessConfigPrepare_RegionRestrictd(t *testing.T) {
+	c := testAccessConfig()
+
+	// Create a Session with a custom region
+	c.session = session.Must(session.NewSession(&aws.Config{
+		Region: aws.String("us-gov-west-1"),
+	}))
+
+	if err := c.Prepare(nil); err != nil {
+		t.Fatalf("shouldn't have err: %s", err)
+	}
+
+	if !c.IsGovCloud() {
+		t.Fatal("We should be in gov region.")
+	}
+}

builder/amazon/common/ami_config.go

@@ -17,7 +17,7 @@ type AMIConfig struct {
 	AMIProductCodes         []string          `mapstructure:"ami_product_codes"`
 	AMIRegions              []string          `mapstructure:"ami_regions"`
 	AMISkipRegionValidation bool              `mapstructure:"skip_region_validation"`
-	AMITags                 map[string]string `mapstructure:"tags"`
+	AMITags                 TagMap            `mapstructure:"tags"`
 	AMIENASupport           bool              `mapstructure:"ena_support"`
 	AMISriovNetSupport      bool              `mapstructure:"sriov_support"`
 	AMIForceDeregister      bool              `mapstructure:"force_deregister"`
@@ -25,7 +25,7 @@ type AMIConfig struct {
 	AMIEncryptBootVolume    bool              `mapstructure:"encrypt_boot"`
 	AMIKmsKeyId             string            `mapstructure:"kms_key_id"`
 	AMIRegionKMSKeyIDs      map[string]string `mapstructure:"region_kms_key_ids"`
-	SnapshotTags            map[string]string `mapstructure:"snapshot_tags"`
+	SnapshotTags            TagMap            `mapstructure:"snapshot_tags"`
 	SnapshotUsers           []string          `mapstructure:"snapshot_users"`
 	SnapshotGroups          []string          `mapstructure:"snapshot_groups"`
 }

builder/amazon/common/step_create_tags.go

@@ -15,8 +15,8 @@ import (
 )
 
 type StepCreateTags struct {
-	Tags         map[string]string
-	SnapshotTags map[string]string
+	Tags         TagMap
+	SnapshotTags TagMap
 	Ctx          interpolate.Context
 }
 
@@ -33,7 +33,7 @@ func (s *StepCreateTags) Run(_ context.Context, state multistep.StateBag) multis
 		sourceAMI = ""
 	}
 
-	if len(s.Tags) == 0 && len(s.SnapshotTags) == 0 {
+	if !s.Tags.IsSet() && !s.SnapshotTags.IsSet() {
 		return multistep.ActionContinue
 	}
 
@@ -79,22 +79,22 @@ func (s *StepCreateTags) Run(_ context.Context, state multistep.StateBag) multis
 
 		// Convert tags to ec2.Tag format
 		ui.Say("Creating AMI tags")
-		amiTags, err := ConvertToEC2Tags(s.Tags, *ec2conn.Config.Region, sourceAMI, s.Ctx)
+		amiTags, err := s.Tags.EC2Tags(s.Ctx, *ec2conn.Config.Region, sourceAMI)
 		if err != nil {
 			state.Put("error", err)
 			ui.Error(err.Error())
 			return multistep.ActionHalt
 		}
-		ReportTags(ui, amiTags)
+		amiTags.Report(ui)
 
 		ui.Say("Creating snapshot tags")
-		snapshotTags, err := ConvertToEC2Tags(s.SnapshotTags, *ec2conn.Config.Region, sourceAMI, s.Ctx)
+		snapshotTags, err := s.SnapshotTags.EC2Tags(s.Ctx, *ec2conn.Config.Region, sourceAMI)
 		if err != nil {
 			state.Put("error", err)
 			ui.Error(err.Error())
 			return multistep.ActionHalt
 		}
-		ReportTags(ui, snapshotTags)
+		snapshotTags.Report(ui)
 
 		// Retry creating tags for about 2.5 minutes
 		err = retry.Retry(0.2, 30, 11, func(_ uint) (bool, error) {
@@ -142,36 +142,3 @@ func (s *StepCreateTags) Run(_ context.Context, state multistep.StateBag) multis
 func (s *StepCreateTags) Cleanup(state multistep.StateBag) {
 	// No cleanup...
 }
-
-func ReportTags(ui packer.Ui, tags []*ec2.Tag) {
-	for _, tag := range tags {
-		ui.Message(fmt.Sprintf("Adding tag: \"%s\": \"%s\"",
-			aws.StringValue(tag.Key), aws.StringValue(tag.Value)))
-	}
-}
-
-func ConvertToEC2Tags(tags map[string]string, region, sourceAmiId string, ctx interpolate.Context) ([]*ec2.Tag, error) {
-	var ec2Tags []*ec2.Tag
-	for key, value := range tags {
-
-		ctx.Data = &BuildInfoTemplate{
-			SourceAMI:   sourceAmiId,
-			BuildRegion: region,
-		}
-		interpolatedKey, err := interpolate.Render(key, &ctx)
-		if err != nil {
-			return ec2Tags, fmt.Errorf("Error processing tag: %s:%s - %s", key, value, err)
-		}
-		interpolatedValue, err := interpolate.Render(value, &ctx)
-		if err != nil {
-			return ec2Tags, fmt.Errorf("Error processing tag: %s:%s - %s", key, value, err)
-		}
-
-		ec2Tags = append(ec2Tags, &ec2.Tag{
-			Key:   aws.String(interpolatedKey),
-			Value: aws.String(interpolatedValue),
-		})
-	}
-
-	return ec2Tags, nil
-}

builder/amazon/common/step_run_source_instance.go

@@ -8,8 +8,10 @@ import (
 	"log"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/ec2"
 
+	retry "github.com/hashicorp/packer/common"
 	"github.com/hashicorp/packer/helper/multistep"
 	"github.com/hashicorp/packer/packer"
 	"github.com/hashicorp/packer/template/interpolate"
@@ -19,19 +21,20 @@ type StepRunSourceInstance struct {
 	AssociatePublicIpAddress          bool
 	AvailabilityZone                  string
 	BlockDevices                      BlockDevices
+	Ctx                               interpolate.Context
 	Debug                             bool
 	EbsOptimized                      bool
 	ExpectedRootDevice                string
 	IamInstanceProfile                string
 	InstanceInitiatedShutdownBehavior string
 	InstanceType                      string
+	IsRestricted                      bool
 	SourceAMI                         string
 	SubnetId                          string
-	Tags                              map[string]string
-	VolumeTags                        map[string]string
+	Tags                              TagMap
 	UserData                          string
 	UserDataFile                      string
-	Ctx                               interpolate.Context
+	VolumeTags                        TagMap
 
 	instanceId string
 }
@@ -85,16 +88,15 @@ func (s *StepRunSourceInstance) Run(_ context.Context, state multistep.StateBag)
 		s.Tags["Name"] = "Packer Builder"
 	}
 
-	ec2Tags, err := ConvertToEC2Tags(s.Tags, *ec2conn.Config.Region, s.SourceAMI, s.Ctx)
+	ec2Tags, err := s.Tags.EC2Tags(s.Ctx, *ec2conn.Config.Region, s.SourceAMI)
 	if err != nil {
 		err := fmt.Errorf("Error tagging source instance: %s", err)
 		state.Put("error", err)
 		ui.Error(err.Error())
 		return multistep.ActionHalt
 	}
-	ReportTags(ui, ec2Tags)
 
-	volTags, err := ConvertToEC2Tags(s.VolumeTags, *ec2conn.Config.Region, s.SourceAMI, s.Ctx)
+	volTags, err := s.VolumeTags.EC2Tags(s.Ctx, *ec2conn.Config.Region, s.SourceAMI)
 	if err != nil {
 		err := fmt.Errorf("Error tagging volumes: %s", err)
 		state.Put("error", err)
@@ -114,6 +116,7 @@ func (s *StepRunSourceInstance) Run(_ context.Context, state multistep.StateBag)
 		EbsOptimized:        &s.EbsOptimized,
 	}
 
+	// Collect tags for tagging on resource creation
 	var tagSpecs []*ec2.TagSpecification
 
 	if len(ec2Tags) > 0 {
@@ -134,8 +137,11 @@ func (s *StepRunSourceInstance) Run(_ context.Context, state multistep.StateBag)
 		tagSpecs = append(tagSpecs, runVolTags)
 	}
 
-	if len(tagSpecs) > 0 {
+	// If our region supports it, set tag specifications
+	if len(tagSpecs) > 0 && !s.IsRestricted {
 		runOpts.SetTagSpecifications(tagSpecs)
+		ec2Tags.Report(ui)
+		volTags.Report(ui)
 	}
 
 	if keyName != "" {
@@ -212,6 +218,70 @@ func (s *StepRunSourceInstance) Run(_ context.Context, state multistep.StateBag)
 
 	state.Put("instance", instance)
 
+	// If we're in a region that doesn't support tagging on instance creation,
+	// do that now.
+
+	if s.IsRestricted {
+		ec2Tags.Report(ui)
+		// Retry creating tags for about 2.5 minutes
+		err = retry.Retry(0.2, 30, 11, func(_ uint) (bool, error) {
+			_, err := ec2conn.CreateTags(&ec2.CreateTagsInput{
+				Tags:      ec2Tags,
+				Resources: []*string{instance.InstanceId},
+			})
+			if err == nil {
+				return true, nil
+			}
+			if awsErr, ok := err.(awserr.Error); ok {
+				if awsErr.Code() == "InvalidInstanceID.NotFound" {
+					return false, nil
+				}
+			}
+			return true, err
+		})
+
+		if err != nil {
+			err := fmt.Errorf("Error tagging source instance: %s", err)
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
+		}
+
+		// Now tag volumes
+
+		volumeIds := make([]*string, 0)
+		for _, v := range instance.BlockDeviceMappings {
+			if ebs := v.Ebs; ebs != nil {
+				volumeIds = append(volumeIds, ebs.VolumeId)
+			}
+		}
+
+		if len(volumeIds) > 0 && s.VolumeTags.IsSet() {
+			ui.Say("Adding tags to source EBS Volumes")
+
+			volumeTags, err := s.VolumeTags.EC2Tags(s.Ctx, *ec2conn.Config.Region, s.SourceAMI)
+			if err != nil {
+				err := fmt.Errorf("Error tagging source EBS Volumes on %s: %s", *instance.InstanceId, err)
+				state.Put("error", err)
+				ui.Error(err.Error())
+				return multistep.ActionHalt
+			}
+			volumeTags.Report(ui)
+
+			_, err = ec2conn.CreateTags(&ec2.CreateTagsInput{
+				Resources: volumeIds,
+				Tags:      volumeTags,
+			})
+
+			if err != nil {
+				err := fmt.Errorf("Error tagging source EBS Volumes on %s: %s", *instance.InstanceId, err)
+				state.Put("error", err)
+				ui.Error(err.Error())
+				return multistep.ActionHalt
+			}
+		}
+	}
+
 	return multistep.ActionContinue
 }
 

builder/amazon/common/step_run_spot_instance.go

@@ -33,8 +33,8 @@ type StepRunSpotInstance struct {
 	SpotPrice                         string
 	SpotPriceProduct                  string
 	SubnetId                          string
-	Tags                              map[string]string
-	VolumeTags                        map[string]string
+	Tags                              TagMap
+	VolumeTags                        TagMap
 	UserData                          string
 	UserDataFile                      string
 	Ctx                               interpolate.Context
@@ -143,14 +143,14 @@ func (s *StepRunSpotInstance) Run(_ context.Context, state multistep.StateBag) m
 		s.Tags["Name"] = "Packer Builder"
 	}
 
-	ec2Tags, err := ConvertToEC2Tags(s.Tags, *ec2conn.Config.Region, s.SourceAMI, s.Ctx)
+	ec2Tags, err := s.Tags.EC2Tags(s.Ctx, *ec2conn.Config.Region, s.SourceAMI)
 	if err != nil {
 		err := fmt.Errorf("Error tagging source instance: %s", err)
 		state.Put("error", err)
 		ui.Error(err.Error())
 		return multistep.ActionHalt
 	}
-	ReportTags(ui, ec2Tags)
+	ec2Tags.Report(ui)
 
 	ui.Message(fmt.Sprintf(
 		"Requesting spot instance '%s' for: %s",
@@ -284,21 +284,21 @@ func (s *StepRunSpotInstance) Run(_ context.Context, state multistep.StateBag) m
 		}
 	}
 
-	if len(volumeIds) > 0 && len(s.VolumeTags) > 0 {
+	if len(volumeIds) > 0 && s.VolumeTags.IsSet() {
 		ui.Say("Adding tags to source EBS Volumes")
-		tags, err := ConvertToEC2Tags(s.VolumeTags, *ec2conn.Config.Region, s.SourceAMI, s.Ctx)
+
+		volumeTags, err := s.VolumeTags.EC2Tags(s.Ctx, *ec2conn.Config.Region, s.SourceAMI)
 		if err != nil {
 			err := fmt.Errorf("Error tagging source EBS Volumes on %s: %s", *instance.InstanceId, err)
 			state.Put("error", err)
 			ui.Error(err.Error())
 			return multistep.ActionHalt
 		}
-
-		ReportTags(ui, tags)
+		volumeTags.Report(ui)
 
 		_, err = ec2conn.CreateTags(&ec2.CreateTagsInput{
 			Resources: volumeIds,
-			Tags:      tags,
+			Tags:      volumeTags,
 		})
 
 		if err != nil {

builder/amazon/common/tags.go

@@ -0,0 +1,47 @@
+package common
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/hashicorp/packer/packer"
+	"github.com/hashicorp/packer/template/interpolate"
+)
+
+type TagMap map[string]string
+type EC2Tags []*ec2.Tag
+
+func (t EC2Tags) Report(ui packer.Ui) {
+	for _, tag := range t {
+		ui.Message(fmt.Sprintf("Adding tag: \"%s\": \"%s\"",
+			aws.StringValue(tag.Key), aws.StringValue(tag.Value)))
+	}
+}
+
+func (t TagMap) IsSet() bool {
+	return len(t) > 0
+}
+
+func (t TagMap) EC2Tags(ctx interpolate.Context, region, sourceAMIID string) (EC2Tags, error) {
+	var ec2Tags []*ec2.Tag
+	ctx.Data = &BuildInfoTemplate{
+		SourceAMI:   sourceAMIID,
+		BuildRegion: region,
+	}
+	for key, value := range t {
+		interpolatedKey, err := interpolate.Render(key, &ctx)
+		if err != nil {
+			return nil, fmt.Errorf("Error processing tag: %s:%s - %s", key, value, err)
+		}
+		interpolatedValue, err := interpolate.Render(value, &ctx)
+		if err != nil {
+			return nil, fmt.Errorf("Error processing tag: %s:%s - %s", key, value, err)
+		}
+		ec2Tags = append(ec2Tags, &ec2.Tag{
+			Key:   aws.String(interpolatedKey),
+			Value: aws.String(interpolatedValue),
+		})
+	}
+	return ec2Tags, nil
+}

builder/amazon/ebs/builder.go

@@ -28,7 +28,7 @@ type Config struct {
 	awscommon.AMIConfig    `mapstructure:",squash"`
 	awscommon.BlockDevices `mapstructure:",squash"`
 	awscommon.RunConfig    `mapstructure:",squash"`
-	VolumeRunTags          map[string]string `mapstructure:"run_volume_tags"`
+	VolumeRunTags          awscommon.TagMap `mapstructure:"run_volume_tags"`
 
 	ctx interpolate.Context
 }
@@ -152,6 +152,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 			IamInstanceProfile:                b.config.IamInstanceProfile,
 			InstanceInitiatedShutdownBehavior: b.config.InstanceInitiatedShutdownBehavior,
 			InstanceType:                      b.config.InstanceType,
+			IsRestricted:                      b.config.IsChinaCloud() || b.config.IsGovCloud(),
 			SourceAMI:                         b.config.SourceAmi,
 			SubnetId:                          b.config.SubnetId,
 			Tags:                              b.config.RunTags,

builder/amazon/ebssurrogate/builder.go

@@ -26,8 +26,8 @@ type Config struct {
 	awscommon.BlockDevices `mapstructure:",squash"`
 	awscommon.AMIConfig    `mapstructure:",squash"`
 
-	RootDevice    RootBlockDevice   `mapstructure:"ami_root_device"`
-	VolumeRunTags map[string]string `mapstructure:"run_volume_tags"`
+	RootDevice    RootBlockDevice  `mapstructure:"ami_root_device"`
+	VolumeRunTags awscommon.TagMap `mapstructure:"run_volume_tags"`
 
 	ctx interpolate.Context
 }
@@ -166,6 +166,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 			IamInstanceProfile:                b.config.IamInstanceProfile,
 			InstanceInitiatedShutdownBehavior: b.config.InstanceInitiatedShutdownBehavior,
 			InstanceType:                      b.config.InstanceType,
+			IsRestricted:                      b.config.IsChinaCloud() || b.config.IsGovCloud(),
 			SourceAMI:                         b.config.SourceAmi,
 			SubnetId:                          b.config.SubnetId,
 			Tags:                              b.config.RunTags,

builder/amazon/ebsvolume/block_device.go

@@ -7,7 +7,7 @@ import (
 
 type BlockDevice struct {
 	awscommon.BlockDevice `mapstructure:"-,squash"`
-	Tags                  map[string]string `mapstructure:"tags"`
+	Tags                  awscommon.TagMap `mapstructure:"tags"`
 }
 
 func commonBlockDevices(mappings []BlockDevice, ctx *interpolate.Context) (awscommon.BlockDevices, error) {

builder/amazon/ebsvolume/builder.go

@@ -149,6 +149,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 			IamInstanceProfile:                b.config.IamInstanceProfile,
 			InstanceInitiatedShutdownBehavior: b.config.InstanceInitiatedShutdownBehavior,
 			InstanceType:                      b.config.InstanceType,
+			IsRestricted:                      b.config.IsChinaCloud() || b.config.IsGovCloud(),
 			SourceAMI:                         b.config.SourceAmi,
 			SubnetId:                          b.config.SubnetId,
 			Tags:                              b.config.RunTags,

builder/amazon/ebsvolume/step_tag_ebs_volumes.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 
 	"github.com/aws/aws-sdk-go/service/ec2"
-	awscommon "github.com/hashicorp/packer/builder/amazon/common"
 	"github.com/hashicorp/packer/helper/multistep"
 	"github.com/hashicorp/packer/packer"
 	"github.com/hashicorp/packer/template/interpolate"
@@ -44,14 +43,14 @@ func (s *stepTagEBSVolumes) Run(_ context.Context, state multistep.StateBag) mul
 				continue
 			}
 
-			tags, err := awscommon.ConvertToEC2Tags(mapping.Tags, *ec2conn.Config.Region, *sourceAMI.ImageId, s.Ctx)
+			tags, err := mapping.Tags.EC2Tags(s.Ctx, *ec2conn.Config.Region, *sourceAMI.ImageId)
 			if err != nil {
 				err := fmt.Errorf("Error tagging device %s with %s", mapping.DeviceName, err)
 				state.Put("error", err)
 				ui.Error(err.Error())
 				return multistep.ActionHalt
 			}
-			awscommon.ReportTags(ui, tags)
+			tags.Report(ui)
 
 			for _, v := range instance.BlockDeviceMappings {
 				if *v.DeviceName == mapping.DeviceName {

builder/amazon/instance/builder.go

@@ -232,6 +232,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
 			EbsOptimized:             b.config.EbsOptimized,
 			IamInstanceProfile:       b.config.IamInstanceProfile,
 			InstanceType:             b.config.InstanceType,
+			IsRestricted:             b.config.IsChinaCloud() || b.config.IsGovCloud(),
 			SourceAMI:                b.config.SourceAmi,
 			SubnetId:                 b.config.SubnetId,
 			Tags:                     b.config.RunTags,