From 5e8b697a762341ae806c63d5ef3bc43243816d2e Mon Sep 17 00:00:00 2001 From: Jeremy Asher Date: Wed, 10 Aug 2016 17:24:30 -0700 Subject: [PATCH] add from_scratch option to amazon-chroot builder This provides an alternate mode for the amazon-chroot builder which uses a blank volume to build the image. It adds StepPreMountCommands to permit partitioning and format commands to be executed before mounting the new volume. --- builder/amazon/chroot/builder.go | 84 ++++++++++++++----- builder/amazon/chroot/run_local_commands.go | 37 ++++++++ builder/amazon/chroot/step_create_volume.go | 64 ++++++++------ builder/amazon/chroot/step_mount_device.go | 13 ++- .../amazon/chroot/step_pre_mount_commands.go | 39 +++++++++ builder/amazon/chroot/step_register_ami.go | 37 ++++++-- 6 files changed, 216 insertions(+), 58 deletions(-) create mode 100644 builder/amazon/chroot/run_local_commands.go create mode 100644 builder/amazon/chroot/step_pre_mount_commands.go diff --git a/builder/amazon/chroot/builder.go b/builder/amazon/chroot/builder.go index e0173dadb..681581107 100644 --- a/builder/amazon/chroot/builder.go +++ b/builder/amazon/chroot/builder.go @@ -25,19 +25,23 @@ const BuilderId = "mitchellh.amazon.chroot" // Config is the configuration that is chained through the steps and // settable from the template. type Config struct { - common.PackerConfig `mapstructure:",squash"` - awscommon.AccessConfig `mapstructure:",squash"` - awscommon.AMIConfig `mapstructure:",squash"` - - ChrootMounts [][]string `mapstructure:"chroot_mounts"` - CommandWrapper string `mapstructure:"command_wrapper"` - CopyFiles []string `mapstructure:"copy_files"` - DevicePath string `mapstructure:"device_path"` - MountPath string `mapstructure:"mount_path"` - SourceAmi string `mapstructure:"source_ami"` - RootVolumeSize int64 `mapstructure:"root_volume_size"` - MountOptions []string `mapstructure:"mount_options"` - MountPartition int `mapstructure:"mount_partition"` + common.PackerConfig `mapstructure:",squash"` + awscommon.AMIBlockDevices `mapstructure:",squash"` + awscommon.AMIConfig `mapstructure:",squash"` + awscommon.AccessConfig `mapstructure:",squash"` + + ChrootMounts [][]string `mapstructure:"chroot_mounts"` + CommandWrapper string `mapstructure:"command_wrapper"` + CopyFiles []string `mapstructure:"copy_files"` + DevicePath string `mapstructure:"device_path"` + FromScratch bool `mapstructure:"from_scratch"` + MountOptions []string `mapstructure:"mount_options"` + MountPartition int `mapstructure:"mount_partition"` + MountPath string `mapstructure:"mount_path"` + PreMountCommands []string `mapstructure:"pre_mount_commands"` + RootDeviceName string `mapstructure:"root_device_name"` + RootVolumeSize int64 `mapstructure:"root_volume_size"` + SourceAmi string `mapstructure:"source_ami"` ctx interpolate.Context } @@ -59,6 +63,7 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { InterpolateFilter: &interpolate.RenderFilter{ Exclude: []string{ "command_wrapper", + "pre_mount_commands", "mount_path", }, }, @@ -86,7 +91,7 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { } } - if len(b.config.CopyFiles) == 0 { + if len(b.config.CopyFiles) == 0 && !b.config.FromScratch { b.config.CopyFiles = []string{"/etc/resolv.conf"} } @@ -115,8 +120,32 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) { } } - if b.config.SourceAmi == "" { - errs = packer.MultiErrorAppend(errs, errors.New("source_ami is required.")) + if b.config.FromScratch { + if b.config.RootVolumeSize == 0 { + errs = packer.MultiErrorAppend( + errs, errors.New("root_volume_size is required with from_scratch.")) + } + if len(b.config.PreMountCommands) == 0 { + errs = packer.MultiErrorAppend( + errs, errors.New("pre_mount_commands is required with from_scratch.")) + } + if b.config.AMIVirtType == "" { + errs = packer.MultiErrorAppend( + errs, errors.New("ami_virtualization_type is required with from_scratch.")) + } + if b.config.RootDeviceName == "" { + errs = packer.MultiErrorAppend( + errs, errors.New("root_device_name is required with from_scratch.")) + } + if len(b.config.AMIMappings) == 0 { + errs = packer.MultiErrorAppend( + errs, errors.New("ami_block_device_mappings is required with from_scratch.")) + } + } else { + if b.config.SourceAmi == "" { + errs = packer.MultiErrorAppend( + errs, errors.New("source_ami is required.")) + } } if errs != nil && len(errs.Errors) > 0 { @@ -161,11 +190,19 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe ForceDeregister: b.config.AMIForceDeregister, }, &StepInstanceInfo{}, - &awscommon.StepSourceAMIInfo{ - SourceAmi: b.config.SourceAmi, - EnhancedNetworking: b.config.AMIEnhancedNetworking, - }, - &StepCheckRootDevice{}, + } + + if !b.config.FromScratch { + steps = append(steps, + &awscommon.StepSourceAMIInfo{ + SourceAmi: b.config.SourceAmi, + EnhancedNetworking: b.config.AMIEnhancedNetworking, + }, + &StepCheckRootDevice{}, + ) + } + + steps = append(steps, &StepFlock{}, &StepPrepareDevice{}, &StepCreateVolume{ @@ -173,6 +210,9 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe }, &StepAttachVolume{}, &StepEarlyUnflock{}, + &StepPreMountCommands{ + Commands: b.config.PreMountCommands, + }, &StepMountDevice{ MountOptions: b.config.MountOptions, MountPartition: b.config.MountPartition, @@ -203,7 +243,7 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe &awscommon.StepCreateTags{ Tags: b.config.AMITags, }, - } + ) // Run! if b.config.PackerDebug { diff --git a/builder/amazon/chroot/run_local_commands.go b/builder/amazon/chroot/run_local_commands.go new file mode 100644 index 000000000..bc339ed11 --- /dev/null +++ b/builder/amazon/chroot/run_local_commands.go @@ -0,0 +1,37 @@ +package chroot + +import ( + "fmt" + + "github.com/mitchellh/packer/packer" + "github.com/mitchellh/packer/post-processor/shell-local" + "github.com/mitchellh/packer/template/interpolate" +) + +func RunLocalCommands(commands []string, wrappedCommand CommandWrapper, ctx interpolate.Context, ui packer.Ui) error { + for _, rawCmd := range commands { + intCmd, err := interpolate.Render(rawCmd, &ctx) + if err != nil { + return fmt.Errorf("Error interpolating: %s", err) + } + + command, err := wrappedCommand(intCmd) + if err != nil { + return fmt.Errorf("Error wrapping command: %s", err) + } + + ui.Say(fmt.Sprintf("Executing command: %s", command)) + comm := &shell_local.Communicator{} + cmd := &packer.RemoteCmd{Command: command} + if err := cmd.StartWithUi(comm, ui); err != nil { + return fmt.Errorf("Error executing command: %s", err) + } + if cmd.ExitStatus != 0 { + return fmt.Errorf( + "Received non-zero exit code %d from command: %s", + cmd.ExitStatus, + command) + } + } + return nil +} diff --git a/builder/amazon/chroot/step_create_volume.go b/builder/amazon/chroot/step_create_volume.go index b190fbed1..a527d86dd 100644 --- a/builder/amazon/chroot/step_create_volume.go +++ b/builder/amazon/chroot/step_create_volume.go @@ -22,40 +22,52 @@ type StepCreateVolume struct { } func (s *StepCreateVolume) Run(state multistep.StateBag) multistep.StepAction { + config := state.Get("config").(*Config) ec2conn := state.Get("ec2").(*ec2.EC2) - image := state.Get("source_image").(*ec2.Image) instance := state.Get("instance").(*ec2.Instance) ui := state.Get("ui").(packer.Ui) - // Determine the root device snapshot - log.Printf("Searching for root device of the image (%s)", *image.RootDeviceName) - var rootDevice *ec2.BlockDeviceMapping - for _, device := range image.BlockDeviceMappings { - if *device.DeviceName == *image.RootDeviceName { - rootDevice = device - break + var createVolume *ec2.CreateVolumeInput + if config.FromScratch { + createVolume = &ec2.CreateVolumeInput{ + AvailabilityZone: instance.Placement.AvailabilityZone, + Size: aws.Int64(s.RootVolumeSize), + VolumeType: aws.String(ec2.VolumeTypeGp2), + } + } else { + // Determine the root device snapshot + image := state.Get("source_image").(*ec2.Image) + log.Printf("Searching for root device of the image (%s)", *image.RootDeviceName) + var rootDevice *ec2.BlockDeviceMapping + for _, device := range image.BlockDeviceMappings { + if *device.DeviceName == *image.RootDeviceName { + rootDevice = device + break + } } - } - if rootDevice == nil { - err := fmt.Errorf("Couldn't find root device!") - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } + if rootDevice == nil { + err := fmt.Errorf("Couldn't find root device!") + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } - ui.Say("Creating the root volume...") - vs := *rootDevice.Ebs.VolumeSize - if s.RootVolumeSize > *rootDevice.Ebs.VolumeSize { - vs = s.RootVolumeSize - } - createVolume := &ec2.CreateVolumeInput{ - AvailabilityZone: instance.Placement.AvailabilityZone, - Size: aws.Int64(vs), - SnapshotId: rootDevice.Ebs.SnapshotId, - VolumeType: rootDevice.Ebs.VolumeType, - Iops: rootDevice.Ebs.Iops, + ui.Say("Creating the root volume...") + vs := *rootDevice.Ebs.VolumeSize + if s.RootVolumeSize > *rootDevice.Ebs.VolumeSize { + vs = s.RootVolumeSize + } + + createVolume = &ec2.CreateVolumeInput{ + AvailabilityZone: instance.Placement.AvailabilityZone, + Size: aws.Int64(vs), + SnapshotId: rootDevice.Ebs.SnapshotId, + VolumeType: rootDevice.Ebs.VolumeType, + Iops: rootDevice.Ebs.Iops, + } } + log.Printf("Create args: %+v", createVolume) createVolumeResp, err := ec2conn.CreateVolume(createVolume) diff --git a/builder/amazon/chroot/step_mount_device.go b/builder/amazon/chroot/step_mount_device.go index 69878b2d3..c7b74b9e6 100644 --- a/builder/amazon/chroot/step_mount_device.go +++ b/builder/amazon/chroot/step_mount_device.go @@ -33,10 +33,18 @@ type StepMountDevice struct { func (s *StepMountDevice) Run(state multistep.StateBag) multistep.StepAction { config := state.Get("config").(*Config) ui := state.Get("ui").(packer.Ui) - image := state.Get("source_image").(*ec2.Image) device := state.Get("device").(string) wrappedCommand := state.Get("wrappedCommand").(CommandWrapper) + var virtualizationType string + if config.FromScratch { + virtualizationType = config.AMIVirtType + } else { + image := state.Get("source_image").(*ec2.Image) + virtualizationType = *image.VirtualizationType + log.Printf("Source image virtualization type is: %s", virtualizationType) + } + ctx := config.ctx ctx.Data = &mountPathData{Device: filepath.Base(device)} mountPath, err := interpolate.Render(config.MountPath, &ctx) @@ -65,9 +73,8 @@ func (s *StepMountDevice) Run(state multistep.StateBag) multistep.StepAction { return multistep.ActionHalt } - log.Printf("Source image virtualization type is: %s", *image.VirtualizationType) deviceMount := device - if *image.VirtualizationType == "hvm" { + if virtualizationType == "hvm" { deviceMount = fmt.Sprintf("%s%d", device, s.MountPartition) } state.Put("deviceMount", deviceMount) diff --git a/builder/amazon/chroot/step_pre_mount_commands.go b/builder/amazon/chroot/step_pre_mount_commands.go new file mode 100644 index 000000000..e4561c2d7 --- /dev/null +++ b/builder/amazon/chroot/step_pre_mount_commands.go @@ -0,0 +1,39 @@ +package chroot + +import ( + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" +) + +type preMountCommandsData struct { + Device string +} + +// StepPreMountCommands sets up the a new block device when building from scratch +type StepPreMountCommands struct { + Commands []string +} + +func (s *StepPreMountCommands) Run(state multistep.StateBag) multistep.StepAction { + config := state.Get("config").(*Config) + device := state.Get("device").(string) + ui := state.Get("ui").(packer.Ui) + wrappedCommand := state.Get("wrappedCommand").(CommandWrapper) + + if len(s.Commands) == 0 { + return multistep.ActionContinue + } + + ctx := config.ctx + ctx.Data = &preMountCommandsData{Device: device} + + ui.Say("Running device setup commands...") + if err := RunLocalCommands(s.Commands, wrappedCommand, ctx, ui); err != nil { + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + return multistep.ActionContinue +} + +func (s *StepPreMountCommands) Cleanup(state multistep.StateBag) {} diff --git a/builder/amazon/chroot/step_register_ami.go b/builder/amazon/chroot/step_register_ami.go index f2a59ae01..afef9ad75 100644 --- a/builder/amazon/chroot/step_register_ami.go +++ b/builder/amazon/chroot/step_register_ami.go @@ -18,22 +18,36 @@ type StepRegisterAMI struct { func (s *StepRegisterAMI) Run(state multistep.StateBag) multistep.StepAction { config := state.Get("config").(*Config) ec2conn := state.Get("ec2").(*ec2.EC2) - image := state.Get("source_image").(*ec2.Image) snapshotId := state.Get("snapshot_id").(string) ui := state.Get("ui").(packer.Ui) ui.Say("Registering the AMI...") - blockDevices := make([]*ec2.BlockDeviceMapping, len(image.BlockDeviceMappings)) - for i, device := range image.BlockDeviceMappings { + + var ( + registerOpts *ec2.RegisterImageInput + blockDevices []*ec2.BlockDeviceMapping + image *ec2.Image + rootDeviceName string + ) + + if config.FromScratch { + blockDevices = config.AMIBlockDevices.BuildAMIDevices() + rootDeviceName = config.RootDeviceName + } else { + image = state.Get("source_image").(*ec2.Image) + blockDevices = make([]*ec2.BlockDeviceMapping, len(image.BlockDeviceMappings)) + rootDeviceName = *image.RootDeviceName + } + for i, device := range blockDevices { newDevice := device - if *newDevice.DeviceName == *image.RootDeviceName { + if *newDevice.DeviceName == rootDeviceName { if newDevice.Ebs != nil { newDevice.Ebs.SnapshotId = aws.String(snapshotId) } else { newDevice.Ebs = &ec2.EbsBlockDevice{SnapshotId: aws.String(snapshotId)} } - if s.RootVolumeSize > *newDevice.Ebs.VolumeSize { + if config.FromScratch || s.RootVolumeSize > *newDevice.Ebs.VolumeSize { newDevice.Ebs.VolumeSize = aws.Int64(s.RootVolumeSize) } } @@ -47,7 +61,17 @@ func (s *StepRegisterAMI) Run(state multistep.StateBag) multistep.StepAction { blockDevices[i] = newDevice } - registerOpts := buildRegisterOpts(config, image, blockDevices) + if config.FromScratch { + registerOpts = &ec2.RegisterImageInput{ + Name: &config.AMIName, + Architecture: aws.String(ec2.ArchitectureValuesX8664), + RootDeviceName: aws.String(rootDeviceName), + VirtualizationType: aws.String(config.AMIVirtType), + BlockDeviceMappings: blockDevices, + } + } else { + registerOpts = buildRegisterOpts(config, image, blockDevices) + } // Set SriovNetSupport to "simple". See http://goo.gl/icuXh5 if config.AMIEnhancedNetworking { @@ -105,6 +129,5 @@ func buildRegisterOpts(config *Config, image *ec2.Image, blockDevices []*ec2.Blo registerOpts.KernelId = image.KernelId registerOpts.RamdiskId = image.RamdiskId } - return registerOpts }