From d5a6781fb7cd0cadab9c9ebaa24779758b283a53 Mon Sep 17 00:00:00 2001 From: Gennady Lipenkov Date: Mon, 27 Apr 2020 02:20:30 +0300 Subject: [PATCH] Add new 'yandex-export' post-processor --- command/plugin.go | 2 + post-processor/yandex-export/artifact.go | 37 ++++ .../yandex-export/post-processor.go | 198 ++++++++++++++++++ .../yandex-export/post-processor.hcl2spec.go | 60 ++++++ post-processor/yandex-export/script.go | 130 ++++++++++++ 5 files changed, 427 insertions(+) create mode 100644 post-processor/yandex-export/artifact.go create mode 100644 post-processor/yandex-export/post-processor.go create mode 100644 post-processor/yandex-export/post-processor.hcl2spec.go create mode 100644 post-processor/yandex-export/script.go diff --git a/command/plugin.go b/command/plugin.go index fa57737a7..96d28ae45 100644 --- a/command/plugin.go +++ b/command/plugin.go @@ -83,6 +83,7 @@ import ( vagrantcloudpostprocessor "github.com/hashicorp/packer/post-processor/vagrant-cloud" vspherepostprocessor "github.com/hashicorp/packer/post-processor/vsphere" vspheretemplatepostprocessor "github.com/hashicorp/packer/post-processor/vsphere-template" + yandexexportpostprocessor "github.com/hashicorp/packer/post-processor/yandex-export" ansibleprovisioner "github.com/hashicorp/packer/provisioner/ansible" ansiblelocalprovisioner "github.com/hashicorp/packer/provisioner/ansible-local" azuredtlartifactprovisioner "github.com/hashicorp/packer/provisioner/azure-dtlartifact" @@ -202,6 +203,7 @@ var PostProcessors = map[string]packer.PostProcessor{ "vagrant-cloud": new(vagrantcloudpostprocessor.PostProcessor), "vsphere": new(vspherepostprocessor.PostProcessor), "vsphere-template": new(vspheretemplatepostprocessor.PostProcessor), + "yandex-export": new(yandexexportpostprocessor.PostProcessor), } var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner)-(.+)") diff --git a/post-processor/yandex-export/artifact.go b/post-processor/yandex-export/artifact.go new file mode 100644 index 000000000..3c85f572c --- /dev/null +++ b/post-processor/yandex-export/artifact.go @@ -0,0 +1,37 @@ +package yandexexport + +import ( + "fmt" +) + +const BuilderId = "packer.post-processor.yandex-export" + +type Artifact struct { + paths []string +} + +func (*Artifact) BuilderId() string { + return BuilderId +} + +func (*Artifact) Id() string { + return "" +} + +func (a *Artifact) Files() []string { + pathsCopy := make([]string, len(a.paths)) + copy(pathsCopy, a.paths) + return pathsCopy +} + +func (a *Artifact) String() string { + return fmt.Sprintf("Exported artifacts in: %s", a.paths) +} + +func (*Artifact) State(name string) interface{} { + return nil +} + +func (a *Artifact) Destroy() error { + return nil +} diff --git a/post-processor/yandex-export/post-processor.go b/post-processor/yandex-export/post-processor.go new file mode 100644 index 000000000..6b9dc9947 --- /dev/null +++ b/post-processor/yandex-export/post-processor.go @@ -0,0 +1,198 @@ +//go:generate struct-markdown +//go:generate mapstructure-to-hcl2 -type Config + +package yandexexport + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/hashicorp/packer/builder/yandex" + "github.com/hashicorp/packer/common" + "github.com/hashicorp/packer/helper/config" + "github.com/hashicorp/packer/helper/multistep" + "github.com/hashicorp/packer/packer" + "github.com/hashicorp/packer/template/interpolate" +) + +type Config struct { + common.PackerConfig `mapstructure:",squash"` + + // Paths to Yandex Object Storage where exported image will be uploaded + Paths []string `mapstructure:"paths" required:"true"` + // The folder ID that will be used to launch a temporary instance. + // Alternatively you may set value by environment variable YC_FOLDER_ID. + FolderID string `mapstructure:"folder_id" required:"true"` + // Service Account ID with proper permission to modify an instance, create and attach disk and + // make upload to specific Yandex Object Storage paths + ServiceAccountID string `mapstructure:"service_account_id" required:"true"` + // The size of the disk in GB. This defaults to `100`, which is 100GB. + DiskSizeGb int `mapstructure:"disk_size" required:"false"` + // Specify disk type for the launched instance. Defaults to `network-ssd`. + DiskType string `mapstructure:"disk_type" required:"false"` + // Identifier of the hardware platform configuration for the instance. This defaults to `standard-v2`. + PlatformID string `mapstructure:"platform_id" required:"false"` + // The Yandex VPC subnet id to use for + // the launched instance. Note, the zone of the subnet must match the + // zone in which the VM is launched. + SubnetID string `mapstructure:"subnet_id" required:"false"` + // The name of the zone to launch the instance. This defaults to `ru-central1-a`. + Zone string `mapstructure:"zone" required:"false"` + // OAuth token to use to authenticate to Yandex.Cloud. Alternatively you may set + // value by environment variable YC_TOKEN. + Token string `mapstructure:"token" required:"false"` + + ctx interpolate.Context +} + +type PostProcessor struct { + config Config + runner multistep.Runner +} + +func (p *PostProcessor) ConfigSpec() hcldec.ObjectSpec { return p.config.FlatMapstructure().HCL2Spec() } + +func (p *PostProcessor) Configure(raws ...interface{}) error { + err := config.Decode(&p.config, &config.DecodeOpts{ + Interpolate: true, + InterpolateContext: &p.config.ctx, + }, raws...) + if err != nil { + return err + } + + errs := new(packer.MultiError) + + if len(p.config.Paths) == 0 { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("paths must be specified")) + } + + // provision config by OS environment variables + if p.config.Token == "" { + p.config.Token = os.Getenv("YC_TOKEN") + } + + if p.config.FolderID == "" { + p.config.FolderID = os.Getenv("YC_FOLDER_ID") + } + + // Set defaults. + if p.config.DiskSizeGb == 0 { + p.config.DiskSizeGb = 100 + } + + if p.config.DiskType == "" { + p.config.DiskType = "network-ssd" + } + + if p.config.PlatformID == "" { + p.config.PlatformID = "standard-v2" + } + + if p.config.Zone == "" { + p.config.Zone = "ru-central1-a" + } + + if len(errs.Errors) > 0 { + return errs + } + + return nil +} + +func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) { + if artifact.BuilderId() != yandex.BuilderID { + err := fmt.Errorf( + "Unknown artifact typs\nCan only export from Yandex Cloud builder artifacts.", + artifact.BuilderId()) + return nil, false, false, err + } + + builderID := artifact.State("ImageID").(string) + + ui.Say(fmt.Sprintf("Exporting image %v to destination: %v", builderID, p.config.Paths)) + + // Set up exporter instance configuration. + exporterName := fmt.Sprintf("%s-exporter", artifact.Id()) + exporterMetadata := map[string]string{ + "image_id": builderID, + "name": exporterName, + "paths": strings.Join(p.config.Paths, " "), + "user-data": CloudInitScript, + "zone": p.config.Zone, + } + + yandexConfig := ycSaneDefaults() + yandexConfig.Token = p.config.Token + yandexConfig.DiskName = exporterName + yandexConfig.InstanceName = exporterName + yandexConfig.DiskSizeGb = p.config.DiskSizeGb + yandexConfig.Metadata = exporterMetadata + yandexConfig.SubnetID = p.config.SubnetID + yandexConfig.FolderID = p.config.FolderID + yandexConfig.Zone = p.config.Zone + + if p.config.ServiceAccountID != "" { + yandexConfig.ServiceAccountID = p.config.ServiceAccountID + } + if p.config.PlatformID != "" { + yandexConfig.ServiceAccountID = p.config.ServiceAccountID + } + + driver, err := yandex.NewDriverYC(ui, &yandexConfig) + if err != nil { + return nil, false, false, err + } + + // Set up the state. + state := new(multistep.BasicStateBag) + state.Put("config", &yandexConfig) + state.Put("driver", driver) + state.Put("sdk", driver.SDK()) + state.Put("ui", ui) + + // Build the steps. + steps := []multistep.Step{ + &yandex.StepCreateSSHKey{ + Debug: p.config.PackerDebug, + DebugKeyPath: fmt.Sprintf("yc_pp_%s.pem", p.config.PackerBuildName), + }, + &yandex.StepCreateInstance{ + Debug: p.config.PackerDebug, + }, + new(yandex.StepWaitCloudInitScript), + new(yandex.StepTeardownInstance), + } + + // Run the steps. + p.runner = common.NewRunner(steps, p.config.PackerConfig, ui) + p.runner.Run(ctx, state) + + result := &Artifact{paths: p.config.Paths} + + return result, false, false, nil +} + +func ycSaneDefaults() yandex.Config { + return yandex.Config{ + DiskType: "network-ssd", + InstanceCores: 2, + InstanceMemory: 2, + Labels: map[string]string{ + "role": "exporter", + "target": "object-storage", + }, + PlatformID: "standard-v2", + Preemptible: true, + SourceImageFamily: "ubuntu-1604-lts", + SourceImageFolderID: yandex.StandardImagesFolderID, + UseIPv4Nat: true, + Zone: "ru-central1-a", + StateTimeout: 3 * time.Minute, + } +} diff --git a/post-processor/yandex-export/post-processor.hcl2spec.go b/post-processor/yandex-export/post-processor.hcl2spec.go new file mode 100644 index 000000000..099bad0f2 --- /dev/null +++ b/post-processor/yandex-export/post-processor.hcl2spec.go @@ -0,0 +1,60 @@ +// Code generated by "mapstructure-to-hcl2 -type Config"; DO NOT EDIT. +package yandexexport + +import ( + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/zclconf/go-cty/cty" +) + +// FlatConfig is an auto-generated flat version of Config. +// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up. +type FlatConfig struct { + PackerBuildName *string `mapstructure:"packer_build_name" cty:"packer_build_name"` + PackerBuilderType *string `mapstructure:"packer_builder_type" cty:"packer_builder_type"` + PackerDebug *bool `mapstructure:"packer_debug" cty:"packer_debug"` + PackerForce *bool `mapstructure:"packer_force" cty:"packer_force"` + PackerOnError *string `mapstructure:"packer_on_error" cty:"packer_on_error"` + PackerUserVars map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables"` + PackerSensitiveVars []string `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables"` + Paths []string `mapstructure:"paths" required:"true" cty:"paths"` + FolderID *string `mapstructure:"folder_id" required:"true" cty:"folder_id"` + ServiceAccountID *string `mapstructure:"service_account_id" required:"true" cty:"service_account_id"` + DiskSizeGb *int `mapstructure:"disk_size" required:"false" cty:"disk_size"` + DiskType *string `mapstructure:"disk_type" required:"false" cty:"disk_type"` + PlatformID *string `mapstructure:"platform_id" required:"false" cty:"platform_id"` + SubnetID *string `mapstructure:"subnet_id" required:"false" cty:"subnet_id"` + Zone *string `mapstructure:"zone" required:"false" cty:"zone"` + Token *string `mapstructure:"token" required:"false" cty:"token"` +} + +// FlatMapstructure returns a new FlatConfig. +// FlatConfig is an auto-generated flat version of Config. +// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up. +func (*Config) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } { + return new(FlatConfig) +} + +// HCL2Spec returns the hcl spec of a Config. +// This spec is used by HCL to read the fields of Config. +// The decoded values from this spec will then be applied to a FlatConfig. +func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec { + s := map[string]hcldec.Spec{ + "packer_build_name": &hcldec.AttrSpec{Name: "packer_build_name", Type: cty.String, Required: false}, + "packer_builder_type": &hcldec.AttrSpec{Name: "packer_builder_type", Type: cty.String, Required: false}, + "packer_debug": &hcldec.AttrSpec{Name: "packer_debug", Type: cty.Bool, Required: false}, + "packer_force": &hcldec.AttrSpec{Name: "packer_force", Type: cty.Bool, Required: false}, + "packer_on_error": &hcldec.AttrSpec{Name: "packer_on_error", Type: cty.String, Required: false}, + "packer_user_variables": &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false}, + "packer_sensitive_variables": &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false}, + "paths": &hcldec.AttrSpec{Name: "paths", Type: cty.List(cty.String), Required: false}, + "folder_id": &hcldec.AttrSpec{Name: "folder_id", Type: cty.String, Required: false}, + "service_account_id": &hcldec.AttrSpec{Name: "service_account_id", Type: cty.String, Required: false}, + "disk_size": &hcldec.AttrSpec{Name: "disk_size", Type: cty.Number, Required: false}, + "disk_type": &hcldec.AttrSpec{Name: "disk_type", Type: cty.String, Required: false}, + "platform_id": &hcldec.AttrSpec{Name: "platform_id", Type: cty.String, Required: false}, + "subnet_id": &hcldec.AttrSpec{Name: "subnet_id", Type: cty.String, Required: false}, + "zone": &hcldec.AttrSpec{Name: "zone", Type: cty.String, Required: false}, + "token": &hcldec.AttrSpec{Name: "token", Type: cty.String, Required: false}, + } + return s +} diff --git a/post-processor/yandex-export/script.go b/post-processor/yandex-export/script.go new file mode 100644 index 000000000..de30d7844 --- /dev/null +++ b/post-processor/yandex-export/script.go @@ -0,0 +1,130 @@ +package yandexexport + +var CloudInitScript string = `#!/usr/bin/env bash +GetMetadata () { + echo "$(curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/attributes/$1 2> /dev/null)" +} + +GetInstanceId () { + echo "$(curl -f -H "Metadata-Flavor: Google" http://169.254.169.254/computeMetadata/v1/instance/id 2> /dev/null)" +} + +GetServiceAccountId () { + yc compute instance get ${INSTANCE_ID} | grep service_account | cut -f2 -d' ' +} + +InstallYc () { + curl -s https://storage.yandexcloud.net/yandexcloud-yc/install.sh | sudo bash -s -- -n -i /usr/local +} + +InstallAwsCli () { + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip -o awscliv2.zip > /dev/null + sudo ./aws/install +} + +InstallPackages () { + sudo apt-get update -qq && sudo apt-get install -y unzip jq qemu-utils +} + +InstallTools () { + InstallPackages + InstallYc + InstallAwsCli +} + +IMAGE_ID=$(GetMetadata image_id) +INSTANCE_ID=$(GetInstanceId) +DISKNAME=${INSTANCE_ID}-toexport +PATHS=$(GetMetadata paths) +ZONE=$(GetMetadata zone) + +Exit () { + for i in ${PATHS}; do + LOGDEST="${i}.exporter.log" + echo "Uploading exporter log to ${LOGDEST}..." + aws s3 --endpoint-url=https://storage.yandexcloud.net cp /var/log/syslog ${LOGDEST} + done + exit $1 +} + +InstallTools + +echo "####### Export configuration #######" +echo "Image ID - ${IMAGE_ID}" +echo "Instance ID - ${INSTANCE_ID}" +echo "Instance zone - ${ZONE}" +echo "Disk name - ${DISKNAME}" +echo "Export paths - ${PATHS}" +echo "####################################" + +echo "Creating disk from image to be exported..." +if ! yc compute disk create --name ${DISKNAME} --source-image-id ${IMAGE_ID} --zone ${ZONE}; then + echo "Failed to create disk." + Exit 1 +fi + +echo "Attaching disk..." +if ! yc compute instance attach-disk ${INSTANCE_ID} --disk-name ${DISKNAME} --device-name doexport --auto-delete ; then + echo "Failed to attach disk." + Exit 1 +fi + +echo "Dumping disk..." +if ! qemu-img convert -O qcow2 -o cluster_size=2M /dev/disk/by-id/virtio-doexport disk.qcow2 ; then + echo "Failed to dump disk to qcow2 image." + Exit 1 +fi + +echo "Detaching disk..." +if ! yc compute instance detach-disk ${INSTANCE_ID} --disk-name ${DISKNAME} ; then + echo "Failed to detach disk." +fi + +echo "Detect Service Account ID..." +SERVICE_ACCOUNT_ID=$(GetServiceAccountId) +echo "Use Service Account ID: ${SERVICE_ACCOUNT_ID}" + +echo "Create static access key..." +SEC_json=$(yc iam access-key create --service-account-id ${SERVICE_ACCOUNT_ID} \ + --description "this key is for export image to storage" --format json) + +if [ $? -ne 0 ]; then + echo "Failed to create static access key." + exit 1 +fi + +echo "Setup env variables to access storage..." +eval "$(jq -r '@sh "export YC_SK_ID=\(.access_key.id); export AWS_ACCESS_KEY_ID=\(.access_key.key_id); export AWS_SECRET_ACCESS_KEY=\(.secret)"' <<<${SEC_json} )" + +echo "Check access to storage..." +if ! aws s3 --endpoint-url=https://storage.yandexcloud.net ls > /dev/null ; then + echo "Failed to access storage." +fi + +FAIL=0 +echo "Deleting disk..." +if ! yc compute disk delete --name ${DISKNAME} ; then + echo "Failed to delete disk." + FAIL=1 +fi +for i in ${PATHS}; do + echo "Uploading qcow2 disk image to ${i}..." + if ! aws s3 --endpoint-url=https://storage.yandexcloud.net cp disk.qcow2 ${i}; then + echo "Failed to upload image to ${i}." + FAIL=1 + fi +done + +echo "Delete static access key..." +if ! yc iam access-key delete ${YC_SK_ID} ; then + echo "Failed to delete static access key." +fi + +echo "Set metadata key to 'cloud-init-status' to 'cloud-init-done' value" +if ! yc compute instance update ${INSTANCE_ID} --metadata cloud-init-status=cloud-init-done ; then + echo "Failed to attach disk." + Exit 1 +fi + +Exit ${FAIL}`