googlecompute-export: add exporter instance configs

post-processor/googlecompute-export/post-processor.go

@@ -15,10 +15,19 @@ import (
 type Config struct {
 	common.PackerConfig `mapstructure:",squash"`
 
-	Paths             []string `mapstructure:"paths"`
+	AccountFile string `mapstructure:"account_file"`
+
+	DiskSizeGb        int64    `mapstructure:"disk_size"`
+	DiskType          string   `mapstructure:"disk_type"`
 	KeepOriginalImage bool     `mapstructure:"keep_input_artifact"`
+	MachineType       string   `mapstructure:"machine_type"`
+	Network           string   `mapstructure:"network"`
+	Paths             []string `mapstructure:"paths"`
+	Subnetwork        string   `mapstructure:"subnetwork"`
+	Zone              string   `mapstructure:"zone"`
 
-	ctx interpolate.Context
+	Account googlecompute.AccountFile
+	ctx     interpolate.Context
 }
 
 type PostProcessor struct {
@@ -35,12 +44,38 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
 		return err
 	}
 
+	errs := new(packer.MultiError)
+
+	if len(p.config.Paths) == 0 {
+		errs = packer.MultiErrorAppend(
+			errs, fmt.Errorf("paths must be specified"))
+	}
+
+	// Set defaults.
+	if p.config.DiskSizeGb == 0 {
+		p.config.DiskSizeGb = 200
+	}
+
+	if p.config.DiskType == "" {
+		p.config.DiskType = "pd-ssd"
+	}
+
+	if p.config.MachineType == "" {
+		p.config.MachineType = "n1-highcpu-4"
+	}
+
+	if p.config.Network == "" && p.config.Subnetwork == "" {
+		p.config.Network = "default"
+	}
+
+	if len(errs.Errors) > 0 {
+		return errs
+	}
+
 	return nil
 }
 
 func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
-	ui.Say("Starting googlecompute-export...")
-	ui.Say(fmt.Sprintf("Exporting image to destinations: %v", p.config.Paths))
 	if artifact.BuilderId() != googlecompute.BuilderId {
 		err := fmt.Errorf(
 			"Unknown artifact type: %s\nCan only export from Google Compute Engine builder artifacts.",
@@ -48,79 +83,90 @@ func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (pac
 		return nil, p.config.KeepOriginalImage, err
 	}
 
-	result := &Artifact{paths: p.config.Paths}
+	builderAccountFile := artifact.State("AccountFilePath").(string)
+	builderImageName := artifact.State("ImageName").(string)
+	builderProjectId := artifact.State("ProjectId").(string)
+	builderZone := artifact.State("BuildZone").(string)
 
-	if len(p.config.Paths) > 0 {
-		accountKeyFilePath := artifact.State("AccountFilePath").(string)
-		imageName := artifact.State("ImageName").(string)
-		imageSizeGb := artifact.State("ImageSizeGb").(int64)
-		projectId := artifact.State("ProjectId").(string)
-		zone := artifact.State("BuildZone").(string)
-
-		// Set up instance configuration.
-		instanceName := fmt.Sprintf("%s-exporter", artifact.Id())
-		metadata := map[string]string{
-			"image_name":     imageName,
-			"name":           instanceName,
-			"paths":          strings.Join(p.config.Paths, " "),
-			"startup-script": StartupScript,
-			"zone":           zone,
-		}
-		exporterConfig := googlecompute.Config{
-			InstanceName:         instanceName,
-			SourceImageProjectId: "debian-cloud",
-			SourceImage:          "debian-8-jessie-v20160629",
-			DiskName:             instanceName,
-			DiskSizeGb:           imageSizeGb + 10,
-			DiskType:             "pd-standard",
-			Metadata:             metadata,
-			MachineType:          "n1-standard-4",
-			Zone:                 zone,
-			Network:              "default",
-			RawStateTimeout:      "5m",
-			Scopes: []string{
-				"https://www.googleapis.com/auth/userinfo.email",
-				"https://www.googleapis.com/auth/compute",
-				"https://www.googleapis.com/auth/devstorage.full_control",
-			},
-		}
-		exporterConfig.CalcTimeout()
-
-		// Set up credentials and GCE driver.
-		if accountKeyFilePath != "" {
-			err := googlecompute.ProcessAccountFile(&exporterConfig.Account, accountKeyFilePath)
-			if err != nil {
-				return nil, p.config.KeepOriginalImage, err
-			}
+	ui.Say(fmt.Sprintf("Exporting image %v to destination: %v", builderImageName, p.config.Paths))
+
+	if p.config.Zone == "" {
+		p.config.Zone = builderZone
+	}
+
+	// Set up credentials for GCE driver.
+	if builderAccountFile != "" {
+		err := googlecompute.ProcessAccountFile(&p.config.Account, builderAccountFile)
+		if err != nil {
+			return nil, p.config.KeepOriginalImage, err
 		}
-		driver, err := googlecompute.NewDriverGCE(ui, projectId, &exporterConfig.Account)
+	}
+	if p.config.AccountFile != "" {
+		err := googlecompute.ProcessAccountFile(&p.config.Account, p.config.AccountFile)
 		if err != nil {
 			return nil, p.config.KeepOriginalImage, err
 		}
+	}
 
-		// Set up the state.
-		state := new(multistep.BasicStateBag)
-		state.Put("config", &exporterConfig)
-		state.Put("driver", driver)
-		state.Put("ui", ui)
-
-		// Build the steps.
-		steps := []multistep.Step{
-			&googlecompute.StepCreateSSHKey{
-				Debug:        p.config.PackerDebug,
-				DebugKeyPath: fmt.Sprintf("gce_%s.pem", p.config.PackerBuildName),
-			},
-			&googlecompute.StepCreateInstance{
-				Debug: p.config.PackerDebug,
-			},
-			new(googlecompute.StepWaitStartupScript),
-			new(googlecompute.StepTeardownInstance),
-		}
+	// Set up exporter instance configuration.
+	exporterName := fmt.Sprintf("%s-exporter", artifact.Id())
+	exporterMetadata := map[string]string{
+		"image_name":     builderImageName,
+		"name":           exporterName,
+		"paths":          strings.Join(p.config.Paths, " "),
+		"startup-script": StartupScript,
+		"zone":           p.config.Zone,
+	}
+	exporterConfig := googlecompute.Config{
+		DiskName:             exporterName,
+		DiskSizeGb:           p.config.DiskSizeGb,
+		DiskType:             p.config.DiskType,
+		InstanceName:         exporterName,
+		MachineType:          p.config.MachineType,
+		Metadata:             exporterMetadata,
+		Network:              p.config.Network,
+		RawStateTimeout:      "5m",
+		SourceImageFamily:    "debian-9-worker",
+		SourceImageProjectId: "compute-image-tools",
+		Subnetwork:           p.config.Subnetwork,
+		Zone:                 p.config.Zone,
+		Scopes: []string{
+			"https://www.googleapis.com/auth/compute",
+			"https://www.googleapis.com/auth/devstorage.full_control",
+			"https://www.googleapis.com/auth/userinfo.email",
+		},
+	}
+	exporterConfig.CalcTimeout()
 
-		// Run the steps.
-		p.runner = common.NewRunner(steps, p.config.PackerConfig, ui)
-		p.runner.Run(state)
+	driver, err := googlecompute.NewDriverGCE(ui, builderProjectId, &p.config.Account)
+	if err != nil {
+		return nil, p.config.KeepOriginalImage, err
 	}
 
+	// Set up the state.
+	state := new(multistep.BasicStateBag)
+	state.Put("config", &exporterConfig)
+	state.Put("driver", driver)
+	state.Put("ui", ui)
+
+	// Build the steps.
+	steps := []multistep.Step{
+		&googlecompute.StepCreateSSHKey{
+			Debug:        p.config.PackerDebug,
+			DebugKeyPath: fmt.Sprintf("gce_%s.pem", p.config.PackerBuildName),
+		},
+		&googlecompute.StepCreateInstance{
+			Debug: p.config.PackerDebug,
+		},
+		new(googlecompute.StepWaitStartupScript),
+		new(googlecompute.StepTeardownInstance),
+	}
+
+	// Run the steps.
+	p.runner = common.NewRunner(steps, p.config.PackerConfig, ui)
+	p.runner.Run(state)
+
+	result := &Artifact{paths: p.config.Paths}
+
 	return result, p.config.KeepOriginalImage, nil
 }

post-processor/googlecompute-export/startup.go

@@ -1,6 +1,6 @@
 package googlecomputeexport
 
-var StartupScript string = `#!/bin/sh
+var StartupScript string = `#!/bin/bash
 
 GetMetadata () {
   echo "$(curl -f -H "Metadata-Flavor: Google" http://metadata/computeMetadata/v1/instance/attributes/$1 2> /dev/null)"
@@ -8,11 +8,11 @@ GetMetadata () {
 IMAGENAME=$(GetMetadata image_name)
 NAME=$(GetMetadata name)
 DISKNAME=${NAME}-toexport
-PATHS=$(GetMetadata paths)
+PATHS=($(GetMetadata paths))
 ZONE=$(GetMetadata zone)
 
 Exit () {
-  for i in ${PATHS}; do
+  for i in ${PATHS[@]}; do
     LOGDEST="${i}.exporter.log"
     echo "Uploading exporter log to ${LOGDEST}..."
     gsutil -h "Content-Type:text/plain" cp /var/log/daemon.log ${LOGDEST}
@@ -40,17 +40,15 @@ if ! gcloud compute instances attach-disk ${NAME} --disk ${DISKNAME} --device-na
   Exit 1
 fi
 
-echo "Dumping disk..."
-if ! dd if=/dev/disk/by-id/google-toexport of=disk.raw bs=4096 conv=sparse; then
-  echo "Failed to dump disk to image."
+echo "GCEExport: Running export tool."
+gce_export -gcs_path "${PATHS[0]}" -disk /dev/disk/by-id/google-toexport -y
+if [ $? -ne 0 ]; then
+  echo "ExportFailed: Failed to export disk source to ${PATHS[0]}."
   Exit 1
 fi
 
-echo "Compressing and tar'ing disk image..."
-if ! tar -czf root.tar.gz disk.raw; then
-  echo "Failed to tar disk image."
-  Exit 1
-fi
+echo "ExportSuccess"
+sync
 
 echo "Detaching disk..."
 if ! gcloud compute instances detach-disk ${NAME} --disk ${DISKNAME} --zone ${ZONE}; then
@@ -64,10 +62,10 @@ if ! gcloud compute disks delete ${DISKNAME} --zone ${ZONE}; then
   FAIL=1
 fi
 
-for i in ${PATHS}; do
-  echo "Uploading tar'ed disk image to ${i}..."
-  if ! gsutil -o GSUtil:parallel_composite_upload_threshold=100M cp root.tar.gz ${i}; then
-    echo "Failed to upload image to ${i}."
+for i in ${PATHS[@]:1}; do
+  echo "Copying archive image to ${i}..."
+  if ! gsutil -o GSUtil:parallel_composite_upload_threshold=100M cp ${PATHS[0]} ${i}; then
+    echo "Failed to copy image to ${i}."
     FAIL=1
   fi
 done

post-processor/googlecompute-import/post-processor.go

@@ -10,7 +10,7 @@ import (
 	"google.golang.org/api/compute/v1"
 	"google.golang.org/api/storage/v1"
 
-	googlecompute "github.com/hashicorp/packer/builder/googlecompute"
+	"github.com/hashicorp/packer/builder/googlecompute"
 	"github.com/hashicorp/packer/common"
 	"github.com/hashicorp/packer/helper/config"
 	"github.com/hashicorp/packer/packer"

website/source/docs/post-processors/googlecompute-export.html.md

@@ -34,9 +34,39 @@ permissions to the GCS `paths`.
 
 ### Optional
 
+-   `account_file` (string) - The JSON file containing your account
+    credentials. If specified, this take precedence over `googlecompute`
+    builder authentication method.
+
+-   `disk_size` (number) - The size of the export instances disk, this disk
+    is unused for the export but a larger size increase `pd-ssd` read speed.
+    This defaults to `200`, which is 200GB.
+
+-   `disk_type` (string) - Type of disk used to back export instance, like
+    `pd-ssd` or `pd-standard`. Defaults to `pd-ssd`.
+
 -   `keep_input_artifact` (boolean) - If true, do not delete the Google Compute
     Engine (GCE) image being exported.
 
+-   `machine_type` (string) - The export instance machine type. Defaults
+    to `"n1-highcpu-4"`.
+
+-   `network` (string) - The Google Compute network id or URL to use for the
+    export instance. Defaults to `"default"`. If the value is not a URL, it
+    will be interpolated to
+    `projects/((network_project_id))/global/networks/((network))`. This value
+    is not required if a `subnet` is specified.
+
+-   `subnetwork` (string) - The Google Compute subnetwork id or URL to use for
+    the export instance. Only required if the `network` has been created with
+    custom subnetting. Note, the region of the subnetwork must match the
+    `zone` in which the VM is launched. If the value is not a URL,
+    it will be interpolated to
+    `projects/((network_project_id))/regions/((region))/subnetworks/((subnetwork))`
+
+-   `zone` (string) - The zone in which to launch the export instance. Defaults
+    to `googlecompute` builder zone. Example: `"us-central1-a"`
+
 ## Basic Example
 
 The following example builds a GCE image in the project, `my-project`, with an