Merge pull request #9601 from GennadySpb/yndx-import-pp-url-input

yandex-import: support work with URL from yandex-export pp

post-processor/yandex-export/artifact.go

@@ -8,14 +8,15 @@ const BuilderId = "packer.post-processor.yandex-export"
 
 type Artifact struct {
 	paths []string
+	urls  []string
 }
 
 func (*Artifact) BuilderId() string {
 	return BuilderId
 }
 
-func (*Artifact) Id() string {
-	return ""
+func (a *Artifact) Id() string {
+	return a.urls[0]
 }
 
 func (a *Artifact) Files() []string {

post-processor/yandex-export/post-processor.go

@@ -24,6 +24,8 @@ import (
 	"github.com/hashicorp/packer/template/interpolate"
 )
 
+const defaultStorageEndpoint = "storage.yandexcloud.net"
+
 type Config struct {
 	common.PackerConfig `mapstructure:",squash"`
 
@@ -31,12 +33,13 @@ type Config struct {
 	// Please be aware that use of space char inside path not supported.
 	// Also this param support [build](/docs/templates/engine) template function.
 	// Check available template data for [Yandex](/docs/builders/yandex#build-template-data) builder.
+	// Paths to Yandex Object Storage where exported image will be uploaded.
 	Paths []string `mapstructure:"paths" required:"true"`
 	// The folder ID that will be used to launch a temporary instance.
 	// Alternatively you may set value by environment variable YC_FOLDER_ID.
 	FolderID string `mapstructure:"folder_id" required:"true"`
 	// Service Account ID with proper permission to modify an instance, create and attach disk and
-	// make upload to specific Yandex Object Storage paths
+	// make upload to specific Yandex Object Storage paths.
 	ServiceAccountID string `mapstructure:"service_account_id" required:"true"`
 	// The size of the disk in GB. This defaults to `100`, which is 100GB.
 	DiskSizeGb int `mapstructure:"disk_size" required:"false"`
@@ -234,7 +237,7 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
 	steps := []multistep.Step{
 		&yandex.StepCreateSSHKey{
 			Debug:        p.config.PackerDebug,
-			DebugKeyPath: fmt.Sprintf("yc_pp_%s.pem", p.config.PackerBuildName),
+			DebugKeyPath: fmt.Sprintf("yc_export_pp_%s.pem", p.config.PackerBuildName),
 		},
 		&yandex.StepCreateInstance{
 			Debug:         p.config.PackerDebug,
@@ -248,7 +251,10 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
 	p.runner = common.NewRunner(steps, p.config.PackerConfig, ui)
 	p.runner.Run(ctx, state)
 
-	result := &Artifact{paths: p.config.Paths}
+	result := &Artifact{
+		paths: p.config.Paths,
+		urls:  formUrls(p.config.Paths),
+	}
 
 	return result, false, false, nil
 }
@@ -271,3 +277,12 @@ func ycSaneDefaults() yandex.Config {
 		StateTimeout:        3 * time.Minute,
 	}
 }
+
+func formUrls(paths []string) []string {
+	result := []string{}
+	for _, path := range paths {
+		url := fmt.Sprintf("https://%s/%s", defaultStorageEndpoint, strings.TrimPrefix(path, "s3://"))
+		result = append(result, url)
+	}
+	return result
+}

post-processor/yandex-export/post-processor_test.go

@@ -3,6 +3,8 @@ package yandexexport
 import (
 	"testing"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/hashicorp/packer/helper/multistep"
 )
 
@@ -64,3 +66,49 @@ func TestPostProcessor_Configure(t *testing.T) {
 		})
 	}
 }
+
+func Test_formUrls(t *testing.T) {
+	type args struct {
+		paths []string
+	}
+	tests := []struct {
+		name       string
+		args       args
+		wantResult []string
+	}{
+		{
+			name: "empty list",
+			args: args{
+				paths: []string{},
+			},
+			wantResult: []string{},
+		},
+		{
+			name: "one element",
+			args: args{
+				paths: []string{"s3://bucket1/object1"},
+			},
+			wantResult: []string{"https://" + defaultStorageEndpoint + "/bucket1/object1"},
+		},
+		{
+			name: "several elements",
+			args: args{
+				paths: []string{
+					"s3://bucket1/object1",
+					"s3://bucket-name/object-with/prefix/filename.blob",
+					"s3://bucket-too/foo/bar.test",
+				},
+			},
+			wantResult: []string{
+				"https://" + defaultStorageEndpoint + "/bucket1/object1",
+				"https://" + defaultStorageEndpoint + "/bucket-name/object-with/prefix/filename.blob",
+				"https://" + defaultStorageEndpoint + "/bucket-too/foo/bar.test",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			require.Equal(t, tt.wantResult, formUrls(tt.args.paths))
+		})
+	}
+}

post-processor/yandex-import/artifact.go

@@ -0,0 +1,36 @@
+package yandeximport
+
+import (
+	"fmt"
+)
+
+const BuilderId = "packer.post-processor.yandex-import"
+
+type Artifact struct {
+	imageID   string
+	sourceURL string
+}
+
+func (*Artifact) BuilderId() string {
+	return BuilderId
+}
+
+func (a *Artifact) Id() string {
+	return a.sourceURL
+}
+
+func (a *Artifact) Files() []string {
+	return nil
+}
+
+func (a *Artifact) String() string {
+	return fmt.Sprintf("Create image %v from URL %v", a.imageID, a.sourceURL)
+}
+
+func (*Artifact) State(name string) interface{} {
+	return nil
+}
+
+func (a *Artifact) Destroy() error {
+	return nil
+}

post-processor/yandex-import/post-processor.go

@@ -8,14 +8,13 @@ import (
 	"fmt"
 	"os"
 	"strings"
-	"time"
-
-	"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/hashicorp/packer/builder/yandex"
 	"github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1"
+	"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility"
+	"github.com/yandex-cloud/go-sdk/iamkey"
 
 	"github.com/hashicorp/hcl/v2/hcldec"
 	"github.com/hashicorp/packer/builder/file"
@@ -24,6 +23,7 @@ import (
 	"github.com/hashicorp/packer/packer"
 	"github.com/hashicorp/packer/post-processor/artifice"
 	"github.com/hashicorp/packer/post-processor/compress"
+	yandexexport "github.com/hashicorp/packer/post-processor/yandex-export"
 	"github.com/hashicorp/packer/template/interpolate"
 )
 
@@ -42,12 +42,15 @@ type Config struct {
 	// is an alternative method to authenticate to Yandex.Cloud.
 	ServiceAccountKeyFile string `mapstructure:"service_account_key_file" required:"false"`
 
-	// The name of the bucket where the qcow2 file will be copied to for import.
+	// The name of the bucket where the qcow2 file will be uploaded to for import.
 	// This bucket must exist when the post-processor is run.
-	Bucket string `mapstructure:"bucket" required:"true"`
-	// The name of the object key in
-	//  `bucket` where the qcow2 file will be copied to import. This is a [template engine](/docs/templates/engine).
-	//  Therefore, you may use user variables and template functions in this field.
+	//
+	// If import occurred after Yandex-Export post-processor, artifact already
+	// in storage service and first paths (URL) is used to, so no need to set this param.
+	Bucket string `mapstructure:"bucket" required:"false"`
+	// The name of the object key in `bucket` where the qcow2 file will be copied to import.
+	// This is a [template engine](/docs/templates/engine).
+	// Therefore, you may use user variables and template functions in this field.
 	ObjectName string `mapstructure:"object_name" required:"false"`
 	// Whether skip removing the qcow2 file uploaded to Storage
 	// after the import process has completed. Possible values are: `true` to
@@ -59,7 +62,7 @@ type Config struct {
 	ImageName string `mapstructure:"image_name" required:"false"`
 	// The description of the image.
 	ImageDescription string `mapstructure:"image_description" required:"false"`
-	//  The family name of the imported image.
+	// The family name of the imported image.
 	ImageFamily string `mapstructure:"image_family" required:"false"`
 	// Key/value pair labels to apply to the imported image.
 	ImageLabels map[string]string `mapstructure:"image_labels" required:"false"`
@@ -89,6 +92,30 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
 
 	errs := new(packer.MultiError)
 
+	// provision config by OS environment variables
+	if p.config.Token == "" {
+		p.config.Token = os.Getenv("YC_TOKEN")
+	}
+
+	if p.config.ServiceAccountKeyFile == "" {
+		p.config.ServiceAccountKeyFile = os.Getenv("YC_SERVICE_ACCOUNT_KEY_FILE")
+	}
+
+	if p.config.Token != "" {
+		packer.LogSecretFilter.Set(p.config.Token)
+	}
+
+	if p.config.ServiceAccountKeyFile != "" {
+		if _, err := iamkey.ReadFromJSONFile(p.config.ServiceAccountKeyFile); err != nil {
+			errs = packer.MultiErrorAppend(
+				errs, fmt.Errorf("fail to read service account key file: %s", err))
+		}
+	}
+
+	if p.config.FolderID == "" {
+		p.config.FolderID = os.Getenv("YC_FOLDER_ID")
+	}
+
 	// Set defaults
 	if p.config.ObjectName == "" {
 		p.config.ObjectName = "packer-import-{{timestamp}}.qcow2"
@@ -103,7 +130,6 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
 	// TODO: make common code to check and prepare Yandex.Cloud auth configuration data
 
 	templates := map[string]*string{
-		"bucket":      &p.config.Bucket,
 		"object_name": &p.config.ObjectName,
 		"folder_id":   &p.config.FolderID,
 	}
@@ -145,47 +171,71 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
 		return nil, false, false, fmt.Errorf("error rendering object_name template: %s", err)
 	}
 
+	var url string
+	var fileSource bool
+
+	// Create temporary storage Access Key
 	respWithKey, err := client.SDK().IAM().AWSCompatibility().AccessKey().Create(ctx, &awscompatibility.CreateAccessKeyRequest{
 		ServiceAccountId: p.config.ServiceAccountID,
-		Description:      "this key is for upload image to storage",
+		Description:      "this temporary key is for upload image to storage; created by Packer",
 	})
 	if err != nil {
 		return nil, false, false, err
 	}
 
+	storageClient, err := newYCStorageClient("", respWithKey.GetAccessKey().GetKeyId(), respWithKey.GetSecret())
+	if err != nil {
+		return nil, false, false, fmt.Errorf("error create object storage client: %s", err)
+	}
+
 	switch artifact.BuilderId() {
 	case compress.BuilderId, artifice.BuilderId, file.BuilderId:
-		break
+		// Artifact as a file, need to be uploaded to storage before create Compute Image
+		fileSource = true
+
+		// As `bucket` option validate input here
+		if p.config.Bucket == "" {
+			return nil, false, false, fmt.Errorf("To upload artfact you need to specify `bucket` value")
+		}
+
+		url, err = uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName)
+		if err != nil {
+			return nil, false, false, err
+		}
+
+	case yandexexport.BuilderId:
+		// Artifact already in storage, just get URL
+		url = artifact.Id()
+
+	case BuilderId:
+		// Artifact from prev yandex-import PP, reuse URL
+		url = artifact.Id()
+
 	default:
 		err := fmt.Errorf(
-			"Unknown artifact type: %s\nCan only import from Compress, Artifice and File post-processor artifacts.",
+			"Unknown artifact type: %s\nCan only import from Yandex-Export, Yandex-Import, Compress, Artifice and File post-processor artifacts.",
 			artifact.BuilderId())
 		return nil, false, false, err
 	}
 
-	storageClient, err := newYCStorageClient("", respWithKey.GetAccessKey().GetKeyId(), respWithKey.GetSecret())
-	if err != nil {
-		return nil, false, false, fmt.Errorf("error create object_storage client: %s", err)
-	}
-
-	rawImageUrl, err := uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName)
+	presignedUrl, err := presignUrl(storageClient, ui, url)
 	if err != nil {
 		return nil, false, false, err
 	}
 
-	ycImageArtifact, err := createYCImage(ctx, client, ui, p.config.FolderID, rawImageUrl, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels)
+	ycImage, err := createYCImage(ctx, client, ui, p.config.FolderID, presignedUrl, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels)
 	if err != nil {
 		return nil, false, false, err
 	}
 
-	if !p.config.SkipClean {
-		err = deleteFromBucket(storageClient, ui, p.config.Bucket, p.config.ObjectName)
+	if fileSource && !p.config.SkipClean {
+		err = deleteFromBucket(storageClient, ui, url)
 		if err != nil {
 			return nil, false, false, err
 		}
 	}
 
-	// cleanup static access keys
+	// Delete temporary storage Access Key
 	_, err = client.SDK().IAM().AWSCompatibility().AccessKey().Delete(ctx, &awscompatibility.DeleteAccessKeyRequest{
 		AccessKeyId: respWithKey.GetAccessKey().GetId(),
 	})
@@ -193,7 +243,10 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
 		return nil, false, false, fmt.Errorf("error delete static access key: %s", err)
 	}
 
-	return ycImageArtifact, false, false, nil
+	return &Artifact{
+		imageID:   ycImage.GetId(),
+		sourceURL: url,
+	}, false, false, nil
 }
 
 func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucket string, objectName string) (string, error) {
@@ -238,16 +291,10 @@ func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucke
 	// Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process
 	req.Config.S3ForcePathStyle = aws.Bool(true)
 
-	urlStr, _, err := req.PresignRequest(15 * time.Minute)
-	if err != nil {
-		ui.Say(fmt.Sprintf("Failed to presign url: %s", err))
-		return "", err
-	}
-
-	return urlStr, nil
+	return req.HTTPRequest.URL.String(), nil
 }
 
-func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, folderID string, rawImageURL string, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string) (packer.Artifact, error) {
+func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, folderID string, rawImageURL string, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string) (*compute.Image, error) {
 	op, err := driver.SDK().WrapOperation(driver.SDK().Compute().Image().Create(ctx, &compute.CreateImageRequest{
 		FolderId:    folderID,
 		Name:        imageName,
@@ -290,15 +337,19 @@ func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, fold
 		return nil, fmt.Errorf("error while image get request: %s", err)
 	}
 
-	return &yandex.Artifact{
-		Image: image,
-	}, nil
+	return image, nil
+
 }
 
-func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, bucket string, objectName string) error {
+func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, url string) error {
+	bucket, objectName, err := s3URLToBucketKey(url)
+	if err != nil {
+		return err
+	}
+
 	ui.Say(fmt.Sprintf("Deleting import source from Object Storage %s/%s...", bucket, objectName))
 
-	_, err := s3conn.DeleteObject(&s3.DeleteObjectInput{
+	_, err = s3conn.DeleteObject(&s3.DeleteObjectInput{
 		Bucket: aws.String(bucket),
 		Key:    aws.String(objectName),
 	})

post-processor/yandex-import/post-processor.hcl2spec.go

@@ -20,7 +20,7 @@ type FlatConfig struct {
 	ServiceAccountID      *string           `mapstructure:"service_account_id" required:"true" cty:"service_account_id" hcl:"service_account_id"`
 	Token                 *string           `mapstructure:"token" required:"false" cty:"token" hcl:"token"`
 	ServiceAccountKeyFile *string           `mapstructure:"service_account_key_file" required:"false" cty:"service_account_key_file" hcl:"service_account_key_file"`
-	Bucket                *string           `mapstructure:"bucket" required:"true" cty:"bucket" hcl:"bucket"`
+	Bucket                *string           `mapstructure:"bucket" required:"false" cty:"bucket" hcl:"bucket"`
 	ObjectName            *string           `mapstructure:"object_name" required:"false" cty:"object_name" hcl:"object_name"`
 	SkipClean             *bool             `mapstructure:"skip_clean" required:"false" cty:"skip_clean" hcl:"skip_clean"`
 	ImageName             *string           `mapstructure:"image_name" required:"false" cty:"image_name" hcl:"image_name"`

post-processor/yandex-import/storage.go

@@ -2,11 +2,15 @@ package yandeximport
 
 import (
 	"fmt"
+	"net/url"
+	"strings"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/hashicorp/packer/packer"
 )
 
 const defaultS3Region = "ru-central1"
@@ -40,3 +44,53 @@ func newYCStorageClient(storageEndpoint, accessKey, secretKey string) (*s3.S3, e
 
 	return s3.New(newSession), nil
 }
+
+// Get path-style S3 URL and return presigned URL
+func presignUrl(s3conn *s3.S3, ui packer.Ui, fullUrl string) (string, error) {
+	bucket, key, err := s3URLToBucketKey(fullUrl)
+	if err != nil {
+		return "", err
+	}
+
+	req, _ := s3conn.GetObjectRequest(&s3.GetObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	})
+
+	// Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process
+	req.Config.S3ForcePathStyle = aws.Bool(true)
+
+	urlStr, _, err := req.PresignRequest(30 * time.Minute)
+	if err != nil {
+		ui.Say(fmt.Sprintf("Failed to presign url: %s", err))
+		return "", err
+	}
+
+	return urlStr, nil
+}
+
+func s3URLToBucketKey(storageURL string) (bucket string, key string, err error) {
+	u, err := url.Parse(storageURL)
+	if err != nil {
+		return
+	}
+
+	if u.Scheme == "s3" {
+		// s3://bucket/key
+		bucket = u.Host
+		key = strings.TrimLeft(u.Path, "/")
+	} else if u.Scheme == "https" {
+		// https://***.storage.yandexcloud.net/...
+		if u.Host == defaultStorageEndpoint {
+			// No bucket name in the host part
+			path := strings.SplitN(u.Path, "/", 3)
+			bucket = path[1]
+			key = path[2]
+		} else {
+			// Bucket name in host
+			bucket = strings.TrimSuffix(u.Host, "."+defaultStorageEndpoint)
+			key = strings.TrimLeft(u.Path, "/")
+		}
+	}
+	return
+}

post-processor/yandex-import/storage_test.go

@@ -0,0 +1,57 @@
+package yandeximport
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_s3URLToBucketKey(t *testing.T) {
+	tests := []struct {
+		name       string
+		storageURL string
+		wantBucket string
+		wantKey    string
+		wantErr    bool
+	}{
+		{
+			name:       "path-style url #1",
+			storageURL: "https://storage.yandexcloud.net/bucket1/key1/foobar.txt",
+			wantBucket: "bucket1",
+			wantKey:    "key1/foobar.txt",
+			wantErr:    false,
+		},
+		{
+			name:       "path-style url #2",
+			storageURL: "https://storage.yandexcloud.net/bucket1.with.dots/key1/foobar.txt",
+			wantBucket: "bucket1.with.dots",
+			wantKey:    "key1/foobar.txt",
+			wantErr:    false,
+		},
+		{
+			name:       "host-style url #1",
+			storageURL: "https://bucket1.with.dots.storage.yandexcloud.net/key1/foobar.txt",
+			wantBucket: "bucket1.with.dots",
+			wantKey:    "key1/foobar.txt",
+			wantErr:    false,
+		},
+		{
+			name:       "host-style url #2",
+			storageURL: "https://bucket-with-dash.storage.yandexcloud.net/key2/foobar.txt",
+			wantBucket: "bucket-with-dash",
+			wantKey:    "key2/foobar.txt",
+			wantErr:    false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotBucket, gotKey, err := s3URLToBucketKey(tt.storageURL)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("s3URLToBucketKey() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			assert.Equal(t, tt.wantBucket, gotBucket)
+			assert.Equal(t, tt.wantKey, gotKey)
+		})
+	}
+}

website/pages/partials/post-processor/yandex-export/Config-required.mdx

@@ -4,9 +4,10 @@
   Please be aware that use of space char inside path not supported.
   Also this param support [build](/docs/templates/engine) template function.
   Check available template data for [Yandex](/docs/builders/yandex#build-template-data) builder.
+  Paths to Yandex Object Storage where exported image will be uploaded.
 
 - `folder_id` (string) - The folder ID that will be used to launch a temporary instance.
   Alternatively you may set value by environment variable YC_FOLDER_ID.
 
 - `service_account_id` (string) - Service Account ID with proper permission to modify an instance, create and attach disk and
-  make upload to specific Yandex Object Storage paths
+  make upload to specific Yandex Object Storage paths.

website/pages/partials/post-processor/yandex-import/Config-not-required.mdx

@@ -5,9 +5,15 @@
 - `service_account_key_file` (string) - Path to file with Service Account key in json format. This
   is an alternative method to authenticate to Yandex.Cloud.
 
-- `object_name` (string) - The name of the object key in
-   `bucket` where the qcow2 file will be copied to import. This is a [template engine](/docs/templates/engine).
-   Therefore, you may use user variables and template functions in this field.
+- `bucket` (string) - The name of the bucket where the qcow2 file will be uploaded to for import.
+  This bucket must exist when the post-processor is run.
+  
+  If import occurred after Yandex-Export post-processor, artifact already
+  in storage service and first paths (URL) is used to, so no need to set this param.
+
+- `object_name` (string) - The name of the object key in `bucket` where the qcow2 file will be copied to import.
+  This is a [template engine](/docs/templates/engine).
+  Therefore, you may use user variables and template functions in this field.
 
 - `skip_clean` (bool) - Whether skip removing the qcow2 file uploaded to Storage
   after the import process has completed. Possible values are: `true` to

website/pages/partials/post-processor/yandex-import/Config-required.mdx

@@ -4,6 +4,3 @@
 
 - `service_account_id` (string) - Service Account ID with proper permission to use Storage service
   for operations 'upload' and 'delete' object to `bucket`
-
-- `bucket` (string) - The name of the bucket where the qcow2 file will be copied to for import.
-  This bucket must exist when the post-processor is run.