diff --git a/post-processor/yandex-export/artifact.go b/post-processor/yandex-export/artifact.go index 3c85f572c..413c19875 100644 --- a/post-processor/yandex-export/artifact.go +++ b/post-processor/yandex-export/artifact.go @@ -8,14 +8,15 @@ const BuilderId = "packer.post-processor.yandex-export" type Artifact struct { paths []string + urls []string } func (*Artifact) BuilderId() string { return BuilderId } -func (*Artifact) Id() string { - return "" +func (a *Artifact) Id() string { + return a.urls[0] } func (a *Artifact) Files() []string { diff --git a/post-processor/yandex-export/post-processor.go b/post-processor/yandex-export/post-processor.go index 6aa1252b8..0fd40ca72 100644 --- a/post-processor/yandex-export/post-processor.go +++ b/post-processor/yandex-export/post-processor.go @@ -24,6 +24,8 @@ import ( "github.com/hashicorp/packer/template/interpolate" ) +const defaultStorageEndpoint = "storage.yandexcloud.net" + type Config struct { common.PackerConfig `mapstructure:",squash"` @@ -31,12 +33,13 @@ type Config struct { // Please be aware that use of space char inside path not supported. // Also this param support [build](/docs/templates/engine) template function. // Check available template data for [Yandex](/docs/builders/yandex#build-template-data) builder. + // Paths to Yandex Object Storage where exported image will be uploaded. Paths []string `mapstructure:"paths" required:"true"` // The folder ID that will be used to launch a temporary instance. // Alternatively you may set value by environment variable YC_FOLDER_ID. FolderID string `mapstructure:"folder_id" required:"true"` // Service Account ID with proper permission to modify an instance, create and attach disk and - // make upload to specific Yandex Object Storage paths + // make upload to specific Yandex Object Storage paths. ServiceAccountID string `mapstructure:"service_account_id" required:"true"` // The size of the disk in GB. This defaults to `100`, which is 100GB. DiskSizeGb int `mapstructure:"disk_size" required:"false"` @@ -234,7 +237,7 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact steps := []multistep.Step{ &yandex.StepCreateSSHKey{ Debug: p.config.PackerDebug, - DebugKeyPath: fmt.Sprintf("yc_pp_%s.pem", p.config.PackerBuildName), + DebugKeyPath: fmt.Sprintf("yc_export_pp_%s.pem", p.config.PackerBuildName), }, &yandex.StepCreateInstance{ Debug: p.config.PackerDebug, @@ -248,7 +251,10 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact p.runner = common.NewRunner(steps, p.config.PackerConfig, ui) p.runner.Run(ctx, state) - result := &Artifact{paths: p.config.Paths} + result := &Artifact{ + paths: p.config.Paths, + urls: formUrls(p.config.Paths), + } return result, false, false, nil } @@ -271,3 +277,12 @@ func ycSaneDefaults() yandex.Config { StateTimeout: 3 * time.Minute, } } + +func formUrls(paths []string) []string { + result := []string{} + for _, path := range paths { + url := fmt.Sprintf("https://%s/%s", defaultStorageEndpoint, strings.TrimPrefix(path, "s3://")) + result = append(result, url) + } + return result +} diff --git a/post-processor/yandex-export/post-processor_test.go b/post-processor/yandex-export/post-processor_test.go index d538504b0..e35381b7a 100644 --- a/post-processor/yandex-export/post-processor_test.go +++ b/post-processor/yandex-export/post-processor_test.go @@ -3,6 +3,8 @@ package yandexexport import ( "testing" + "github.com/stretchr/testify/require" + "github.com/hashicorp/packer/helper/multistep" ) @@ -64,3 +66,49 @@ func TestPostProcessor_Configure(t *testing.T) { }) } } + +func Test_formUrls(t *testing.T) { + type args struct { + paths []string + } + tests := []struct { + name string + args args + wantResult []string + }{ + { + name: "empty list", + args: args{ + paths: []string{}, + }, + wantResult: []string{}, + }, + { + name: "one element", + args: args{ + paths: []string{"s3://bucket1/object1"}, + }, + wantResult: []string{"https://" + defaultStorageEndpoint + "/bucket1/object1"}, + }, + { + name: "several elements", + args: args{ + paths: []string{ + "s3://bucket1/object1", + "s3://bucket-name/object-with/prefix/filename.blob", + "s3://bucket-too/foo/bar.test", + }, + }, + wantResult: []string{ + "https://" + defaultStorageEndpoint + "/bucket1/object1", + "https://" + defaultStorageEndpoint + "/bucket-name/object-with/prefix/filename.blob", + "https://" + defaultStorageEndpoint + "/bucket-too/foo/bar.test", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.wantResult, formUrls(tt.args.paths)) + }) + } +} diff --git a/post-processor/yandex-import/artifact.go b/post-processor/yandex-import/artifact.go new file mode 100644 index 000000000..c1112019a --- /dev/null +++ b/post-processor/yandex-import/artifact.go @@ -0,0 +1,36 @@ +package yandeximport + +import ( + "fmt" +) + +const BuilderId = "packer.post-processor.yandex-import" + +type Artifact struct { + imageID string + sourceURL string +} + +func (*Artifact) BuilderId() string { + return BuilderId +} + +func (a *Artifact) Id() string { + return a.sourceURL +} + +func (a *Artifact) Files() []string { + return nil +} + +func (a *Artifact) String() string { + return fmt.Sprintf("Create image %v from URL %v", a.imageID, a.sourceURL) +} + +func (*Artifact) State(name string) interface{} { + return nil +} + +func (a *Artifact) Destroy() error { + return nil +} diff --git a/post-processor/yandex-import/post-processor.go b/post-processor/yandex-import/post-processor.go index be4260a4a..32e5c71c9 100644 --- a/post-processor/yandex-import/post-processor.go +++ b/post-processor/yandex-import/post-processor.go @@ -8,14 +8,13 @@ import ( "fmt" "os" "strings" - "time" - - "github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/hashicorp/packer/builder/yandex" "github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1" + "github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility" + "github.com/yandex-cloud/go-sdk/iamkey" "github.com/hashicorp/hcl/v2/hcldec" "github.com/hashicorp/packer/builder/file" @@ -24,6 +23,7 @@ import ( "github.com/hashicorp/packer/packer" "github.com/hashicorp/packer/post-processor/artifice" "github.com/hashicorp/packer/post-processor/compress" + yandexexport "github.com/hashicorp/packer/post-processor/yandex-export" "github.com/hashicorp/packer/template/interpolate" ) @@ -42,12 +42,15 @@ type Config struct { // is an alternative method to authenticate to Yandex.Cloud. ServiceAccountKeyFile string `mapstructure:"service_account_key_file" required:"false"` - // The name of the bucket where the qcow2 file will be copied to for import. + // The name of the bucket where the qcow2 file will be uploaded to for import. // This bucket must exist when the post-processor is run. - Bucket string `mapstructure:"bucket" required:"true"` - // The name of the object key in - // `bucket` where the qcow2 file will be copied to import. This is a [template engine](/docs/templates/engine). - // Therefore, you may use user variables and template functions in this field. + // + // If import occurred after Yandex-Export post-processor, artifact already + // in storage service and first paths (URL) is used to, so no need to set this param. + Bucket string `mapstructure:"bucket" required:"false"` + // The name of the object key in `bucket` where the qcow2 file will be copied to import. + // This is a [template engine](/docs/templates/engine). + // Therefore, you may use user variables and template functions in this field. ObjectName string `mapstructure:"object_name" required:"false"` // Whether skip removing the qcow2 file uploaded to Storage // after the import process has completed. Possible values are: `true` to @@ -59,7 +62,7 @@ type Config struct { ImageName string `mapstructure:"image_name" required:"false"` // The description of the image. ImageDescription string `mapstructure:"image_description" required:"false"` - // The family name of the imported image. + // The family name of the imported image. ImageFamily string `mapstructure:"image_family" required:"false"` // Key/value pair labels to apply to the imported image. ImageLabels map[string]string `mapstructure:"image_labels" required:"false"` @@ -89,6 +92,30 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { errs := new(packer.MultiError) + // provision config by OS environment variables + if p.config.Token == "" { + p.config.Token = os.Getenv("YC_TOKEN") + } + + if p.config.ServiceAccountKeyFile == "" { + p.config.ServiceAccountKeyFile = os.Getenv("YC_SERVICE_ACCOUNT_KEY_FILE") + } + + if p.config.Token != "" { + packer.LogSecretFilter.Set(p.config.Token) + } + + if p.config.ServiceAccountKeyFile != "" { + if _, err := iamkey.ReadFromJSONFile(p.config.ServiceAccountKeyFile); err != nil { + errs = packer.MultiErrorAppend( + errs, fmt.Errorf("fail to read service account key file: %s", err)) + } + } + + if p.config.FolderID == "" { + p.config.FolderID = os.Getenv("YC_FOLDER_ID") + } + // Set defaults if p.config.ObjectName == "" { p.config.ObjectName = "packer-import-{{timestamp}}.qcow2" @@ -103,7 +130,6 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { // TODO: make common code to check and prepare Yandex.Cloud auth configuration data templates := map[string]*string{ - "bucket": &p.config.Bucket, "object_name": &p.config.ObjectName, "folder_id": &p.config.FolderID, } @@ -145,47 +171,71 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact return nil, false, false, fmt.Errorf("error rendering object_name template: %s", err) } + var url string + var fileSource bool + + // Create temporary storage Access Key respWithKey, err := client.SDK().IAM().AWSCompatibility().AccessKey().Create(ctx, &awscompatibility.CreateAccessKeyRequest{ ServiceAccountId: p.config.ServiceAccountID, - Description: "this key is for upload image to storage", + Description: "this temporary key is for upload image to storage; created by Packer", }) if err != nil { return nil, false, false, err } + storageClient, err := newYCStorageClient("", respWithKey.GetAccessKey().GetKeyId(), respWithKey.GetSecret()) + if err != nil { + return nil, false, false, fmt.Errorf("error create object storage client: %s", err) + } + switch artifact.BuilderId() { case compress.BuilderId, artifice.BuilderId, file.BuilderId: - break + // Artifact as a file, need to be uploaded to storage before create Compute Image + fileSource = true + + // As `bucket` option validate input here + if p.config.Bucket == "" { + return nil, false, false, fmt.Errorf("To upload artfact you need to specify `bucket` value") + } + + url, err = uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName) + if err != nil { + return nil, false, false, err + } + + case yandexexport.BuilderId: + // Artifact already in storage, just get URL + url = artifact.Id() + + case BuilderId: + // Artifact from prev yandex-import PP, reuse URL + url = artifact.Id() + default: err := fmt.Errorf( - "Unknown artifact type: %s\nCan only import from Compress, Artifice and File post-processor artifacts.", + "Unknown artifact type: %s\nCan only import from Yandex-Export, Yandex-Import, Compress, Artifice and File post-processor artifacts.", artifact.BuilderId()) return nil, false, false, err } - storageClient, err := newYCStorageClient("", respWithKey.GetAccessKey().GetKeyId(), respWithKey.GetSecret()) - if err != nil { - return nil, false, false, fmt.Errorf("error create object_storage client: %s", err) - } - - rawImageUrl, err := uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName) + presignedUrl, err := presignUrl(storageClient, ui, url) if err != nil { return nil, false, false, err } - ycImageArtifact, err := createYCImage(ctx, client, ui, p.config.FolderID, rawImageUrl, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels) + ycImage, err := createYCImage(ctx, client, ui, p.config.FolderID, presignedUrl, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels) if err != nil { return nil, false, false, err } - if !p.config.SkipClean { - err = deleteFromBucket(storageClient, ui, p.config.Bucket, p.config.ObjectName) + if fileSource && !p.config.SkipClean { + err = deleteFromBucket(storageClient, ui, url) if err != nil { return nil, false, false, err } } - // cleanup static access keys + // Delete temporary storage Access Key _, err = client.SDK().IAM().AWSCompatibility().AccessKey().Delete(ctx, &awscompatibility.DeleteAccessKeyRequest{ AccessKeyId: respWithKey.GetAccessKey().GetId(), }) @@ -193,7 +243,10 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact return nil, false, false, fmt.Errorf("error delete static access key: %s", err) } - return ycImageArtifact, false, false, nil + return &Artifact{ + imageID: ycImage.GetId(), + sourceURL: url, + }, false, false, nil } func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucket string, objectName string) (string, error) { @@ -238,16 +291,10 @@ func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucke // Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process req.Config.S3ForcePathStyle = aws.Bool(true) - urlStr, _, err := req.PresignRequest(15 * time.Minute) - if err != nil { - ui.Say(fmt.Sprintf("Failed to presign url: %s", err)) - return "", err - } - - return urlStr, nil + return req.HTTPRequest.URL.String(), nil } -func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, folderID string, rawImageURL string, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string) (packer.Artifact, error) { +func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, folderID string, rawImageURL string, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string) (*compute.Image, error) { op, err := driver.SDK().WrapOperation(driver.SDK().Compute().Image().Create(ctx, &compute.CreateImageRequest{ FolderId: folderID, Name: imageName, @@ -290,15 +337,19 @@ func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, fold return nil, fmt.Errorf("error while image get request: %s", err) } - return &yandex.Artifact{ - Image: image, - }, nil + return image, nil + } -func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, bucket string, objectName string) error { +func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, url string) error { + bucket, objectName, err := s3URLToBucketKey(url) + if err != nil { + return err + } + ui.Say(fmt.Sprintf("Deleting import source from Object Storage %s/%s...", bucket, objectName)) - _, err := s3conn.DeleteObject(&s3.DeleteObjectInput{ + _, err = s3conn.DeleteObject(&s3.DeleteObjectInput{ Bucket: aws.String(bucket), Key: aws.String(objectName), }) diff --git a/post-processor/yandex-import/post-processor.hcl2spec.go b/post-processor/yandex-import/post-processor.hcl2spec.go index 91c5570c3..eff818fa7 100644 --- a/post-processor/yandex-import/post-processor.hcl2spec.go +++ b/post-processor/yandex-import/post-processor.hcl2spec.go @@ -20,7 +20,7 @@ type FlatConfig struct { ServiceAccountID *string `mapstructure:"service_account_id" required:"true" cty:"service_account_id" hcl:"service_account_id"` Token *string `mapstructure:"token" required:"false" cty:"token" hcl:"token"` ServiceAccountKeyFile *string `mapstructure:"service_account_key_file" required:"false" cty:"service_account_key_file" hcl:"service_account_key_file"` - Bucket *string `mapstructure:"bucket" required:"true" cty:"bucket" hcl:"bucket"` + Bucket *string `mapstructure:"bucket" required:"false" cty:"bucket" hcl:"bucket"` ObjectName *string `mapstructure:"object_name" required:"false" cty:"object_name" hcl:"object_name"` SkipClean *bool `mapstructure:"skip_clean" required:"false" cty:"skip_clean" hcl:"skip_clean"` ImageName *string `mapstructure:"image_name" required:"false" cty:"image_name" hcl:"image_name"` diff --git a/post-processor/yandex-import/storage.go b/post-processor/yandex-import/storage.go index 006afcaca..e218be3ff 100644 --- a/post-processor/yandex-import/storage.go +++ b/post-processor/yandex-import/storage.go @@ -2,11 +2,15 @@ package yandeximport import ( "fmt" + "net/url" + "strings" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" + "github.com/hashicorp/packer/packer" ) const defaultS3Region = "ru-central1" @@ -40,3 +44,53 @@ func newYCStorageClient(storageEndpoint, accessKey, secretKey string) (*s3.S3, e return s3.New(newSession), nil } + +// Get path-style S3 URL and return presigned URL +func presignUrl(s3conn *s3.S3, ui packer.Ui, fullUrl string) (string, error) { + bucket, key, err := s3URLToBucketKey(fullUrl) + if err != nil { + return "", err + } + + req, _ := s3conn.GetObjectRequest(&s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }) + + // Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process + req.Config.S3ForcePathStyle = aws.Bool(true) + + urlStr, _, err := req.PresignRequest(30 * time.Minute) + if err != nil { + ui.Say(fmt.Sprintf("Failed to presign url: %s", err)) + return "", err + } + + return urlStr, nil +} + +func s3URLToBucketKey(storageURL string) (bucket string, key string, err error) { + u, err := url.Parse(storageURL) + if err != nil { + return + } + + if u.Scheme == "s3" { + // s3://bucket/key + bucket = u.Host + key = strings.TrimLeft(u.Path, "/") + } else if u.Scheme == "https" { + // https://***.storage.yandexcloud.net/... + if u.Host == defaultStorageEndpoint { + // No bucket name in the host part + path := strings.SplitN(u.Path, "/", 3) + bucket = path[1] + key = path[2] + } else { + // Bucket name in host + bucket = strings.TrimSuffix(u.Host, "."+defaultStorageEndpoint) + key = strings.TrimLeft(u.Path, "/") + } + } + return +} diff --git a/post-processor/yandex-import/storage_test.go b/post-processor/yandex-import/storage_test.go new file mode 100644 index 000000000..7cfeed976 --- /dev/null +++ b/post-processor/yandex-import/storage_test.go @@ -0,0 +1,57 @@ +package yandeximport + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_s3URLToBucketKey(t *testing.T) { + tests := []struct { + name string + storageURL string + wantBucket string + wantKey string + wantErr bool + }{ + { + name: "path-style url #1", + storageURL: "https://storage.yandexcloud.net/bucket1/key1/foobar.txt", + wantBucket: "bucket1", + wantKey: "key1/foobar.txt", + wantErr: false, + }, + { + name: "path-style url #2", + storageURL: "https://storage.yandexcloud.net/bucket1.with.dots/key1/foobar.txt", + wantBucket: "bucket1.with.dots", + wantKey: "key1/foobar.txt", + wantErr: false, + }, + { + name: "host-style url #1", + storageURL: "https://bucket1.with.dots.storage.yandexcloud.net/key1/foobar.txt", + wantBucket: "bucket1.with.dots", + wantKey: "key1/foobar.txt", + wantErr: false, + }, + { + name: "host-style url #2", + storageURL: "https://bucket-with-dash.storage.yandexcloud.net/key2/foobar.txt", + wantBucket: "bucket-with-dash", + wantKey: "key2/foobar.txt", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotBucket, gotKey, err := s3URLToBucketKey(tt.storageURL) + if (err != nil) != tt.wantErr { + t.Errorf("s3URLToBucketKey() error = %v, wantErr %v", err, tt.wantErr) + return + } + assert.Equal(t, tt.wantBucket, gotBucket) + assert.Equal(t, tt.wantKey, gotKey) + }) + } +} diff --git a/website/pages/partials/post-processor/yandex-export/Config-required.mdx b/website/pages/partials/post-processor/yandex-export/Config-required.mdx index b86fb95c7..7202eaa72 100644 --- a/website/pages/partials/post-processor/yandex-export/Config-required.mdx +++ b/website/pages/partials/post-processor/yandex-export/Config-required.mdx @@ -4,9 +4,10 @@ Please be aware that use of space char inside path not supported. Also this param support [build](/docs/templates/engine) template function. Check available template data for [Yandex](/docs/builders/yandex#build-template-data) builder. + Paths to Yandex Object Storage where exported image will be uploaded. - `folder_id` (string) - The folder ID that will be used to launch a temporary instance. Alternatively you may set value by environment variable YC_FOLDER_ID. - `service_account_id` (string) - Service Account ID with proper permission to modify an instance, create and attach disk and - make upload to specific Yandex Object Storage paths + make upload to specific Yandex Object Storage paths. diff --git a/website/pages/partials/post-processor/yandex-import/Config-not-required.mdx b/website/pages/partials/post-processor/yandex-import/Config-not-required.mdx index 28352f5c3..6f5923b77 100644 --- a/website/pages/partials/post-processor/yandex-import/Config-not-required.mdx +++ b/website/pages/partials/post-processor/yandex-import/Config-not-required.mdx @@ -5,9 +5,15 @@ - `service_account_key_file` (string) - Path to file with Service Account key in json format. This is an alternative method to authenticate to Yandex.Cloud. -- `object_name` (string) - The name of the object key in - `bucket` where the qcow2 file will be copied to import. This is a [template engine](/docs/templates/engine). - Therefore, you may use user variables and template functions in this field. +- `bucket` (string) - The name of the bucket where the qcow2 file will be uploaded to for import. + This bucket must exist when the post-processor is run. + + If import occurred after Yandex-Export post-processor, artifact already + in storage service and first paths (URL) is used to, so no need to set this param. + +- `object_name` (string) - The name of the object key in `bucket` where the qcow2 file will be copied to import. + This is a [template engine](/docs/templates/engine). + Therefore, you may use user variables and template functions in this field. - `skip_clean` (bool) - Whether skip removing the qcow2 file uploaded to Storage after the import process has completed. Possible values are: `true` to diff --git a/website/pages/partials/post-processor/yandex-import/Config-required.mdx b/website/pages/partials/post-processor/yandex-import/Config-required.mdx index 3ce5a6242..86bda22ce 100644 --- a/website/pages/partials/post-processor/yandex-import/Config-required.mdx +++ b/website/pages/partials/post-processor/yandex-import/Config-required.mdx @@ -4,6 +4,3 @@ - `service_account_id` (string) - Service Account ID with proper permission to use Storage service for operations 'upload' and 'delete' object to `bucket` - -- `bucket` (string) - The name of the bucket where the qcow2 file will be copied to for import. - This bucket must exist when the post-processor is run.