aherman
/
packer
mirror of https://github.com/hashicorp/packer
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Separate workflow for file on disk and prepared URL

Browse Source
pull/9601/head
Gennady Lipenkov 6 years ago
parent 59aaaf7a91
commit 29a6687475
5 changed files with 62 additions and 41 deletions
Show Stats Download Patch File Download Diff File

84
post-processor/yandex-import/post-processor.go
Unescape View File

@ -6,17 +6,15 @@ package yandeximport
import (
"context"
"fmt"
yandexexport "github.com/hashicorp/packer/post-processor/yandex-export"
"os"
"strings"
"time"
"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/hashicorp/packer/builder/yandex"
"github.com/yandex-cloud/go-genproto/yandex/cloud/compute/v1"
"github.com/yandex-cloud/go-genproto/yandex/cloud/iam/v1/awscompatibility"
"github.com/yandex-cloud/go-sdk/iamkey"
"github.com/hashicorp/hcl/v2/hcldec"
"github.com/hashicorp/packer/builder/file"
@ -25,6 +23,7 @@ import (
"github.com/hashicorp/packer/packer"
"github.com/hashicorp/packer/post-processor/artifice"
"github.com/hashicorp/packer/post-processor/compress"
"github.com/hashicorp/packer/post-processor/yandex-export"
"github.com/hashicorp/packer/template/interpolate"
)
@ -43,12 +42,15 @@ type Config struct {
// is an alternative method to authenticate to Yandex.Cloud.
ServiceAccountKeyFile string `mapstructure:"service_account_key_file" required:"false"`
// The name of the bucket where the qcow2 file will be copied to for import.
// The name of the bucket where the qcow2 file will be uploaded to for import.
// This bucket must exist when the post-processor is run.
Bucket string `mapstructure:"bucket" required:"true"`
// The name of the object key in
// `bucket` where the qcow2 file will be copied to import. This is a [template engine](/docs/templates/engine).
// Therefore, you may use user variables and template functions in this field.
//
// If import occurred after Yandex-Export post-processor, artifact already
// in storage service and first paths (URL) is used to, so no need to set this param.
Bucket string `mapstructure:"bucket" required:"false"`
// The name of the object key in `bucket` where the qcow2 file will be copied to import.
// This is a [template engine](/docs/templates/engine).
// Therefore, you may use user variables and template functions in this field.
ObjectName string `mapstructure:"object_name" required:"false"`
// Whether skip removing the qcow2 file uploaded to Storage
// after the import process has completed. Possible values are: `true` to
@ -60,7 +62,7 @@ type Config struct {
ImageName string `mapstructure:"image_name" required:"false"`
// The description of the image.
ImageDescription string `mapstructure:"image_description" required:"false"`
// The family name of the imported image.
// The family name of the imported image.
ImageFamily string `mapstructure:"image_family" required:"false"`
// Key/value pair labels to apply to the imported image.
ImageLabels map[string]string `mapstructure:"image_labels" required:"false"`
@ -128,7 +130,6 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
// TODO: make common code to check and prepare Yandex.Cloud auth configuration data
templates := map[string]*string{
"bucket": &p.config.Bucket,
"object_name": &p.config.ObjectName,
"folder_id": &p.config.FolderID,
}
@ -170,17 +171,40 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
return nil, false, false, fmt.Errorf("error rendering object_name template: %s", err)
}
var url string
// Create temporary storage Access Key
respWithKey, err := client.SDK().IAM().AWSCompatibility().AccessKey().Create(ctx, &awscompatibility.CreateAccessKeyRequest{
ServiceAccountId: p.config.ServiceAccountID,
Description: "this key is for upload image to storage",
Description: "this temporary key is for upload image to storage; created by Packer",
})
if err != nil {
return nil, false, false, err
}
storageClient, err := newYCStorageClient("", respWithKey.GetAccessKey().GetKeyId(), respWithKey.GetSecret())
if err != nil {
return nil, false, false, fmt.Errorf("error create object storage client: %s", err)
}
switch artifact.BuilderId() {
case compress.BuilderId, artifice.BuilderId, file.BuilderId, yandexexport.BuilderId:
break
case compress.BuilderId, artifice.BuilderId, file.BuilderId:
// Artifact as a file, need to be uploaded to storage before create Compute Image
// As `bucket` option validate input here
if p.config.Bucket == "" {
return nil, false, false, fmt.Errorf("To upload artfact you need to specify `bucket` value")
}
url, err = uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName)
if err != nil {
return nil, false, false, err
}
case yandexexport.BuilderId:
// Artifact already in storage, just get URL
url = artifact.Id()
default:
err := fmt.Errorf(
"Unknown artifact type: %s\nCan only import from Yandex-Export, Compress, Artifice and File post-processor artifacts.",
@ -188,29 +212,24 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
return nil, false, false, err
}
storageClient, err := newYCStorageClient("", respWithKey.GetAccessKey().GetKeyId(), respWithKey.GetSecret())
if err != nil {
return nil, false, false, fmt.Errorf("error create object_storage client: %s", err)
}
rawImageUrl, err := uploadToBucket(storageClient, ui, artifact, p.config.Bucket, p.config.ObjectName)
presignedUrl, err := presignUrl(storageClient, ui, url)
if err != nil {
return nil, false, false, err
}
ycImageArtifact, err := createYCImage(ctx, client, ui, p.config.FolderID, rawImageUrl, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels)
ycImage, err := createYCImage(ctx, client, ui, p.config.FolderID, presignedUrl, p.config.ImageName, p.config.ImageDescription, p.config.ImageFamily, p.config.ImageLabels)
if err != nil {
return nil, false, false, err
}
if !p.config.SkipClean {
err = deleteFromBucket(storageClient, ui, p.config.Bucket, p.config.ObjectName)
err = deleteFromBucket(storageClient, ui, url)
if err != nil {
return nil, false, false, err
}
}
// cleanup static access keys
// Delete temporary storage Access Key
_, err = client.SDK().IAM().AWSCompatibility().AccessKey().Delete(ctx, &awscompatibility.DeleteAccessKeyRequest{
AccessKeyId: respWithKey.GetAccessKey().GetId(),
})
@ -218,7 +237,7 @@ func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact
return nil, false, false, fmt.Errorf("error delete static access key: %s", err)
}
return ycImageArtifact, false, false, nil
return ycImage, false, false, nil
}
func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucket string, objectName string) (string, error) {
@ -263,13 +282,7 @@ func uploadToBucket(s3conn *s3.S3, ui packer.Ui, artifact packer.Artifact, bucke
// Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process
req.Config.S3ForcePathStyle = aws.Bool(true)
urlStr, _, err := req.PresignRequest(15 * time.Minute)
if err != nil {
ui.Say(fmt.Sprintf("Failed to presign url: %s", err))
return "", err
}
return urlStr, nil
return req.HTTPRequest.URL.String(), nil
}
func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, folderID string, rawImageURL string, imageName string, imageDescription string, imageFamily string, imageLabels map[string]string) (packer.Artifact, error) {
@ -320,10 +333,15 @@ func createYCImage(ctx context.Context, driver yandex.Driver, ui packer.Ui, fold
}, nil
}
func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, bucket string, objectName string) error {
func deleteFromBucket(s3conn *s3.S3, ui packer.Ui, url string) error {
bucket, objectName, err := s3URLToBucketKey(url)
if err != nil {
return err
}
ui.Say(fmt.Sprintf("Deleting import source from Object Storage %s/%s...", bucket, objectName))
_, err := s3conn.DeleteObject(&s3.DeleteObjectInput{
_, err = s3conn.DeleteObject(&s3.DeleteObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(objectName),
})

2
post-processor/yandex-import/post-processor.hcl2spec.go
Unescape View File

@ -20,7 +20,7 @@ type FlatConfig struct {
ServiceAccountID *string `mapstructure:"service_account_id" required:"true" cty:"service_account_id" hcl:"service_account_id"`
Token *string `mapstructure:"token" required:"false" cty:"token" hcl:"token"`
ServiceAccountKeyFile *string `mapstructure:"service_account_key_file" required:"false" cty:"service_account_key_file" hcl:"service_account_key_file"`
Bucket *string `mapstructure:"bucket" required:"true" cty:"bucket" hcl:"bucket"`
Bucket *string `mapstructure:"bucket" required:"false" cty:"bucket" hcl:"bucket"`
ObjectName *string `mapstructure:"object_name" required:"false" cty:"object_name" hcl:"object_name"`
SkipClean *bool `mapstructure:"skip_clean" required:"false" cty:"skip_clean" hcl:"skip_clean"`
ImageName *string `mapstructure:"image_name" required:"false" cty:"image_name" hcl:"image_name"`

2
post-processor/yandex-import/storage.go
Unescape View File

@ -60,7 +60,7 @@ func presignUrl(s3conn *s3.S3, ui packer.Ui, fullUrl string) (string, error) {
// Compute service allow only `https://storage.yandexcloud.net/...` URLs for Image create process
req.Config.S3ForcePathStyle = aws.Bool(true)
urlStr, _, err := req.PresignRequest(15 * time.Minute)
urlStr, _, err := req.PresignRequest(30 * time.Minute)
if err != nil {
ui.Say(fmt.Sprintf("Failed to presign url: %s", err))
return "", err

12
website/pages/partials/post-processor/yandex-import/Config-not-required.mdx
Unescape View File

@ -5,9 +5,15 @@
- `service_account_key_file` (string) - Path to file with Service Account key in json format. This
is an alternative method to authenticate to Yandex.Cloud.
- `object_name` (string) - The name of the object key in
`bucket` where the qcow2 file will be copied to import. This is a [template engine](/docs/templates/engine).
Therefore, you may use user variables and template functions in this field.
- `bucket` (string) - The name of the bucket where the qcow2 file will be uploaded to for import.
This bucket must exist when the post-processor is run.
If import occurred after Yandex-Export post-processor, artifact already
in storage service and first paths (URL) is used to, so no need to set this param.
- `object_name` (string) - The name of the object key in `bucket` where the qcow2 file will be copied to import.
This is a [template engine](/docs/templates/engine).
Therefore, you may use user variables and template functions in this field.
- `skip_clean` (bool) - Whether skip removing the qcow2 file uploaded to Storage
after the import process has completed. Possible values are: `true` to

3
website/pages/partials/post-processor/yandex-import/Config-required.mdx
Unescape View File

@ -4,6 +4,3 @@
- `service_account_id` (string) - Service Account ID with proper permission to use Storage service
for operations 'upload' and 'delete' object to `bucket`
- `bucket` (string) - The name of the bucket where the qcow2 file will be copied to for import.
This bucket must exist when the post-processor is run.

Write Preview
Loading…
Cancel
Save