From 8885a5ef312a30ab6818041ca89262497da3e6f8 Mon Sep 17 00:00:00 2001 From: Marin Salinas Date: Tue, 25 Aug 2020 18:02:11 -0500 Subject: [PATCH 1/2] refactor: update step_create_omi to new OSC SDK --- builder/osc/bsu/builder.go | 4 +- builder/osc/bsu/step_create_omi.go | 61 +++++++++++++---------- builder/osc/common/block_device.go | 46 +++++++++++++++++ builder/osc/common/state.go | 39 +++++++++++++++ builder/osc/common/step_deregister_omi.go | 2 - 5 files changed, 123 insertions(+), 29 deletions(-) diff --git a/builder/osc/bsu/builder.go b/builder/osc/bsu/builder.go index e81c9d084..c9f38e70b 100644 --- a/builder/osc/bsu/builder.go +++ b/builder/osc/bsu/builder.go @@ -194,7 +194,9 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack OMIName: b.config.OMIName, Regions: b.config.OMIRegions, }, - &stepCreateOMI{}, + &stepCreateOMI{ + RawRegion: b.config.RawRegion, + }, &osccommon.StepUpdateOMIAttributes{ AccountIds: b.config.OMIAccountIDs, SnapshotAccountIds: b.config.SnapshotAccountIDs, diff --git a/builder/osc/bsu/step_create_omi.go b/builder/osc/bsu/step_create_omi.go index cb935d89d..015a1008f 100644 --- a/builder/osc/bsu/step_create_omi.go +++ b/builder/osc/bsu/step_create_omi.go @@ -5,20 +5,21 @@ import ( "fmt" "log" + "github.com/antihax/optional" osccommon "github.com/hashicorp/packer/builder/osc/common" "github.com/hashicorp/packer/helper/multistep" "github.com/hashicorp/packer/packer" - "github.com/outscale/osc-go/oapi" "github.com/outscale/osc-sdk-go/osc" ) type stepCreateOMI struct { - image *oapi.Image + image *osc.Image + RawRegion string } func (s *stepCreateOMI) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { config := state.Get("config").(*Config) - oapiconn := state.Get("oapi").(*oapi.Client) + oscconn := state.Get("osc").(*osc.APIClient) vm := state.Get("vm").(osc.Vm) ui := state.Get("ui").(packer.Ui) @@ -26,42 +27,46 @@ func (s *stepCreateOMI) Run(ctx context.Context, state multistep.StateBag) multi omiName := config.OMIName ui.Say(fmt.Sprintf("Creating OMI %s from vm %s", omiName, vm.VmId)) - createOpts := oapi.CreateImageRequest{ + createOpts := osc.CreateImageRequest{ VmId: vm.VmId, ImageName: omiName, - BlockDeviceMappings: config.BlockDevices.BuildOMIDevices(), + BlockDeviceMappings: config.BlockDevices.BuildOscOMIDevices(), } - resp, err := oapiconn.POST_CreateImage(createOpts) - if err != nil || resp.OK == nil { + resp, _, err := oscconn.ImageApi.CreateImage(context.Background(), &osc.CreateImageOpts{ + CreateImageRequest: optional.NewInterface(createOpts), + }) + if err != nil || resp.Image.ImageId == "" { err := fmt.Errorf("Error creating OMI: %s", err) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } - image := resp.OK.Image + image := resp.Image // Set the OMI ID in the state ui.Message(fmt.Sprintf("OMI: %s", image.ImageId)) omis := make(map[string]string) - omis[oapiconn.GetConfig().Region] = image.ImageId + omis[s.RawRegion] = image.ImageId state.Put("omis", omis) // Wait for the image to become ready ui.Say("Waiting for OMI to become ready...") - if err := osccommon.WaitUntilImageAvailable(oapiconn, image.ImageId); err != nil { + if err := osccommon.WaitUntilOscImageAvailable(oscconn, image.ImageId); err != nil { log.Printf("Error waiting for OMI: %s", err) - imagesResp, err := oapiconn.POST_ReadImages(oapi.ReadImagesRequest{ - Filters: oapi.FiltersImage{ - ImageIds: []string{image.ImageId}, - }, + imagesResp, _, err := oscconn.ImageApi.ReadImages(context.Background(), &osc.ReadImagesOpts{ + ReadImagesRequest: optional.NewInterface(osc.ReadImagesRequest{ + Filters: osc.FiltersImage{ + ImageIds: []string{image.ImageId}, + }, + }), }) if err != nil { log.Printf("Unable to determine reason waiting for OMI failed: %s", err) - err = fmt.Errorf("Unknown error waiting for OMI.") + err = fmt.Errorf("Unknown error waiting for OMI") } else { - stateReason := imagesResp.OK.Images[0].StateComment + stateReason := imagesResp.Images[0].StateComment err = fmt.Errorf("Error waiting for OMI. Reason: %s", stateReason) } @@ -70,10 +75,12 @@ func (s *stepCreateOMI) Run(ctx context.Context, state multistep.StateBag) multi return multistep.ActionHalt } - imagesResp, err := oapiconn.POST_ReadImages(oapi.ReadImagesRequest{ - Filters: oapi.FiltersImage{ - ImageIds: []string{image.ImageId}, - }, + imagesResp, _, err := oscconn.ImageApi.ReadImages(context.Background(), &osc.ReadImagesOpts{ + ReadImagesRequest: optional.NewInterface(osc.ReadImagesRequest{ + Filters: osc.FiltersImage{ + ImageIds: []string{image.ImageId}, + }, + }), }) if err != nil { err := fmt.Errorf("Error searching for OMI: %s", err) @@ -81,12 +88,12 @@ func (s *stepCreateOMI) Run(ctx context.Context, state multistep.StateBag) multi ui.Error(err.Error()) return multistep.ActionHalt } - s.image = &imagesResp.OK.Images[0] + s.image = &imagesResp.Images[0] snapshots := make(map[string][]string) - for _, blockDeviceMapping := range imagesResp.OK.Images[0].BlockDeviceMappings { + for _, blockDeviceMapping := range imagesResp.Images[0].BlockDeviceMappings { if blockDeviceMapping.Bsu.SnapshotId != "" { - snapshots[oapiconn.GetConfig().Region] = append(snapshots[oapiconn.GetConfig().Region], blockDeviceMapping.Bsu.SnapshotId) + snapshots[s.RawRegion] = append(snapshots[s.RawRegion], blockDeviceMapping.Bsu.SnapshotId) } } state.Put("snapshots", snapshots) @@ -105,12 +112,14 @@ func (s *stepCreateOMI) Cleanup(state multistep.StateBag) { return } - oapiconn := state.Get("oapi").(*oapi.Client) + oscconn := state.Get("osc").(*osc.APIClient) ui := state.Get("ui").(packer.Ui) ui.Say("Deregistering the OMI because cancellation or error...") - DeleteOpts := oapi.DeleteImageRequest{ImageId: s.image.ImageId} - if _, err := oapiconn.POST_DeleteImage(DeleteOpts); err != nil { + DeleteOpts := osc.DeleteImageRequest{ImageId: s.image.ImageId} + if _, _, err := oscconn.ImageApi.DeleteImage(context.Background(), &osc.DeleteImageOpts{ + DeleteImageRequest: optional.NewInterface(DeleteOpts), + }); err != nil { ui.Error(fmt.Sprintf("Error Deleting OMI, may still be around: %s", err)) return } diff --git a/builder/osc/common/block_device.go b/builder/osc/common/block_device.go index 7c0c90034..1f229c896 100644 --- a/builder/osc/common/block_device.go +++ b/builder/osc/common/block_device.go @@ -78,6 +78,48 @@ func buildBlockDevicesImage(b []BlockDevice) []oapi.BlockDeviceMappingImage { return blockDevices } +func buildOscBlockDevicesImage(b []BlockDevice) []osc.BlockDeviceMappingImage { + var blockDevices []osc.BlockDeviceMappingImage + + for _, blockDevice := range b { + mapping := osc.BlockDeviceMappingImage{ + DeviceName: blockDevice.DeviceName, + } + + if blockDevice.VirtualName != "" { + if strings.HasPrefix(blockDevice.VirtualName, "ephemeral") { + mapping.VirtualDeviceName = blockDevice.VirtualName + } + } else { + bsu := osc.BsuToCreate{ + DeleteOnVmDeletion: blockDevice.DeleteOnVmDeletion, + } + + if blockDevice.VolumeType != "" { + bsu.VolumeType = blockDevice.VolumeType + } + + if blockDevice.VolumeSize > 0 { + bsu.VolumeSize = int32(blockDevice.VolumeSize) + } + + // IOPS is only valid for io1 type + if blockDevice.VolumeType == "io1" { + bsu.Iops = int32(blockDevice.IOPS) + } + + if blockDevice.SnapshotId != "" { + bsu.SnapshotId = blockDevice.SnapshotId + } + + mapping.Bsu = bsu + } + + blockDevices = append(blockDevices, mapping) + } + return blockDevices +} + func buildBlockDevicesVmCreation(b []BlockDevice) []oapi.BlockDeviceMappingVmCreation { log.Printf("[DEBUG] Launch Block Device %#v", b) @@ -196,6 +238,10 @@ func (b *OMIBlockDevices) BuildOMIDevices() []oapi.BlockDeviceMappingImage { return buildBlockDevicesImage(b.OMIMappings) } +func (b *OMIBlockDevices) BuildOscOMIDevices() []osc.BlockDeviceMappingImage { + return buildOscBlockDevicesImage(b.OMIMappings) +} + func (b *LaunchBlockDevices) BuildLaunchDevices() []oapi.BlockDeviceMappingVmCreation { return buildBlockDevicesVmCreation(b.LaunchMappings) } diff --git a/builder/osc/common/state.go b/builder/osc/common/state.go index 62e763043..d75c6aae8 100644 --- a/builder/osc/common/state.go +++ b/builder/osc/common/state.go @@ -70,6 +70,12 @@ func WaitUntilImageAvailable(conn *oapi.Client, imageID string) error { return <-errCh } +func WaitUntilOscImageAvailable(conn *osc.APIClient, imageID string) error { + errCh := make(chan error, 1) + go waitForState(errCh, "available", waitUntilOscImageStateFunc(conn, imageID)) + return <-errCh +} + func WaitUntilVolumeAvailable(conn *oapi.Client, volumeID string) error { errCh := make(chan error, 1) go waitForState(errCh, "available", volumeWaitFunc(conn, volumeID)) @@ -285,6 +291,39 @@ func waitUntilImageStateFunc(conn *oapi.Client, id string) stateRefreshFunc { } } +func waitUntilOscImageStateFunc(conn *osc.APIClient, id string) stateRefreshFunc { + return func() (string, error) { + log.Printf("[Debug] Check if Image with id %s exists", id) + resp, _, err := conn.ImageApi.ReadImages(context.Background(), &osc.ReadImagesOpts{ + ReadImagesRequest: optional.NewInterface(osc.ReadImagesRequest{ + Filters: osc.FiltersImage{ + ImageIds: []string{id}, + }, + }), + }) + + log.Printf("[Debug] Read Response %+v", resp) + + if err != nil { + return "", err + } + + // if resp.Images[] == "" { + // return "", fmt.Errorf("Vm with ID %s. Not Found", id) + // } + + if len(resp.Images) == 0 { + return "pending", nil + } + + if resp.Images[0].State == "failed" { + return resp.Images[0].State, fmt.Errorf("Image (%s) creation is failed", id) + } + + return resp.Images[0].State, nil + } +} + func securityGroupWaitFunc(conn *oapi.Client, id string) stateRefreshFunc { return func() (string, error) { log.Printf("[Debug] Check if SG with id %s exists", id) diff --git a/builder/osc/common/step_deregister_omi.go b/builder/osc/common/step_deregister_omi.go index 807f5f142..ba9025e69 100644 --- a/builder/osc/common/step_deregister_omi.go +++ b/builder/osc/common/step_deregister_omi.go @@ -23,8 +23,6 @@ type StepDeregisterOMI struct { func (s *StepDeregisterOMI) Run(_ context.Context, state multistep.StateBag) multistep.StepAction { // Check for force deregister if !s.ForceDeregister { - log.Println("ENTRO 0 ") - return multistep.ActionContinue } From e0badb3fb7a41ac5ef307a4e7bfe65c063c0f8e2 Mon Sep 17 00:00:00 2001 From: Marin Salinas Date: Wed, 26 Aug 2020 12:25:46 -0500 Subject: [PATCH 2/2] refactor: update step_update_omi_attributes and create_tags to new OSC SDK --- builder/osc/bsu/builder.go | 3 +- builder/osc/common/step_create_tags.go | 58 ++++++++----------- builder/osc/common/step_update_omi.go | 79 ++++++++++++-------------- 3 files changed, 61 insertions(+), 79 deletions(-) diff --git a/builder/osc/bsu/builder.go b/builder/osc/bsu/builder.go index c9f38e70b..2a32f40ff 100644 --- a/builder/osc/bsu/builder.go +++ b/builder/osc/bsu/builder.go @@ -106,7 +106,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack state.Put("config", &b.config) state.Put("oapi", oapiconn) state.Put("osc", oscConn) - state.Put("clientConfig", clientConfig) + state.Put("accessConfig", &b.config.AccessConfig) state.Put("hook", hook) state.Put("ui", ui) @@ -200,6 +200,7 @@ func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (pack &osccommon.StepUpdateOMIAttributes{ AccountIds: b.config.OMIAccountIDs, SnapshotAccountIds: b.config.SnapshotAccountIDs, + RawRegion: b.config.RawRegion, Ctx: b.config.ctx, }, &osccommon.StepCreateTags{ diff --git a/builder/osc/common/step_create_tags.go b/builder/osc/common/step_create_tags.go index 44bdcc78c..f9a8e46b9 100644 --- a/builder/osc/common/step_create_tags.go +++ b/builder/osc/common/step_create_tags.go @@ -2,16 +2,15 @@ package common import ( "context" - "crypto/tls" "fmt" - "net/http" + "github.com/antihax/optional" "github.com/aws/aws-sdk-go/aws/awserr" retry "github.com/hashicorp/packer/common" "github.com/hashicorp/packer/helper/multistep" "github.com/hashicorp/packer/packer" "github.com/hashicorp/packer/template/interpolate" - "github.com/outscale/osc-go/oapi" + "github.com/outscale/osc-sdk-go/osc" ) type StepCreateTags struct { @@ -21,8 +20,7 @@ type StepCreateTags struct { } func (s *StepCreateTags) Run(_ context.Context, state multistep.StateBag) multistep.StepAction { - oapiconn := state.Get("oapi").(*oapi.Client) - config := state.Get("clientConfig").(*oapi.Config) + config := state.Get("accessConfig").(*AccessConfig) ui := state.Get("ui").(packer.Ui) omis := state.Get("omis").(map[string]string) @@ -34,28 +32,16 @@ func (s *StepCreateTags) Run(_ context.Context, state multistep.StateBag) multis for region, ami := range omis { ui.Say(fmt.Sprintf("Adding tags to OMI (%s)...", ami)) - newConfig := &oapi.Config{ - UserAgent: config.UserAgent, - SecretKey: config.SecretKey, - Service: config.Service, - Region: region, //New region - URL: config.URL, - } - - skipClient := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }, - } - - regionConn := oapi.NewClient(newConfig, skipClient) + regionconn := config.NewOSCClientByRegion(region) // Retrieve image list for given OMI resourceIds := []string{ami} - imageResp, err := regionConn.POST_ReadImages(oapi.ReadImagesRequest{ - Filters: oapi.FiltersImage{ - ImageIds: resourceIds, - }, + imageResp, _, err := regionconn.ImageApi.ReadImages(context.Background(), &osc.ReadImagesOpts{ + ReadImagesRequest: optional.NewInterface(osc.ReadImagesRequest{ + Filters: osc.FiltersImage{ + ImageIds: resourceIds, + }, + }), }) if err != nil { @@ -65,14 +51,14 @@ func (s *StepCreateTags) Run(_ context.Context, state multistep.StateBag) multis return multistep.ActionHalt } - if len(imageResp.OK.Images) == 0 { + if len(imageResp.Images) == 0 { err := fmt.Errorf("Error retrieving details for OMI (%s), no images found", ami) state.Put("error", err) ui.Error(err.Error()) return multistep.ActionHalt } - image := imageResp.OK.Images[0] + image := imageResp.Images[0] snapshotIds := []string{} // Add only those with a Snapshot ID, i.e. not Ephemeral @@ -86,7 +72,7 @@ func (s *StepCreateTags) Run(_ context.Context, state multistep.StateBag) multis // Convert tags to oapi.Tag format ui.Say("Creating OMI tags") - amiTags, err := s.Tags.OAPITags(s.Ctx, oapiconn.GetConfig().Region, state) + amiTags, err := s.Tags.OSCTags(s.Ctx, config.RawRegion, state) if err != nil { state.Put("error", err) ui.Error(err.Error()) @@ -95,7 +81,7 @@ func (s *StepCreateTags) Run(_ context.Context, state multistep.StateBag) multis amiTags.Report(ui) ui.Say("Creating snapshot tags") - snapshotTags, err := s.SnapshotTags.OAPITags(s.Ctx, oapiconn.GetConfig().Region, state) + snapshotTags, err := s.SnapshotTags.OSCTags(s.Ctx, config.RawRegion, state) if err != nil { state.Put("error", err) ui.Error(err.Error()) @@ -106,9 +92,11 @@ func (s *StepCreateTags) Run(_ context.Context, state multistep.StateBag) multis // Retry creating tags for about 2.5 minutes err = retry.Retry(0.2, 30, 11, func(_ uint) (bool, error) { // Tag images and snapshots - _, err := regionConn.POST_CreateTags(oapi.CreateTagsRequest{ - ResourceIds: resourceIds, - Tags: amiTags, + _, _, err := regionconn.TagApi.CreateTags(context.Background(), &osc.CreateTagsOpts{ + CreateTagsRequest: optional.NewInterface(osc.CreateTagsRequest{ + ResourceIds: resourceIds, + Tags: amiTags, + }), }) if awsErr, ok := err.(awserr.Error); ok { if awsErr.Code() == "InvalidOMIID.NotFound" || @@ -119,9 +107,11 @@ func (s *StepCreateTags) Run(_ context.Context, state multistep.StateBag) multis // Override tags on snapshots if len(snapshotTags) > 0 { - _, err = regionConn.POST_CreateTags(oapi.CreateTagsRequest{ - ResourceIds: snapshotIds, - Tags: snapshotTags, + _, _, err = regionconn.TagApi.CreateTags(context.Background(), &osc.CreateTagsOpts{ + CreateTagsRequest: optional.NewInterface(osc.CreateTagsRequest{ + ResourceIds: snapshotIds, + Tags: snapshotTags, + }), }) } if err == nil { diff --git a/builder/osc/common/step_update_omi.go b/builder/osc/common/step_update_omi.go index 41ce69a89..2be6b4e4f 100644 --- a/builder/osc/common/step_update_omi.go +++ b/builder/osc/common/step_update_omi.go @@ -2,25 +2,24 @@ package common import ( "context" - "crypto/tls" "fmt" - "net/http" + "github.com/antihax/optional" "github.com/hashicorp/packer/helper/multistep" "github.com/hashicorp/packer/packer" "github.com/hashicorp/packer/template/interpolate" - "github.com/outscale/osc-go/oapi" + "github.com/outscale/osc-sdk-go/osc" ) type StepUpdateOMIAttributes struct { AccountIds []string SnapshotAccountIds []string + RawRegion string Ctx interpolate.Context } func (s *StepUpdateOMIAttributes) Run(_ context.Context, state multistep.StateBag) multistep.StepAction { - oapiconn := state.Get("oapi").(*oapi.Client) - config := state.Get("clientConfig").(*oapi.Config) + config := state.Get("accessConfig").(*AccessConfig) ui := state.Get("ui").(packer.Ui) omis := state.Get("omis").(map[string]string) snapshots := state.Get("snapshots").(map[string][]string) @@ -34,20 +33,20 @@ func (s *StepUpdateOMIAttributes) Run(_ context.Context, state multistep.StateBa return multistep.ActionContinue } - s.Ctx.Data = extractBuildInfo(oapiconn.GetConfig().Region, state) + s.Ctx.Data = extractBuildInfo(s.RawRegion, state) - updateSnapshoptRequest := oapi.UpdateSnapshotRequest{ - PermissionsToCreateVolume: oapi.PermissionsOnResourceCreation{ - Additions: oapi.PermissionsOnResource{ + updateSnapshoptRequest := osc.UpdateSnapshotRequest{ + PermissionsToCreateVolume: osc.PermissionsOnResourceCreation{ + Additions: osc.PermissionsOnResource{ AccountIds: s.AccountIds, GlobalPermission: false, }, }, } - updateImageRequest := oapi.UpdateImageRequest{ - PermissionsToLaunch: oapi.PermissionsOnResourceCreation{ - Additions: oapi.PermissionsOnResource{ + updateImageRequest := osc.UpdateImageRequest{ + PermissionsToLaunch: osc.PermissionsOnResourceCreation{ + Additions: osc.PermissionsOnResource{ AccountIds: s.AccountIds, GlobalPermission: false, }, @@ -57,26 +56,31 @@ func (s *StepUpdateOMIAttributes) Run(_ context.Context, state multistep.StateBa // Updating image attributes for region, omi := range omis { ui.Say(fmt.Sprintf("Updating attributes on OMI (%s)...", omi)) - newConfig := &oapi.Config{ - UserAgent: config.UserAgent, - AccessKey: config.AccessKey, - SecretKey: config.SecretKey, - Service: config.Service, - Region: region, //New region - URL: config.URL, - } + regionconn := config.NewOSCClientByRegion(region) - skipClient := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }, - } + // newConfig := &osc.Configuration{ + // UserAgent: config.UserAgent, + // AccessKey: config.AccessKey, + // SecretKey: config.SecretKey, + // Service: config.Service, + // Region: region, //New region + // URL: config.URL, + // } - regionconn := oapi.NewClient(newConfig, skipClient) + // skipClient := &http.Client{ + // Transport: &http.Transport{ + // TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + // }, + // } + + //regionconn := oapi.NewClient(newConfig, skipClient) ui.Message(fmt.Sprintf("Updating: %s", omi)) updateImageRequest.ImageId = omi - _, err := regionconn.POST_UpdateImage(updateImageRequest) + _, _, err := regionconn.ImageApi.UpdateImage(context.Background(), &osc.UpdateImageOpts{ + UpdateImageRequest: optional.NewInterface(updateImageRequest), + }) + if err != nil { err := fmt.Errorf("Error updating OMI: %s", err) state.Put("error", err) @@ -89,26 +93,13 @@ func (s *StepUpdateOMIAttributes) Run(_ context.Context, state multistep.StateBa for region, region_snapshots := range snapshots { for _, snapshot := range region_snapshots { ui.Say(fmt.Sprintf("Updating attributes on snapshot (%s)...", snapshot)) - newConfig := &oapi.Config{ - UserAgent: config.UserAgent, - AccessKey: config.AccessKey, - SecretKey: config.SecretKey, - Service: config.Service, - Region: region, //New region - URL: config.URL, - } - - skipClient := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }, - } - - regionconn := oapi.NewClient(newConfig, skipClient) + regionconn := config.NewOSCClientByRegion(region) ui.Message(fmt.Sprintf("Updating: %s", snapshot)) updateSnapshoptRequest.SnapshotId = snapshot - _, err := regionconn.POST_UpdateSnapshot(updateSnapshoptRequest) + _, _, err := regionconn.SnapshotApi.UpdateSnapshot(context.Background(), &osc.UpdateSnapshotOpts{ + UpdateSnapshotRequest: optional.NewInterface(updateSnapshoptRequest), + }) if err != nil { err := fmt.Errorf("Error updating snapshot: %s", err) state.Put("error", err)