From 8e5b04810c69893d613db31e0a0b072d8e16709f Mon Sep 17 00:00:00 2001 From: sylviamoss Date: Mon, 17 Jul 2023 16:27:44 +0200 Subject: [PATCH] wip --- go.mod | 9 +- go.sum | 14 +- internal/hcp/api/client.go | 218 ++++++++++++++++++++++---------- internal/hcp/api/client_test.go | 166 ------------------------ 4 files changed, 161 insertions(+), 246 deletions(-) delete mode 100644 internal/hcp/api/client_test.go diff --git a/go.mod b/go.mod index fa48d7d58..757b64dec 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/hashicorp/go-uuid v1.0.2 github.com/hashicorp/go-version v1.6.0 github.com/hashicorp/hcl/v2 v2.14.1 - github.com/hashicorp/hcp-sdk-go v0.36.0 + github.com/hashicorp/hcp-sdk-go v0.51.0 github.com/hashicorp/packer-plugin-amazon v1.2.1 github.com/hashicorp/packer-plugin-sdk v0.4.0 github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 @@ -48,7 +48,7 @@ require ( golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect golang.org/x/mod v0.8.0 golang.org/x/net v0.8.0 - golang.org/x/oauth2 v0.1.0 + golang.org/x/oauth2 v0.6.0 golang.org/x/sync v0.1.0 golang.org/x/sys v0.7.0 // indirect golang.org/x/term v0.6.0 // indirect @@ -59,7 +59,6 @@ require ( ) require ( - github.com/go-openapi/strfmt v0.21.3 github.com/hashicorp/packer-plugin-ansible v1.0.3 github.com/hashicorp/packer-plugin-azure v1.4.0 github.com/hashicorp/packer-plugin-docker v1.0.8 @@ -74,8 +73,7 @@ require ( ) require ( - cloud.google.com/go/compute v1.12.1 // indirect - cloud.google.com/go/compute/metadata v0.1.1 // indirect + cloud.google.com/go/compute/metadata v0.2.0 // indirect cloud.google.com/go/iam v0.6.0 // indirect cloud.google.com/go/storage v1.27.0 // indirect github.com/Azure/azure-sdk-for-go v64.0.0+incompatible // indirect @@ -127,6 +125,7 @@ require ( github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/loads v0.21.2 // indirect github.com/go-openapi/spec v0.20.8 // indirect + github.com/go-openapi/strfmt v0.21.3 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/go-openapi/validate v0.22.1 // indirect github.com/gofrs/uuid v4.0.0+incompatible // indirect diff --git a/go.sum b/go.sum index 96c6e63f9..e1008a111 100644 --- a/go.sum +++ b/go.sum @@ -2,10 +2,8 @@ bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxo cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= -cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0= -cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= -cloud.google.com/go/compute/metadata v0.1.1 h1:/sxEbyrm6cw+XOUw1YxBHlatV71z4vpnmO7z2IZ0h3I= -cloud.google.com/go/compute/metadata v0.1.1/go.mod h1:Z1VN+bulIf6bt4P/C37K4DyZYZEXYonfTBHHFPO/4UU= +cloud.google.com/go/compute/metadata v0.2.0 h1:nBbNSZyDpkNlo3DepaaLKVuO7ClyifSAmNloSCZrHnQ= +cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/iam v0.6.0 h1:nsqQC88kT5Iwlm4MeNGTpfMWddp6NB/UOLFTH6m1QfQ= cloud.google.com/go/iam v0.6.0/go.mod h1:+1AH33ueBne5MzYccyMHtEKqLE4/kJOibtffMHDMFMc= cloud.google.com/go/longrunning v0.1.1 h1:y50CXG4j0+qvEukslYFBCrzaXX0qpFbBzc3PchSu/LE= @@ -441,8 +439,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl/v2 v2.14.1 h1:x0BpjfZ+CYdbiz+8yZTQ+gdLO7IXvOut7Da+XJayx34= github.com/hashicorp/hcl/v2 v2.14.1/go.mod h1:e4z5nxYlWNPdDSNYX+ph14EvWYMFm3eP0zIUqPc2jr0= -github.com/hashicorp/hcp-sdk-go v0.36.0 h1:B1qvnsStyYQyuEudgPrpMdheEC/zTeHRhyF1UwpYTJQ= -github.com/hashicorp/hcp-sdk-go v0.36.0/go.mod h1:mJHPFD1Rs62bieKNVXUiFQlF76NCGACKqHu9a8ihcFk= +github.com/hashicorp/hcp-sdk-go v0.51.0 h1:T9kbgkVwIsKkI7MMAueUHpIswpfILifeW90WYik2wXE= +github.com/hashicorp/hcp-sdk-go v0.51.0/go.mod h1:hZqky4HEzsKwvLOt4QJlZUrjeQmb4UCZUhDP2HyQFfc= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/memberlist v0.2.2 h1:5+RffWKwqJ71YPu9mWsF7ZOscZmwfasdA8kbdC7AO2g= @@ -845,8 +843,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.1.0 h1:isLCZuhj4v+tYv7eskaN4v/TM+A1begWWgyVJDdl1+Y= -golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= +golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/internal/hcp/api/client.go b/internal/hcp/api/client.go index 21a07156c..e4a4a4ce7 100644 --- a/internal/hcp/api/client.go +++ b/internal/hcp/api/client.go @@ -7,14 +7,12 @@ package api import ( "fmt" "log" - "os" "time" packerSvc "github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/stable/2021-04-30/client/packer_service" - organizationSvc "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/client/organization_service" - projectSvc "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/client/project_service" - "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/models" - rmmodels "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/models" + organizationSvc "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/stable/2019-12-10/client/organization_service" + projectSvc "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/stable/2019-12-10/client/project_service" + "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/stable/2019-12-10/models" "github.com/hashicorp/hcp-sdk-go/httpclient" "github.com/hashicorp/packer/internal/hcp/env" "github.com/hashicorp/packer/version" @@ -60,95 +58,181 @@ func NewClient() (*Client, error) { Project: projectSvc.New(cl, nil), } - if err := client.loadOrganizationID(); err != nil { - return nil, &ClientError{ - StatusCode: InvalidClientConfig, - Err: err, + if client.ProjectID != "" && client.OrganizationID == "" { + getProjParams := projectSvc.NewProjectServiceGetParams() + getProjParams.ID = client.ProjectID + project, err := RetryProjectServiceGet(client, getProjParams) + if err != nil { + return nil, fmt.Errorf("unable to fetch project %q: %v", client.ProjectID, err) } + + client.ProjectID = project.Payload.Project.ID + client.OrganizationID = project.Payload.Project.Parent.ID + } - if err := client.loadProjectID(); err != nil { - return nil, &ClientError{ - StatusCode: InvalidClientConfig, - Err: err, + + if client.ProjectID == "" { + // For the initial release of the HCP TFP, since only one project was allowed per organization at the time, + // the provider handled used the single organization's single project by default, instead of requiring the + // user to set it. Once multiple projects are available, this helper issues a warning: when multiple projects exist within the org, + // a project ID should be set on the provider or on each resource. Otherwise, the oldest project will be used by default. + // This helper will eventually be deprecated after a migration period. + project, err := getProjectFromCredentials(client) + if err != nil { + return nil, fmt.Errorf("unable to get project from credentials: %v", err) } + + client.ProjectID = project.ID + client.OrganizationID = project.Parent.ID } return client, nil } -func (c *Client) loadOrganizationID() error { - // Get the organization ID. - listOrgParams := organizationSvc.NewOrganizationServiceListParams() - listOrgResp, err := c.Organization.OrganizationServiceList(listOrgParams, nil) - if err != nil { - return fmt.Errorf("unable to fetch organization list: %v", err) - } - orgLen := len(listOrgResp.Payload.Organizations) - if orgLen != 1 { - return fmt.Errorf("unexpected number of organizations: expected 1, actual: %v", orgLen) +const ( + retryCount = 10 + retryDelay = 10 + counterStart = 1 +) + +var errorCodesToRetry = [...]int{502, 503, 504} + +// Helper to check what requests to retry based on the response HTTP code +func shouldRetryErrorCode(errorCode int, errorCodesToRetry []int) bool { + for i := range errorCodesToRetry { + if errorCodesToRetry[i] == errorCode { + return true + } } - c.OrganizationID = listOrgResp.Payload.Organizations[0].ID - return nil + return false } -func (c *Client) loadProjectID() error { - // Get the project using the organization ID. - listProjParams := projectSvc.NewProjectServiceListParams() - listProjParams.ScopeID = &c.OrganizationID - scopeType := string(rmmodels.HashicorpCloudResourcemanagerResourceIDResourceTypeORGANIZATION) - listProjParams.ScopeType = &scopeType - listProjResp, err := c.Project.ProjectServiceList(listProjParams, nil) - if err != nil { - return fmt.Errorf("unable to fetch project id: %v", err) - } +// RetryProjectServiceGet wraps the ProjectServiceGet function in a loop that supports retrying the GET request +func RetryProjectServiceGet(client *Client, params *projectSvc.ProjectServiceGetParams) (*projectSvc.ProjectServiceGetOK, error) { + resp, err := client.Project.ProjectServiceGet(params, nil) - if env.HasProjectID() { - proj, err := findProjectByID(os.Getenv(env.HCPProjectID), listProjResp.Payload.Projects) - if err != nil { - return err + if err != nil { + serviceErr, ok := err.(*projectSvc.ProjectServiceGetDefault) + if !ok { + return nil, err } - c.ProjectID = proj.ID - } else { - if len(listProjResp.Payload.Projects) > 1 { - log.Printf("[WARNING] Multiple HCP projects found, will pick the oldest one by default\n" + - "To specify which project to use, set the HCP_PROJECT_ID environment variable to the one you want to use.") + counter := counterStart + for shouldRetryErrorCode(serviceErr.Code(), errorCodesToRetry[:]) && counter < retryCount { + resp, err = client.Project.ProjectServiceGet(params, nil) + if err == nil { + break + } + // Avoid wasting time if we're not going to retry next loop cycle + if (counter + 1) != retryCount { + fmt.Printf("Error trying to get configured project. Retrying in %d seconds...", retryDelay*counter) + time.Sleep(time.Duration(retryDelay*counter) * time.Second) + } + counter++ } + } + return resp, err +} - proj, err := findOldestProject(listProjResp.Payload.Projects) - if err != nil { - return err - } +// RetryOrganizationServiceList wraps the OrganizationServiceList function in a loop that supports retrying the GET request +func RetryOrganizationServiceList(client *Client, params *organizationSvc.OrganizationServiceListParams) (*organizationSvc.OrganizationServiceListOK, error) { + resp, err := client.Organization.OrganizationServiceList(params, nil) - c.ProjectID = proj.ID + if err != nil { + serviceErr, ok := err.(*organizationSvc.OrganizationServiceListDefault) + if !ok { + return nil, err + } + counter := counterStart + for shouldRetryErrorCode(serviceErr.Code(), errorCodesToRetry[:]) && counter < retryCount { + resp, err = client.Organization.OrganizationServiceList(params, nil) + if err == nil { + break + } + // Avoid wasting time if we're not going to retry next loop cycle + if (counter + 1) != retryCount { + fmt.Printf("Error trying to get list of organizations. Retrying in %d seconds...", retryDelay*counter) + time.Sleep(time.Duration(retryDelay*counter) * time.Second) + } + counter++ + } } - - return nil + return resp, err } -func findOldestProject(projs []*models.HashicorpCloudResourcemanagerProject) (*models.HashicorpCloudResourcemanagerProject, error) { - if len(projs) == 0 { - return nil, fmt.Errorf("no project found") - } +// RetryProjectServiceList wraps the ProjectServiceList function in a loop that supports retrying the GET request +func RetryProjectServiceList(client *Client, params *projectSvc.ProjectServiceListParams) (*projectSvc.ProjectServiceListOK, error) { + resp, err := client.Project.ProjectServiceList(params, nil) - proj := projs[0] - for i := 1; i < len(projs); i++ { - nxtProj := projs[i] + if err != nil { + serviceErr, ok := err.(*projectSvc.ProjectServiceListDefault) + if !ok { + return nil, err + } - if time.Time(nxtProj.CreatedAt).Before(time.Time(proj.CreatedAt)) { - proj = nxtProj + counter := counterStart + for shouldRetryErrorCode(serviceErr.Code(), errorCodesToRetry[:]) && counter < retryCount { + resp, err = client.Project.ProjectServiceList(params, nil) + if err == nil { + break + } + // Avoid wasting time if we're not going to retry next loop cycle + if (counter + 1) != retryCount { + fmt.Printf("Error trying to get list of projects. Retrying in %d seconds...", retryDelay*counter) + time.Sleep(time.Duration(retryDelay*counter) * time.Second) + } + counter++ } } - - return proj, nil + return resp, err } -func findProjectByID(projID string, projs []*models.HashicorpCloudResourcemanagerProject) (*models.HashicorpCloudResourcemanagerProject, error) { - for _, proj := range projs { - if proj.ID == projID { - return proj, nil +// getProjectFromCredentials uses the configured client credentials to +// fetch the associated organization and returns that organization's +// single project. +func getProjectFromCredentials(client *Client) (project *models.ResourcemanagerProject, err error) { + if client.OrganizationID == "" { + // Get the organization ID. + listOrgParams := organizationSvc.NewOrganizationServiceListParams() + listOrgResp, err := RetryOrganizationServiceList(client, listOrgParams) + if err != nil { + return nil, fmt.Errorf("unable to fetch organization list: %v", err) } + orgLen := len(listOrgResp.Payload.Organizations) + if orgLen != 1 { + return nil, fmt.Errorf("unexpected number of organizations: expected 1, actual: %v", orgLen) + } + client.OrganizationID = listOrgResp.Payload.Organizations[0].ID + } + + // Get the project using the organization ID. + listProjParams := projectSvc.NewProjectServiceListParams() + listProjParams.ScopeID = &client.OrganizationID + scopeType := string(models.ResourceIDResourceTypeORGANIZATION) + listProjParams.ScopeType = &scopeType + listProjResp, err := RetryProjectServiceList(client, listProjParams) + if err != nil { + return nil, fmt.Errorf("unable to fetch project id: %v", err) + } + if len(listProjResp.Payload.Projects) > 1 { + log.Printf("[WARNING] Multiple HCP projects found, will pick the oldest one by default\n" + + "To specify which project to use, set the HCP_PROJECT_ID environment variable to the one you want to use.") + return getOldestProject(listProjResp.Payload.Projects), nil } + project = listProjResp.Payload.Projects[0] + return project, nil +} - return nil, fmt.Errorf("No project %q found", projID) +// getOldestProject retrieves the oldest project from a list based on its created_at time. +func getOldestProject(projects []*models.ResourcemanagerProject) (oldestProj *models.ResourcemanagerProject) { + oldestTime := time.Now() + + for _, proj := range projects { + projTime := time.Time(proj.CreatedAt) + if projTime.Before(oldestTime) { + oldestProj = proj + oldestTime = projTime + } + } + return oldestProj } diff --git a/internal/hcp/api/client_test.go b/internal/hcp/api/client_test.go deleted file mode 100644 index b1cdbff61..000000000 --- a/internal/hcp/api/client_test.go +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 - -package api - -import ( - "testing" - "time" - - "github.com/go-openapi/strfmt" - "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/models" -) - -func TestFindProjectID(t *testing.T) { - testcases := []struct { - Name string - ProjectID string - ProjectList []*models.HashicorpCloudResourcemanagerProject - ExpectProjectID string - ExpectErr bool - }{ - { - "Only one project, project exists, success", - "test-project-exists", - []*models.HashicorpCloudResourcemanagerProject{ - { - ID: "test-project-exists", - }, - }, - "test-project-exists", - false, - }, - { - "Multiple projects, project exists, success", - "test-project-exists", - []*models.HashicorpCloudResourcemanagerProject{ - { - ID: "other-project-exists", - }, - { - ID: "test-project-exists", - }, - }, - "test-project-exists", - false, - }, - { - "One project, no id match, fail", - "test-project-exists", - []*models.HashicorpCloudResourcemanagerProject{ - { - ID: "other-project-exists", - }, - }, - "", - true, - }, - { - "Multiple projects, no id match, fail", - "test-project-exists", - []*models.HashicorpCloudResourcemanagerProject{ - { - ID: "other-project-exists", - }, - { - ID: "yet-another-project-exists", - }, - }, - "", - true, - }, - { - "No projects, no id match, fail", - "test-project-exists", - []*models.HashicorpCloudResourcemanagerProject{}, - "", - true, - }, - } - - for _, tt := range testcases { - t.Run(tt.Name, func(t *testing.T) { - proj, err := findProjectByID(tt.ProjectID, tt.ProjectList) - if (err != nil) != tt.ExpectErr { - t.Errorf("test findProjectByID, expected %t, got %t", - tt.ExpectErr, - err != nil) - } - - if proj != nil && proj.ID != tt.ExpectProjectID { - t.Errorf("expected to select project %q, got %q", tt.ExpectProjectID, proj.ID) - } - }) - } -} - -func TestFindOldestProject(t *testing.T) { - testcases := []struct { - Name string - ProjectList []*models.HashicorpCloudResourcemanagerProject - ExpectProjectID string - ExpectErr bool - }{ - { - "Only one project, project exists, success", - []*models.HashicorpCloudResourcemanagerProject{ - { - ID: "test-project-exists", - }, - }, - "test-project-exists", - false, - }, - { - "Multiple projects, pick the oldest", - []*models.HashicorpCloudResourcemanagerProject{ - { - ID: "test-project-exists", - CreatedAt: strfmt.DateTime(time.Date(2023, 1, 1, 1, 0, 0, 0, time.UTC)), - }, - { - ID: "test-oldest-project", - CreatedAt: strfmt.DateTime(time.Date(2022, 1, 1, 1, 0, 0, 0, time.UTC)), - }, - }, - "test-oldest-project", - false, - }, - { - "Multiple projects, different order, pick the oldest", - []*models.HashicorpCloudResourcemanagerProject{ - { - ID: "test-oldest-project", - CreatedAt: strfmt.DateTime(time.Date(2022, 1, 1, 1, 0, 0, 0, time.UTC)), - }, - { - ID: "test-project-exists", - CreatedAt: strfmt.DateTime(time.Date(2023, 1, 1, 1, 0, 0, 0, time.UTC)), - }, - }, - "test-oldest-project", - false, - }, - { - "No projects, should error", - []*models.HashicorpCloudResourcemanagerProject{}, - "", - true, - }, - } - - for _, tt := range testcases { - t.Run(tt.Name, func(t *testing.T) { - proj, err := findOldestProject(tt.ProjectList) - if (err != nil) != tt.ExpectErr { - t.Errorf("test findProjectByID, expected %t, got %t", - tt.ExpectErr, - err != nil) - } - - if proj != nil && proj.ID != tt.ExpectProjectID { - t.Errorf("expected to select project %q, got %q", tt.ExpectProjectID, proj.ID) - } - }) - } -}