From 3cfc83e335cbbf4c7b15d8865deaae4eff646ab9 Mon Sep 17 00:00:00 2001 From: Sander van Harmelen Date: Thu, 30 Aug 2018 22:18:43 +0200 Subject: [PATCH] backend/remote: take working directories into account In TFE you can configure a workspace to use a custom working directory. When determining which directory that needs to be uploaded to TFE, this working directory should be taken into account to make sure we are uploading the correct root directory for the workspace. --- backend/remote/backend_mock.go | 146 +++++++++++++----- backend/remote/backend_plan.go | 22 ++- backend/remote/backend_plan_test.go | 37 +++++ .../terraform/main.tf | 1 + .../terraform/output.log | 29 ++++ backend/remote/testing.go | 17 +- 6 files changed, 207 insertions(+), 45 deletions(-) create mode 100644 backend/remote/test-fixtures/plan-with-working-directory/terraform/main.tf create mode 100644 backend/remote/test-fixtures/plan-with-working-directory/terraform/output.log diff --git a/backend/remote/backend_mock.go b/backend/remote/backend_mock.go index 9aea17168a..e51682eb7c 100644 --- a/backend/remote/backend_mock.go +++ b/backend/remote/backend_mock.go @@ -9,21 +9,45 @@ import ( "io" "io/ioutil" "math/rand" + "os" + "path/filepath" tfe "github.com/hashicorp/go-tfe" ) +type mockClient struct { + ConfigurationVersions *mockConfigurationVersions + Organizations *mockOrganizations + Plans *mockPlans + Runs *mockRuns + StateVersions *mockStateVersions + Workspaces *mockWorkspaces +} + +func newMockClient() *mockClient { + c := &mockClient{} + c.ConfigurationVersions = newMockConfigurationVersions(c) + c.Organizations = newMockOrganizations(c) + c.Plans = newMockPlans(c) + c.Runs = newMockRuns(c) + c.StateVersions = newMockStateVersions(c) + c.Workspaces = newMockWorkspaces(c) + return c +} + type mockConfigurationVersions struct { + client *mockClient configVersions map[string]*tfe.ConfigurationVersion + uploadPaths map[string]string uploadURLs map[string]*tfe.ConfigurationVersion - workspaces map[string]*tfe.ConfigurationVersion } -func newMockConfigurationVersions() *mockConfigurationVersions { +func newMockConfigurationVersions(client *mockClient) *mockConfigurationVersions { return &mockConfigurationVersions{ + client: client, configVersions: make(map[string]*tfe.ConfigurationVersion), + uploadPaths: make(map[string]string), uploadURLs: make(map[string]*tfe.ConfigurationVersion), - workspaces: make(map[string]*tfe.ConfigurationVersion), } } @@ -47,7 +71,6 @@ func (m *mockConfigurationVersions) Create(ctx context.Context, workspaceID stri m.configVersions[cv.ID] = cv m.uploadURLs[url] = cv - m.workspaces[workspaceID] = cv return cv, nil } @@ -65,16 +88,19 @@ func (m *mockConfigurationVersions) Upload(ctx context.Context, url, path string if !ok { return errors.New("404 not found") } + m.uploadPaths[cv.ID] = path cv.Status = tfe.ConfigurationUploaded return nil } type mockOrganizations struct { + client *mockClient organizations map[string]*tfe.Organization } -func newMockOrganizations() *mockOrganizations { +func newMockOrganizations(client *mockClient) *mockOrganizations { return &mockOrganizations{ + client: client, organizations: make(map[string]*tfe.Organization), } } @@ -117,32 +143,53 @@ func (m *mockOrganizations) Delete(ctx context.Context, name string) error { } type mockPlans struct { - logs map[string]string - plans map[string]*tfe.Plan + client *mockClient + logs map[string]string + plans map[string]*tfe.Plan } -func newMockPlans() *mockPlans { +func newMockPlans(client *mockClient) *mockPlans { return &mockPlans{ - logs: make(map[string]string), - plans: make(map[string]*tfe.Plan), + client: client, + logs: make(map[string]string), + plans: make(map[string]*tfe.Plan), } } -func (m *mockPlans) Read(ctx context.Context, planID string) (*tfe.Plan, error) { - p, ok := m.plans[planID] - if !ok { - url := fmt.Sprintf("https://app.terraform.io/_archivist/%s", planID) +// create is a helper function to create a mock plan that uses the configured +// working directory to find the logfile. This enables us to test if we are +// using the +func (m *mockPlans) create(cvID, workspaceID string) (*tfe.Plan, error) { + id := generateID("plan-") + url := fmt.Sprintf("https://app.terraform.io/_archivist/%s", id) - p = &tfe.Plan{ - ID: planID, - LogReadURL: url, - Status: tfe.PlanFinished, - } + p := &tfe.Plan{ + ID: id, + LogReadURL: url, + Status: tfe.PlanPending, + } - m.logs[url] = "plan/output.log" - m.plans[p.ID] = p + w, ok := m.client.Workspaces.workspaceIDs[workspaceID] + if !ok { + return nil, tfe.ErrResourceNotFound } + m.logs[url] = filepath.Join( + m.client.ConfigurationVersions.uploadPaths[cvID], + w.WorkingDirectory, + "output.log", + ) + m.plans[p.ID] = p + + return p, nil +} + +func (m *mockPlans) Read(ctx context.Context, planID string) (*tfe.Plan, error) { + p, ok := m.plans[planID] + if !ok { + return nil, tfe.ErrResourceNotFound + } + p.Status = tfe.PlanFinished return p, nil } @@ -157,7 +204,11 @@ func (m *mockPlans) Logs(ctx context.Context, planID string) (io.Reader, error) return nil, tfe.ErrResourceNotFound } - logs, err := ioutil.ReadFile("./test-fixtures/" + logfile) + if _, err := os.Stat(logfile); os.IsNotExist(err) { + return bytes.NewBufferString("logfile does not exist"), nil + } + + logs, err := ioutil.ReadFile(logfile) if err != nil { return nil, err } @@ -166,34 +217,39 @@ func (m *mockPlans) Logs(ctx context.Context, planID string) (io.Reader, error) } type mockRuns struct { + client *mockClient runs map[string]*tfe.Run workspaces map[string][]*tfe.Run } -func newMockRuns() *mockRuns { +func newMockRuns(client *mockClient) *mockRuns { return &mockRuns{ + client: client, runs: make(map[string]*tfe.Run), workspaces: make(map[string][]*tfe.Run), } } func (m *mockRuns) List(ctx context.Context, workspaceID string, options tfe.RunListOptions) ([]*tfe.Run, error) { + w, ok := m.client.Workspaces.workspaceIDs[workspaceID] + if !ok { + return nil, tfe.ErrResourceNotFound + } var rs []*tfe.Run - for _, r := range m.workspaces[workspaceID] { + for _, r := range m.workspaces[w.ID] { rs = append(rs, r) } return rs, nil } func (m *mockRuns) Create(ctx context.Context, options tfe.RunCreateOptions) (*tfe.Run, error) { - id := generateID("run-") - p := &tfe.Plan{ - ID: generateID("plan-"), - Status: tfe.PlanPending, + p, err := m.client.Plans.create(options.ConfigurationVersion.ID, options.Workspace.ID) + if err != nil { + return nil, err } r := &tfe.Run{ - ID: id, + ID: generateID("run-"), Plan: p, Status: tfe.RunPending, } @@ -225,13 +281,15 @@ func (m *mockRuns) Discard(ctx context.Context, runID string, options tfe.RunDis } type mockStateVersions struct { + client *mockClient states map[string][]byte stateVersions map[string]*tfe.StateVersion workspaces map[string][]string } -func newMockStateVersions() *mockStateVersions { +func newMockStateVersions(client *mockClient) *mockStateVersions { return &mockStateVersions{ + client: client, states: make(map[string][]byte), stateVersions: make(map[string]*tfe.StateVersion), workspaces: make(map[string][]string), @@ -277,14 +335,21 @@ func (m *mockStateVersions) Read(ctx context.Context, svID string) (*tfe.StateVe } func (m *mockStateVersions) Current(ctx context.Context, workspaceID string) (*tfe.StateVersion, error) { - svs, ok := m.workspaces[workspaceID] + w, ok := m.client.Workspaces.workspaceIDs[workspaceID] + if !ok { + return nil, tfe.ErrResourceNotFound + } + + svs, ok := m.workspaces[w.ID] if !ok || len(svs) == 0 { return nil, tfe.ErrResourceNotFound } + sv, ok := m.stateVersions[svs[len(svs)-1]] if !ok { return nil, tfe.ErrResourceNotFound } + return sv, nil } @@ -297,12 +362,14 @@ func (m *mockStateVersions) Download(ctx context.Context, url string) ([]byte, e } type mockWorkspaces struct { + client *mockClient workspaceIDs map[string]*tfe.Workspace workspaceNames map[string]*tfe.Workspace } -func newMockWorkspaces() *mockWorkspaces { +func newMockWorkspaces(client *mockClient) *mockWorkspaces { return &mockWorkspaces{ + client: client, workspaceIDs: make(map[string]*tfe.Workspace), workspaceNames: make(map[string]*tfe.Workspace), } @@ -317,9 +384,8 @@ func (m *mockWorkspaces) List(ctx context.Context, organization string, options } func (m *mockWorkspaces) Create(ctx context.Context, organization string, options tfe.WorkspaceCreateOptions) (*tfe.Workspace, error) { - id := generateID("ws-") w := &tfe.Workspace{ - ID: id, + ID: generateID("ws-"), Name: *options.Name, } m.workspaceIDs[w.ID] = w @@ -340,8 +406,16 @@ func (m *mockWorkspaces) Update(ctx context.Context, organization, workspace str if !ok { return nil, tfe.ErrResourceNotFound } - w.Name = *options.Name - w.TerraformVersion = *options.TerraformVersion + + if options.Name != nil { + w.Name = *options.Name + } + if options.TerraformVersion != nil { + w.TerraformVersion = *options.TerraformVersion + } + if options.WorkingDirectory != nil { + w.WorkingDirectory = *options.WorkingDirectory + } delete(m.workspaceNames, workspace) m.workspaceNames[w.Name] = w diff --git a/backend/remote/backend_plan.go b/backend/remote/backend_plan.go index 3928301255..a3a5c06c4f 100644 --- a/backend/remote/backend_plan.go +++ b/backend/remote/backend_plan.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "log" "os" + "path/filepath" "strings" "time" @@ -64,15 +65,32 @@ func (b *Remote) opPlan(stopCtx, cancelCtx context.Context, op *backend.Operatio var configDir string if op.Module != nil && op.Module.Config().Dir != "" { - configDir = op.Module.Config().Dir + // Make sure to take the working directory into account by removing + // the working directory from the current path. This will result in + // a path that points to the expected root of the workspace. + configDir = filepath.Clean(strings.TrimSuffix( + filepath.Clean(op.Module.Config().Dir), + filepath.Clean(w.WorkingDirectory), + )) } else { + // We did a check earlier to make sure we either have a config dir, + // or the plan is run with -destroy. So this else clause will only + // be executed when we are destroying and doesn't need the config. configDir, err = ioutil.TempDir("", "tf") if err != nil { runningOp.Err = fmt.Errorf(strings.TrimSpace(fmt.Sprintf( - generalErr, "error creating temp directory", err))) + generalErr, "error creating temporary directory", err))) return } defer os.RemoveAll(configDir) + + // Make sure the configured working directory exists. + err = os.MkdirAll(filepath.Join(configDir, w.WorkingDirectory), 0700) + if err != nil { + runningOp.Err = fmt.Errorf(strings.TrimSpace(fmt.Sprintf( + generalErr, "error creating temporary working directory", err))) + return + } } err = b.client.ConfigurationVersions.Upload(stopCtx, cv.UploadURL, configDir) diff --git a/backend/remote/backend_plan_test.go b/backend/remote/backend_plan_test.go index cf34057294..c005e2c335 100644 --- a/backend/remote/backend_plan_test.go +++ b/backend/remote/backend_plan_test.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + tfe "github.com/hashicorp/go-tfe" "github.com/hashicorp/terraform/backend" "github.com/hashicorp/terraform/config/module" "github.com/hashicorp/terraform/terraform" @@ -179,3 +180,39 @@ func TestRemote_planDestroyNoConfig(t *testing.T) { t.Fatalf("unexpected plan error: %v", run.Err) } } + +func TestRemote_planWithWorkingDirectory(t *testing.T) { + b := testBackendDefault(t) + + options := tfe.WorkspaceUpdateOptions{ + WorkingDirectory: tfe.String("terraform"), + } + + // Configure the workspace to use a custom working direcrtory. + _, err := b.client.Workspaces.Update(context.Background(), b.organization, b.workspace, options) + if err != nil { + t.Fatalf("error configuring working directory: %v", err) + } + + mod, modCleanup := module.TestTree(t, "./test-fixtures/plan-with-working-directory/terraform") + defer modCleanup() + + op := testOperationPlan() + op.Module = mod + op.Workspace = backend.DefaultStateName + + run, err := b.Operation(context.Background(), op) + if err != nil { + t.Fatalf("error starting operation: %v", err) + } + + <-run.Done() + if run.Err != nil { + t.Fatalf("error running operation: %v", run.Err) + } + + output := b.CLI.(*cli.MockUi).OutputWriter.String() + if !strings.Contains(output, "1 to add, 0 to change, 0 to destroy") { + t.Fatalf("missing plan summery in output: %s", output) + } +} diff --git a/backend/remote/test-fixtures/plan-with-working-directory/terraform/main.tf b/backend/remote/test-fixtures/plan-with-working-directory/terraform/main.tf new file mode 100644 index 0000000000..3911a2a9b2 --- /dev/null +++ b/backend/remote/test-fixtures/plan-with-working-directory/terraform/main.tf @@ -0,0 +1 @@ +resource "null_resource" "foo" {} diff --git a/backend/remote/test-fixtures/plan-with-working-directory/terraform/output.log b/backend/remote/test-fixtures/plan-with-working-directory/terraform/output.log new file mode 100644 index 0000000000..d9fe98082d --- /dev/null +++ b/backend/remote/test-fixtures/plan-with-working-directory/terraform/output.log @@ -0,0 +1,29 @@ +Running plan in the remote backend. Output will stream here. Pressing Ctrl-C +will stop streaming the logs, but will not stop the plan running remotely. +To view this plan in a browser, visit: +https://atlas.local/app/demo1/my-app-web/runs/run-cPK6EnfTpqwy6ucU + +Waiting for the plan to start... + +Terraform v0.11.7 + +Configuring remote state backend... +Initializing Terraform configuration... +Refreshing Terraform state in-memory prior to plan... +The refreshed state will be used to calculate this plan, but will not be +persisted to local or remote state storage. + + +------------------------------------------------------------------------ + +An execution plan has been generated and is shown below. +Resource actions are indicated with the following symbols: + + create + +Terraform will perform the following actions: + + + null_resource.foo + id: + + +Plan: 1 to add, 0 to change, 0 to destroy. diff --git a/backend/remote/testing.go b/backend/remote/testing.go index 253272f76f..7df847a502 100644 --- a/backend/remote/testing.go +++ b/backend/remote/testing.go @@ -69,14 +69,17 @@ func testBackend(t *testing.T, c map[string]interface{}) *Remote { // Configure the backend so the client is created. backend.TestBackendConfig(t, b, c) - // Once the client exists, mock the services we use.. + // Get a new mock client. + mc := newMockClient() + + // Replace the services we use with our mock services. b.CLI = cli.NewMockUi() - b.client.ConfigurationVersions = newMockConfigurationVersions() - b.client.Organizations = newMockOrganizations() - b.client.Plans = newMockPlans() - b.client.Runs = newMockRuns() - b.client.StateVersions = newMockStateVersions() - b.client.Workspaces = newMockWorkspaces() + b.client.ConfigurationVersions = mc.ConfigurationVersions + b.client.Organizations = mc.Organizations + b.client.Plans = mc.Plans + b.client.Runs = mc.Runs + b.client.StateVersions = mc.StateVersions + b.client.Workspaces = mc.Workspaces ctx := context.Background()