Merge pull request #18773 from hashicorp/b-working-directories

backend/remote: take working directories into account

backend/remote/backend_mock.go

@@ -9,21 +9,45 @@ import (
 	"io"
 	"io/ioutil"
 	"math/rand"
+	"os"
+	"path/filepath"
 
 	tfe "github.com/hashicorp/go-tfe"
 )
 
+type mockClient struct {
+	ConfigurationVersions *mockConfigurationVersions
+	Organizations         *mockOrganizations
+	Plans                 *mockPlans
+	Runs                  *mockRuns
+	StateVersions         *mockStateVersions
+	Workspaces            *mockWorkspaces
+}
+
+func newMockClient() *mockClient {
+	c := &mockClient{}
+	c.ConfigurationVersions = newMockConfigurationVersions(c)
+	c.Organizations = newMockOrganizations(c)
+	c.Plans = newMockPlans(c)
+	c.Runs = newMockRuns(c)
+	c.StateVersions = newMockStateVersions(c)
+	c.Workspaces = newMockWorkspaces(c)
+	return c
+}
+
 type mockConfigurationVersions struct {
+	client         *mockClient
 	configVersions map[string]*tfe.ConfigurationVersion
+	uploadPaths    map[string]string
 	uploadURLs     map[string]*tfe.ConfigurationVersion
-	workspaces     map[string]*tfe.ConfigurationVersion
 }
 
-func newMockConfigurationVersions() *mockConfigurationVersions {
+func newMockConfigurationVersions(client *mockClient) *mockConfigurationVersions {
 	return &mockConfigurationVersions{
+		client:         client,
 		configVersions: make(map[string]*tfe.ConfigurationVersion),
+		uploadPaths:    make(map[string]string),
 		uploadURLs:     make(map[string]*tfe.ConfigurationVersion),
-		workspaces:     make(map[string]*tfe.ConfigurationVersion),
 	}
 }
 
@@ -47,7 +71,6 @@ func (m *mockConfigurationVersions) Create(ctx context.Context, workspaceID stri
 
 	m.configVersions[cv.ID] = cv
 	m.uploadURLs[url] = cv
-	m.workspaces[workspaceID] = cv
 
 	return cv, nil
 }
@@ -65,16 +88,19 @@ func (m *mockConfigurationVersions) Upload(ctx context.Context, url, path string
 	if !ok {
 		return errors.New("404 not found")
 	}
+	m.uploadPaths[cv.ID] = path
 	cv.Status = tfe.ConfigurationUploaded
 	return nil
 }
 
 type mockOrganizations struct {
+	client        *mockClient
 	organizations map[string]*tfe.Organization
 }
 
-func newMockOrganizations() *mockOrganizations {
+func newMockOrganizations(client *mockClient) *mockOrganizations {
 	return &mockOrganizations{
+		client:        client,
 		organizations: make(map[string]*tfe.Organization),
 	}
 }
@@ -117,32 +143,53 @@ func (m *mockOrganizations) Delete(ctx context.Context, name string) error {
 }
 
 type mockPlans struct {
-	logs  map[string]string
-	plans map[string]*tfe.Plan
+	client *mockClient
+	logs   map[string]string
+	plans  map[string]*tfe.Plan
 }
 
-func newMockPlans() *mockPlans {
+func newMockPlans(client *mockClient) *mockPlans {
 	return &mockPlans{
-		logs:  make(map[string]string),
-		plans: make(map[string]*tfe.Plan),
+		client: client,
+		logs:   make(map[string]string),
+		plans:  make(map[string]*tfe.Plan),
 	}
 }
 
-func (m *mockPlans) Read(ctx context.Context, planID string) (*tfe.Plan, error) {
-	p, ok := m.plans[planID]
-	if !ok {
-		url := fmt.Sprintf("https://app.terraform.io/_archivist/%s", planID)
+// create is a helper function to create a mock plan that uses the configured
+// working directory to find the logfile. This enables us to test if we are
+// using the
+func (m *mockPlans) create(cvID, workspaceID string) (*tfe.Plan, error) {
+	id := generateID("plan-")
+	url := fmt.Sprintf("https://app.terraform.io/_archivist/%s", id)
 
-		p = &tfe.Plan{
-			ID:         planID,
-			LogReadURL: url,
-			Status:     tfe.PlanFinished,
-		}
+	p := &tfe.Plan{
+		ID:         id,
+		LogReadURL: url,
+		Status:     tfe.PlanPending,
+	}
 
-		m.logs[url] = "plan/output.log"
-		m.plans[p.ID] = p
+	w, ok := m.client.Workspaces.workspaceIDs[workspaceID]
+	if !ok {
+		return nil, tfe.ErrResourceNotFound
 	}
 
+	m.logs[url] = filepath.Join(
+		m.client.ConfigurationVersions.uploadPaths[cvID],
+		w.WorkingDirectory,
+		"output.log",
+	)
+	m.plans[p.ID] = p
+
+	return p, nil
+}
+
+func (m *mockPlans) Read(ctx context.Context, planID string) (*tfe.Plan, error) {
+	p, ok := m.plans[planID]
+	if !ok {
+		return nil, tfe.ErrResourceNotFound
+	}
+	p.Status = tfe.PlanFinished
 	return p, nil
 }
 
@@ -157,7 +204,11 @@ func (m *mockPlans) Logs(ctx context.Context, planID string) (io.Reader, error)
 		return nil, tfe.ErrResourceNotFound
 	}
 
-	logs, err := ioutil.ReadFile("./test-fixtures/" + logfile)
+	if _, err := os.Stat(logfile); os.IsNotExist(err) {
+		return bytes.NewBufferString("logfile does not exist"), nil
+	}
+
+	logs, err := ioutil.ReadFile(logfile)
 	if err != nil {
 		return nil, err
 	}
@@ -166,34 +217,39 @@ func (m *mockPlans) Logs(ctx context.Context, planID string) (io.Reader, error)
 }
 
 type mockRuns struct {
+	client     *mockClient
 	runs       map[string]*tfe.Run
 	workspaces map[string][]*tfe.Run
 }
 
-func newMockRuns() *mockRuns {
+func newMockRuns(client *mockClient) *mockRuns {
 	return &mockRuns{
+		client:     client,
 		runs:       make(map[string]*tfe.Run),
 		workspaces: make(map[string][]*tfe.Run),
 	}
 }
 
 func (m *mockRuns) List(ctx context.Context, workspaceID string, options tfe.RunListOptions) ([]*tfe.Run, error) {
+	w, ok := m.client.Workspaces.workspaceIDs[workspaceID]
+	if !ok {
+		return nil, tfe.ErrResourceNotFound
+	}
 	var rs []*tfe.Run
-	for _, r := range m.workspaces[workspaceID] {
+	for _, r := range m.workspaces[w.ID] {
 		rs = append(rs, r)
 	}
 	return rs, nil
 }
 
 func (m *mockRuns) Create(ctx context.Context, options tfe.RunCreateOptions) (*tfe.Run, error) {
-	id := generateID("run-")
-	p := &tfe.Plan{
-		ID:     generateID("plan-"),
-		Status: tfe.PlanPending,
+	p, err := m.client.Plans.create(options.ConfigurationVersion.ID, options.Workspace.ID)
+	if err != nil {
+		return nil, err
 	}
 
 	r := &tfe.Run{
-		ID:     id,
+		ID:     generateID("run-"),
 		Plan:   p,
 		Status: tfe.RunPending,
 	}
@@ -225,13 +281,15 @@ func (m *mockRuns) Discard(ctx context.Context, runID string, options tfe.RunDis
 }
 
 type mockStateVersions struct {
+	client        *mockClient
 	states        map[string][]byte
 	stateVersions map[string]*tfe.StateVersion
 	workspaces    map[string][]string
 }
 
-func newMockStateVersions() *mockStateVersions {
+func newMockStateVersions(client *mockClient) *mockStateVersions {
 	return &mockStateVersions{
+		client:        client,
 		states:        make(map[string][]byte),
 		stateVersions: make(map[string]*tfe.StateVersion),
 		workspaces:    make(map[string][]string),
@@ -277,14 +335,21 @@ func (m *mockStateVersions) Read(ctx context.Context, svID string) (*tfe.StateVe
 }
 
 func (m *mockStateVersions) Current(ctx context.Context, workspaceID string) (*tfe.StateVersion, error) {
-	svs, ok := m.workspaces[workspaceID]
+	w, ok := m.client.Workspaces.workspaceIDs[workspaceID]
+	if !ok {
+		return nil, tfe.ErrResourceNotFound
+	}
+
+	svs, ok := m.workspaces[w.ID]
 	if !ok || len(svs) == 0 {
 		return nil, tfe.ErrResourceNotFound
 	}
+
 	sv, ok := m.stateVersions[svs[len(svs)-1]]
 	if !ok {
 		return nil, tfe.ErrResourceNotFound
 	}
+
 	return sv, nil
 }
 
@@ -297,12 +362,14 @@ func (m *mockStateVersions) Download(ctx context.Context, url string) ([]byte, e
 }
 
 type mockWorkspaces struct {
+	client         *mockClient
 	workspaceIDs   map[string]*tfe.Workspace
 	workspaceNames map[string]*tfe.Workspace
 }
 
-func newMockWorkspaces() *mockWorkspaces {
+func newMockWorkspaces(client *mockClient) *mockWorkspaces {
 	return &mockWorkspaces{
+		client:         client,
 		workspaceIDs:   make(map[string]*tfe.Workspace),
 		workspaceNames: make(map[string]*tfe.Workspace),
 	}
@@ -317,9 +384,8 @@ func (m *mockWorkspaces) List(ctx context.Context, organization string, options
 }
 
 func (m *mockWorkspaces) Create(ctx context.Context, organization string, options tfe.WorkspaceCreateOptions) (*tfe.Workspace, error) {
-	id := generateID("ws-")
 	w := &tfe.Workspace{
-		ID:   id,
+		ID:   generateID("ws-"),
 		Name: *options.Name,
 	}
 	m.workspaceIDs[w.ID] = w
@@ -340,8 +406,16 @@ func (m *mockWorkspaces) Update(ctx context.Context, organization, workspace str
 	if !ok {
 		return nil, tfe.ErrResourceNotFound
 	}
-	w.Name = *options.Name
-	w.TerraformVersion = *options.TerraformVersion
+
+	if options.Name != nil {
+		w.Name = *options.Name
+	}
+	if options.TerraformVersion != nil {
+		w.TerraformVersion = *options.TerraformVersion
+	}
+	if options.WorkingDirectory != nil {
+		w.WorkingDirectory = *options.WorkingDirectory
+	}
 
 	delete(m.workspaceNames, workspace)
 	m.workspaceNames[w.Name] = w

backend/remote/backend_plan.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -64,15 +65,32 @@ func (b *Remote) opPlan(stopCtx, cancelCtx context.Context, op *backend.Operatio
 
 	var configDir string
 	if op.Module != nil && op.Module.Config().Dir != "" {
-		configDir = op.Module.Config().Dir
+		// Make sure to take the working directory into account by removing
+		// the working directory from the current path. This will result in
+		// a path that points to the expected root of the workspace.
+		configDir = filepath.Clean(strings.TrimSuffix(
+			filepath.Clean(op.Module.Config().Dir),
+			filepath.Clean(w.WorkingDirectory),
+		))
 	} else {
+		// We did a check earlier to make sure we either have a config dir,
+		// or the plan is run with -destroy. So this else clause will only
+		// be executed when we are destroying and doesn't need the config.
 		configDir, err = ioutil.TempDir("", "tf")
 		if err != nil {
 			runningOp.Err = fmt.Errorf(strings.TrimSpace(fmt.Sprintf(
-				generalErr, "error creating temp directory", err)))
+				generalErr, "error creating temporary directory", err)))
 			return
 		}
 		defer os.RemoveAll(configDir)
+
+		// Make sure the configured working directory exists.
+		err = os.MkdirAll(filepath.Join(configDir, w.WorkingDirectory), 0700)
+		if err != nil {
+			runningOp.Err = fmt.Errorf(strings.TrimSpace(fmt.Sprintf(
+				generalErr, "error creating temporary working directory", err)))
+			return
+		}
 	}
 
 	err = b.client.ConfigurationVersions.Upload(stopCtx, cv.UploadURL, configDir)

backend/remote/backend_plan_test.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 	"testing"
 
+	tfe "github.com/hashicorp/go-tfe"
 	"github.com/hashicorp/terraform/backend"
 	"github.com/hashicorp/terraform/config/module"
 	"github.com/hashicorp/terraform/terraform"
@@ -179,3 +180,39 @@ func TestRemote_planDestroyNoConfig(t *testing.T) {
 		t.Fatalf("unexpected plan error: %v", run.Err)
 	}
 }
+
+func TestRemote_planWithWorkingDirectory(t *testing.T) {
+	b := testBackendDefault(t)
+
+	options := tfe.WorkspaceUpdateOptions{
+		WorkingDirectory: tfe.String("terraform"),
+	}
+
+	// Configure the workspace to use a custom working direcrtory.
+	_, err := b.client.Workspaces.Update(context.Background(), b.organization, b.workspace, options)
+	if err != nil {
+		t.Fatalf("error configuring working directory: %v", err)
+	}
+
+	mod, modCleanup := module.TestTree(t, "./test-fixtures/plan-with-working-directory/terraform")
+	defer modCleanup()
+
+	op := testOperationPlan()
+	op.Module = mod
+	op.Workspace = backend.DefaultStateName
+
+	run, err := b.Operation(context.Background(), op)
+	if err != nil {
+		t.Fatalf("error starting operation: %v", err)
+	}
+
+	<-run.Done()
+	if run.Err != nil {
+		t.Fatalf("error running operation: %v", run.Err)
+	}
+
+	output := b.CLI.(*cli.MockUi).OutputWriter.String()
+	if !strings.Contains(output, "1 to add, 0 to change, 0 to destroy") {
+		t.Fatalf("missing plan summery in output: %s", output)
+	}
+}

backend/remote/test-fixtures/plan-with-working-directory/terraform/main.tf

@@ -0,0 +1 @@
+resource "null_resource" "foo" {}

backend/remote/test-fixtures/plan-with-working-directory/terraform/output.log

@@ -0,0 +1,29 @@
+Running plan in the remote backend. Output will stream here. Pressing Ctrl-C
+will stop streaming the logs, but will not stop the plan running remotely.
+To view this plan in a browser, visit:
+https://atlas.local/app/demo1/my-app-web/runs/run-cPK6EnfTpqwy6ucU
+
+Waiting for the plan to start...
+
+Terraform v0.11.7
+
+Configuring remote state backend...
+Initializing Terraform configuration...
+Refreshing Terraform state in-memory prior to plan...
+The refreshed state will be used to calculate this plan, but will not be
+persisted to local or remote state storage.
+
+
+------------------------------------------------------------------------
+
+An execution plan has been generated and is shown below.
+Resource actions are indicated with the following symbols:
+  + create
+
+Terraform will perform the following actions:
+
+  + null_resource.foo
+      id: <computed>
+
+
+Plan: 1 to add, 0 to change, 0 to destroy.

backend/remote/testing.go

@@ -69,14 +69,17 @@ func testBackend(t *testing.T, c map[string]interface{}) *Remote {
 	// Configure the backend so the client is created.
 	backend.TestBackendConfig(t, b, c)
 
-	// Once the client exists, mock the services we use..
+	// Get a new mock client.
+	mc := newMockClient()
+
+	// Replace the services we use with our mock services.
 	b.CLI = cli.NewMockUi()
-	b.client.ConfigurationVersions = newMockConfigurationVersions()
-	b.client.Organizations = newMockOrganizations()
-	b.client.Plans = newMockPlans()
-	b.client.Runs = newMockRuns()
-	b.client.StateVersions = newMockStateVersions()
-	b.client.Workspaces = newMockWorkspaces()
+	b.client.ConfigurationVersions = mc.ConfigurationVersions
+	b.client.Organizations = mc.Organizations
+	b.client.Plans = mc.Plans
+	b.client.Runs = mc.Runs
+	b.client.StateVersions = mc.StateVersions
+	b.client.Workspaces = mc.Workspaces
 
 	ctx := context.Background()