terraform/internal/cloud/test.go

// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1

package cloud

import (
	"bufio"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"path/filepath"
	"strings"
	"time"

	"github.com/hashicorp/go-tfe"
	tfaddr "github.com/hashicorp/terraform-registry-address"
	svchost "github.com/hashicorp/terraform-svchost"
	"github.com/hashicorp/terraform-svchost/disco"
	"github.com/zclconf/go-cty/cty"

	"github.com/hashicorp/terraform/internal/backend/backendrun"
	"github.com/hashicorp/terraform/internal/command/format"
	"github.com/hashicorp/terraform/internal/command/jsonformat"
	"github.com/hashicorp/terraform/internal/command/views"
	"github.com/hashicorp/terraform/internal/configs"
	"github.com/hashicorp/terraform/internal/logging"
	"github.com/hashicorp/terraform/internal/moduletest"
	"github.com/hashicorp/terraform/internal/plans"
	"github.com/hashicorp/terraform/internal/terminal"
	"github.com/hashicorp/terraform/internal/tfdiags"
	tfversion "github.com/hashicorp/terraform/version"
)

// TestSuiteRunner executes any tests found in the relevant directories in TFC.
//
// It uploads the configuration and uses go-tfe to execute a .
//
// We keep this separate from Cloud, as the tests don't execute with a
// particular workspace in mind but instead with a specific module from a
// private registry. Many things within Cloud assume the existence of a
// workspace when initialising so it isn't pratical to share this for tests.
type TestSuiteRunner struct {

	// ConfigDirectory and TestingDirectory are the paths to the directory
	// that contains our configuration and our testing files.
	ConfigDirectory  string
	TestingDirectory string

	// Config is the actual loaded config.
	Config *configs.Config

	Services *disco.Disco

	// Source is the private registry module we should be sending the tests
	// to when they execute.
	Source string

	// GlobalVariables are the variables provided by the TF_VAR_* environment
	// variables and -var and -var-file flags.
	GlobalVariables map[string]backendrun.UnparsedVariableValue

	// Stopped and Cancelled track whether the user requested the testing
	// process to be interrupted. Stopped is a nice graceful exit, we'll still
	// tidy up any state that was created and mark the tests with relevant
	// `skipped` status updates. Cancelled is a hard stop right now exit, we
	// won't attempt to clean up any state left hanging, and tests will just
	// be left showing `pending` as the status. We will still print out the
	// destroy summary diagnostics that tell the user what state has been left
	// behind and needs manual clean up.
	Stopped   bool
	Cancelled bool

	// StoppedCtx and CancelledCtx allow in progress Terraform operations to
	// respond to external calls from the test command.
	StoppedCtx   context.Context
	CancelledCtx context.Context

	// Verbose tells the runner to print out plan files during each test run.
	Verbose bool

	// Filters restricts which test files will be executed.
	Filters []string

	// Renderer knows how to convert JSON logs retrieved from TFE back into
	// human-readable.
	//
	// If this is nil, the runner will print the raw logs directly to Streams.
	Renderer *jsonformat.Renderer

	// View and Streams provide alternate ways to output raw data to the
	// user.
	View    views.Test
	Streams *terminal.Streams

	// clientOverride allows tests to specify the client instead of letting the
	// system initialise one itself.
	clientOverride *tfe.Client
}

func (runner *TestSuiteRunner) Stop() {
	runner.Stopped = true
}

func (runner *TestSuiteRunner) Cancel() {
	runner.Cancelled = true
}

func (runner *TestSuiteRunner) Test() (moduletest.Status, tfdiags.Diagnostics) {
	var diags tfdiags.Diagnostics

	configDirectory, err := filepath.Abs(runner.ConfigDirectory)
	if err != nil {
		diags = diags.Append(fmt.Errorf("Failed to get absolute path of the configuration directory: %v", err))
		return moduletest.Error, diags
	}

	variables, variableDiags := ParseCloudRunTestVariables(runner.GlobalVariables)
	diags = diags.Append(variableDiags)
	if variableDiags.HasErrors() {
		// Stop early if we couldn't parse the global variables.
		return moduletest.Error, diags
	}

	addr, err := tfaddr.ParseModuleSource(runner.Source)
	if err != nil {
		if parserError, ok := err.(*tfaddr.ParserError); ok {
			diags = diags.Append(tfdiags.AttributeValue(
				tfdiags.Error,
				parserError.Summary,
				parserError.Detail,
				cty.Path{cty.GetAttrStep{Name: "source"}}))
		} else {
			diags = diags.Append(err)
		}
		return moduletest.Error, diags
	}

	if addr.Package.Host == tfaddr.DefaultModuleRegistryHost {
		// Then they've reference something from the public registry. We can't
		// run tests against that in this way yet.
		diags = diags.Append(tfdiags.AttributeValue(
			tfdiags.Error,
			"Module source points to the public registry",
			"Terraform Cloud can only execute tests for modules held within private registries.",
			cty.Path{cty.GetAttrStep{Name: "source"}}))
		return moduletest.Error, diags
	}

	id := tfe.RegistryModuleID{
		Organization: addr.Package.Namespace,
		Name:         addr.Package.Name,
		Provider:     addr.Package.TargetSystem,
		Namespace:    addr.Package.Namespace,
		RegistryName: tfe.PrivateRegistry,
	}

	client, module, clientDiags := runner.client(addr, id)
	diags = diags.Append(clientDiags)
	if clientDiags.HasErrors() {
		return moduletest.Error, diags
	}

	configurationVersion, err := client.ConfigurationVersions.CreateForRegistryModule(runner.StoppedCtx, id)
	if err != nil {
		diags = diags.Append(generalError("Failed to create configuration version", err))
		return moduletest.Error, diags
	}

	if runner.Stopped || runner.Cancelled {
		return moduletest.Error, diags
	}

	if err := client.ConfigurationVersions.Upload(runner.StoppedCtx, configurationVersion.UploadURL, configDirectory); err != nil {
		diags = diags.Append(generalError("Failed to upload configuration version", err))
		return moduletest.Error, diags
	}

	if runner.Stopped || runner.Cancelled {
		return moduletest.Error, diags
	}

	// From here, we'll pass any cancellation signals into the test run instead
	// of cancelling things locally. The reason for this is we want to make sure
	// the test run tidies up any state properly. This means, we'll send the
	// cancellation signals and then still wait for and process the logs.
	//
	// This also means that all calls to TFC will use context.Background()
	// instead of the stopped or cancelled context as we want them to finish and
	// the run to be cancelled by TFC properly.

	opts := tfe.TestRunCreateOptions{
		Filters:       runner.Filters,
		TestDirectory: tfe.String(runner.TestingDirectory),
		Verbose:       tfe.Bool(runner.Verbose),
		Variables: func() []*tfe.RunVariable {
			runVariables := make([]*tfe.RunVariable, 0, len(variables))
			for name, value := range variables {
				runVariables = append(runVariables, &tfe.RunVariable{
					Key:   name,
					Value: value,
				})
			}
			return runVariables
		}(),
		ConfigurationVersion: configurationVersion,
		RegistryModule:       module,
	}

	run, err := client.TestRuns.Create(context.Background(), opts)
	if err != nil {
		diags = diags.Append(generalError("Failed to create test run", err))
		return moduletest.Error, diags
	}

	runningCtx, done := context.WithCancel(context.Background())

	go func() {
		defer logging.PanicHandler()
		defer done()

		// Let's wait for the test run to start separately, so we can provide
		// some nice updates while we wait.

		completed := false
		started := time.Now()
		updated := started
		for i := 0; !completed; i++ {
			run, err := client.TestRuns.Read(context.Background(), id, run.ID)
			if err != nil {
				diags = diags.Append(generalError("Failed to retrieve test run", err))
				return // exit early
			}

			if run.Status != tfe.TestRunQueued {
				// We block as long as the test run is still queued.
				completed = true
				continue // We can render the logs now.
			}

			current := time.Now()
			if i == 0 || current.Sub(updated).Seconds() > 30 {
				updated = current

				// TODO: Provide better updates based on queue status etc.
				// We could look through the queue to find out exactly where the
				// test run is and give a count down. Other stuff like that.
				// For now, we'll just print a simple status updated.

				runner.View.TFCStatusUpdate(run.Status, current.Sub(started))
			}
		}

		// The test run has actually started now, so let's render the logs.

		logDiags := runner.renderLogs(client, run, id)
		diags = diags.Append(logDiags)
	}()

	// We're doing a couple of things in the wait function. Firstly, waiting
	// for the test run to actually finish. Secondly, listening for interrupt
	// signals and forwarding them onto TFC.
	waitDiags := runner.wait(runningCtx, client, run, id)
	diags = diags.Append(waitDiags)

	if diags.HasErrors() {
		return moduletest.Error, diags
	}

	// Refresh the run now we know it is finished.
	run, err = client.TestRuns.Read(context.Background(), id, run.ID)
	if err != nil {
		diags = diags.Append(generalError("Failed to retrieve completed test run", err))
		return moduletest.Error, diags
	}

	if run.Status != tfe.TestRunFinished {
		// The only reason we'd get here without the run being finished properly
		// is because the run errored outside the scope of the tests, or because
		// the run was cancelled. Either way, we can just mark it has having
		// errored for the purpose of our return code.
		return moduletest.Error, diags
	}

	// Otherwise the run has finished successfully, and we can look at the
	// actual status of the test instead of the run to figure out what status we
	// should return.

	switch run.TestStatus {
	case tfe.TestError:
		return moduletest.Error, diags
	case tfe.TestFail:
		return moduletest.Fail, diags
	case tfe.TestPass:
		return moduletest.Pass, diags
	case tfe.TestPending:
		return moduletest.Pending, diags
	case tfe.TestSkip:
		return moduletest.Skip, diags
	default:
		panic("found unrecognized test status: " + run.TestStatus)
	}
}

// discover the TFC/E API service URL
func discoverTfeURL(hostname svchost.Hostname, services *disco.Disco) (*url.URL, error) {
	host, err := services.Discover(hostname)
	if err != nil {
		var serviceDiscoErr *disco.ErrServiceDiscoveryNetworkRequest

		switch {
		case errors.As(err, &serviceDiscoErr):
			err = fmt.Errorf("a network issue prevented cloud configuration; %w", err)
			return nil, err
		default:
			return nil, err
		}
	}

	return host.ServiceURL(tfeServiceID)
}

func (runner *TestSuiteRunner) client(addr tfaddr.Module, id tfe.RegistryModuleID) (*tfe.Client, *tfe.RegistryModule, tfdiags.Diagnostics) {
	var diags tfdiags.Diagnostics

	var client *tfe.Client
	if runner.clientOverride != nil {
		client = runner.clientOverride
	} else {
		service, err := discoverTfeURL(addr.Package.Host, runner.Services)
		if err != nil {
			diags = diags.Append(tfdiags.AttributeValue(
				tfdiags.Error,
				strings.ToUpper(err.Error()[:1])+err.Error()[1:],
				"", // no description is needed here, the error is clear
				cty.Path{cty.GetAttrStep{Name: "hostname"}},
			))
			return nil, nil, diags
		}

		token, err := cliConfigToken(addr.Package.Host, runner.Services)
		if err != nil {
			diags = diags.Append(tfdiags.AttributeValue(
				tfdiags.Error,
				strings.ToUpper(err.Error()[:1])+err.Error()[1:],
				"", // no description is needed here, the error is clear
				cty.Path{cty.GetAttrStep{Name: "hostname"}},
			))
			return nil, nil, diags
		}

		if token == "" {
			hostname := addr.Package.Host.ForDisplay()

			loginCommand := "terraform login"
			if hostname != defaultHostname {
				loginCommand = loginCommand + " " + hostname
			}
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Error,
				"Required token could not be found",
				fmt.Sprintf(
					"Run the following command to generate a token for %s:\n    %s",
					hostname,
					loginCommand,
				),
			))
			return nil, nil, diags
		}

		cfg := &tfe.Config{
			Address:      service.String(),
			BasePath:     service.Path,
			Token:        token,
			Headers:      make(http.Header),
			RetryLogHook: runner.View.TFCRetryHook,
		}

		// Set the version header to the current version.
		cfg.Headers.Set(tfversion.Header, tfversion.Version)
		cfg.Headers.Set(headerSourceKey, headerSourceValue)

		if client, err = tfe.NewClient(cfg); err != nil {
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Error,
				"Failed to create the Terraform Cloud/Enterprise client",
				fmt.Sprintf(
					`Encountered an unexpected error while creating the `+
						`Terraform Cloud/Enterprise client: %s.`, err,
				),
			))
			return nil, nil, diags
		}
	}

	module, err := client.RegistryModules.Read(runner.StoppedCtx, id)
	if err != nil {
		// Then the module doesn't exist, and we can't run tests against it.
		if err == tfe.ErrResourceNotFound {
			err = fmt.Errorf("module %q was not found.\n\nPlease ensure that the organization and hostname are correct and that your API token for %s is valid.", addr.ForDisplay(), addr.Package.Host.ForDisplay())
		}
		diags = diags.Append(tfdiags.AttributeValue(
			tfdiags.Error,
			fmt.Sprintf("Failed to read module %q", addr.ForDisplay()),
			fmt.Sprintf("Encountered an unexpected error while the module: %s", err),
			cty.Path{cty.GetAttrStep{Name: "source"}}))
		return client, nil, diags
	}

	// Enable retries for server errors.
	client.RetryServerErrors(true)

	// Aaaaand I'm done.
	return client, module, diags
}

func (runner *TestSuiteRunner) wait(ctx context.Context, client *tfe.Client, run *tfe.TestRun, moduleId tfe.RegistryModuleID) tfdiags.Diagnostics {
	var diags tfdiags.Diagnostics

	handleCancelled := func() {
		if err := client.TestRuns.Cancel(context.Background(), moduleId, run.ID); err != nil {
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Error,
				"Could not cancel the test run",
				fmt.Sprintf("Terraform could not cancel the test run, you will have to navigate to the Terraform Cloud console and cancel the test run manually.\n\nThe error message received when cancelling the test run was %s", err)))
			return
		}

		// At this point we've requested a force cancel, and we know that
		// Terraform locally is just going to quit after some amount of time so
		// we'll just wait for that to happen or for TFC to finish, whichever
		// happens first.
		<-ctx.Done()
	}

	handleStopped := func() {
		if err := client.TestRuns.Cancel(context.Background(), moduleId, run.ID); err != nil {
			diags = diags.Append(tfdiags.Sourceless(
				tfdiags.Error,
				"Could not stop the test run",
				fmt.Sprintf("Terraform could not stop the test run, you will have to navigate to the Terraform Cloud console and cancel the test run manually.\n\nThe error message received when stopping the test run was %s", err)))
			return
		}

		// We've request a cancel, we're happy to just wait for TFC to cancel
		// the run appropriately.
		select {
		case <-runner.CancelledCtx.Done():
			// We got more pushy, let's force cancel.
			handleCancelled()
		case <-ctx.Done():
			// It finished normally after we request the cancel. Do nothing.
		}
	}

	select {
	case <-runner.StoppedCtx.Done():
		// The StoppedCtx is passed in from the command package, which is
		// listening for interrupts from the user. After the first interrupt the
		// StoppedCtx is triggered.
		handleStopped()
	case <-ctx.Done():
		// The remote run finished normally! Do nothing.
	}

	return diags
}

func (runner *TestSuiteRunner) renderLogs(client *tfe.Client, run *tfe.TestRun, moduleId tfe.RegistryModuleID) tfdiags.Diagnostics {
	var diags tfdiags.Diagnostics

	logs, err := client.TestRuns.Logs(context.Background(), moduleId, run.ID)
	if err != nil {
		diags = diags.Append(generalError("Failed to retrieve logs", err))
		return diags
	}

	reader := bufio.NewReaderSize(logs, 64*1024)

	for next := true; next; {
		var l, line []byte
		var err error

		for isPrefix := true; isPrefix; {
			l, isPrefix, err = reader.ReadLine()
			if err != nil {
				if err != io.EOF {
					diags = diags.Append(generalError("Failed to read logs", err))
					return diags
				}
				next = false
			}

			line = append(line, l...)
		}

		if next || len(line) > 0 {

			if runner.Renderer != nil {
				log := jsonformat.JSONLog{}
				if err := json.Unmarshal(line, &log); err != nil {
					runner.Streams.Println(string(line)) // Just print the raw line so the user can still try and interpret the information.
					continue
				}

				// Most of the log types can be rendered with just the
				// information they contain. We just pass these straight into
				// the renderer. Others, however, need additional context that
				// isn't available within the renderer so we process them first.

				switch log.Type {
				case jsonformat.LogTestInterrupt:
					interrupt := log.TestFatalInterrupt

					runner.Streams.Eprintln(format.WordWrap(log.Message, runner.Streams.Stderr.Columns()))
					if len(interrupt.State) > 0 {
						runner.Streams.Eprint(format.WordWrap("\nTerraform has already created the following resources from the module under test:\n", runner.Streams.Stderr.Columns()))
						for _, resource := range interrupt.State {
							if len(resource.DeposedKey) > 0 {
								runner.Streams.Eprintf(" - %s (%s)\n", resource.Instance, resource.DeposedKey)
							} else {
								runner.Streams.Eprintf(" - %s\n", resource.Instance)
							}
						}
					}

					if len(interrupt.States) > 0 {
						for run, resources := range interrupt.States {
							runner.Streams.Eprint(format.WordWrap(fmt.Sprintf("\nTerraform has already created the following resources for %q:\n", run), runner.Streams.Stderr.Columns()))

							for _, resource := range resources {
								if len(resource.DeposedKey) > 0 {
									runner.Streams.Eprintf(" - %s (%s)\n", resource.Instance, resource.DeposedKey)
								} else {
									runner.Streams.Eprintf(" - %s\n", resource.Instance)
								}
							}
						}
					}

					if len(interrupt.Planned) > 0 {
						module := "the module under test"
						for _, run := range runner.Config.Module.Tests[log.TestFile].Runs {
							if run.Name == log.TestRun && run.ConfigUnderTest != nil {
								module = fmt.Sprintf("%q", run.Module.Source.String())
							}
						}

						runner.Streams.Eprint(format.WordWrap(fmt.Sprintf("\nTerraform was in the process of creating the following resources for %q from %s, and they may not have been destroyed:\n", log.TestRun, module), runner.Streams.Stderr.Columns()))
						for _, resource := range interrupt.Planned {
							runner.Streams.Eprintf("  - %s\n", resource)
						}
					}

				case jsonformat.LogTestPlan:
					var uimode plans.Mode
					for _, run := range runner.Config.Module.Tests[log.TestFile].Runs {
						if run.Name == log.TestRun {
							switch run.Options.Mode {
							case configs.RefreshOnlyTestMode:
								uimode = plans.RefreshOnlyMode
							case configs.NormalTestMode:
								uimode = plans.NormalMode
							}

							// Don't keep searching the runs.
							break
						}
					}
					runner.Renderer.RenderHumanPlan(*log.TestPlan, uimode)

				case jsonformat.LogTestState:
					runner.Renderer.RenderHumanState(*log.TestState)

				default:
					// For all the rest we can just hand over to the renderer
					// to handle directly.
					if err := runner.Renderer.RenderLog(&log); err != nil {
						runner.Streams.Println(string(line)) // Just print the raw line so the can still try and interpret the information.
						continue
					}
				}

			} else {
				runner.Streams.Println(string(line)) // If the renderer is null, it means the user just wants to see the raw JSON outputs anyway.
			}
		}
	}

	return diags
}