terraform/internal/moduletest/graph/eval_context.go

// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1

package graph

import (
	"context"
	"fmt"
	"log"
	"sort"
	"sync"

	"github.com/hashicorp/hcl/v2"
	"github.com/zclconf/go-cty/cty"
	"github.com/zclconf/go-cty/cty/convert"

	"github.com/hashicorp/terraform/internal/addrs"
	"github.com/hashicorp/terraform/internal/command/views"
	"github.com/hashicorp/terraform/internal/configs"
	"github.com/hashicorp/terraform/internal/didyoumean"
	"github.com/hashicorp/terraform/internal/lang"
	"github.com/hashicorp/terraform/internal/lang/langrefs"
	"github.com/hashicorp/terraform/internal/moduletest"
	hcltest "github.com/hashicorp/terraform/internal/moduletest/hcl"
	"github.com/hashicorp/terraform/internal/terraform"
	"github.com/hashicorp/terraform/internal/tfdiags"
)

// EvalContext is a container for context relating to the evaluation of a
// particular .tftest.hcl file.
// This context is used to track the various values that are available to the
// test suite, both from the test suite itself and from the results of the runs
// within the suite.
// The struct provides concurrency-safe access to the various maps it contains.
type EvalContext struct {
	VariableCaches *hcltest.VariableCaches

	// runOutputs is a mapping from run addresses to cty object values
	// representing the collected output values from the module under test.
	//
	// This is used to allow run blocks to refer back to the output values of
	// previous run blocks. It is passed into the Evaluate functions that
	// validate the test assertions, and used when calculating values for
	// variables within run blocks.
	runOutputs  map[addrs.Run]cty.Value
	outputsLock sync.Mutex

	// configProviders is a cache of config keys mapped to all the providers
	// referenced by the given config.
	//
	// The config keys are globally unique across an entire test suite, so we
	// store this at the suite runner level to get maximum efficiency.
	configProviders map[string]map[string]bool
	providersLock   sync.Mutex

	// FileStates is a mapping of module keys to it's last applied state
	// file.
	//
	// This is used to clean up the infrastructure created during the test after
	// the test has finished.
	FileStates map[string]*TestFileState
	stateLock  sync.Mutex

	// cancelContext and stopContext can be used to terminate the evaluation of the
	// test suite when a cancellation or stop signal is received.
	// cancelFunc and stopFunc are the corresponding functions to call to signal
	// the termination.
	cancelContext context.Context
	cancelFunc    context.CancelFunc
	stopContext   context.Context
	stopFunc      context.CancelFunc

	renderer views.Test
	verbose  bool
}

type EvalContextOpts struct {
	Verbose   bool
	Render    views.Test
	CancelCtx context.Context
	StopCtx   context.Context
}

// NewEvalContext constructs a new graph evaluation context for use in
// evaluating the runs within a test suite.
// The context is initialized with the provided cancel and stop contexts, and
// these contexts can be used from external commands to signal the termination of the test suite.
func NewEvalContext(opts *EvalContextOpts) *EvalContext {
	cancelCtx, cancel := context.WithCancel(opts.CancelCtx)
	stopCtx, stop := context.WithCancel(opts.StopCtx)
	return &EvalContext{
		runOutputs:      make(map[addrs.Run]cty.Value),
		outputsLock:     sync.Mutex{},
		configProviders: make(map[string]map[string]bool),
		providersLock:   sync.Mutex{},
		FileStates:      make(map[string]*TestFileState),
		stateLock:       sync.Mutex{},
		VariableCaches:  hcltest.NewVariableCaches(),
		cancelContext:   cancelCtx,
		cancelFunc:      cancel,
		stopContext:     stopCtx,
		stopFunc:        stop,
		verbose:         opts.Verbose,
		renderer:        opts.Render,
	}
}

// Renderer returns the renderer for the test suite.
func (ec *EvalContext) Renderer() views.Test {
	return ec.renderer
}

// Cancel signals to the runs in the test suite that they should stop evaluating
// the test suite, and return immediately.
func (ec *EvalContext) Cancel() {
	ec.cancelFunc()
}

// Cancelled returns true if the context has been stopped. The default cause
// of the error is context.Canceled.
func (ec *EvalContext) Cancelled() bool {
	return ec.cancelContext.Err() != nil
}

// Stop signals to the runs in the test suite that they should stop evaluating
// the test suite, and just skip.
func (ec *EvalContext) Stop() {
	ec.stopFunc()
}

func (ec *EvalContext) Stopped() bool {
	return ec.stopContext.Err() != nil
}

// Verbose returns true if the context is in verbose mode.
func (ec *EvalContext) Verbose() bool {
	return ec.verbose
}

// EvaluateRun processes the assertions inside the provided configs.TestRun against
// the run results, returning a status, an object value representing the output
// values from the module under test, and diagnostics describing any problems.
//
// extraVariableVals, if provided, overlays the input variables that are
// already available in resultScope in case there are additional input
// variables that were defined only for use in the test suite. Any variable
// not defined in extraVariableVals will be evaluated through resultScope instead.
func (ec *EvalContext) EvaluateRun(run *moduletest.Run, resultScope *lang.Scope, extraVariableVals terraform.InputValues) (moduletest.Status, cty.Value, tfdiags.Diagnostics) {
	var diags tfdiags.Diagnostics
	if run.ModuleConfig == nil {
		// This should never happen, but if it does, we can't evaluate the run
		return moduletest.Error, cty.NilVal, tfdiags.Diagnostics{}
	}

	mod := run.ModuleConfig.Module
	// We need a derived evaluation scope that also supports referring to
	// the prior run output values using the "run.NAME" syntax.
	evalData := &evaluationData{
		ctx:       ec,
		module:    mod,
		current:   resultScope.Data,
		extraVars: extraVariableVals,
	}
	scope := &lang.Scope{
		Data:          evalData,
		ParseRef:      addrs.ParseRefFromTestingScope,
		SourceAddr:    resultScope.SourceAddr,
		BaseDir:       resultScope.BaseDir,
		PureOnly:      resultScope.PureOnly,
		PlanTimestamp: resultScope.PlanTimestamp,
		ExternalFuncs: resultScope.ExternalFuncs,
	}

	log.Printf("[TRACE] EvalContext.Evaluate for %s", run.Addr())

	// We're going to assume the run has passed, and then if anything fails this
	// value will be updated.
	status := run.Status.Merge(moduletest.Pass)

	// Now validate all the assertions within this run block.
	for i, rule := range run.Config.CheckRules {
		var ruleDiags tfdiags.Diagnostics

		refs, moreDiags := langrefs.ReferencesInExpr(addrs.ParseRefFromTestingScope, rule.Condition)
		ruleDiags = ruleDiags.Append(moreDiags)
		moreRefs, moreDiags := langrefs.ReferencesInExpr(addrs.ParseRefFromTestingScope, rule.ErrorMessage)
		ruleDiags = ruleDiags.Append(moreDiags)
		refs = append(refs, moreRefs...)

		// We want to emit diagnostics if users are using ephemeral resources in their checks
		// as they are not supported since they are closed before this is evaluated.
		// We do not remove the diagnostic about the ephemeral resource being closed already as it
		// might be useful to the user.
		ruleDiags = ruleDiags.Append(diagsForEphemeralResources(refs))

		hclCtx, moreDiags := scope.EvalContext(refs)
		ruleDiags = ruleDiags.Append(moreDiags)
		if moreDiags.HasErrors() {
			// if we can't evaluate the context properly, we can't evaulate the rule
			// we add the diagnostics to the main diags and continue to the next rule
			log.Printf("[TRACE] EvalContext.Evaluate: check rule %d for %s is invalid, could not evalaute the context, so cannot evaluate it", i, run.Addr())
			status = status.Merge(moduletest.Error)
			diags = diags.Append(ruleDiags)
			continue
		}

		errorMessage, moreDiags := lang.EvalCheckErrorMessage(rule.ErrorMessage, hclCtx, nil)
		ruleDiags = ruleDiags.Append(moreDiags)

		runVal, hclDiags := rule.Condition.Value(hclCtx)
		ruleDiags = ruleDiags.Append(hclDiags)

		diags = diags.Append(ruleDiags)
		if ruleDiags.HasErrors() {
			log.Printf("[TRACE] EvalContext.Evaluate: check rule %d for %s is invalid, so cannot evaluate it", i, run.Addr())
			status = status.Merge(moduletest.Error)
			continue
		}

		if runVal.IsNull() {
			status = status.Merge(moduletest.Error)
			diags = diags.Append(&hcl.Diagnostic{
				Severity:    hcl.DiagError,
				Summary:     "Invalid condition run",
				Detail:      "Condition expression must return either true or false, not null.",
				Subject:     rule.Condition.Range().Ptr(),
				Expression:  rule.Condition,
				EvalContext: hclCtx,
			})
			log.Printf("[TRACE] EvalContext.Evaluate: check rule %d for %s has null condition result", i, run.Addr())
			continue
		}

		if !runVal.IsKnown() {
			status = status.Merge(moduletest.Error)
			diags = diags.Append(&hcl.Diagnostic{
				Severity:    hcl.DiagError,
				Summary:     "Unknown condition value",
				Detail:      "Condition expression could not be evaluated at this time. This means you have executed a `run` block with `command = plan` and one of the values your condition depended on is not known until after the plan has been applied. Either remove this value from your condition, or execute an `apply` command from this `run` block. Alternatively, if there is an override for this value, you can make it available during the plan phase by setting `override_during = plan` in the `override_` block.",
				Subject:     rule.Condition.Range().Ptr(),
				Expression:  rule.Condition,
				EvalContext: hclCtx,
			})
			log.Printf("[TRACE] EvalContext.Evaluate: check rule %d for %s has unknown condition result", i, run.Addr())
			continue
		}

		var err error
		if runVal, err = convert.Convert(runVal, cty.Bool); err != nil {
			status = status.Merge(moduletest.Error)
			diags = diags.Append(&hcl.Diagnostic{
				Severity:    hcl.DiagError,
				Summary:     "Invalid condition run",
				Detail:      fmt.Sprintf("Invalid condition run value: %s.", tfdiags.FormatError(err)),
				Subject:     rule.Condition.Range().Ptr(),
				Expression:  rule.Condition,
				EvalContext: hclCtx,
			})
			log.Printf("[TRACE] EvalContext.Evaluate: check rule %d for %s has non-boolean condition result", i, run.Addr())
			continue
		}

		// If the runVal refers to any sensitive values, then we'll have a
		// sensitive mark on the resulting value.
		runVal, _ = runVal.Unmark()

		if runVal.False() {
			log.Printf("[TRACE] EvalContext.Evaluate: test assertion failed for %s assertion %d", run.Addr(), i)
			status = status.Merge(moduletest.Fail)
			diags = diags.Append(&hcl.Diagnostic{
				Severity:    hcl.DiagError,
				Summary:     "Test assertion failed",
				Detail:      errorMessage,
				Subject:     rule.Condition.Range().Ptr(),
				Expression:  rule.Condition,
				EvalContext: hclCtx,
				// Diagnostic can be identified as originating from a failing test assertion.
				// Also, values that are ephemeral, sensitive, or unknown are replaced with
				// redacted values in renderings of the diagnostic.
				Extra: DiagnosticCausedByTestFailure{Verbose: ec.verbose},
			})
			continue
		} else {
			log.Printf("[TRACE] EvalContext.Evaluate: test assertion succeeded for %s assertion %d", run.Addr(), i)
		}
	}

	// Our result includes an object representing all of the output values
	// from the module we've just tested, which will then be available in
	// any subsequent test cases in the same test suite.
	outputVals := make(map[string]cty.Value, len(mod.Outputs))
	runRng := tfdiags.SourceRangeFromHCL(run.Config.DeclRange)
	for _, oc := range mod.Outputs {
		addr := oc.Addr()
		v, moreDiags := scope.Data.GetOutput(addr, runRng)
		diags = diags.Append(moreDiags)
		if v == cty.NilVal {
			v = cty.NullVal(cty.DynamicPseudoType)
		}
		outputVals[addr.Name] = v
	}

	return status, cty.ObjectVal(outputVals), diags
}

func (ec *EvalContext) SetOutput(run *moduletest.Run, output cty.Value) {
	ec.outputsLock.Lock()
	defer ec.outputsLock.Unlock()
	ec.runOutputs[run.Addr()] = output
}

func (ec *EvalContext) GetOutputs() map[addrs.Run]cty.Value {
	ec.outputsLock.Lock()
	defer ec.outputsLock.Unlock()
	outputCopy := make(map[addrs.Run]cty.Value, len(ec.runOutputs))
	for k, v := range ec.runOutputs {
		outputCopy[k] = v
	}
	return outputCopy
}

func (ec *EvalContext) GetCache(run *moduletest.Run) *hcltest.VariableCache {
	return ec.VariableCaches.GetCache(run.Name, run.ModuleConfig)
}

// ProviderExists returns true if the provider exists for the run inside the context.
func (ec *EvalContext) ProviderExists(run *moduletest.Run, key string) bool {
	ec.providersLock.Lock()
	defer ec.providersLock.Unlock()
	runProviders, ok := ec.configProviders[run.GetModuleConfigID()]
	if !ok {
		return false
	}

	found, ok := runProviders[key]
	return ok && found
}

func (ec *EvalContext) SetProviders(run *moduletest.Run, providers map[string]bool) {
	ec.providersLock.Lock()
	defer ec.providersLock.Unlock()
	ec.configProviders[run.GetModuleConfigID()] = providers
}

func diagsForEphemeralResources(refs []*addrs.Reference) (diags tfdiags.Diagnostics) {
	for _, ref := range refs {
		switch v := ref.Subject.(type) {
		case addrs.ResourceInstance:
			if v.Resource.Mode == addrs.EphemeralResourceMode {
				diags = diags.Append(&hcl.Diagnostic{
					Severity: hcl.DiagError,
					Summary:  "Ephemeral resources cannot be asserted",
					Detail:   "Ephemeral resources are closed when the test is finished, and are not available within the test context for assertions.",
					Subject:  ref.SourceRange.ToHCL().Ptr(),
				})
			}
		}
	}
	return diags
}

func (ec *EvalContext) SetFileState(key string, state *TestFileState) {
	ec.stateLock.Lock()
	defer ec.stateLock.Unlock()
	ec.FileStates[key] = &TestFileState{
		Run:   state.Run,
		State: state.State,
	}
}

func (ec *EvalContext) GetFileState(key string) *TestFileState {
	ec.stateLock.Lock()
	defer ec.stateLock.Unlock()
	return ec.FileStates[key]
}

// evaluationData augments an underlying lang.Data -- presumably resulting
// from a terraform.Context.PlanAndEval or terraform.Context.ApplyAndEval call --
// with results from prior runs that should therefore be available when
// evaluating expressions written inside a "run" block.
type evaluationData struct {
	ctx       *EvalContext
	module    *configs.Module
	current   lang.Data
	extraVars terraform.InputValues
}

var _ lang.Data = (*evaluationData)(nil)

// GetCheckBlock implements lang.Data.
func (d *evaluationData) GetCheckBlock(addr addrs.Check, rng tfdiags.SourceRange) (cty.Value, tfdiags.Diagnostics) {
	return d.current.GetCheckBlock(addr, rng)
}

// GetCountAttr implements lang.Data.
func (d *evaluationData) GetCountAttr(addr addrs.CountAttr, rng tfdiags.SourceRange) (cty.Value, tfdiags.Diagnostics) {
	return d.current.GetCountAttr(addr, rng)
}

// GetForEachAttr implements lang.Data.
func (d *evaluationData) GetForEachAttr(addr addrs.ForEachAttr, rng tfdiags.SourceRange) (cty.Value, tfdiags.Diagnostics) {
	return d.current.GetForEachAttr(addr, rng)
}

// GetInputVariable implements lang.Data.
func (d *evaluationData) GetInputVariable(addr addrs.InputVariable, rng tfdiags.SourceRange) (cty.Value, tfdiags.Diagnostics) {
	if extra, exists := d.extraVars[addr.Name]; exists {
		return extra.Value, nil
	}
	return d.current.GetInputVariable(addr, rng)
}

// GetLocalValue implements lang.Data.
func (d *evaluationData) GetLocalValue(addr addrs.LocalValue, rng tfdiags.SourceRange) (cty.Value, tfdiags.Diagnostics) {
	return d.current.GetLocalValue(addr, rng)
}

// GetModule implements lang.Data.
func (d *evaluationData) GetModule(addr addrs.ModuleCall, rng tfdiags.SourceRange) (cty.Value, tfdiags.Diagnostics) {
	return d.current.GetModule(addr, rng)
}

// GetOutput implements lang.Data.
func (d *evaluationData) GetOutput(addr addrs.OutputValue, rng tfdiags.SourceRange) (cty.Value, tfdiags.Diagnostics) {
	return d.current.GetOutput(addr, rng)
}

// GetPathAttr implements lang.Data.
func (d *evaluationData) GetPathAttr(addr addrs.PathAttr, rng tfdiags.SourceRange) (cty.Value, tfdiags.Diagnostics) {
	return d.current.GetPathAttr(addr, rng)
}

// GetResource implements lang.Data.
func (d *evaluationData) GetResource(addr addrs.Resource, rng tfdiags.SourceRange) (cty.Value, tfdiags.Diagnostics) {
	return d.current.GetResource(addr, rng)
}

// GetRunBlock implements lang.Data.
func (d *evaluationData) GetRunBlock(addr addrs.Run, rng tfdiags.SourceRange) (cty.Value, tfdiags.Diagnostics) {
	var diags tfdiags.Diagnostics
	ret, exists := d.ctx.GetOutputs()[addr]
	if !exists {
		ret = cty.DynamicVal
		diags = diags.Append(&hcl.Diagnostic{
			Severity: hcl.DiagError,
			Summary:  "Reference to undeclared run block",
			Detail:   fmt.Sprintf("There is no run %q declared in this test suite.", addr.Name),
			Subject:  rng.ToHCL().Ptr(),
		})
	}
	if ret == cty.NilVal {
		// An explicit nil value indicates that the block was declared but
		// hasn't yet been visited.
		ret = cty.DynamicVal
		diags = diags.Append(&hcl.Diagnostic{
			Severity: hcl.DiagError,
			Summary:  "Reference to unevaluated run block",
			Detail:   fmt.Sprintf("The run %q block has not yet been evaluated, so its results are not available here.", addr.Name),
			Subject:  rng.ToHCL().Ptr(),
		})
	}
	return ret, diags
}

// GetTerraformAttr implements lang.Data.
func (d *evaluationData) GetTerraformAttr(addr addrs.TerraformAttr, rng tfdiags.SourceRange) (cty.Value, tfdiags.Diagnostics) {
	return d.current.GetTerraformAttr(addr, rng)
}

// StaticValidateReferences implements lang.Data.
func (d *evaluationData) StaticValidateReferences(refs []*addrs.Reference, self addrs.Referenceable, source addrs.Referenceable) tfdiags.Diagnostics {
	// We only handle addrs.Run directly here, with everything else delegated
	// to the underlying Data object to deal with.
	var diags tfdiags.Diagnostics
	for _, ref := range refs {
		switch ref.Subject.(type) {
		case addrs.Run:
			diags = diags.Append(d.staticValidateRunRef(ref))
		default:
			diags = diags.Append(d.current.StaticValidateReferences([]*addrs.Reference{ref}, self, source))
		}
	}
	return diags
}

func (d *evaluationData) staticValidateRunRef(ref *addrs.Reference) tfdiags.Diagnostics {
	var diags tfdiags.Diagnostics

	addr := ref.Subject.(addrs.Run)
	outputs := d.ctx.GetOutputs()
	_, exists := outputs[addr]
	if !exists {
		var suggestions []string
		for altAddr := range outputs {
			suggestions = append(suggestions, altAddr.Name)
		}
		sort.Strings(suggestions)
		suggestion := didyoumean.NameSuggestion(addr.Name, suggestions)
		if suggestion != "" {
			suggestion = fmt.Sprintf(" Did you mean %q?", suggestion)
		}
		// A totally absent priorVals means that there is no run block with
		// the given name at all. If it was declared but hasn't yet been
		// evaluated then it would have an entry set to cty.NilVal.
		diags = diags.Append(&hcl.Diagnostic{
			Severity: hcl.DiagError,
			Summary:  "Reference to undeclared run block",
			Detail:   fmt.Sprintf("There is no run %q declared in this test suite.%s", addr.Name, suggestion),
			Subject:  ref.SourceRange.ToHCL().Ptr(),
		})
	}

	return diags
}