From 2e911132249d1db9fa63765bd356d60abf91bab4 Mon Sep 17 00:00:00 2001 From: Samsondeen <40821565+dsa0x@users.noreply.github.com> Date: Tue, 11 Feb 2025 14:48:48 +0100 Subject: [PATCH] Terraform test: Consolidate test execution procedure (#36459) --- internal/backend/local/test.go | 970 +----------------- internal/command/test_test.go | 99 +- .../testdata/test/invalid_config/main.tf | 11 + .../test/invalid_config/main.tftest.hcl | 2 + .../main.tf | 17 + .../main.tftest.hcl | 24 + .../test/parallel_divided/main.tftest.hcl | 2 +- internal/configs/test_file.go | 34 +- internal/moduletest/graph/apply.go | 196 ++++ internal/moduletest/graph/eval_context.go | 56 +- .../moduletest/graph/eval_context_test.go | 5 +- .../moduletest/graph/node_state_cleanup.go | 150 +++ internal/moduletest/graph/node_test_run.go | 137 ++- internal/moduletest/graph/plan.go | 125 +++ .../moduletest/graph/test_graph_builder.go | 45 +- internal/moduletest/graph/transform_config.go | 78 +- .../moduletest/graph/transform_config_test.go | 7 +- .../moduletest/graph/transform_providers.go | 38 +- .../graph/transform_state_cleanup.go | 83 ++ .../moduletest/graph/transform_test_run.go | 12 +- internal/moduletest/graph/variables.go | 239 +++++ internal/moduletest/graph/wait.go | 163 +++ internal/terraform/graph_builder.go | 3 + 23 files changed, 1462 insertions(+), 1034 deletions(-) create mode 100644 internal/command/testdata/test/invalid_config/main.tf create mode 100644 internal/command/testdata/test/invalid_config/main.tftest.hcl create mode 100644 internal/command/testdata/test/missing-provider-definition-in-file/main.tf create mode 100644 internal/command/testdata/test/missing-provider-definition-in-file/main.tftest.hcl create mode 100644 internal/moduletest/graph/apply.go create mode 100644 internal/moduletest/graph/node_state_cleanup.go create mode 100644 internal/moduletest/graph/plan.go create mode 100644 internal/moduletest/graph/transform_state_cleanup.go create mode 100644 internal/moduletest/graph/variables.go create mode 100644 internal/moduletest/graph/wait.go diff --git a/internal/backend/local/test.go b/internal/backend/local/test.go index 72541d4260..e5c1a1fb52 100644 --- a/internal/backend/local/test.go +++ b/internal/backend/local/test.go @@ -8,28 +8,19 @@ import ( "fmt" "log" "path/filepath" - "slices" "sort" - "time" - "github.com/hashicorp/hcl/v2" "github.com/zclconf/go-cty/cty" - "github.com/hashicorp/terraform/internal/addrs" "github.com/hashicorp/terraform/internal/backend/backendrun" "github.com/hashicorp/terraform/internal/command/junit" "github.com/hashicorp/terraform/internal/command/views" "github.com/hashicorp/terraform/internal/configs" "github.com/hashicorp/terraform/internal/dag" - "github.com/hashicorp/terraform/internal/lang" - "github.com/hashicorp/terraform/internal/lang/langrefs" "github.com/hashicorp/terraform/internal/logging" "github.com/hashicorp/terraform/internal/moduletest" "github.com/hashicorp/terraform/internal/moduletest/graph" hcltest "github.com/hashicorp/terraform/internal/moduletest/hcl" - "github.com/hashicorp/terraform/internal/moduletest/mocking" - "github.com/hashicorp/terraform/internal/plans" - "github.com/hashicorp/terraform/internal/states" "github.com/hashicorp/terraform/internal/terraform" "github.com/hashicorp/terraform/internal/tfdiags" ) @@ -130,10 +121,12 @@ func (runner *TestSuiteRunner) Test() (moduletest.Status, tfdiags.Diagnostics) { } file := suite.Files[name] - // The eval context inherits the cancelled context from the runner. - // This allows the eval context to stop the graph walk if the runner - // requests a hard stop. - evalCtx := graph.NewEvalContext(runner.CancelledCtx) + evalCtx := graph.NewEvalContext(&graph.EvalContextOpts{ + CancelCtx: runner.CancelledCtx, + StopCtx: runner.StoppedCtx, + Verbose: runner.Verbose, + Render: runner.View, + }) for _, run := range file.Runs { // Pre-initialise the prior outputs, so we can easily tell between @@ -163,8 +156,6 @@ func (runner *TestSuiteRunner) Test() (moduletest.Status, tfdiags.Diagnostics) { runner.View.File(file, moduletest.Starting) fileRunner.Test(file) - runner.View.File(file, moduletest.TearDown) - fileRunner.cleanup(file) runner.View.File(file, moduletest.Complete) suite.Status = suite.Status.Merge(file.Status) } @@ -270,19 +261,23 @@ func (runner *TestFileRunner) Test(file *moduletest.File) { if len(file.Runs) == 0 { // If we have zero run blocks then we'll just mark the file as passed. file.Status = file.Status.Merge(moduletest.Pass) + return } // Build the graph for the file. - b := graph.TestGraphBuilder{File: file, GlobalVars: runner.EvalContext.VariableCaches.GlobalVariables} - graph, diags := b.Build() + b := graph.TestGraphBuilder{ + File: file, + GlobalVars: runner.EvalContext.VariableCaches.GlobalVariables, + ContextOpts: runner.Suite.Opts, + } + g, diags := b.Build() file.Diagnostics = file.Diagnostics.Append(diags) - if diags.HasErrors() { - file.Status = file.Status.Merge(moduletest.Error) + if walkCancelled := runner.renderPreWalkDiags(file); walkCancelled { return } // walk and execute the graph - diags = runner.walkGraph(graph, file) + diags = runner.walkGraph(g) // If the graph walk was terminated, we don't want to add the diagnostics. // The error the user receives will just be: @@ -298,7 +293,7 @@ func (runner *TestFileRunner) Test(file *moduletest.File) { } // walkGraph goes through the graph and execute each run it finds. -func (runner *TestFileRunner) walkGraph(g *terraform.Graph, file *moduletest.File) tfdiags.Diagnostics { +func (runner *TestFileRunner) walkGraph(g *terraform.Graph) tfdiags.Diagnostics { sem := runner.Suite.semaphore // Walk the graph. @@ -345,46 +340,8 @@ func (runner *TestFileRunner) walkGraph(g *terraform.Graph, file *moduletest.Fil sem.Acquire() defer sem.Release() - switch v := v.(type) { - case *graph.NodeTestRun: // NodeTestRun is also executable, so it has to be first. - file := v.File() - run := v.Run() - if file.GetStatus() == moduletest.Error { - // If the overall test file has errored, we don't keep trying to - // execute tests. Instead, we mark all remaining run blocks as - // skipped, print the status, and move on. - run.Status = moduletest.Skip - runner.Suite.View.Run(run, file, moduletest.Complete, 0) - return - } - if runner.Suite.Stopped { - // Then the test was requested to be stopped, so we just mark each - // following test as skipped, print the status, and move on. - run.Status = moduletest.Skip - runner.Suite.View.Run(run, file, moduletest.Complete, 0) - return - } - - // TODO: The execution of a NodeTestRun is currently split between - // its Execute method and the continuation of the walk callback. - // Eventually, we should move all the logic related to a test run into - // its Execute method, effectively ensuring that the Execute method is - // enough to execute a test run in the graph. - diags = v.Execute(runner.EvalContext) - if diags.HasErrors() { - return diags - } - - startTime := time.Now().UTC() - runner.run(run, file, startTime) - runner.Suite.View.Run(run, file, moduletest.Complete, 0) - file.UpdateStatus(run.Status) - case graph.GraphNodeExecutable: - diags = v.Execute(runner.EvalContext) - return diags - default: - // If the vertex isn't a test run or executable, we'll just skip it. - return + if executable, ok := v.(graph.GraphNodeExecutable); ok { + diags = executable.Execute(runner.EvalContext) } return } @@ -392,883 +349,24 @@ func (runner *TestFileRunner) walkGraph(g *terraform.Graph, file *moduletest.Fil return g.AcyclicGraph.Walk(walkFn) } -func (runner *TestFileRunner) run(run *moduletest.Run, file *moduletest.File, startTime time.Time) { - log.Printf("[TRACE] TestFileRunner: executing run block %s/%s", file.Name, run.Name) - defer func() { - // If we got far enough to actually execute the run then we'll give - // the view some additional metadata about the execution. - run.ExecutionMeta = &moduletest.RunExecutionMeta{ - Start: startTime, - Duration: time.Since(startTime), - } - - }() - - key := run.GetStateKey() - if run.Config.ConfigUnderTest != nil { - if key == moduletest.MainStateIdentifier { - // This is bad. It means somehow the module we're loading has - // the same key as main state and we're about to corrupt things. - - run.Diagnostics = run.Diagnostics.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "Invalid module source", - Detail: fmt.Sprintf("The source for the selected module evaluated to %s which should not be possible. This is a bug in Terraform - please report it!", key), - Subject: run.Config.Module.DeclRange.Ptr(), - }) - - run.Status = moduletest.Error - file.UpdateStatus(moduletest.Error) - return - } - } - state := runner.EvalContext.GetFileState(key).State - - config := run.ModuleConfig - if runner.Suite.Cancelled { - // Don't do anything, just give up and return immediately. - // The surrounding functions should stop this even being called, but in - // case of race conditions or something we can still verify this. - return - } - - if runner.Suite.Stopped { - // Basically the same as above, except we'll be a bit nicer. - run.Status = moduletest.Skip - return - } - - start := time.Now().UTC().UnixMilli() - runner.Suite.View.Run(run, file, moduletest.Starting, 0) - - run.Diagnostics = run.Diagnostics.Append(run.Config.Validate(config)) - if run.Diagnostics.HasErrors() { - run.Status = moduletest.Error - return - } - - configDiags := graph.TransformConfigForTest(runner.EvalContext, run, file) - run.Diagnostics = run.Diagnostics.Append(configDiags) - if configDiags.HasErrors() { - run.Status = moduletest.Error - return - } - - validateDiags := runner.validate(run, file, start) - run.Diagnostics = run.Diagnostics.Append(validateDiags) - if validateDiags.HasErrors() { - run.Status = moduletest.Error - return - } - - // already validated during static analysis - references, _ := run.GetReferences() - - variables, variableDiags := runner.GetVariables(run, references, true) - run.Diagnostics = run.Diagnostics.Append(variableDiags) - if variableDiags.HasErrors() { - run.Status = moduletest.Error - return - } - - // FilterVariablesToModule only returns warnings, so we don't check the - // returned diags for errors. - setVariables, testOnlyVariables, setVariableDiags := runner.FilterVariablesToModule(run, variables) - run.Diagnostics = run.Diagnostics.Append(setVariableDiags) - - tfCtx, ctxDiags := terraform.NewContext(runner.Suite.Opts) - run.Diagnostics = run.Diagnostics.Append(ctxDiags) - if ctxDiags.HasErrors() { - return - } - - planScope, plan, planDiags := runner.plan(tfCtx, config, state, run, file, setVariables, references, start) - if run.Config.Command == configs.PlanTestCommand { - // Then we want to assess our conditions and diagnostics differently. - planDiags = run.ValidateExpectedFailures(planDiags) - run.Diagnostics = run.Diagnostics.Append(planDiags) - if planDiags.HasErrors() { - run.Status = moduletest.Error - return - } - - runner.AddVariablesToConfig(run, variables) - - if runner.Suite.Verbose { - schemas, diags := tfCtx.Schemas(config, plan.PriorState) - - // If we're going to fail to render the plan, let's not fail the overall - // test. It can still have succeeded. So we'll add the diagnostics, but - // still report the test status as a success. - if diags.HasErrors() { - // This is very unlikely. - diags = diags.Append(tfdiags.Sourceless( - tfdiags.Warning, - "Failed to print verbose output", - fmt.Sprintf("Terraform failed to print the verbose output for %s, other diagnostics will contain more details as to why.", filepath.Join(file.Name, run.Name)))) - } else { - run.Verbose = &moduletest.Verbose{ - Plan: plan, - State: nil, // We don't have a state to show in plan mode. - Config: config, - Providers: schemas.Providers, - Provisioners: schemas.Provisioners, - } - } - - run.Diagnostics = run.Diagnostics.Append(diags) - } - - // Evaluate the run block directly in the graph context to validate the assertions - // of the run. We also pass in all the - // previous contexts so this run block can refer to outputs from - // previous run blocks. - newStatus, outputVals, moreDiags := runner.EvalContext.EvaluateRun(run, planScope, testOnlyVariables) - run.Status = newStatus - run.Diagnostics = run.Diagnostics.Append(moreDiags) - - // Now we've successfully validated this run block, lets add it into - // our prior run outputs so future run blocks can access it. - runner.EvalContext.SetOutput(run, outputVals) - return - } - - // Otherwise any error during the planning prevents our apply from - // continuing which is an error. - planDiags = run.ExplainExpectedFailures(planDiags) - run.Diagnostics = run.Diagnostics.Append(planDiags) - if planDiags.HasErrors() { - run.Status = moduletest.Error - return - } - - // Since we're carrying on an executing the apply operation as well, we're - // just going to do some post processing of the diagnostics. We remove the - // warnings generated from check blocks, as the apply operation will either - // reproduce them or fix them and we don't want fixed diagnostics to be - // reported and we don't want duplicates either. - var filteredDiags tfdiags.Diagnostics - for _, diag := range run.Diagnostics { - if rule, ok := addrs.DiagnosticOriginatesFromCheckRule(diag); ok && rule.Container.CheckableKind() == addrs.CheckableCheck { - continue - } - filteredDiags = filteredDiags.Append(diag) - } - run.Diagnostics = filteredDiags - - applyScope, updated, applyDiags := runner.apply(tfCtx, plan, state, run, file, moduletest.Running, start, variables) - - // Remove expected diagnostics, and add diagnostics in case anything that should have failed didn't. - // We'll also update the run status based on the presence of errors or missing expected failures. - failOrErr := runner.checkForMissingExpectedFailures(run, applyDiags) - if failOrErr { - // Even though the apply operation failed, the graph may have done - // partial updates and the returned state should reflect this. - runner.EvalContext.SetFileState(key, &graph.TestFileState{ - Run: run, - State: updated, - }) - return - } - - runner.AddVariablesToConfig(run, variables) - - if runner.Suite.Verbose { - schemas, diags := tfCtx.Schemas(config, updated) - - // If we're going to fail to render the plan, let's not fail the overall - // test. It can still have succeeded. So we'll add the diagnostics, but - // still report the test status as a success. - if diags.HasErrors() { - // This is very unlikely. - diags = diags.Append(tfdiags.Sourceless( - tfdiags.Warning, - "Failed to print verbose output", - fmt.Sprintf("Terraform failed to print the verbose output for %s, other diagnostics will contain more details as to why.", filepath.Join(file.Name, run.Name)))) - } else { - run.Verbose = &moduletest.Verbose{ - Plan: nil, // We don't have a plan to show in apply mode. - State: updated, - Config: config, - Providers: schemas.Providers, - Provisioners: schemas.Provisioners, - } - } - - run.Diagnostics = run.Diagnostics.Append(diags) - } - - // Evaluate the run block directly in the graph context to validate the assertions - // of the run. We also pass in all the - // previous contexts so this run block can refer to outputs from - // previous run blocks. - newStatus, outputVals, moreDiags := runner.EvalContext.EvaluateRun(run, applyScope, testOnlyVariables) - run.Status = newStatus - run.Diagnostics = run.Diagnostics.Append(moreDiags) - - // Now we've successfully validated this run block, lets add it into - // our prior run outputs so future run blocks can access it. - runner.EvalContext.SetOutput(run, outputVals) - - // Only update the most recent run and state if the state was - // actually updated by this change. We want to use the run that - // most recently updated the tracked state as the cleanup - // configuration. - runner.EvalContext.SetFileState(key, &graph.TestFileState{ - Run: run, - State: updated, - }) -} - -// checkForMissingExpectedFailures checks for missing expected failures in the diagnostics. -// It updates the run status based on the presence of errors or missing expected failures. -func (runner *TestFileRunner) checkForMissingExpectedFailures(run *moduletest.Run, diags tfdiags.Diagnostics) (failOrErr bool) { - // Retrieve and append diagnostics that are either unrelated to expected failures - // or report missing expected failures. - unexpectedDiags := run.ValidateExpectedFailures(diags) - run.Diagnostics = run.Diagnostics.Append(unexpectedDiags) - for _, diag := range unexpectedDiags { - // // If any diagnostic indicates a missing expected failure, set the run status to fail. - if ok := moduletest.DiagnosticFromMissingExpectedFailure(diag); ok { - run.Status = run.Status.Merge(moduletest.Fail) - continue - } - - // upgrade the run status to error if there still are other errors in the diagnostics - if diag.Severity() == tfdiags.Error { - run.Status = run.Status.Merge(moduletest.Error) - break - } - } - return run.Status > moduletest.Pass -} - -func (runner *TestFileRunner) validate(run *moduletest.Run, file *moduletest.File, start int64) tfdiags.Diagnostics { - log.Printf("[TRACE] TestFileRunner: called validate for %s/%s", file.Name, run.Name) - - var diags tfdiags.Diagnostics - config := run.ModuleConfig - - tfCtx, ctxDiags := terraform.NewContext(runner.Suite.Opts) - diags = diags.Append(ctxDiags) - if ctxDiags.HasErrors() { - return diags - } - - runningCtx, done := context.WithCancel(context.Background()) - - var validateDiags tfdiags.Diagnostics - go func() { - defer logging.PanicHandler() - defer done() - - log.Printf("[DEBUG] TestFileRunner: starting validate for %s/%s", file.Name, run.Name) - validateDiags = tfCtx.Validate(config, nil) - log.Printf("[DEBUG] TestFileRunner: completed validate for %s/%s", file.Name, run.Name) - }() - waitDiags, cancelled := runner.wait(tfCtx, runningCtx, run, file, nil, moduletest.Running, start) - - if cancelled { - diags = diags.Append(tfdiags.Sourceless(tfdiags.Error, "Test interrupted", "The test operation could not be completed due to an interrupt signal. Please read the remaining diagnostics carefully for any sign of failed state cleanup or dangling resources.")) - } - - diags = diags.Append(waitDiags) - diags = diags.Append(validateDiags) - - return diags -} - -func (runner *TestFileRunner) destroy(state *states.State, run *moduletest.Run, file *moduletest.File) (*states.State, tfdiags.Diagnostics) { - log.Printf("[TRACE] TestFileRunner: called destroy for %s/%s", file.Name, run.Name) - - if state.Empty() { - // Nothing to do! - return state, nil - } - - var diags tfdiags.Diagnostics - - variables, variableDiags := runner.GetVariables(run, nil, false) - diags = diags.Append(variableDiags) - - if diags.HasErrors() { - return state, diags - } - - // During the destroy operation, we don't add warnings from this operation. - // Anything that would have been reported here was already reported during - // the original plan, and a successful destroy operation is the only thing - // we care about. - setVariables, _, _ := runner.FilterVariablesToModule(run, variables) - - planOpts := &terraform.PlanOpts{ - Mode: plans.DestroyMode, - SetVariables: setVariables, - Overrides: mocking.PackageOverrides(run.Config, file.Config, run.ModuleConfig), - } - - tfCtx, ctxDiags := terraform.NewContext(runner.Suite.Opts) - diags = diags.Append(ctxDiags) - if ctxDiags.HasErrors() { - return state, diags - } - - runningCtx, done := context.WithCancel(context.Background()) - - start := time.Now().UTC().UnixMilli() - runner.Suite.View.Run(run, file, moduletest.TearDown, 0) - - var plan *plans.Plan - var planDiags tfdiags.Diagnostics - go func() { - defer logging.PanicHandler() - defer done() - - log.Printf("[DEBUG] TestFileRunner: starting destroy plan for %s/%s", file.Name, run.Name) - plan, planDiags = tfCtx.Plan(run.ModuleConfig, state, planOpts) - log.Printf("[DEBUG] TestFileRunner: completed destroy plan for %s/%s", file.Name, run.Name) - }() - waitDiags, cancelled := runner.wait(tfCtx, runningCtx, run, file, nil, moduletest.TearDown, start) - - if cancelled { - diags = diags.Append(tfdiags.Sourceless(tfdiags.Error, "Test interrupted", "The test operation could not be completed due to an interrupt signal. Please read the remaining diagnostics carefully for any sign of failed state cleanup or dangling resources.")) - } - - diags = diags.Append(waitDiags) - diags = diags.Append(planDiags) - - if diags.HasErrors() { - return state, diags - } - - _, updated, applyDiags := runner.apply(tfCtx, plan, state, run, file, moduletest.TearDown, start, variables) - diags = diags.Append(applyDiags) - return updated, diags -} - -func (runner *TestFileRunner) plan(tfCtx *terraform.Context, config *configs.Config, state *states.State, run *moduletest.Run, file *moduletest.File, variables terraform.InputValues, references []*addrs.Reference, start int64) (*lang.Scope, *plans.Plan, tfdiags.Diagnostics) { - log.Printf("[TRACE] TestFileRunner: called plan for %s/%s", file.Name, run.Name) - - var diags tfdiags.Diagnostics - - targets, targetDiags := run.GetTargets() - diags = diags.Append(targetDiags) - - replaces, replaceDiags := run.GetReplaces() - diags = diags.Append(replaceDiags) - - if diags.HasErrors() { - return nil, nil, diags - } - - planOpts := &terraform.PlanOpts{ - Mode: func() plans.Mode { - switch run.Config.Options.Mode { - case configs.RefreshOnlyTestMode: - return plans.RefreshOnlyMode - default: - return plans.NormalMode - } - }(), - Targets: targets, - ForceReplace: replaces, - SkipRefresh: !run.Config.Options.Refresh, - SetVariables: variables, - ExternalReferences: references, - Overrides: mocking.PackageOverrides(run.Config, file.Config, config), - } - - runningCtx, done := context.WithCancel(context.Background()) - - var plan *plans.Plan - var planDiags tfdiags.Diagnostics - var planScope *lang.Scope - go func() { - defer logging.PanicHandler() - defer done() - - log.Printf("[DEBUG] TestFileRunner: starting plan for %s/%s", file.Name, run.Name) - plan, planScope, planDiags = tfCtx.PlanAndEval(config, state, planOpts) - log.Printf("[DEBUG] TestFileRunner: completed plan for %s/%s", file.Name, run.Name) - }() - waitDiags, cancelled := runner.wait(tfCtx, runningCtx, run, file, nil, moduletest.Running, start) - - if cancelled { - diags = diags.Append(tfdiags.Sourceless(tfdiags.Error, "Test interrupted", "The test operation could not be completed due to an interrupt signal. Please read the remaining diagnostics carefully for any sign of failed state cleanup or dangling resources.")) - } - - diags = diags.Append(waitDiags) - diags = diags.Append(planDiags) - - return planScope, plan, diags -} - -func (runner *TestFileRunner) apply(tfCtx *terraform.Context, plan *plans.Plan, state *states.State, run *moduletest.Run, file *moduletest.File, progress moduletest.Progress, start int64, variables terraform.InputValues) (*lang.Scope, *states.State, tfdiags.Diagnostics) { - log.Printf("[TRACE] TestFileRunner: called apply for %s/%s", file.Name, run.Name) - - var diags tfdiags.Diagnostics - config := run.ModuleConfig - - // If things get cancelled while we are executing the apply operation below - // we want to print out all the objects that we were creating so the user - // can verify we managed to tidy everything up possibly. - // - // Unfortunately, this creates a race condition as the apply operation can - // edit the plan (by removing changes once they are applied) while at the - // same time our cancellation process will try to read the plan. - // - // We take a quick copy of the changes we care about here, which will then - // be used in place of the plan when we print out the objects to be created - // as part of the cancellation process. - var created []*plans.ResourceInstanceChangeSrc - for _, change := range plan.Changes.Resources { - if change.Action != plans.Create { - continue - } - created = append(created, change) - } - - runningCtx, done := context.WithCancel(context.Background()) - - var updated *states.State - var applyDiags tfdiags.Diagnostics - var newScope *lang.Scope - - // We only need to pass ephemeral variables to the apply operation, as the - // plan has already been evaluated with the full set of variables. - ephemeralVariables := make(terraform.InputValues) - for k, v := range config.Root.Module.Variables { - if v.EphemeralSet { - if value, ok := variables[k]; ok { - ephemeralVariables[k] = value - } - } - } - - applyOpts := &terraform.ApplyOpts{ - SetVariables: ephemeralVariables, - } - - go func() { - defer logging.PanicHandler() - defer done() - log.Printf("[DEBUG] TestFileRunner: starting apply for %s/%s", file.Name, run.Name) - updated, newScope, applyDiags = tfCtx.ApplyAndEval(plan, config, applyOpts) - log.Printf("[DEBUG] TestFileRunner: completed apply for %s/%s", file.Name, run.Name) - }() - waitDiags, cancelled := runner.wait(tfCtx, runningCtx, run, file, created, progress, start) - - if cancelled { - diags = diags.Append(tfdiags.Sourceless(tfdiags.Error, "Test interrupted", "The test operation could not be completed due to an interrupt signal. Please read the remaining diagnostics carefully for any sign of failed state cleanup or dangling resources.")) - } - - diags = diags.Append(waitDiags) - diags = diags.Append(applyDiags) - - return newScope, updated, diags -} - -func (runner *TestFileRunner) wait(ctx *terraform.Context, runningCtx context.Context, run *moduletest.Run, file *moduletest.File, created []*plans.ResourceInstanceChangeSrc, progress moduletest.Progress, start int64) (diags tfdiags.Diagnostics, cancelled bool) { - var identifier string - if file == nil { - identifier = "validate" - } else { - identifier = file.Name - if run != nil { - identifier = fmt.Sprintf("%s/%s", identifier, run.Name) - } - } - log.Printf("[TRACE] TestFileRunner: waiting for execution during %s", identifier) - - // Keep track of when the execution is actually finished. - finished := false - - // This function handles what happens when the user presses the second - // interrupt. This is a "hard cancel", we are going to stop doing whatever - // it is we're doing. This means even if we're halfway through creating or - // destroying infrastructure we just give up. - handleCancelled := func() { - log.Printf("[DEBUG] TestFileRunner: test execution cancelled during %s", identifier) - - states := make(map[*moduletest.Run]*states.State) - mainKey := moduletest.MainStateIdentifier - states[nil] = runner.EvalContext.GetFileState(mainKey).State - for key, module := range runner.EvalContext.FileStates { - if key == mainKey { - continue - } - states[module.Run] = module.State - } - runner.Suite.View.FatalInterruptSummary(run, file, states, created) - - cancelled = true - go ctx.Stop() - - for !finished { - select { - case <-time.After(2 * time.Second): - // Print an update while we're waiting. - now := time.Now().UTC().UnixMilli() - runner.Suite.View.Run(run, file, progress, now-start) - case <-runningCtx.Done(): - // Just wait for things to finish now, the overall test execution will - // exit early if this takes too long. - finished = true - } - } - - } - - // This function handles what happens when the user presses the first - // interrupt. This is essentially a "soft cancel", we're not going to do - // anything but just wait for things to finish safely. But, we do listen - // for the crucial second interrupt which will prompt a hard stop / cancel. - handleStopped := func() { - log.Printf("[DEBUG] TestFileRunner: test execution stopped during %s", identifier) - - for !finished { - select { - case <-time.After(2 * time.Second): - // Print an update while we're waiting. - now := time.Now().UTC().UnixMilli() - runner.Suite.View.Run(run, file, progress, now-start) - case <-runner.Suite.CancelledCtx.Done(): - // We've been asked again. This time we stop whatever we're doing - // and abandon all attempts to do anything reasonable. - handleCancelled() - case <-runningCtx.Done(): - // Do nothing, we finished safely and skipping the remaining tests - // will be handled elsewhere. - finished = true - } - } - - } - - for !finished { - select { - case <-time.After(2 * time.Second): - // Print an update while we're waiting. - now := time.Now().UTC().UnixMilli() - runner.Suite.View.Run(run, file, progress, now-start) - case <-runner.Suite.StoppedCtx.Done(): - handleStopped() - case <-runner.Suite.CancelledCtx.Done(): - handleCancelled() - case <-runningCtx.Done(): - // The operation exited normally. - finished = true - } - } - - return diags, cancelled -} - -func (runner *TestFileRunner) cleanup(file *moduletest.File) { - log.Printf("[TRACE] TestStateManager: cleaning up state for %s", file.Name) - - if runner.Suite.Cancelled { - // Don't try and clean anything up if the execution has been cancelled. - log.Printf("[DEBUG] TestStateManager: skipping state cleanup for %s due to cancellation", file.Name) - return - } - - var states []*graph.TestFileState - for key, state := range runner.EvalContext.FileStates { - - empty := true - for _, module := range state.State.Modules { - for _, resource := range module.Resources { - if resource.Addr.Resource.Mode == addrs.ManagedResourceMode { - empty = false - break - } - } - } - - if empty { - // The state can be empty for a run block that just executed a plan - // command, or a run block that only read data sources. We'll just - // skip empty run blocks. - continue - } - - if state.Run == nil { - log.Printf("[ERROR] TestFileRunner: found inconsistent run block and state file in %s for module %s", file.Name, key) - - // The state can have a nil run block if it only executed a plan - // command. In which case, we shouldn't have reached here as the - // state should also have been empty and this will have been skipped - // above. If we do reach here, then something has gone badly wrong - // and we can't really recover from it. - - var diags tfdiags.Diagnostics - diags = diags.Append(tfdiags.Sourceless(tfdiags.Error, "Inconsistent state", fmt.Sprintf("Found inconsistent state while cleaning up %s. This is a bug in Terraform - please report it", file.Name))) - file.UpdateStatus(moduletest.Error) - runner.Suite.View.DestroySummary(diags, nil, file, state.State) - continue - } - - states = append(states, state) - } - - slices.SortFunc(states, func(a, b *graph.TestFileState) int { - // We want to clean up later run blocks first. So, we'll sort this in - // reverse according to index. This means larger indices first. - return b.Run.Index - a.Run.Index - }) - - // Then we'll clean up the additional states for custom modules in reverse - // order. - for _, state := range states { - log.Printf("[DEBUG] TestStateManager: cleaning up state for %s/%s", file.Name, state.Run.Name) - - if runner.Suite.Cancelled { - // In case the cancellation came while a previous state was being - // destroyed. - log.Printf("[DEBUG] TestStateManager: skipping state cleanup for %s/%s due to cancellation", file.Name, state.Run.Name) - return - } - - var diags tfdiags.Diagnostics - - configDiags := graph.TransformConfigForTest(runner.EvalContext, state.Run, file) - diags = diags.Append(configDiags) - - updated := state.State - if !diags.HasErrors() { - var destroyDiags tfdiags.Diagnostics - updated, destroyDiags = runner.destroy(state.State, state.Run, file) - diags = diags.Append(destroyDiags) - } - - if !updated.Empty() { - // Then we failed to adequately clean up the state, so mark success - // as false. - file.UpdateStatus(moduletest.Error) - } - runner.Suite.View.DestroySummary(diags, state.Run, file, updated) - } -} - -// GetVariables builds the terraform.InputValues required for the provided run -// block. It pulls the relevant variables (ie. the variables needed for the -// run block) from the total pool of all available variables, and converts them -// into input values. -// -// As a run block can reference variables defined within the file and are not -// actually defined within the configuration, this function actually returns -// more variables than are required by the config. FilterVariablesToConfig -// should be called before trying to use these variables within a Terraform -// plan, apply, or destroy operation. -func (runner *TestFileRunner) GetVariables(run *moduletest.Run, references []*addrs.Reference, includeWarnings bool) (terraform.InputValues, tfdiags.Diagnostics) { - var diags tfdiags.Diagnostics - - // relevantVariables contains the variables that are of interest to this - // run block. This is a combination of the variables declared within the - // configuration for this run block, and the variables referenced by the - // run block assertions. - relevantVariables := make(map[string]bool) - - // First, we'll check to see which variables the run block assertions - // reference. - for _, reference := range references { - if addr, ok := reference.Subject.(addrs.InputVariable); ok { - relevantVariables[addr.Name] = true - } - } - - // And check to see which variables the run block configuration references. - for name := range run.ModuleConfig.Module.Variables { - relevantVariables[name] = true - } - - // We'll put the parsed values into this map. - values := make(terraform.InputValues) - - // First, let's step through the expressions within the run block and work - // them out. - for name, expr := range run.Config.Variables { - requiredValues := make(map[string]cty.Value) - - refs, refDiags := langrefs.ReferencesInExpr(addrs.ParseRefFromTestingScope, expr) - for _, ref := range refs { - if addr, ok := ref.Subject.(addrs.InputVariable); ok { - cache := runner.EvalContext.GetCache(run) - - value, valueDiags := cache.GetFileVariable(addr.Name) - diags = diags.Append(valueDiags) - if value != nil { - requiredValues[addr.Name] = value.Value - continue - } - - // Otherwise, it might be a global variable. - value, valueDiags = cache.GetGlobalVariable(addr.Name) - diags = diags.Append(valueDiags) - if value != nil { - requiredValues[addr.Name] = value.Value - continue - } - } - } - diags = diags.Append(refDiags) - - ctx, ctxDiags := hcltest.EvalContext(hcltest.TargetRunBlock, map[string]hcl.Expression{name: expr}, requiredValues, runner.EvalContext.GetOutputs()) - diags = diags.Append(ctxDiags) - - value := cty.DynamicVal - if !ctxDiags.HasErrors() { - var valueDiags hcl.Diagnostics - value, valueDiags = expr.Value(ctx) - diags = diags.Append(valueDiags) - } - - // We do this late on so we still validate whatever it was that the user - // wrote in the variable expression. But, we don't want to actually use - // it if it's not actually relevant. - if _, exists := relevantVariables[name]; !exists { - // Do not display warnings during cleanup phase - if includeWarnings { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagWarning, - Summary: "Value for undeclared variable", - Detail: fmt.Sprintf("The module under test does not declare a variable named %q, but it is declared in run block %q.", name, run.Name), - Subject: expr.Range().Ptr(), - }) - } - continue // Don't add it to our final set of variables. - } - - values[name] = &terraform.InputValue{ - Value: value, - SourceType: terraform.ValueFromConfig, - SourceRange: tfdiags.SourceRangeFromHCL(expr.Range()), - } - } - - for variable := range relevantVariables { - if _, exists := values[variable]; exists { - // Then we've already got a value for this variable. - continue - } - - // Otherwise, we'll get it from the cache as a file-level or global - // variable. - cache := runner.EvalContext.GetCache(run) - - value, valueDiags := cache.GetFileVariable(variable) - diags = diags.Append(valueDiags) - if value != nil { - values[variable] = value - continue - } - - value, valueDiags = cache.GetGlobalVariable(variable) - diags = diags.Append(valueDiags) - if value != nil { - values[variable] = value - continue - } - } - - // Finally, we check the configuration again. This is where we'll discover - // if there's any missing variables and fill in any optional variables that - // don't have a value already. - - for name, variable := range run.ModuleConfig.Module.Variables { - if _, exists := values[name]; exists { - // Then we've provided a variable for this. It's all good. - continue - } - - // Otherwise, we're going to give these variables a value. They'll be - // processed by the Terraform graph and provided a default value later - // if they have one. - - if variable.Required() { - diags = diags.Append(&hcl.Diagnostic{ - Severity: hcl.DiagError, - Summary: "No value for required variable", - Detail: fmt.Sprintf("The module under test for run block %q has a required variable %q with no set value. Use a -var or -var-file command line argument or add this variable into a \"variables\" block within the test file or run block.", - run.Name, variable.Name), - Subject: variable.DeclRange.Ptr(), - }) - - values[name] = &terraform.InputValue{ - Value: cty.DynamicVal, - SourceType: terraform.ValueFromConfig, - SourceRange: tfdiags.SourceRangeFromHCL(variable.DeclRange), - } - } else { - values[name] = &terraform.InputValue{ - Value: cty.NilVal, - SourceType: terraform.ValueFromConfig, - SourceRange: tfdiags.SourceRangeFromHCL(variable.DeclRange), - } - } - } - - return values, diags -} - -// FilterVariablesToModule splits the provided values into two disjoint maps: -// moduleVars contains the ones that correspond with declarations in the root -// module of the given configuration, while testOnlyVars contains any others -// that are presumably intended only for use in the test configuration file. -// -// This function is essentially the opposite of AddVariablesToConfig which -// makes the config match the variables rather than the variables match the -// config. -// -// This function can only return warnings, and the callers can rely on this so -// please check the callers of this function if you add any error diagnostics. -func (runner *TestFileRunner) FilterVariablesToModule(run *moduletest.Run, values terraform.InputValues) (moduleVars, testOnlyVars terraform.InputValues, diags tfdiags.Diagnostics) { - moduleVars = make(terraform.InputValues) - testOnlyVars = make(terraform.InputValues) - for name, value := range values { - _, exists := run.ModuleConfig.Module.Variables[name] - if !exists { - // If it's not in the configuration then it's a test-only variable. - testOnlyVars[name] = value - continue +func (runner *TestFileRunner) renderPreWalkDiags(file *moduletest.File) (walkCancelled bool) { + errored := file.Diagnostics.HasErrors() + // Some runs may have errored during the graph build, but we didn't fail immediately + // as we still wanted to gather all the diagnostics. + // Now we go through the runs and if there are any errors, we'll update the + // file status to be errored. + for _, run := range file.Runs { + if run.Status == moduletest.Error { + errored = true + runner.Suite.View.Run(run, file, moduletest.Complete, 0) } - - moduleVars[name] = value } - return moduleVars, testOnlyVars, diags -} - -// AddVariablesToConfig extends the provided config to ensure it has definitions -// for all specified variables. -// -// This function is essentially the opposite of FilterVariablesToConfig which -// makes the variables match the config rather than the config match the -// variables. -func (runner *TestFileRunner) AddVariablesToConfig(run *moduletest.Run, variables terraform.InputValues) { - - // If we have got variable values from the test file we need to make sure - // they have an equivalent entry in the configuration. We're going to do - // that dynamically here. - - // First, take a backup of the existing configuration so we can easily - // restore it later. - currentVars := make(map[string]*configs.Variable) - for name, variable := range run.ModuleConfig.Module.Variables { - currentVars[name] = variable - } - - for name, value := range variables { - if _, exists := run.ModuleConfig.Module.Variables[name]; exists { - continue - } - - run.ModuleConfig.Module.Variables[name] = &configs.Variable{ - Name: name, - Type: value.Value.Type(), - ConstraintType: value.Value.Type(), - DeclRange: value.SourceRange.ToHCL(), - } + if errored { + // print a teardown message even though there was no teardown to run + runner.Suite.View.File(file, moduletest.TearDown) + file.Status = file.Status.Merge(moduletest.Error) + return true } + return false } diff --git a/internal/command/test_test.go b/internal/command/test_test.go index af11ed9621..fd44e233f2 100644 --- a/internal/command/test_test.go +++ b/internal/command/test_test.go @@ -10,6 +10,7 @@ import ( "os" "path" "regexp" + "runtime" "strings" "testing" @@ -801,7 +802,7 @@ func TestTest_ProviderAlias(t *testing.T) { output := done(t) - if code := init.Run(nil); code != 0 { + if code := init.Run([]string{"-no-color"}); code != 0 { t.Fatalf("expected status code 0 but got %d: %s", code, output.All()) } @@ -814,7 +815,7 @@ func TestTest_ProviderAlias(t *testing.T) { Meta: meta, } - code := command.Run(nil) + code := command.Run([]string{"-no-color"}) output = done(t) printedOutput := false @@ -1127,6 +1128,23 @@ it has been removed. This occurs when a provider configuration is removed while objects created by that provider still exist in the state. Re-add the provider configuration to destroy test_resource.secondary, after which you can remove the provider configuration again. +`, + }, + "missing-provider-definition-in-file": { + expectedOut: `main.tftest.hcl... in progress + run "passes_validation"... fail +main.tftest.hcl... tearing down +main.tftest.hcl... fail + +Failure! 0 passed, 1 failed. +`, + expectedErr: ` +Error: Missing provider definition for test + + on main.tftest.hcl line 12, in run "passes_validation": + 12: test = test + +This provider block references a provider definition that does not exist. `, }, "missing-provider-in-test-module": { @@ -2465,6 +2483,83 @@ Success! 2 passed, 0 failed. } } +func TestTest_InvalidConfig(t *testing.T) { + td := t.TempDir() + testCopyDir(t, testFixturePath(path.Join("test", "invalid_config")), td) + defer testChdir(t, td)() + + provider := testing_command.NewProvider(nil) + + providerSource, close := newMockProviderSource(t, map[string][]string{ + "test": {"1.0.0"}, + }) + defer close() + + streams, done := terminal.StreamsForTesting(t) + view := views.NewView(streams) + ui := new(cli.MockUi) + + meta := Meta{ + Ui: ui, + View: view, + Streams: streams, + ProviderSource: providerSource, + } + + init := &InitCommand{ + Meta: meta, + } + + output := done(t) + + if code := init.Run(nil); code != 0 { + t.Fatalf("expected status code 0 but got %d: %s", code, output.All()) + } + + // Reset the streams for the next command. + streams, done = terminal.StreamsForTesting(t) + meta.Streams = streams + meta.View = views.NewView(streams) + + c := &TestCommand{ + Meta: meta, + } + + code := c.Run([]string{"-no-color"}) + output = done(t) + + if code != 1 { + t.Errorf("expected status code ! but got %d", code) + } + + expected := `main.tftest.hcl... in progress + run "test"... fail + +Error: Failed to load plugin schemas + +Error while loading schemas for plugin components: Failed to obtain provider +schema: Could not load the schema for provider +registry.terraform.io/hashicorp/test: failed to instantiate provider +"registry.terraform.io/hashicorp/test" to obtain schema: fork/exec +.terraform/providers/registry.terraform.io/hashicorp/test/1.0.0/%s/terraform-provider-test_1.0.0: +permission denied.. +main.tftest.hcl... tearing down +main.tftest.hcl... fail + +Failure! 0 passed, 1 failed. +` + expected = fmt.Sprintf(expected, runtime.GOOS+"_"+runtime.GOARCH) + actual := output.All() + + if diff := cmp.Diff(actual, expected); len(diff) > 0 { + t.Errorf("output didn't match expected:\nexpected:\n%s\nactual:\n%s\ndiff:\n%s", expected, actual, diff) + } + + if provider.ResourceCount() > 0 { + t.Errorf("should have deleted all resources on completion but left %v", provider.ResourceString()) + } +} + func TestTest_RunBlocksInProviders(t *testing.T) { td := t.TempDir() testCopyDir(t, testFixturePath(path.Join("test", "provider_runs")), td) diff --git a/internal/command/testdata/test/invalid_config/main.tf b/internal/command/testdata/test/invalid_config/main.tf new file mode 100644 index 0000000000..65dd87f156 --- /dev/null +++ b/internal/command/testdata/test/invalid_config/main.tf @@ -0,0 +1,11 @@ +terraform { + required_providers { + test = { + source = "hashicorp/test" + } + } +} + +resource "test_resource" "foo" { + nein = "foo" +} diff --git a/internal/command/testdata/test/invalid_config/main.tftest.hcl b/internal/command/testdata/test/invalid_config/main.tftest.hcl new file mode 100644 index 0000000000..d3995cae55 --- /dev/null +++ b/internal/command/testdata/test/invalid_config/main.tftest.hcl @@ -0,0 +1,2 @@ + +run "test" {} diff --git a/internal/command/testdata/test/missing-provider-definition-in-file/main.tf b/internal/command/testdata/test/missing-provider-definition-in-file/main.tf new file mode 100644 index 0000000000..32a4e744bc --- /dev/null +++ b/internal/command/testdata/test/missing-provider-definition-in-file/main.tf @@ -0,0 +1,17 @@ +terraform { + required_providers { + test = { + source = "hashicorp/test" + configuration_aliases = [ test.secondary, test ] + } + } +} + +resource "test_resource" "primary" { + value = "foo" +} + +resource "test_resource" "secondary" { + provider = test.secondary + value = "bar" +} diff --git a/internal/command/testdata/test/missing-provider-definition-in-file/main.tftest.hcl b/internal/command/testdata/test/missing-provider-definition-in-file/main.tftest.hcl new file mode 100644 index 0000000000..68a5f2beba --- /dev/null +++ b/internal/command/testdata/test/missing-provider-definition-in-file/main.tftest.hcl @@ -0,0 +1,24 @@ + +# provider "test" {} + +# provider "test" { +# alias = "secondary" +# } + +run "passes_validation" { + +// references a provider that is not defined in the test file + providers = { + test = test + } + + assert { + condition = test_resource.primary.value == "foo" + error_message = "primary contains invalid value" + } + + assert { + condition = test_resource.secondary.value == "bar" + error_message = "secondary contains invalid value" + } +} diff --git a/internal/command/testdata/test/parallel_divided/main.tftest.hcl b/internal/command/testdata/test/parallel_divided/main.tftest.hcl index 190e36c512..977bf9068a 100644 --- a/internal/command/testdata/test/parallel_divided/main.tftest.hcl +++ b/internal/command/testdata/test/parallel_divided/main.tftest.hcl @@ -84,7 +84,7 @@ run "main_fifth" { } run "main_sixth" { - state_key = "uniq_5" + state_key = "uniq_6" variables { input = "foo" } diff --git a/internal/configs/test_file.go b/internal/configs/test_file.go index b894b2d513..fb2cdce6c6 100644 --- a/internal/configs/test_file.go +++ b/internal/configs/test_file.go @@ -138,6 +138,9 @@ type TestRun struct { // configuration load process and should be used when the test is executed. ConfigUnderTest *Config + // File is a reference to the parent TestFile that contains this run block. + File *TestFile + // ExpectFailures should be a list of checkable objects that are expected // to report a failure from their custom conditions as part of this test // run. @@ -271,6 +274,23 @@ func (run *TestRun) Validate(config *Config) tfdiags.Diagnostics { } } + // All the providers defined within a run block should target an existing + // provider block within the test file. + for _, ref := range run.Providers { + _, ok := run.File.Providers[ref.InParent.String()] + if !ok { + // Then this reference was invalid as we didn't have the + // specified provider in the parent. This should have been + // caught earlier in validation anyway so is unlikely to happen. + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: fmt.Sprintf("Missing provider definition for %s", ref.InParent.String()), + Detail: "This provider block references a provider definition that does not exist.", + Subject: ref.InParent.NameRange.Ptr(), + }) + } + } + return diags } @@ -306,7 +326,7 @@ type TestRunOptions struct { func loadTestFile(body hcl.Body) (*TestFile, hcl.Diagnostics) { var diags hcl.Diagnostics - tf := TestFile{ + tf := &TestFile{ Providers: make(map[string]*Provider), Overrides: addrs.MakeMap[addrs.Targetable, *Override](), } @@ -333,7 +353,7 @@ func loadTestFile(body hcl.Body) (*TestFile, hcl.Diagnostics) { for _, block := range content.Blocks { switch block.Type { case "run": - run, runDiags := decodeTestRunBlock(block, tf.Config) + run, runDiags := decodeTestRunBlock(block, tf) diags = append(diags, runDiags...) if !runDiags.HasErrors() { tf.Runs = append(tf.Runs, run) @@ -455,7 +475,7 @@ func loadTestFile(body hcl.Body) (*TestFile, hcl.Diagnostics) { } } - return &tf, diags + return tf, diags } func decodeFileConfigBlock(fileContent *hcl.BodyContent) (*TestFileConfig, hcl.Diagnostics) { @@ -495,19 +515,19 @@ func decodeFileConfigBlock(fileContent *hcl.BodyContent) (*TestFileConfig, hcl.D return ret, diags } -func decodeTestRunBlock(block *hcl.Block, fileConfig *TestFileConfig) (*TestRun, hcl.Diagnostics) { +func decodeTestRunBlock(block *hcl.Block, file *TestFile) (*TestRun, hcl.Diagnostics) { var diags hcl.Diagnostics content, contentDiags := block.Body.Content(testRunBlockSchema) diags = append(diags, contentDiags...) r := TestRun{ - Overrides: addrs.MakeMap[addrs.Targetable, *Override](), - + Overrides: addrs.MakeMap[addrs.Targetable, *Override](), + File: file, Name: block.Labels[0], NameDeclRange: block.LabelRanges[0], DeclRange: block.DefRange, - Parallel: fileConfig != nil && fileConfig.Parallel, + Parallel: file.Config != nil && file.Config.Parallel, } if !hclsyntax.ValidIdentifier(r.Name) { diff --git a/internal/moduletest/graph/apply.go b/internal/moduletest/graph/apply.go new file mode 100644 index 0000000000..632545bcbf --- /dev/null +++ b/internal/moduletest/graph/apply.go @@ -0,0 +1,196 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package graph + +import ( + "fmt" + "log" + "path/filepath" + + "github.com/hashicorp/terraform/internal/addrs" + "github.com/hashicorp/terraform/internal/lang" + "github.com/hashicorp/terraform/internal/moduletest" + "github.com/hashicorp/terraform/internal/plans" + "github.com/hashicorp/terraform/internal/states" + "github.com/hashicorp/terraform/internal/terraform" + "github.com/hashicorp/terraform/internal/tfdiags" +) + +func (n *NodeTestRun) testApply(ctx *EvalContext, variables terraform.InputValues, waiter *operationWaiter) { + file, run := n.File(), n.run + config := run.ModuleConfig + key := n.run.GetStateKey() + + // FilterVariablesToModule only returns warnings, so we don't check the + // returned diags for errors. + setVariables, testOnlyVariables, setVariableDiags := n.FilterVariablesToModule(variables) + run.Diagnostics = run.Diagnostics.Append(setVariableDiags) + + // ignore diags because validate has covered it + tfCtx, _ := terraform.NewContext(n.opts.ContextOpts) + + // execute the terraform plan operation + _, plan, planDiags := n.plan(ctx, tfCtx, setVariables, waiter) + + // Any error during the planning prevents our apply from + // continuing which is an error. + planDiags = run.ExplainExpectedFailures(planDiags) + run.Diagnostics = run.Diagnostics.Append(planDiags) + if planDiags.HasErrors() { + run.Status = moduletest.Error + return + } + + // Since we're carrying on an executing the apply operation as well, we're + // just going to do some post processing of the diagnostics. We remove the + // warnings generated from check blocks, as the apply operation will either + // reproduce them or fix them and we don't want fixed diagnostics to be + // reported and we don't want duplicates either. + var filteredDiags tfdiags.Diagnostics + for _, diag := range run.Diagnostics { + if rule, ok := addrs.DiagnosticOriginatesFromCheckRule(diag); ok && rule.Container.CheckableKind() == addrs.CheckableCheck { + continue + } + filteredDiags = filteredDiags.Append(diag) + } + run.Diagnostics = filteredDiags + + // execute the apply operation + applyScope, updated, applyDiags := n.apply(tfCtx, plan, moduletest.Running, variables, waiter) + + // Remove expected diagnostics, and add diagnostics in case anything that should have failed didn't. + // We'll also update the run status based on the presence of errors or missing expected failures. + failOrErr := n.checkForMissingExpectedFailures(run, applyDiags) + if failOrErr { + // Even though the apply operation failed, the graph may have done + // partial updates and the returned state should reflect this. + ctx.SetFileState(key, &TestFileState{ + Run: run, + State: updated, + }) + return + } + + n.AddVariablesToConfig(variables) + + if ctx.Verbose() { + schemas, diags := tfCtx.Schemas(config, updated) + + // If we're going to fail to render the plan, let's not fail the overall + // test. It can still have succeeded. So we'll add the diagnostics, but + // still report the test status as a success. + if diags.HasErrors() { + // This is very unlikely. + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Warning, + "Failed to print verbose output", + fmt.Sprintf("Terraform failed to print the verbose output for %s, other diagnostics will contain more details as to why.", filepath.Join(file.Name, run.Name)))) + } else { + run.Verbose = &moduletest.Verbose{ + Plan: nil, // We don't have a plan to show in apply mode. + State: updated, + Config: config, + Providers: schemas.Providers, + Provisioners: schemas.Provisioners, + } + } + + run.Diagnostics = run.Diagnostics.Append(diags) + } + + // Evaluate the run block directly in the graph context to validate the assertions + // of the run. We also pass in all the + // previous contexts so this run block can refer to outputs from + // previous run blocks. + newStatus, outputVals, moreDiags := ctx.EvaluateRun(run, applyScope, testOnlyVariables) + run.Status = newStatus + run.Diagnostics = run.Diagnostics.Append(moreDiags) + + // Now we've successfully validated this run block, lets add it into + // our prior run outputs so future run blocks can access it. + ctx.SetOutput(run, outputVals) + + // Only update the most recent run and state if the state was + // actually updated by this change. We want to use the run that + // most recently updated the tracked state as the cleanup + // configuration. + ctx.SetFileState(key, &TestFileState{ + Run: run, + State: updated, + }) +} + +func (n *NodeTestRun) apply(tfCtx *terraform.Context, plan *plans.Plan, progress moduletest.Progress, variables terraform.InputValues, waiter *operationWaiter) (*lang.Scope, *states.State, tfdiags.Diagnostics) { + run := n.run + file := n.File() + log.Printf("[TRACE] TestFileRunner: called apply for %s/%s", file.Name, run.Name) + + var diags tfdiags.Diagnostics + config := run.ModuleConfig + + // If things get cancelled while we are executing the apply operation below + // we want to print out all the objects that we were creating so the user + // can verify we managed to tidy everything up possibly. + // + // Unfortunately, this creates a race condition as the apply operation can + // edit the plan (by removing changes once they are applied) while at the + // same time our cancellation process will try to read the plan. + // + // We take a quick copy of the changes we care about here, which will then + // be used in place of the plan when we print out the objects to be created + // as part of the cancellation process. + var created []*plans.ResourceInstanceChangeSrc + for _, change := range plan.Changes.Resources { + if change.Action != plans.Create { + continue + } + created = append(created, change) + } + + // We only need to pass ephemeral variables to the apply operation, as the + // plan has already been evaluated with the full set of variables. + ephemeralVariables := make(terraform.InputValues) + for k, v := range config.Root.Module.Variables { + if v.EphemeralSet { + if value, ok := variables[k]; ok { + ephemeralVariables[k] = value + } + } + } + + applyOpts := &terraform.ApplyOpts{ + SetVariables: ephemeralVariables, + } + + waiter.update(tfCtx, progress, created) + log.Printf("[DEBUG] TestFileRunner: starting apply for %s/%s", file.Name, run.Name) + updated, newScope, applyDiags := tfCtx.ApplyAndEval(plan, config, applyOpts) + log.Printf("[DEBUG] TestFileRunner: completed apply for %s/%s", file.Name, run.Name) + diags = diags.Append(applyDiags) + + return newScope, updated, diags +} + +// checkForMissingExpectedFailures checks for missing expected failures in the diagnostics. +// It updates the run status based on the presence of errors or missing expected failures. +func (n *NodeTestRun) checkForMissingExpectedFailures(run *moduletest.Run, diags tfdiags.Diagnostics) (failOrErr bool) { + // Retrieve and append diagnostics that are either unrelated to expected failures + // or report missing expected failures. + unexpectedDiags := run.ValidateExpectedFailures(diags) + run.Diagnostics = run.Diagnostics.Append(unexpectedDiags) + for _, diag := range unexpectedDiags { + // // If any diagnostic indicates a missing expected failure, set the run status to fail. + if ok := moduletest.DiagnosticFromMissingExpectedFailure(diag); ok { + run.Status = run.Status.Merge(moduletest.Fail) + continue + } + + // upgrade the run status to error if there still are other errors in the diagnostics + if diag.Severity() == tfdiags.Error { + run.Status = run.Status.Merge(moduletest.Error) + break + } + } + return run.Status > moduletest.Pass +} diff --git a/internal/moduletest/graph/eval_context.go b/internal/moduletest/graph/eval_context.go index 26dd8cf999..1e5a4a6bbb 100644 --- a/internal/moduletest/graph/eval_context.go +++ b/internal/moduletest/graph/eval_context.go @@ -15,6 +15,7 @@ import ( "github.com/zclconf/go-cty/cty/convert" "github.com/hashicorp/terraform/internal/addrs" + "github.com/hashicorp/terraform/internal/command/views" "github.com/hashicorp/terraform/internal/configs" "github.com/hashicorp/terraform/internal/didyoumean" "github.com/hashicorp/terraform/internal/lang" @@ -60,18 +61,33 @@ type EvalContext struct { FileStates map[string]*TestFileState stateLock sync.Mutex - // cancelContext is a context that can be used to terminate the evaluation of the - // test suite. - // cancelFunc is the conrresponding cancel function that should be called to - // cancel the context. + // cancelContext and stopContext can be used to terminate the evaluation of the + // test suite when a cancellation or stop signal is received. + // cancelFunc and stopFunc are the corresponding functions to call to signal + // the termination. cancelContext context.Context cancelFunc context.CancelFunc + stopContext context.Context + stopFunc context.CancelFunc + + renderer views.Test + verbose bool +} + +type EvalContextOpts struct { + Verbose bool + Render views.Test + CancelCtx context.Context + StopCtx context.Context } // NewEvalContext constructs a new graph evaluation context for use in // evaluating the runs within a test suite. -func NewEvalContext(cancelCtx context.Context) *EvalContext { - cancelCtx, cancel := context.WithCancel(cancelCtx) +// The context is initialized with the provided cancel and stop contexts, and +// these contexts can be used from external commands to signal the termination of the test suite. +func NewEvalContext(opts *EvalContextOpts) *EvalContext { + cancelCtx, cancel := context.WithCancel(opts.CancelCtx) + stopCtx, stop := context.WithCancel(opts.StopCtx) return &EvalContext{ runOutputs: make(map[addrs.Run]cty.Value), outputsLock: sync.Mutex{}, @@ -82,11 +98,20 @@ func NewEvalContext(cancelCtx context.Context) *EvalContext { VariableCaches: hcltest.NewVariableCaches(), cancelContext: cancelCtx, cancelFunc: cancel, + stopContext: stopCtx, + stopFunc: stop, + verbose: opts.Verbose, + renderer: opts.Render, } } -// Cancel cancels the context, which signals to the test suite that it should -// stop evaluating the test suite. +// Renderer returns the renderer for the test suite. +func (ec *EvalContext) Renderer() views.Test { + return ec.renderer +} + +// Cancel signals to the runs in the test suite that they should stop evaluating +// the test suite, and return immediately. func (ec *EvalContext) Cancel() { ec.cancelFunc() } @@ -97,6 +122,21 @@ func (ec *EvalContext) Cancelled() bool { return ec.cancelContext.Err() != nil } +// Stop signals to the runs in the test suite that they should stop evaluating +// the test suite, and just skip. +func (ec *EvalContext) Stop() { + ec.stopFunc() +} + +func (ec *EvalContext) Stopped() bool { + return ec.stopContext.Err() != nil +} + +// Verbose returns true if the context is in verbose mode. +func (ec *EvalContext) Verbose() bool { + return ec.verbose +} + // EvaluateRun processes the assertions inside the provided configs.TestRun against // the run results, returning a status, an object value representing the output // values from the module under test, and diagnostics describing any problems. diff --git a/internal/moduletest/graph/eval_context_test.go b/internal/moduletest/graph/eval_context_test.go index ec7627fcbe..7ed041f7f3 100644 --- a/internal/moduletest/graph/eval_context_test.go +++ b/internal/moduletest/graph/eval_context_test.go @@ -735,7 +735,10 @@ func TestEvalContext_Evaluate(t *testing.T) { priorOutputs[addrs.Run{Name: name}] = val } - testCtx := NewEvalContext(context.Background()) + testCtx := NewEvalContext(&EvalContextOpts{ + CancelCtx: context.Background(), + StopCtx: context.Background(), + }) testCtx.runOutputs = priorOutputs gotStatus, gotOutputs, diags := testCtx.EvaluateRun(run, planScope, test.testOnlyVars) diff --git a/internal/moduletest/graph/node_state_cleanup.go b/internal/moduletest/graph/node_state_cleanup.go new file mode 100644 index 0000000000..35f3b2921b --- /dev/null +++ b/internal/moduletest/graph/node_state_cleanup.go @@ -0,0 +1,150 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package graph + +import ( + "fmt" + "log" + "time" + + "github.com/hashicorp/terraform/internal/addrs" + "github.com/hashicorp/terraform/internal/moduletest" + "github.com/hashicorp/terraform/internal/moduletest/mocking" + "github.com/hashicorp/terraform/internal/plans" + "github.com/hashicorp/terraform/internal/states" + "github.com/hashicorp/terraform/internal/terraform" + "github.com/hashicorp/terraform/internal/tfdiags" +) + +var ( + _ GraphNodeExecutable = (*NodeStateCleanup)(nil) +) + +type NodeStateCleanup struct { + stateKey string + opts *graphOptions +} + +func (n *NodeStateCleanup) Name() string { + return fmt.Sprintf("cleanup.%s", n.stateKey) +} + +// Execute destroys the resources created in the state file. +func (n *NodeStateCleanup) Execute(evalCtx *EvalContext) tfdiags.Diagnostics { + var diags tfdiags.Diagnostics + file := n.opts.File + state := evalCtx.GetFileState(n.stateKey) + log.Printf("[TRACE] TestStateManager: cleaning up state for %s", file.Name) + + if evalCtx.Cancelled() { + // Don't try and clean anything up if the execution has been cancelled. + log.Printf("[DEBUG] TestStateManager: skipping state cleanup for %s due to cancellation", file.Name) + return diags + } + + empty := true + for _, module := range state.State.Modules { + for _, resource := range module.Resources { + if resource.Addr.Resource.Mode == addrs.ManagedResourceMode { + empty = false + break + } + } + } + + if empty { + // The state can be empty for a run block that just executed a plan + // command, or a run block that only read data sources. We'll just + // skip empty run blocks. + return diags + } + + if state.Run == nil { + log.Printf("[ERROR] TestFileRunner: found inconsistent run block and state file in %s for module %s", file.Name, n.stateKey) + + // The state can have a nil run block if it only executed a plan + // command. In which case, we shouldn't have reached here as the + // state should also have been empty and this will have been skipped + // above. If we do reach here, then something has gone badly wrong + // and we can't really recover from it. + + diags = diags.Append(tfdiags.Sourceless(tfdiags.Error, "Inconsistent state", fmt.Sprintf("Found inconsistent state while cleaning up %s. This is a bug in Terraform - please report it", file.Name))) + file.UpdateStatus(moduletest.Error) + evalCtx.Renderer().DestroySummary(diags, nil, file, state.State) + return diags + } + TransformConfigForRun(evalCtx, state.Run, file) + + runNode := &NodeTestRun{run: state.Run, opts: n.opts} + updated := state.State + startTime := time.Now().UTC() + waiter := NewOperationWaiter(nil, evalCtx, runNode, moduletest.Running, startTime.UnixMilli()) + var destroyDiags tfdiags.Diagnostics + cancelled := waiter.Run(func() { + updated, destroyDiags = n.destroy(evalCtx, runNode, waiter) + diags = diags.Append(destroyDiags) + }) + if cancelled { + diags = diags.Append(tfdiags.Sourceless(tfdiags.Error, "Test interrupted", "The test operation could not be completed due to an interrupt signal. Please read the remaining diagnostics carefully for any sign of failed state cleanup or dangling resources.")) + } + + if !updated.Empty() { + // Then we failed to adequately clean up the state, so mark success + // as false. + file.UpdateStatus(moduletest.Error) + } + evalCtx.Renderer().DestroySummary(diags, state.Run, file, updated) + return diags +} + +func (n *NodeStateCleanup) destroy(ctx *EvalContext, runNode *NodeTestRun, waiter *operationWaiter) (*states.State, tfdiags.Diagnostics) { + file := n.opts.File + fileState := ctx.GetFileState(n.stateKey) + state := fileState.State + run := runNode.run + log.Printf("[TRACE] TestFileRunner: called destroy for %s/%s", file.Name, run.Name) + + if state.Empty() { + // Nothing to do! + return state, nil + } + + var diags tfdiags.Diagnostics + variables, variableDiags := runNode.GetVariables(ctx, false) + diags = diags.Append(variableDiags) + + if diags.HasErrors() { + return state, diags + } + + // During the destroy operation, we don't add warnings from this operation. + // Anything that would have been reported here was already reported during + // the original plan, and a successful destroy operation is the only thing + // we care about. + setVariables, _, _ := runNode.FilterVariablesToModule(variables) + + planOpts := &terraform.PlanOpts{ + Mode: plans.DestroyMode, + SetVariables: setVariables, + Overrides: mocking.PackageOverrides(run.Config, file.Config, run.ModuleConfig), + } + + tfCtx, ctxDiags := terraform.NewContext(n.opts.ContextOpts) + diags = diags.Append(ctxDiags) + if ctxDiags.HasErrors() { + return state, diags + } + ctx.Renderer().Run(run, file, moduletest.TearDown, 0) + + waiter.update(tfCtx, moduletest.TearDown, nil) + plan, planDiags := tfCtx.Plan(run.ModuleConfig, state, planOpts) + diags = diags.Append(planDiags) + if diags.HasErrors() { + return state, diags + } + + _, updated, applyDiags := runNode.apply(tfCtx, plan, moduletest.TearDown, variables, waiter) + diags = diags.Append(applyDiags) + return updated, diags +} diff --git a/internal/moduletest/graph/node_test_run.go b/internal/moduletest/graph/node_test_run.go index fbc72cffe1..124d743bc3 100644 --- a/internal/moduletest/graph/node_test_run.go +++ b/internal/moduletest/graph/node_test_run.go @@ -5,19 +5,24 @@ package graph import ( "fmt" + "log" + "time" + "github.com/hashicorp/terraform/internal/addrs" + "github.com/hashicorp/terraform/internal/configs" + "github.com/hashicorp/terraform/internal/logging" "github.com/hashicorp/terraform/internal/moduletest" + "github.com/hashicorp/terraform/internal/terraform" "github.com/hashicorp/terraform/internal/tfdiags" ) -var _ GraphNodeExecutable = (*NodeTestRun)(nil) +var ( + _ GraphNodeExecutable = (*NodeTestRun)(nil) +) type NodeTestRun struct { - file *moduletest.File run *moduletest.Run - - // requiredProviders is a map of provider names that the test run depends on. - requiredProviders map[string]bool + opts *graphOptions } func (n *NodeTestRun) Run() *moduletest.Run { @@ -25,18 +30,126 @@ func (n *NodeTestRun) Run() *moduletest.Run { } func (n *NodeTestRun) File() *moduletest.File { - return n.file + return n.opts.File } func (n *NodeTestRun) Name() string { - return fmt.Sprintf("%s.%s", n.file.Name, n.run.Name) + return fmt.Sprintf("%s.%s", n.opts.File.Name, n.run.Name) } -// Execute adds the providers required by the test run to the context. -// TODO: Eventually, we should move all the logic related to a test run into this method, -// effectively ensuring that the Execute method is enough to execute a test run in the graph. -func (n *NodeTestRun) Execute(ctx *EvalContext) tfdiags.Diagnostics { +func (n *NodeTestRun) References() []*addrs.Reference { + references, _ := n.run.GetReferences() + return references +} + +// Execute executes the test run block and update the status of the run block +// based on the result of the execution. +func (n *NodeTestRun) Execute(evalCtx *EvalContext) tfdiags.Diagnostics { + log.Printf("[TRACE] TestFileRunner: executing run block %s/%s", n.File().Name, n.run.Name) + startTime := time.Now().UTC() var diags tfdiags.Diagnostics - ctx.SetProviders(n.run, n.requiredProviders) + file, run := n.File(), n.run + + // At the end of the function, we'll update the status of the file based on + // the status of the run block, and render the run summary. + defer func() { + evalCtx.Renderer().Run(run, file, moduletest.Complete, 0) + file.UpdateStatus(run.Status) + }() + + if file.GetStatus() == moduletest.Error { + // If the overall test file has errored, we don't keep trying to + // execute tests. Instead, we mark all remaining run blocks as + // skipped, print the status, and move on. + run.Status = moduletest.Skip + return diags + } + if evalCtx.Cancelled() { + // A cancellation signal has been received. + // Don't do anything, just give up and return immediately. + // The surrounding functions should stop this even being called, but in + // case of race conditions or something we can still verify this. + return diags + } + + if evalCtx.Stopped() { + // Then the test was requested to be stopped, so we just mark each + // following test as skipped, print the status, and move on. + run.Status = moduletest.Skip + return diags + } + + // Create a waiter which handles waiting for terraform operations to complete. + // While waiting, the wait will also respond to cancellation signals, and + // handle them appropriately. + // The test progress is updated periodically, and the progress status + // depends on the async operation being waited on. + // Before the terraform operation is started, the operation updates the + // waiter with the cleanup context on cancellation, as well as the + // progress status. + waiter := NewOperationWaiter(nil, evalCtx, n, moduletest.Running, startTime.UnixMilli()) + cancelled := waiter.Run(func() { + defer logging.PanicHandler() + n.execute(evalCtx, waiter) + }) + + if cancelled { + diags = diags.Append(tfdiags.Sourceless(tfdiags.Error, "Test interrupted", "The test operation could not be completed due to an interrupt signal. Please read the remaining diagnostics carefully for any sign of failed state cleanup or dangling resources.")) + } + + // If we got far enough to actually attempt to execute the run then + // we'll give the view some additional metadata about the execution. + n.run.ExecutionMeta = &moduletest.RunExecutionMeta{ + Start: startTime, + Duration: time.Since(startTime), + } return diags } + +func (n *NodeTestRun) execute(ctx *EvalContext, waiter *operationWaiter) { + file, run := n.File(), n.run + ctx.Renderer().Run(run, file, moduletest.Starting, 0) + if run.Config.ConfigUnderTest != nil && run.GetStateKey() == moduletest.MainStateIdentifier { + // This is bad, and should not happen because the state key is derived from the custom module source. + panic(fmt.Sprintf("TestFileRunner: custom module %s has the same key as main state", file.Name)) + } + + n.testValidate(ctx, waiter) + if run.Diagnostics.HasErrors() { + return + } + + variables, variableDiags := n.GetVariables(ctx, true) + run.Diagnostics = run.Diagnostics.Append(variableDiags) + if variableDiags.HasErrors() { + run.Status = moduletest.Error + return + } + + if run.Config.Command == configs.PlanTestCommand { + n.testPlan(ctx, variables, waiter) + } else { + n.testApply(ctx, variables, waiter) + } +} + +// Validating the module config which the run acts on +func (n *NodeTestRun) testValidate(ctx *EvalContext, waiter *operationWaiter) { + run := n.run + file := n.File() + config := run.ModuleConfig + + log.Printf("[TRACE] TestFileRunner: called validate for %s/%s", file.Name, run.Name) + TransformConfigForRun(ctx, run, file) + tfCtx, ctxDiags := terraform.NewContext(n.opts.ContextOpts) + if ctxDiags.HasErrors() { + return + } + waiter.update(tfCtx, moduletest.Running, nil) + validateDiags := tfCtx.Validate(config, nil) + run.Diagnostics = run.Diagnostics.Append(validateDiags) + if validateDiags.HasErrors() { + run.Status = moduletest.Error + return + } +} diff --git a/internal/moduletest/graph/plan.go b/internal/moduletest/graph/plan.go new file mode 100644 index 0000000000..f803d84999 --- /dev/null +++ b/internal/moduletest/graph/plan.go @@ -0,0 +1,125 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package graph + +import ( + "fmt" + "log" + "path/filepath" + + "github.com/hashicorp/terraform/internal/configs" + "github.com/hashicorp/terraform/internal/lang" + "github.com/hashicorp/terraform/internal/moduletest" + "github.com/hashicorp/terraform/internal/moduletest/mocking" + "github.com/hashicorp/terraform/internal/plans" + "github.com/hashicorp/terraform/internal/terraform" + "github.com/hashicorp/terraform/internal/tfdiags" +) + +func (n *NodeTestRun) testPlan(ctx *EvalContext, variables terraform.InputValues, waiter *operationWaiter) { + file, run := n.File(), n.run + config := run.ModuleConfig + + // FilterVariablesToModule only returns warnings, so we don't check the + // returned diags for errors. + setVariables, testOnlyVariables, setVariableDiags := n.FilterVariablesToModule(variables) + run.Diagnostics = run.Diagnostics.Append(setVariableDiags) + + // ignore diags because validate has covered it + tfCtx, _ := terraform.NewContext(n.opts.ContextOpts) + + // execute the terraform plan operation + planScope, plan, planDiags := n.plan(ctx, tfCtx, setVariables, waiter) + // We exclude the diagnostics that are expected to fail from the plan + // diagnostics, and if an expected failure is not found, we add a new error diagnostic. + planDiags = run.ValidateExpectedFailures(planDiags) + run.Diagnostics = run.Diagnostics.Append(planDiags) + if planDiags.HasErrors() { + run.Status = moduletest.Error + return + } + + n.AddVariablesToConfig(variables) + + if ctx.Verbose() { + schemas, diags := tfCtx.Schemas(config, plan.PriorState) + + // If we're going to fail to render the plan, let's not fail the overall + // test. It can still have succeeded. So we'll add the diagnostics, but + // still report the test status as a success. + if diags.HasErrors() { + // This is very unlikely. + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Warning, + "Failed to print verbose output", + fmt.Sprintf("Terraform failed to print the verbose output for %s, other diagnostics will contain more details as to why.", filepath.Join(file.Name, run.Name)))) + } else { + run.Verbose = &moduletest.Verbose{ + Plan: plan, + State: nil, // We don't have a state to show in plan mode. + Config: config, + Providers: schemas.Providers, + Provisioners: schemas.Provisioners, + } + } + + run.Diagnostics = run.Diagnostics.Append(diags) + } + + // Evaluate the run block directly in the graph context to validate the assertions + // of the run. We also pass in all the + // previous contexts so this run block can refer to outputs from + // previous run blocks. + newStatus, outputVals, moreDiags := ctx.EvaluateRun(run, planScope, testOnlyVariables) + run.Status = newStatus + run.Diagnostics = run.Diagnostics.Append(moreDiags) + + // Now we've successfully validated this run block, lets add it into + // our prior run outputs so future run blocks can access it. + ctx.SetOutput(run, outputVals) +} + +func (n *NodeTestRun) plan(ctx *EvalContext, tfCtx *terraform.Context, variables terraform.InputValues, waiter *operationWaiter) (*lang.Scope, *plans.Plan, tfdiags.Diagnostics) { + file, run := n.File(), n.run + config := run.ModuleConfig + log.Printf("[TRACE] TestFileRunner: called plan for %s/%s", file.Name, run.Name) + + var diags tfdiags.Diagnostics + + targets, targetDiags := run.GetTargets() + diags = diags.Append(targetDiags) + + replaces, replaceDiags := run.GetReplaces() + diags = diags.Append(replaceDiags) + + if diags.HasErrors() { + return nil, nil, diags + } + + planOpts := &terraform.PlanOpts{ + Mode: func() plans.Mode { + switch run.Config.Options.Mode { + case configs.RefreshOnlyTestMode: + return plans.RefreshOnlyMode + default: + return plans.NormalMode + } + }(), + Targets: targets, + ForceReplace: replaces, + SkipRefresh: !run.Config.Options.Refresh, + SetVariables: variables, + ExternalReferences: n.References(), + Overrides: mocking.PackageOverrides(run.Config, file.Config, config), + } + + waiter.update(tfCtx, moduletest.Running, nil) + log.Printf("[DEBUG] TestFileRunner: starting plan for %s/%s", file.Name, run.Name) + state := ctx.GetFileState(run.GetStateKey()).State + plan, planScope, planDiags := tfCtx.PlanAndEval(config, state, planOpts) + log.Printf("[DEBUG] TestFileRunner: completed plan for %s/%s", file.Name, run.Name) + diags = diags.Append(planDiags) + + return planScope, plan, diags +} diff --git a/internal/moduletest/graph/test_graph_builder.go b/internal/moduletest/graph/test_graph_builder.go index ac08a54a0a..1ee7a697a1 100644 --- a/internal/moduletest/graph/test_graph_builder.go +++ b/internal/moduletest/graph/test_graph_builder.go @@ -17,8 +17,15 @@ import ( // a terraform test file. The file may contain multiple runs, and each run may have // dependencies on other runs. type TestGraphBuilder struct { - File *moduletest.File - GlobalVars map[string]backendrun.UnparsedVariableValue + File *moduletest.File + GlobalVars map[string]backendrun.UnparsedVariableValue + ContextOpts *terraform.ContextOpts +} + +type graphOptions struct { + File *moduletest.File + GlobalVars map[string]backendrun.UnparsedVariableValue + ContextOpts *terraform.ContextOpts } // See GraphBuilder @@ -32,9 +39,16 @@ func (b *TestGraphBuilder) Build() (*terraform.Graph, tfdiags.Diagnostics) { // See GraphBuilder func (b *TestGraphBuilder) Steps() []terraform.GraphTransformer { + opts := &graphOptions{ + File: b.File, + GlobalVars: b.GlobalVars, + ContextOpts: b.ContextOpts, + } steps := []terraform.GraphTransformer{ - &TestRunTransformer{File: b.File, globalVars: b.GlobalVars}, - &TestConfigTransformer{}, + &TestRunTransformer{opts}, + &TestConfigTransformer{File: b.File}, + &TestStateCleanupTransformer{opts}, + terraform.DynamicTransformer(validateRunConfigs), &TestProvidersTransformer{}, &CloseTestGraphTransformer{}, &terraform.TransitiveReductionTransformer{}, @@ -42,3 +56,26 @@ func (b *TestGraphBuilder) Steps() []terraform.GraphTransformer { return steps } + +func validateRunConfigs(g *terraform.Graph) error { + for _, v := range g.Vertices() { + if node, ok := v.(*NodeTestRun); ok { + diags := node.run.Config.Validate(node.run.ModuleConfig) + node.run.Diagnostics = node.run.Diagnostics.Append(diags) + if diags.HasErrors() { + node.run.Status = moduletest.Error + } + } + } + return nil +} + +// dynamicNode is a helper node which can be added to the graph to execute +// a dynamic function at some desired point in the graph. +type dynamicNode struct { + eval func(*EvalContext) tfdiags.Diagnostics +} + +func (n *dynamicNode) Execute(evalCtx *EvalContext) tfdiags.Diagnostics { + return n.eval(evalCtx) +} diff --git a/internal/moduletest/graph/transform_config.go b/internal/moduletest/graph/transform_config.go index b0938cbda5..317c670452 100644 --- a/internal/moduletest/graph/transform_config.go +++ b/internal/moduletest/graph/transform_config.go @@ -16,75 +16,73 @@ import ( "github.com/hashicorp/terraform/internal/tfdiags" ) +type GraphNodeExecutable interface { + Execute(ctx *EvalContext) tfdiags.Diagnostics +} + +// TestFileState is a helper struct that just maps a run block to the state that +// was produced by the execution of that run block. +type TestFileState struct { + Run *moduletest.Run + State *states.State +} + // TestConfigTransformer is a GraphTransformer that adds all the test runs, // and the variables defined in each run block, to the graph. -type TestConfigTransformer struct{} +type TestConfigTransformer struct { + File *moduletest.File +} func (t *TestConfigTransformer) Transform(g *terraform.Graph) error { // This map tracks the state of each run in the file. If multiple runs // have the same state key, they will share the same state. statesMap := make(map[string]*TestFileState) + + // a root config node that will add the file states to the context + rootConfigNode := t.addRootConfigNode(g, statesMap) + for _, v := range g.Vertices() { node, ok := v.(*NodeTestRun) if !ok { continue } - if _, exists := statesMap[node.run.GetStateKey()]; !exists { - statesMap[node.run.GetStateKey()] = &TestFileState{ + key := node.run.GetStateKey() + if _, exists := statesMap[key]; !exists { + state := &TestFileState{ Run: nil, State: states.NewState(), } + statesMap[key] = state } - } - cfgNode := &nodeConfig{configMap: statesMap} - g.Add(cfgNode) - // Connect all the test runs to the config node, so that the config node - // is executed before any of the test runs. - for _, v := range g.Vertices() { - node, ok := v.(*NodeTestRun) - if !ok { - continue - } - g.Connect(dag.BasicEdge(node, cfgNode)) + // Connect all the test runs to the config node, so that the config node + // is executed before any of the test runs. + g.Connect(dag.BasicEdge(node, rootConfigNode)) } return nil } -type nodeConfig struct { - configMap map[string]*TestFileState -} - -func (n *nodeConfig) Name() string { - return "nodeConfig" -} - -type GraphNodeExecutable interface { - Execute(ctx *EvalContext) tfdiags.Diagnostics -} - -func (n *nodeConfig) Execute(ctx *EvalContext) tfdiags.Diagnostics { - var diags tfdiags.Diagnostics - ctx.FileStates = n.configMap - return diags -} - -// TestFileState is a helper struct that just maps a run block to the state that -// was produced by the execution of that run block. -type TestFileState struct { - Run *moduletest.Run - State *states.State +func (t *TestConfigTransformer) addRootConfigNode(g *terraform.Graph, statesMap map[string]*TestFileState) *dynamicNode { + rootConfigNode := &dynamicNode{ + eval: func(ctx *EvalContext) tfdiags.Diagnostics { + var diags tfdiags.Diagnostics + ctx.FileStates = statesMap + return diags + }, + } + g.Add(rootConfigNode) + return rootConfigNode } -// TransformConfigForTest transforms the provided configuration ready for the -// test execution specified by the provided run block and test file. +// TransformConfigForRun transforms the run's module configuration to include +// the providers and variables from its block and the test file. // // In practice, this actually just means performing some surgery on the // available providers. We want to copy the relevant providers from the test // file into the configuration. We also want to process the providers so they // use variables from the file instead of variables from within the test file. -func TransformConfigForTest(ctx *EvalContext, run *moduletest.Run, file *moduletest.File) hcl.Diagnostics { +func TransformConfigForRun(ctx *EvalContext, run *moduletest.Run, file *moduletest.File) hcl.Diagnostics { var diags hcl.Diagnostics // Currently, we only need to override the provider settings. diff --git a/internal/moduletest/graph/transform_config_test.go b/internal/moduletest/graph/transform_config_test.go index 41c11a24d1..7db4763033 100644 --- a/internal/moduletest/graph/transform_config_test.go +++ b/internal/moduletest/graph/transform_config_test.go @@ -219,12 +219,15 @@ func TestTransformForTest(t *testing.T) { availableProviders[provider] = true } - ctx := NewEvalContext(context.Background()) + ctx := NewEvalContext(&EvalContextOpts{ + CancelCtx: context.Background(), + StopCtx: context.Background(), + }) ctx.configProviders = map[string]map[string]bool{ run.GetModuleConfigID(): availableProviders, } - diags := TransformConfigForTest(ctx, run, file) + diags := TransformConfigForRun(ctx, run, file) var actualErrs []string for _, err := range diags.Errs() { diff --git a/internal/moduletest/graph/transform_providers.go b/internal/moduletest/graph/transform_providers.go index 1c2289c839..793b647f42 100644 --- a/internal/moduletest/graph/transform_providers.go +++ b/internal/moduletest/graph/transform_providers.go @@ -4,11 +4,10 @@ package graph import ( - "errors" - "fmt" - "github.com/hashicorp/terraform/internal/configs" + "github.com/hashicorp/terraform/internal/dag" "github.com/hashicorp/terraform/internal/terraform" + "github.com/hashicorp/terraform/internal/tfdiags" ) // TestProvidersTransformer is a GraphTransformer that gathers all the providers @@ -17,8 +16,11 @@ import ( type TestProvidersTransformer struct{} func (t *TestProvidersTransformer) Transform(g *terraform.Graph) error { - var errs []error configsProviderMap := make(map[string]map[string]bool) + runProviderMap := make(map[*NodeTestRun]map[string]bool) + + // a root provider node that will add the providers to the context + rootProviderNode := t.createRootNode(g, runProviderMap) for _, v := range g.Vertices() { node, ok := v.(*NodeTestRun) @@ -32,18 +34,13 @@ func (t *TestProvidersTransformer) Transform(g *terraform.Graph) error { providers := t.transformSingleConfig(node.run.ModuleConfig) configsProviderMap[configKey] = providers } + runProviderMap[node] = configsProviderMap[configKey] - providers, ok := configsProviderMap[configKey] - if !ok { - // This should not happen - errs = append(errs, fmt.Errorf("missing providers for module config %q", configKey)) - continue - } - - // Add the required providers for the test run node - node.requiredProviders = providers + // Add an edge from the test run node to the root provider node + g.Connect(dag.BasicEdge(v, rootProviderNode)) } - return errors.Join(errs...) + + return nil } func (t *TestProvidersTransformer) transformSingleConfig(config *configs.Config) map[string]bool { @@ -88,3 +85,16 @@ func (t *TestProvidersTransformer) transformSingleConfig(config *configs.Config) return providers } + +func (t *TestProvidersTransformer) createRootNode(g *terraform.Graph, providerMap map[*NodeTestRun]map[string]bool) *dynamicNode { + node := &dynamicNode{ + eval: func(ctx *EvalContext) tfdiags.Diagnostics { + for node, providers := range providerMap { + ctx.SetProviders(node.run, providers) + } + return nil + }, + } + g.Add(node) + return node +} diff --git a/internal/moduletest/graph/transform_state_cleanup.go b/internal/moduletest/graph/transform_state_cleanup.go new file mode 100644 index 0000000000..12cbf14dc8 --- /dev/null +++ b/internal/moduletest/graph/transform_state_cleanup.go @@ -0,0 +1,83 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package graph + +import ( + "slices" + + "github.com/hashicorp/terraform/internal/dag" + "github.com/hashicorp/terraform/internal/moduletest" + "github.com/hashicorp/terraform/internal/terraform" + "github.com/hashicorp/terraform/internal/tfdiags" +) + +// TestStateCleanupTransformer is a GraphTransformer that adds a cleanup node +// for each state that is created by the test runs. +type TestStateCleanupTransformer struct { + opts *graphOptions +} + +func (t *TestStateCleanupTransformer) Transform(g *terraform.Graph) error { + cleanupMap := make(map[string]*NodeStateCleanup) + + for _, v := range g.Vertices() { + node, ok := v.(*NodeTestRun) + if !ok { + continue + } + key := node.run.GetStateKey() + if _, exists := cleanupMap[key]; !exists { + cleanupMap[key] = &NodeStateCleanup{stateKey: key, opts: t.opts} + g.Add(cleanupMap[key]) + } + + // Connect the cleanup node to the test run node. + g.Connect(dag.BasicEdge(cleanupMap[key], node)) + } + + // Add a root cleanup node that runs before cleanup nodes for each state. + // Right now it just simply renders a teardown summary, so as to maintain + // existing CLI output. + rootCleanupNode := t.addRootCleanupNode(g) + + for _, v := range g.Vertices() { + switch node := v.(type) { + case *NodeTestRun: + // All the runs that share the same state, must share the same cleanup node, + // which only executes once after all the dependent runs have completed. + g.Connect(dag.BasicEdge(rootCleanupNode, node)) + } + } + + // connect all cleanup nodes in reverse-sequential order of run index to + // preserve existing behavior, starting from the root cleanup node. + // TODO: Parallelize cleanup nodes execution instead of sequential. + added := make(map[string]bool) + var prev dag.Vertex + for _, v := range slices.Backward(t.opts.File.Runs) { + key := v.GetStateKey() + if _, exists := added[key]; !exists { + node := cleanupMap[key] + if prev != nil { + g.Connect(dag.BasicEdge(node, prev)) + } + prev = node + added[key] = true + } + } + + return nil +} + +func (t *TestStateCleanupTransformer) addRootCleanupNode(g *terraform.Graph) *dynamicNode { + rootCleanupNode := &dynamicNode{ + eval: func(ctx *EvalContext) tfdiags.Diagnostics { + var diags tfdiags.Diagnostics + ctx.Renderer().File(t.opts.File, moduletest.TearDown) + return diags + }, + } + g.Add(rootCleanupNode) + return rootCleanupNode +} diff --git a/internal/moduletest/graph/transform_test_run.go b/internal/moduletest/graph/transform_test_run.go index 3dfb904e64..9e652aacec 100644 --- a/internal/moduletest/graph/transform_test_run.go +++ b/internal/moduletest/graph/transform_test_run.go @@ -8,7 +8,6 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/terraform/internal/addrs" - "github.com/hashicorp/terraform/internal/backend/backendrun" "github.com/hashicorp/terraform/internal/dag" "github.com/hashicorp/terraform/internal/moduletest" "github.com/hashicorp/terraform/internal/terraform" @@ -18,15 +17,14 @@ import ( // TestRunTransformer is a GraphTransformer that adds all the test runs, // and the variables defined in each run block, to the graph. type TestRunTransformer struct { - File *moduletest.File - globalVars map[string]backendrun.UnparsedVariableValue + opts *graphOptions } func (t *TestRunTransformer) Transform(g *terraform.Graph) error { // Create and add nodes for each run var nodes []*NodeTestRun - for _, run := range t.File.Runs { - node := &NodeTestRun{run: run, file: t.File} + for _, run := range t.opts.File.Runs { + node := &NodeTestRun{run: run, opts: t.opts} g.Add(node) nodes = append(nodes, node) } @@ -141,14 +139,14 @@ func (t *TestRunTransformer) connectSameStateRuns(g *terraform.Graph, nodes []*N func (t *TestRunTransformer) getVariableNames(run *moduletest.Run) map[string]struct{} { set := make(map[string]struct{}) - for name := range t.globalVars { + for name := range t.opts.GlobalVars { set[name] = struct{}{} } for name := range run.Config.Variables { set[name] = struct{}{} } - for name := range t.File.Config.Variables { + for name := range t.opts.File.Config.Variables { set[name] = struct{}{} } for name := range run.ModuleConfig.Module.Variables { diff --git a/internal/moduletest/graph/variables.go b/internal/moduletest/graph/variables.go new file mode 100644 index 0000000000..0c19524683 --- /dev/null +++ b/internal/moduletest/graph/variables.go @@ -0,0 +1,239 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package graph + +import ( + "fmt" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/terraform/internal/addrs" + "github.com/hashicorp/terraform/internal/configs" + "github.com/hashicorp/terraform/internal/lang/langrefs" + hcltest "github.com/hashicorp/terraform/internal/moduletest/hcl" + "github.com/hashicorp/terraform/internal/terraform" + "github.com/hashicorp/terraform/internal/tfdiags" + "github.com/zclconf/go-cty/cty" +) + +// GetVariables builds the terraform.InputValues required for the provided run +// block. It pulls the relevant variables (ie. the variables needed for the +// run block) from the total pool of all available variables, and converts them +// into input values. +// +// As a run block can reference variables defined within the file and are not +// actually defined within the configuration, this function actually returns +// more variables than are required by the config. FilterVariablesToConfig +// should be called before trying to use these variables within a Terraform +// plan, apply, or destroy operation. +func (n *NodeTestRun) GetVariables(ctx *EvalContext, includeWarnings bool) (terraform.InputValues, tfdiags.Diagnostics) { + var diags tfdiags.Diagnostics + run := n.run + // relevantVariables contains the variables that are of interest to this + // run block. This is a combination of the variables declared within the + // configuration for this run block, and the variables referenced by the + // run block assertions. + relevantVariables := make(map[string]bool) + + // First, we'll check to see which variables the run block assertions + // reference. + for _, reference := range n.References() { + if addr, ok := reference.Subject.(addrs.InputVariable); ok { + relevantVariables[addr.Name] = true + } + } + + // And check to see which variables the run block configuration references. + for name := range run.ModuleConfig.Module.Variables { + relevantVariables[name] = true + } + + // We'll put the parsed values into this map. + values := make(terraform.InputValues) + + // First, let's step through the expressions within the run block and work + // them out. + for name, expr := range run.Config.Variables { + requiredValues := make(map[string]cty.Value) + + refs, refDiags := langrefs.ReferencesInExpr(addrs.ParseRefFromTestingScope, expr) + for _, ref := range refs { + if addr, ok := ref.Subject.(addrs.InputVariable); ok { + cache := ctx.GetCache(run) + + value, valueDiags := cache.GetFileVariable(addr.Name) + diags = diags.Append(valueDiags) + if value != nil { + requiredValues[addr.Name] = value.Value + continue + } + + // Otherwise, it might be a global variable. + value, valueDiags = cache.GetGlobalVariable(addr.Name) + diags = diags.Append(valueDiags) + if value != nil { + requiredValues[addr.Name] = value.Value + continue + } + } + } + diags = diags.Append(refDiags) + + ctx, ctxDiags := hcltest.EvalContext(hcltest.TargetRunBlock, map[string]hcl.Expression{name: expr}, requiredValues, ctx.GetOutputs()) + diags = diags.Append(ctxDiags) + + value := cty.DynamicVal + if !ctxDiags.HasErrors() { + var valueDiags hcl.Diagnostics + value, valueDiags = expr.Value(ctx) + diags = diags.Append(valueDiags) + } + + // We do this late on so we still validate whatever it was that the user + // wrote in the variable expression. But, we don't want to actually use + // it if it's not actually relevant. + if _, exists := relevantVariables[name]; !exists { + // Do not display warnings during cleanup phase + if includeWarnings { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagWarning, + Summary: "Value for undeclared variable", + Detail: fmt.Sprintf("The module under test does not declare a variable named %q, but it is declared in run block %q.", name, run.Name), + Subject: expr.Range().Ptr(), + }) + } + continue // Don't add it to our final set of variables. + } + + values[name] = &terraform.InputValue{ + Value: value, + SourceType: terraform.ValueFromConfig, + SourceRange: tfdiags.SourceRangeFromHCL(expr.Range()), + } + } + + for variable := range relevantVariables { + if _, exists := values[variable]; exists { + // Then we've already got a value for this variable. + continue + } + + // Otherwise, we'll get it from the cache as a file-level or global + // variable. + cache := ctx.GetCache(run) + + value, valueDiags := cache.GetFileVariable(variable) + diags = diags.Append(valueDiags) + if value != nil { + values[variable] = value + continue + } + + value, valueDiags = cache.GetGlobalVariable(variable) + diags = diags.Append(valueDiags) + if value != nil { + values[variable] = value + continue + } + } + + // Finally, we check the configuration again. This is where we'll discover + // if there's any missing variables and fill in any optional variables that + // don't have a value already. + + for name, variable := range run.ModuleConfig.Module.Variables { + if _, exists := values[name]; exists { + // Then we've provided a variable for this. It's all good. + continue + } + + // Otherwise, we're going to give these variables a value. They'll be + // processed by the Terraform graph and provided a default value later + // if they have one. + + if variable.Required() { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "No value for required variable", + Detail: fmt.Sprintf("The module under test for run block %q has a required variable %q with no set value. Use a -var or -var-file command line argument or add this variable into a \"variables\" block within the test file or run block.", + run.Name, variable.Name), + Subject: variable.DeclRange.Ptr(), + }) + + values[name] = &terraform.InputValue{ + Value: cty.DynamicVal, + SourceType: terraform.ValueFromConfig, + SourceRange: tfdiags.SourceRangeFromHCL(variable.DeclRange), + } + } else { + values[name] = &terraform.InputValue{ + Value: cty.NilVal, + SourceType: terraform.ValueFromConfig, + SourceRange: tfdiags.SourceRangeFromHCL(variable.DeclRange), + } + } + } + + return values, diags +} + +// FilterVariablesToModule splits the provided values into two disjoint maps: +// moduleVars contains the ones that correspond with declarations in the root +// module of the given configuration, while testOnlyVars contains any others +// that are presumably intended only for use in the test configuration file. +// +// This function is essentially the opposite of AddVariablesToConfig which +// makes the config match the variables rather than the variables match the +// config. +// +// This function can only return warnings, and the callers can rely on this so +// please check the callers of this function if you add any error diagnostics. +func (n *NodeTestRun) FilterVariablesToModule(values terraform.InputValues) (moduleVars, testOnlyVars terraform.InputValues, diags tfdiags.Diagnostics) { + moduleVars = make(terraform.InputValues) + testOnlyVars = make(terraform.InputValues) + for name, value := range values { + _, exists := n.run.ModuleConfig.Module.Variables[name] + if !exists { + // If it's not in the configuration then it's a test-only variable. + testOnlyVars[name] = value + continue + } + + moduleVars[name] = value + } + return moduleVars, testOnlyVars, diags +} + +// AddVariablesToConfig extends the provided config to ensure it has definitions +// for all specified variables. +// +// This function is essentially the opposite of FilterVariablesToConfig which +// makes the variables match the config rather than the config match the +// variables. +func (n *NodeTestRun) AddVariablesToConfig(variables terraform.InputValues) { + run := n.run + // If we have got variable values from the test file we need to make sure + // they have an equivalent entry in the configuration. We're going to do + // that dynamically here. + + // First, take a backup of the existing configuration so we can easily + // restore it later. + currentVars := make(map[string]*configs.Variable) + for name, variable := range run.ModuleConfig.Module.Variables { + currentVars[name] = variable + } + + for name, value := range variables { + if _, exists := run.ModuleConfig.Module.Variables[name]; exists { + continue + } + + run.ModuleConfig.Module.Variables[name] = &configs.Variable{ + Name: name, + Type: value.Value.Type(), + ConstraintType: value.Value.Type(), + DeclRange: value.SourceRange.ToHCL(), + } + } + +} diff --git a/internal/moduletest/graph/wait.go b/internal/moduletest/graph/wait.go new file mode 100644 index 0000000000..fd219de478 --- /dev/null +++ b/internal/moduletest/graph/wait.go @@ -0,0 +1,163 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package graph + +import ( + "context" + "fmt" + "log" + "sync/atomic" + "time" + + "github.com/hashicorp/terraform/internal/command/views" + "github.com/hashicorp/terraform/internal/moduletest" + "github.com/hashicorp/terraform/internal/plans" + "github.com/hashicorp/terraform/internal/states" + "github.com/hashicorp/terraform/internal/terraform" +) + +// operationWaiter waits for an operation within +// a test run execution to complete. +type operationWaiter struct { + ctx *terraform.Context + runningCtx context.Context + run *moduletest.Run + file *moduletest.File + created []*plans.ResourceInstanceChangeSrc + progress atomicProgress[moduletest.Progress] + start int64 + identifier string + finished bool + evalCtx *EvalContext + renderer views.Test +} + +type atomicProgress[T moduletest.Progress] struct { + internal atomic.Value +} + +func (a *atomicProgress[T]) Load() T { + return a.internal.Load().(T) +} + +func (a *atomicProgress[T]) Store(progress T) { + a.internal.Store(progress) +} + +// NewOperationWaiter creates a new operation waiter. +func NewOperationWaiter(ctx *terraform.Context, evalCtx *EvalContext, n *NodeTestRun, + progress moduletest.Progress, start int64) *operationWaiter { + identifier := "validate" + if n.File() != nil { + identifier = n.File().Name + if n.run != nil { + identifier = fmt.Sprintf("%s/%s", identifier, n.run.Name) + } + } + + p := atomicProgress[moduletest.Progress]{} + p.Store(progress) + + return &operationWaiter{ + ctx: ctx, + run: n.run, + file: n.File(), + progress: p, + start: start, + identifier: identifier, + evalCtx: evalCtx, + renderer: evalCtx.Renderer(), + } +} + +// Run executes the given function in a goroutine and waits for it to finish. +// If the function finishes, it returns false. If the function is cancelled or +// interrupted, it returns true. +func (w *operationWaiter) Run(fn func()) bool { + runningCtx, doneRunning := context.WithCancel(context.Background()) + w.runningCtx = runningCtx + + go func() { + fn() + doneRunning() + }() + + // either the function finishes or a cancel/stop signal is received + return w.wait() +} + +func (w *operationWaiter) wait() bool { + log.Printf("[TRACE] TestFileRunner: waiting for execution during %s", w.identifier) + + for !w.finished { + select { + case <-time.After(2 * time.Second): + w.updateProgress() + case <-w.evalCtx.stopContext.Done(): + // Soft cancel - wait for completion or hard cancel + for !w.finished { + select { + case <-time.After(2 * time.Second): + w.updateProgress() + case <-w.evalCtx.cancelContext.Done(): + return w.handleCancelled() + case <-w.runningCtx.Done(): + w.finished = true + } + } + case <-w.evalCtx.cancelContext.Done(): + return w.handleCancelled() + case <-w.runningCtx.Done(): + w.finished = true + } + } + + return false +} + +// update refreshes the operationWaiter with the latest terraform context, progress, and any newly created resources. +// This should be called before starting a new Terraform operation. +func (w *operationWaiter) update(ctx *terraform.Context, progress moduletest.Progress, created []*plans.ResourceInstanceChangeSrc) { + w.ctx = ctx + w.progress.Store(progress) + w.created = created +} + +func (w *operationWaiter) updateProgress() { + now := time.Now().UTC().UnixMilli() + progress := w.progress.Load() + w.renderer.Run(w.run, w.file, progress, now-w.start) +} + +// handleCancelled is called when the test execution is hard cancelled. +func (w *operationWaiter) handleCancelled() bool { + log.Printf("[DEBUG] TestFileRunner: test execution cancelled during %s", w.identifier) + states := make(map[*moduletest.Run]*states.State) + mainKey := moduletest.MainStateIdentifier + states[nil] = w.evalCtx.GetFileState(mainKey).State + for key, module := range w.evalCtx.FileStates { + if key == mainKey { + continue + } + states[module.Run] = module.State + } + w.renderer.FatalInterruptSummary(w.run, w.file, states, w.created) + + go func() { + if w.ctx != nil { + w.ctx.Stop() + } + }() + + for !w.finished { + select { + case <-time.After(2 * time.Second): + w.updateProgress() + case <-w.runningCtx.Done(): + w.finished = true + } + } + + return true +} diff --git a/internal/terraform/graph_builder.go b/internal/terraform/graph_builder.go index f8ae346982..4b7d6f781b 100644 --- a/internal/terraform/graph_builder.go +++ b/internal/terraform/graph_builder.go @@ -55,6 +55,9 @@ func (b *BasicGraphBuilder) Build(path addrs.ModuleInstance) (*Graph, tfdiags.Di if err != nil { if nf, isNF := err.(tfdiags.NonFatalError); isNF { diags = diags.Append(nf.Diagnostics) + } else if diag, isDiag := err.(tfdiags.DiagnosticsAsError); isDiag { + diags = diags.Append(diag.Diagnostics) + return g, diags } else { diags = diags.Append(err) return g, diags