diff --git a/.changes/v1.11/BUG FIXES-20250303-125722.yaml b/.changes/v1.11/BUG FIXES-20250303-125722.yaml new file mode 100644 index 0000000000..ca7b008227 --- /dev/null +++ b/.changes/v1.11/BUG FIXES-20250303-125722.yaml @@ -0,0 +1,5 @@ +kind: BUG FIXES +body: 'lang/funcs/transpose: Avoid crash due to map with null values' +time: 2025-03-03T12:57:22.400359Z +custom: + Issue: "36611" diff --git a/.changes/v1.12/ENHANCEMENTS-20250220-120438.yaml b/.changes/v1.12/ENHANCEMENTS-20250220-120438.yaml new file mode 100644 index 0000000000..1bc70e2f84 --- /dev/null +++ b/.changes/v1.12/ENHANCEMENTS-20250220-120438.yaml @@ -0,0 +1,5 @@ +kind: ENHANCEMENTS +body: Produce detailed diagnostic objects when test run assertions fail +time: 2025-02-20T12:04:38.005393+01:00 +custom: + Issue: "34428" diff --git a/go.mod b/go.mod index a40ef97794..35416e3b68 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/go-test/deep v1.0.3 github.com/google/go-cmp v0.6.0 github.com/google/uuid v1.6.0 - github.com/hashicorp/cli v1.1.6 + github.com/hashicorp/cli v1.1.7 github.com/hashicorp/go-checkpoint v0.5.0 github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-getter v1.7.8 diff --git a/go.sum b/go.sum index b1a071080b..b1681d9ddc 100644 --- a/go.sum +++ b/go.sum @@ -1059,8 +1059,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rH github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.58 h1:lf6PxLIHge0UL5LJgt/Szs0K3PYS27yqDEkaOa0P+ZU= github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.58/go.mod h1:9DB57cKw/ZNu1UQJX1YNmaJ7A2/+xCpCUUwbGZy4Qx0= -github.com/hashicorp/cli v1.1.6 h1:CMOV+/LJfL1tXCOKrgAX0uRKnzjj/mpmqNXloRSy2K8= -github.com/hashicorp/cli v1.1.6/go.mod h1:MPon5QYlgjjo0BSoAiN0ESeT5fRzDjVRp+uioJ0piz4= +github.com/hashicorp/cli v1.1.7 h1:/fZJ+hNdwfTSfsxMBa9WWMlfjUZbX8/LnUxgAd7lCVU= +github.com/hashicorp/cli v1.1.7/go.mod h1:e6Mfpga9OCT1vqzFuoGZiiF/KaG9CbUfO5s3ghU3YgU= github.com/hashicorp/consul/api v1.13.0 h1:2hnLQ0GjQvw7f3O61jMO8gbasZviZTrt9R8WzgiirHc= github.com/hashicorp/consul/api v1.13.0/go.mod h1:ZlVrynguJKcYr54zGaDbaL3fOvKC9m72FhPvA8T35KQ= github.com/hashicorp/consul/sdk v0.8.0 h1:OJtKBtEjboEZvG6AOUdh4Z1Zbyu0WcxQ0qatRrZHTVU= diff --git a/internal/command/arguments/apply.go b/internal/command/arguments/apply.go index 30896fd8e6..2965f06736 100644 --- a/internal/command/arguments/apply.go +++ b/internal/command/arguments/apply.go @@ -61,7 +61,7 @@ func ParseApply(args []string) (*Apply, tfdiags.Diagnostics) { diags = diags.Append(tfdiags.Sourceless( tfdiags.Warning, "Deprecated flag: -state", - "Use `path` attribute within the `local` backend instead: https://developer.hashicorp.com/terraform/language/v1.10.x/settings/backends/local#path", + `Use the "path" attribute within the "local" backend to specify a file for state storage`, )) } diff --git a/internal/command/arguments/plan.go b/internal/command/arguments/plan.go index e36ed978f2..d4d2a746a4 100644 --- a/internal/command/arguments/plan.go +++ b/internal/command/arguments/plan.go @@ -66,7 +66,7 @@ func ParsePlan(args []string) (*Plan, tfdiags.Diagnostics) { diags = diags.Append(tfdiags.Sourceless( tfdiags.Warning, "Deprecated flag: -state", - "Use `path` attribute within the `local` backend instead: https://developer.hashicorp.com/terraform/language/v1.10.x/settings/backends/local#path", + `Use the "path" attribute within the "local" backend to specify a file for state storage`, )) } diff --git a/internal/command/arguments/refresh.go b/internal/command/arguments/refresh.go index b55e9e2c20..1bc0c6fb15 100644 --- a/internal/command/arguments/refresh.go +++ b/internal/command/arguments/refresh.go @@ -51,7 +51,7 @@ func ParseRefresh(args []string) (*Refresh, tfdiags.Diagnostics) { diags = diags.Append(tfdiags.Sourceless( tfdiags.Warning, "Deprecated flag: -state", - "Use `path` attribute within the `local` backend instead: https://developer.hashicorp.com/terraform/language/v1.10.x/settings/backends/local#path", + `Use the "path" attribute within the "local" backend to specify a file for state storage`, )) } diff --git a/internal/command/format/diagnostic.go b/internal/command/format/diagnostic.go index 14b402f085..dc7ecd0dda 100644 --- a/internal/command/format/diagnostic.go +++ b/internal/command/format/diagnostic.go @@ -7,6 +7,7 @@ import ( "bufio" "bytes" "fmt" + "iter" "sort" "strings" @@ -77,7 +78,8 @@ func DiagnosticFromJSON(diag *viewsjson.Diagnostic, color *colorstring.Colorize, // be pure text that lends itself well to word-wrapping. fmt.Fprintf(&buf, color.Color("[bold]%s[reset]\n\n"), diag.Summary) - appendSourceSnippets(&buf, diag, color) + f := &snippetFormatter{&buf, diag, color} + f.write() if diag.Detail != "" { paraWidth := width - leftRuleWidth - 1 // leave room for the left rule @@ -151,7 +153,8 @@ func DiagnosticPlainFromJSON(diag *viewsjson.Diagnostic, width int) string { // be pure text that lends itself well to word-wrapping. fmt.Fprintf(&buf, "%s\n\n", diag.Summary) - appendSourceSnippets(&buf, diag, disabledColorize) + f := &snippetFormatter{&buf, diag, disabledColorize} + f.write() if diag.Detail != "" { if width > 1 { @@ -215,7 +218,17 @@ func DiagnosticWarningsCompact(diags tfdiags.Diagnostics, color *colorstring.Col return b.String() } -func appendSourceSnippets(buf *bytes.Buffer, diag *viewsjson.Diagnostic, color *colorstring.Colorize) { +// snippetFormatter handles formatting diagnostic information with source snippets +type snippetFormatter struct { + buf *bytes.Buffer + diag *viewsjson.Diagnostic + color *colorstring.Colorize +} + +func (f *snippetFormatter) write() { + diag := f.diag + buf := f.buf + color := f.color if diag.Address != "" { fmt.Fprintf(buf, " with %s,\n", diag.Address) } @@ -281,7 +294,7 @@ func appendSourceSnippets(buf *bytes.Buffer, diag *viewsjson.Diagnostic, color * ) } - if len(snippet.Values) > 0 || (snippet.FunctionCall != nil && snippet.FunctionCall.Signature != nil) { + if len(snippet.Values) > 0 || (snippet.FunctionCall != nil && snippet.FunctionCall.Signature != nil) || snippet.TestAssertionExpr != nil { // The diagnostic may also have information about the dynamic // values of relevant variables at the point of evaluation. // This is particularly useful for expressions that get evaluated @@ -312,11 +325,139 @@ func appendSourceSnippets(buf *bytes.Buffer, diag *viewsjson.Diagnostic, color * } buf.WriteString(")\n") } - for _, value := range values { - fmt.Fprintf(buf, color.Color(" [dark_gray]│[reset] [bold]%s[reset] %s\n"), value.Traversal, value.Statement) + + // always print the values unless in the case of a test assertion, where we only print them if the user has requested verbose output + printValues := snippet.TestAssertionExpr == nil || snippet.TestAssertionExpr.ShowVerbose + + // The diagnostic may also have information about failures from test assertions + // in a `terraform test` run. This is useful for understanding the values that + // were being compared when the assertion failed. + // Also, we'll print a JSON diff of the two values to make it easier to see the + // differences. + if snippet.TestAssertionExpr != nil { + f.printTestDiagOutput(snippet.TestAssertionExpr) + } + + if printValues { + for _, value := range values { + // if the statement is one line, we'll just print it as is + // otherwise, we have to ensure that each line is indented correctly + // and that the first line has the traversal information + valSlice := strings.Split(value.Statement, "\n") + fmt.Fprintf(buf, color.Color(" [dark_gray]│[reset] [bold]%s[reset] %s\n"), + value.Traversal, valSlice[0]) + + for _, line := range valSlice[1:] { + fmt.Fprintf(buf, color.Color(" [dark_gray]│[reset] %s\n"), line) + } + } } } } buf.WriteByte('\n') } + +func (f *snippetFormatter) printTestDiagOutput(diag *viewsjson.DiagnosticTestBinaryExpr) { + buf := f.buf + color := f.color + // We only print the LHS and RHS if the user has requested verbose output + // for the test assertion. + if diag.ShowVerbose { + fmt.Fprint(buf, color.Color(" [dark_gray]│[reset] [bold]LHS[reset]:\n")) + for line := range strings.SplitSeq(diag.LHS, "\n") { + fmt.Fprintf(buf, color.Color(" [dark_gray]│[reset] %s\n"), line) + } + fmt.Fprint(buf, color.Color(" [dark_gray]│[reset] [bold]RHS[reset]:\n")) + for line := range strings.SplitSeq(diag.RHS, "\n") { + fmt.Fprintf(buf, color.Color(" [dark_gray]│[reset] %s\n"), line) + } + } + if diag.Warning != "" { + fmt.Fprintf(buf, color.Color(" [dark_gray]│[reset] [bold]Warning[reset]: %s\n"), diag.Warning) + } + f.printJSONDiff(diag.LHS, diag.RHS) + buf.WriteByte('\n') +} + +// printJSONDiff prints a colorized line-by-line diff of the JSON values of the LHS and RHS expressions +// in a test assertion. +// It visually distinguishes removed and added lines, helping users identify +// discrepancies between an "actual" (lhsStr) and an "expected" (rhsStr) JSON output. +func (f *snippetFormatter) printJSONDiff(lhsStr, rhsStr string) { + + buf := f.buf + color := f.color + // No visible difference in the JSON, so we'll just return + if lhsStr == rhsStr { + return + } + fmt.Fprint(buf, color.Color(" [dark_gray]│[reset] [bold]Diff[reset]:\n")) + fmt.Fprint(buf, color.Color(" [dark_gray]│[reset] [red][bold]--- actual[reset]\n")) + fmt.Fprint(buf, color.Color(" [dark_gray]│[reset] [green][bold]+++ expected[reset]\n")) + nextLhs, stopLhs := iter.Pull(strings.SplitSeq(lhsStr, "\n")) + nextRhs, stopRhs := iter.Pull(strings.SplitSeq(rhsStr, "\n")) + + printLine := func(prefix, line string) { + var colour string + switch prefix { + case "-": + colour = "[red]" + case "+": + colour = "[green]" + default: + } + msg := fmt.Sprintf(" [dark_gray]│[reset] %s%s[reset] %s\n", colour, prefix, line) + fmt.Fprint(buf, color.Color(msg)) + } + + // Collect differing lines separately for each side + removedLines := []string{} + addedLines := []string{} + + // Function to print collected diffs and reset buffers + printDiffs := func() { + for _, line := range removedLines { + printLine("-", line) + } + for _, line := range addedLines { + printLine("+", line) + } + removedLines = []string{} + addedLines = []string{} + } + + // We'll iterate over both sides of the expression and collect the differences + // along the way. When a match is found, we'll then print all the collected diffs + // and the matching line, and then reset the buffers. + for { + lhsLine, lhsOk := nextLhs() + rhsLine, rhsOk := nextRhs() + + if !lhsOk && !rhsOk { // Both sides are done, so we'll print the diffs and break + printDiffs() + break + } + + // If one side is done, we'll just print the remaining lines from the other side + if !lhsOk { + addedLines = append(addedLines, rhsLine) + continue + } + if !rhsOk { + removedLines = append(removedLines, lhsLine) + continue + } + + if lhsLine == rhsLine { + printDiffs() + printLine(" ", lhsLine) + } else { + removedLines = append(removedLines, lhsLine) + addedLines = append(addedLines, rhsLine) + } + } + + stopLhs() + stopRhs() +} diff --git a/internal/command/format/diagnostic_test.go b/internal/command/format/diagnostic_test.go index 832236a919..0c21375898 100644 --- a/internal/command/format/diagnostic_test.go +++ b/internal/command/format/diagnostic_test.go @@ -4,6 +4,9 @@ package format import ( + "bytes" + "fmt" + "regexp" "strings" "testing" @@ -263,6 +266,82 @@ func TestDiagnostic(t *testing.T) { [red]│[reset] [red]│[reset] Whatever shall we do? [red]╵[reset] +`, + }, + "error origination from failed test assertion": { + &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Test assertion failed", + Detail: "LHS not equal to RHS", + Subject: &hcl.Range{ + Filename: "test.tf", + Start: hcl.Pos{Line: 1, Column: 6, Byte: 5}, + End: hcl.Pos{Line: 1, Column: 12, Byte: 11}, + }, + Expression: &hclsyntax.BinaryOpExpr{ + Op: hclsyntax.OpEqual, + LHS: &hclsyntax.LiteralValueExpr{ + Val: cty.ObjectVal(map[string]cty.Value{ + "inner": cty.StringVal("str1"), + "extra": cty.StringVal("str2"), + }), + }, + RHS: &hclsyntax.LiteralValueExpr{ + Val: cty.ObjectVal(map[string]cty.Value{ + "inner": cty.StringVal("str11"), + "extra": cty.StringVal("str21"), + }), + }, + SrcRange: hcl.Range{ + Filename: "test.tf", + Start: hcl.Pos{Line: 1, Column: 6, Byte: 5}, + End: hcl.Pos{Line: 1, Column: 12, Byte: 11}, + }, + }, + EvalContext: &hcl.EvalContext{ + Variables: map[string]cty.Value{ + "foo": cty.ObjectVal(map[string]cty.Value{ + "inner": cty.StringVal("str1"), + }), + "bar": cty.ObjectVal(map[string]cty.Value{ + "inner": cty.StringVal("str2"), + }), + }, + }, + // This is simulating what the test assertion expression + // type would generate on evaluation, by implementing the + // same interface it uses. + Extra: diagnosticCausedByTestFailure{true}, + }, + `[red]╷[reset] +[red]│[reset] [bold][red]Error: [reset][bold]Test assertion failed[reset] +[red]│[reset] +[red]│[reset] on test.tf line 1: +[red]│[reset] 1: test [underline]source[reset] code +[red]│[reset] [dark_gray]├────────────────[reset] +[red]│[reset] [dark_gray]│[reset] [bold]LHS[reset]: +[red]│[reset] [dark_gray]│[reset] { +[red]│[reset] [dark_gray]│[reset] "extra": "str2", +[red]│[reset] [dark_gray]│[reset] "inner": "str1" +[red]│[reset] [dark_gray]│[reset] } +[red]│[reset] [dark_gray]│[reset] [bold]RHS[reset]: +[red]│[reset] [dark_gray]│[reset] { +[red]│[reset] [dark_gray]│[reset] "extra": "str21", +[red]│[reset] [dark_gray]│[reset] "inner": "str11" +[red]│[reset] [dark_gray]│[reset] } +[red]│[reset] [dark_gray]│[reset] [bold]Diff[reset]: +[red]│[reset] [dark_gray]│[reset] [red][bold]--- actual[reset] +[red]│[reset] [dark_gray]│[reset] [green][bold]+++ expected[reset] +[red]│[reset] [dark_gray]│[reset] [reset] { +[red]│[reset] [dark_gray]│[reset] [red]-[reset] "extra": "str2", +[red]│[reset] [dark_gray]│[reset] [red]-[reset] "inner": "str1" +[red]│[reset] [dark_gray]│[reset] [green]+[reset] "extra": "str21", +[red]│[reset] [dark_gray]│[reset] [green]+[reset] "inner": "str11" +[red]│[reset] [dark_gray]│[reset] [reset] } +[red]│[reset] +[red]│[reset] +[red]│[reset] LHS not equal to RHS +[red]╵[reset] `, }, } @@ -910,6 +989,243 @@ func TestDiagnosticFromJSON_invalid(t *testing.T) { } } +func TestJsonDiff(t *testing.T) { + f := &snippetFormatter{ + buf: &bytes.Buffer{}, + color: &colorstring.Colorize{ + Reset: true, + Disable: true, + }, + } + + tests := []struct { + name string + strA string + strB string + diff string + }{ + { + name: "Basic different fields", + strA: `{ + "field1": "value1", + "field2": "value2", + "field3": "value3", + "field4": "value4" +}`, + strB: `{ + "field1": "value1", + "field2": "different", + "field3": "value3", + "field4": "value4" +}`, + diff: ` │ Diff: + │ --- actual + │ +++ expected + │ { + │ "field1": "value1", + │ - "field2": "value2", + │ + "field2": "different", + │ "field3": "value3", + │ "field4": "value4" + │ } +`, + }, + { + name: "Unequal number of fields", + strA: `{ + "field1": "value1", + "field2": "value2", + "field3": "value3", + "extraField": "extraValue", + "field4": "value4" +}`, + strB: `{ + "field1": "value1", + "fieldX": "valueX", + "fieldY": "valueY", + "field4": "value4" +}`, + diff: ` │ Diff: + │ --- actual + │ +++ expected + │ { + │ "field1": "value1", + │ - "field2": "value2", + │ - "field3": "value3", + │ - "extraField": "extraValue", + │ - "field4": "value4" + │ - } + │ + "fieldX": "valueX", + │ + "fieldY": "valueY", + │ + "field4": "value4" + │ + } +`, + }, + { + name: "Empty vs non-empty JSON", + strA: `{}`, + strB: `{ + "field1": "value1", + "field2": "value2" +}`, + diff: ` │ Diff: + │ --- actual + │ +++ expected + │ - {} + │ + { + │ + "field1": "value1", + │ + "field2": "value2" + │ + } +`, + }, + { + name: "Completely different JSONs", + strA: `{ + "a": 1, + "b": 2 +}`, + strB: `{ + "c": 3, + "d": 4 +}`, + diff: ` │ Diff: + │ --- actual + │ +++ expected + │ { + │ - "a": 1, + │ - "b": 2 + │ + "c": 3, + │ + "d": 4 + │ } +`, + }, + { + name: "Nested objects with differences", + strA: `{ + "outer": { + "inner1": "value1", + "inner2": "value2" + } +}`, + strB: `{ + "outer": { + "inner1": "changed", + "inner2": "value2" + } +}`, + diff: ` │ Diff: + │ --- actual + │ +++ expected + │ { + │ "outer": { + │ - "inner1": "value1", + │ + "inner1": "changed", + │ "inner2": "value2" + │ } + │ } +`, + }, + { + name: "Multiple separate diff blocks", + strA: `{ + "block1": "original1", + "unchanged1": "same", + "block2": "original2", + "unchanged2": "same", + "block3": "original3" +}`, + strB: `{ + "block1": "changed1", + "unchanged1": "same", + "block2": "changed2", + "unchanged2": "same", + "block3": "changed3" +}`, + diff: ` │ Diff: + │ --- actual + │ +++ expected + │ { + │ - "block1": "original1", + │ + "block1": "changed1", + │ "unchanged1": "same", + │ - "block2": "original2", + │ + "block2": "changed2", + │ "unchanged2": "same", + │ - "block3": "original3" + │ + "block3": "changed3" + │ } +`, + }, + { + name: "Large number of differences", + strA: `{ + "item1": "a", + "item2": "b", + "item3": "c", + "item4": "d", + "item5": "e", + "item6": "f", + "item7": "g" +}`, + strB: `{ + "item1": "a", + "item2": "B", + "item3": "C", + "item4": "D", + "item5": "e", + "item6": "F", + "item7": "g" +}`, + diff: ` │ Diff: + │ --- actual + │ +++ expected + │ { + │ "item1": "a", + │ - "item2": "b", + │ - "item3": "c", + │ - "item4": "d", + │ + "item2": "B", + │ + "item3": "C", + │ + "item4": "D", + │ "item5": "e", + │ - "item6": "f", + │ + "item6": "F", + │ "item7": "g" + │ } +`, + }, + { + name: "Identical JSONs", + strA: `{"field": "value"}`, + strB: `{"field": "value"}`, + diff: ``, // No output expected for identical JSONs + }, + { + name: "simple: no matches", + strA: `1`, + strB: `2`, + diff: ` │ Diff: + │ --- actual + │ +++ expected + │ - 1 + │ + 2 +`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + f.buf.Reset() + f.printJSONDiff(test.strA, test.strB) + diff := regexp.MustCompile(`\[[^\]]+\]`).ReplaceAllString(f.buf.String(), "") + fmt.Println(diff) + if d := cmp.Diff(diff, test.diff); d != "" { + t.Errorf("diff mismatch: got %s\n, want %s\n: diff: %s\n", diff, test.diff, d) + } + }) + } +} + // fakeDiagFunctionCallExtra is a fake implementation of the interface that // HCL uses to provide "extra information" associated with diagnostics that // describe errors during a function call. @@ -946,3 +1262,17 @@ var _ tfdiags.DiagnosticExtraBecauseSensitive = diagnosticCausedBySensitive(true func (e diagnosticCausedBySensitive) DiagnosticCausedBySensitive() bool { return bool(e) } + +var _ tfdiags.DiagnosticExtraCausedByTestFailure = diagnosticCausedByTestFailure{} + +type diagnosticCausedByTestFailure struct { + Verbose bool +} + +func (e diagnosticCausedByTestFailure) DiagnosticCausedByTestFailure() bool { + return true +} + +func (e diagnosticCausedByTestFailure) IsTestVerboseMode() bool { + return e.Verbose +} diff --git a/internal/command/junit/junit.go b/internal/command/junit/junit.go index c1a9a855a0..34051ddfa2 100644 --- a/internal/command/junit/junit.go +++ b/internal/command/junit/junit.go @@ -18,10 +18,6 @@ import ( "github.com/hashicorp/terraform/internal/tfdiags" ) -var ( - failedTestSummary = "Test assertion failed" -) - // TestJUnitXMLFile produces a JUnit XML file at the conclusion of a test // run, summarizing the outcome of the test in a form that can then be // interpreted by tools which render JUnit XML result reports. @@ -217,7 +213,7 @@ func junitXMLTestReport(suite *moduletest.Suite, suiteRunnerStopped bool, source // When the test fails we only use error diags that originate from failing assertions var failedAssertions tfdiags.Diagnostics for _, d := range run.Diagnostics { - if d.Severity() == tfdiags.Error && d.Description().Summary == failedTestSummary { + if tfdiags.DiagnosticCausedByTestFailure(d) { failedAssertions = failedAssertions.Append(d) } } @@ -259,7 +255,7 @@ func junitXMLTestReport(suite *moduletest.Suite, suiteRunnerStopped bool, source // Collect diags not due to failed assertions, both errors and warnings for _, d := range run.Diagnostics { - if d.Description().Summary != failedTestSummary { + if !tfdiags.DiagnosticCausedByTestFailure(d) { systemErrDiags = systemErrDiags.Append(d) } } diff --git a/internal/command/test_test.go b/internal/command/test_test.go index a89b61251a..cd5048501b 100644 --- a/internal/command/test_test.go +++ b/internal/command/test_test.go @@ -146,8 +146,9 @@ func TestTest_Runs(t *testing.T) { }, "simple_fail": { expectedOut: []string{"0 passed, 1 failed."}, - expectedErr: []string{"invalid value"}, - code: 1, + expectedErr: []string{"invalid value", `│ - "bar" + │ + "zap"`}, + code: 1, }, "custom_condition_checks": { expectedOut: []string{"0 passed, 1 failed."}, @@ -301,8 +302,10 @@ func TestTest_Runs(t *testing.T) { }, "ephemeral_input_with_error": { expectedOut: []string{"Error message refers to ephemeral values", "1 passed, 1 failed."}, - expectedErr: []string{"Test assertion failed", "has an ephemeral value"}, - code: 1, + expectedErr: []string{"Test assertion failed", + `│ - "(ephemeral value)" + │ + "bar"`}, + code: 1, }, "ephemeral_resource": { expectedOut: []string{"0 passed, 1 failed."}, @@ -837,6 +840,495 @@ func TestTest_ProviderAlias(t *testing.T) { } } +func TestTest_ComplexCondition(t *testing.T) { + td := t.TempDir() + testCopyDir(t, testFixturePath(path.Join("test", "complex_condition")), td) + defer testChdir(t, td)() + + provider := testing_command.NewProvider(nil) + + providerSource, close := newMockProviderSource(t, map[string][]string{"test": {"1.0.0"}}) + defer close() + + streams, done := terminal.StreamsForTesting(t) + view := views.NewView(streams) + ui := new(cli.MockUi) + + meta := Meta{ + testingOverrides: metaOverridesForProvider(provider.Provider), + Ui: ui, + View: view, + Streams: streams, + ProviderSource: providerSource, + } + + init := &InitCommand{ + Meta: meta, + } + + output := done(t) + + if code := init.Run([]string{"-no-color"}); code != 0 { + t.Fatalf("expected status code 0 but got %d: %s", code, output.All()) + } + + // Reset the streams for the next command. + streams, done = terminal.StreamsForTesting(t) + meta.Streams = streams + meta.View = views.NewView(streams) + + command := &TestCommand{ + Meta: meta, + } + + code := command.Run([]string{"-no-color"}) + output = done(t) + + printedOutput := false + + if code != 1 { + printedOutput = true + t.Errorf("expected status code 1 but got %d: %s", code, output.All()) + } + + expectedOut := `main.tftest.hcl... in progress + run "validate_diff_types"... fail + run "validate_output"... fail + run "validate_complex_output"... fail + run "validate_complex_output_sensitive"... fail + run "validate_complex_output_pass"... pass +main.tftest.hcl... tearing down +main.tftest.hcl... fail + +Failure! 1 passed, 4 failed. +` + + expectedErr := ` +Error: Test assertion failed + + on main.tftest.hcl line 37, in run "validate_diff_types": + 37: condition = var.tr1 == var.tr2 + ├──────────────── + │ Warning: LHS and RHS values are of different types + + +expected to fail + +Error: Test assertion failed + + on main.tftest.hcl line 44, in run "validate_output": + 44: condition = output.foo == var.foo + ├──────────────── + │ Diff: + │ --- actual + │ +++ expected + │ { + │ - "bar": "notbaz", + │ + "bar": "baz", + │ "matches": "matches", + │ - "qux": "quux", + │ - "xuq": "xuq" + │ + "qux": "qux", + │ + "xuq": "nope" + │ } + + +expected to fail due to different values + +Error: Test assertion failed + + on main.tftest.hcl line 52, in run "validate_complex_output": + 52: condition = output.complex == var.bar + ├──────────────── + │ Warning: LHS and RHS values are of different types + │ Diff: + │ --- actual + │ +++ expected + │ { + │ "root": [ + │ { + │ "bar": [ + │ 1 + │ ], + │ - "qux": "quux" + │ + "qux": "qux" + │ }, + │ { + │ "bar": [ + │ 2 + │ ], + │ "qux": "quux" + │ } + │ ] + │ } + + +expected to fail + +Error: Test assertion failed + + on main.tftest.hcl line 60, in run "validate_complex_output_sensitive": + 60: condition = output.complex == output.complex_sensitive + ├──────────────── + │ Diff: + │ --- actual + │ +++ expected + │ - { + │ - "root": [ + │ - { + │ - "bar": [ + │ - 1 + │ - ], + │ - "qux": "quux" + │ - }, + │ - { + │ - "bar": [ + │ - 2 + │ - ], + │ - "qux": "quux" + │ - } + │ - ] + │ - } + │ + "(sensitive value)" + + +expected to fail +` + if diff := cmp.Diff(output.Stdout(), expectedOut); len(diff) > 0 { + t.Errorf("\nexpected: \n%s\ngot: %s\ndiff: %s", expectedOut, output.All(), diff) + } + if diff := cmp.Diff(output.Stderr(), expectedErr); len(diff) > 0 { + t.Errorf("\nexpected stderr: \n%s\ngot: %s\ndiff: %s", expectedErr, output.Stderr(), diff) + } + + if provider.ResourceCount() > 0 { + if !printedOutput { + t.Errorf("should have deleted all resources on completion but left %s\n\n%s", provider.ResourceString(), output.All()) + } else { + t.Errorf("should have deleted all resources on completion but left %s", provider.ResourceString()) + } + } +} + +func TestTest_ComplexConditionVerbose(t *testing.T) { + td := t.TempDir() + testCopyDir(t, testFixturePath(path.Join("test", "complex_condition")), td) + defer testChdir(t, td)() + + provider := testing_command.NewProvider(nil) + + providerSource, close := newMockProviderSource(t, map[string][]string{"test": {"1.0.0"}}) + defer close() + + streams, done := terminal.StreamsForTesting(t) + view := views.NewView(streams) + ui := new(cli.MockUi) + + meta := Meta{ + testingOverrides: metaOverridesForProvider(provider.Provider), + Ui: ui, + View: view, + Streams: streams, + ProviderSource: providerSource, + } + + init := &InitCommand{ + Meta: meta, + } + + output := done(t) + + if code := init.Run([]string{"-no-color"}); code != 0 { + t.Fatalf("expected status code 0 but got %d: %s", code, output.All()) + } + + // Reset the streams for the next command. + streams, done = terminal.StreamsForTesting(t) + meta.Streams = streams + meta.View = views.NewView(streams) + + command := &TestCommand{ + Meta: meta, + } + + code := command.Run([]string{"-no-color", "-verbose"}) + output = done(t) + + printedOutput := false + + if code != 1 { + printedOutput = true + t.Errorf("expected status code 1 but got %d: %s", code, output.All()) + } + + expectedErr := ` +Error: Test assertion failed + + on main.tftest.hcl line 37, in run "validate_diff_types": + 37: condition = var.tr1 == var.tr2 + ├──────────────── + │ LHS: + │ { + │ "iops": null, + │ "size": 60 + │ } + │ RHS: + │ { + │ "iops": null, + │ "size": 60 + │ } + │ Warning: LHS and RHS values are of different types + + │ var.tr1 is { + │ "iops": null, + │ "size": 60 + │ } + │ var.tr2 is { + │ "iops": null, + │ "size": 60 + │ } + +expected to fail + +Error: Test assertion failed + + on main.tftest.hcl line 44, in run "validate_output": + 44: condition = output.foo == var.foo + ├──────────────── + │ LHS: + │ { + │ "bar": "notbaz", + │ "matches": "matches", + │ "qux": "quux", + │ "xuq": "xuq" + │ } + │ RHS: + │ { + │ "bar": "baz", + │ "matches": "matches", + │ "qux": "qux", + │ "xuq": "nope" + │ } + │ Diff: + │ --- actual + │ +++ expected + │ { + │ - "bar": "notbaz", + │ + "bar": "baz", + │ "matches": "matches", + │ - "qux": "quux", + │ - "xuq": "xuq" + │ + "qux": "qux", + │ + "xuq": "nope" + │ } + + │ output.foo is { + │ "bar": "notbaz", + │ "matches": "matches", + │ "qux": "quux", + │ "xuq": "xuq" + │ } + │ var.foo is { + │ "bar": "baz", + │ "matches": "matches", + │ "qux": "qux", + │ "xuq": "nope" + │ } + +expected to fail due to different values + +Error: Test assertion failed + + on main.tftest.hcl line 52, in run "validate_complex_output": + 52: condition = output.complex == var.bar + ├──────────────── + │ LHS: + │ { + │ "root": [ + │ { + │ "bar": [ + │ 1 + │ ], + │ "qux": "quux" + │ }, + │ { + │ "bar": [ + │ 2 + │ ], + │ "qux": "quux" + │ } + │ ] + │ } + │ RHS: + │ { + │ "root": [ + │ { + │ "bar": [ + │ 1 + │ ], + │ "qux": "qux" + │ }, + │ { + │ "bar": [ + │ 2 + │ ], + │ "qux": "quux" + │ } + │ ] + │ } + │ Warning: LHS and RHS values are of different types + │ Diff: + │ --- actual + │ +++ expected + │ { + │ "root": [ + │ { + │ "bar": [ + │ 1 + │ ], + │ - "qux": "quux" + │ + "qux": "qux" + │ }, + │ { + │ "bar": [ + │ 2 + │ ], + │ "qux": "quux" + │ } + │ ] + │ } + + │ output.complex is { + │ "root": [ + │ { + │ "bar": [ + │ 1 + │ ], + │ "qux": "quux" + │ }, + │ { + │ "bar": [ + │ 2 + │ ], + │ "qux": "quux" + │ } + │ ] + │ } + │ var.bar is { + │ "root": [ + │ { + │ "bar": [ + │ 1 + │ ], + │ "qux": "qux" + │ }, + │ { + │ "bar": [ + │ 2 + │ ], + │ "qux": "quux" + │ } + │ ] + │ } + +expected to fail + +Error: Test assertion failed + + on main.tftest.hcl line 60, in run "validate_complex_output_sensitive": + 60: condition = output.complex == output.complex_sensitive + ├──────────────── + │ LHS: + │ { + │ "root": [ + │ { + │ "bar": [ + │ 1 + │ ], + │ "qux": "quux" + │ }, + │ { + │ "bar": [ + │ 2 + │ ], + │ "qux": "quux" + │ } + │ ] + │ } + │ RHS: + │ "(sensitive value)" + │ Diff: + │ --- actual + │ +++ expected + │ - { + │ - "root": [ + │ - { + │ - "bar": [ + │ - 1 + │ - ], + │ - "qux": "quux" + │ - }, + │ - { + │ - "bar": [ + │ - 2 + │ - ], + │ - "qux": "quux" + │ - } + │ - ] + │ - } + │ + "(sensitive value)" + + │ output.complex is { + │ "root": [ + │ { + │ "bar": [ + │ 1 + │ ], + │ "qux": "quux" + │ }, + │ { + │ "bar": [ + │ 2 + │ ], + │ "qux": "quux" + │ } + │ ] + │ } + │ output.complex_sensitive is "(sensitive value)" + +expected to fail +` + outputs := []string{ + "main.tftest.hcl... in progress", + " run \"validate_diff_types\"... fail", + " run \"validate_output\"... fail", + " run \"validate_complex_output\"... fail", + " run \"validate_complex_output_sensitive\"... fail", + " run \"validate_complex_output_pass\"... pass", + "main.tftest.hcl... tearing down", + "main.tftest.hcl... fail", + "Failure! 1 passed, 4 failed.", + } + stdout := output.Stdout() + for _, expected := range outputs { + if !strings.Contains(stdout, expected) { + t.Errorf("output didn't contain expected output %q", expected) + } + } + + if diff := cmp.Diff(output.Stderr(), expectedErr); len(diff) > 0 { + t.Errorf("\nexpected stderr: \n%s\ngot: %s\ndiff: %s", expectedErr, output.Stderr(), diff) + } + + if provider.ResourceCount() > 0 { + if !printedOutput { + t.Errorf("should have deleted all resources on completion but left %s\n\n%s", provider.ResourceString(), output.All()) + } else { + t.Errorf("should have deleted all resources on completion but left %s", provider.ResourceString()) + } + } +} + func TestTest_ModuleDependencies(t *testing.T) { td := t.TempDir() testCopyDir(t, testFixturePath(path.Join("test", "with_setup_module")), td) @@ -2027,8 +2519,13 @@ Error: Test assertion failed on main.tftest.hcl line 8, in run "first": 8: condition = test_resource.resource.value == output.null_output ├──────────────── - │ output.null_output is null - │ test_resource.resource.value is "bar" + │ Warning: LHS and RHS values are of different types + │ Diff: + │ --- actual + │ +++ expected + │ - "bar" + │ + null + this is always going to fail `, @@ -2210,8 +2707,8 @@ func TestTest_SensitiveInputValues(t *testing.T) { code := c.Run([]string{"-no-color", "-verbose"}) output = done(t) - if code != 0 { - t.Errorf("expected status code 0 but got %d", code) + if code != 1 { + t.Errorf("expected status code 1 but got %d", code) } expected := `main.tftest.hcl... in progress @@ -2233,22 +2730,77 @@ resource "test_resource" "resource" { } +Outputs: + +password = (sensitive value) + + run "test_failed"... fail + +# test_resource.resource: +resource "test_resource" "resource" { + destroy_fail = false + id = "9ddca5a9" + value = (sensitive value) +} + + Outputs: password = (sensitive value) main.tftest.hcl... tearing down -main.tftest.hcl... pass +main.tftest.hcl... fail -Success! 2 passed, 0 failed. +Failure! 2 passed, 1 failed. ` - actual := output.All() + expectedErr := ` +Error: Test assertion failed + + on main.tftest.hcl line 27, in run "test_failed": + 27: condition = var.complex == { + 28: foo = "bar" + 29: baz = test_resource.resource.id + 30: } + ├──────────────── + │ LHS: + │ { + │ "baz": "(sensitive value)", + │ "foo": "bar" + │ } + │ RHS: + │ { + │ "baz": "9ddca5a9", + │ "foo": "bar" + │ } + │ Diff: + │ --- actual + │ +++ expected + │ { + │ - "baz": "(sensitive value)", + │ + "baz": "9ddca5a9", + │ "foo": "bar" + │ } + + │ test_resource.resource.id is "9ddca5a9" + │ var.complex is { + │ "baz": "(sensitive value)", + │ "foo": "bar" + │ } + +expected to fail +` + + actual := output.Stdout() if diff := cmp.Diff(actual, expected); len(diff) > 0 { t.Errorf("output didn't match expected:\nexpected:\n%s\nactual:\n%s\ndiff:\n%s", expected, actual, diff) } + if diff := cmp.Diff(output.Stderr(), expectedErr); len(diff) > 0 { + t.Errorf("stderr didn't match expected:\nexpected:\n%s\nactual:\n%s\ndiff:\n%s", expectedErr, output.Stderr(), diff) + } + if provider.ResourceCount() > 0 { t.Errorf("should have deleted all resources on completion but left %v", provider.ResourceString()) } @@ -2804,7 +3356,7 @@ func TestTest_JUnitOutput(t *testing.T) { actualOut = timestampRegexp.ReplaceAll(actualOut, []byte("timestamp=\"TIMESTAMP_REDACTED\"")) if !bytes.Equal(actualOut, expectedOutput) { - t.Fatalf("wanted XML:\n%s\n got XML:\n%s\n", string(expectedOutput), string(actualOut)) + t.Fatalf("wanted XML:\n%s\n got XML:\n%s\ndiff:%s\n", string(expectedOutput), string(actualOut), cmp.Diff(expectedOutput, actualOut)) } if provider.ResourceCount() > 0 { diff --git a/internal/command/testdata/apply/output.jsonlog b/internal/command/testdata/apply/output.jsonlog index 96ce709a8c..a742e943da 100644 --- a/internal/command/testdata/apply/output.jsonlog +++ b/internal/command/testdata/apply/output.jsonlog @@ -1,5 +1,5 @@ {"@level":"info","@message":"Terraform 0.15.0-dev","@module":"terraform.ui","terraform":"0.15.0-dev","type":"version","ui":"0.1.0"} -{"@level":"warn","@message":"Warning: Deprecated flag: -state","@module":"terraform.ui","diagnostic":{"detail":"Use `path` attribute within the `local` backend instead: https://developer.hashicorp.com/terraform/language/v1.10.x/settings/backends/local#path","severity":"warning","summary":"Deprecated flag: -state"},"type":"diagnostic"} +{"@level":"warn","@message":"Warning: Deprecated flag: -state","@module":"terraform.ui","diagnostic":{"detail":"Use the \"path\" attribute within the \"local\" backend to specify a file for state storage","severity":"warning","summary":"Deprecated flag: -state"},"type":"diagnostic"} {"@level":"info","@message":"test_instance.foo: Plan to create","@module":"terraform.ui","change":{"resource":{"addr":"test_instance.foo","module":"","resource":"test_instance.foo","implied_provider":"test","resource_type":"test_instance","resource_name":"foo","resource_key":null},"action":"create"},"type":"planned_change"} {"@level":"info","@message":"Plan: 1 to add, 0 to change, 0 to destroy.","@module":"terraform.ui","changes":{"add":1,"import":0,"change":0,"remove":0,"operation":"plan"},"type":"change_summary"} {"@level":"info","@message":"test_instance.foo: Creating...","@module":"terraform.ui","hook":{"resource":{"addr":"test_instance.foo","module":"","resource":"test_instance.foo","implied_provider":"test","resource_type":"test_instance","resource_name":"foo","resource_key":null},"action":"create"},"type":"apply_start"} diff --git a/internal/command/testdata/test/complex_condition/main.tf b/internal/command/testdata/test/complex_condition/main.tf new file mode 100644 index 0000000000..2f61115647 --- /dev/null +++ b/internal/command/testdata/test/complex_condition/main.tf @@ -0,0 +1,58 @@ +resource "test_resource" "foo" { + value = "bar" +} + +output "foo" { + value = { + bar = "notbaz" + qux = "quux" + matches = "matches" + xuq = "xuq" + } +} + +variable "sample" { + type = list(object({ + bar = tuple([number]) + qux = string + })) + + default = [ { + bar = [1] + qux = "quux" + }, + { + bar = [2] + qux = "quux" + }] +} + +variable "sample_sensitive" { + sensitive = true + type = list(object({ + bar = tuple([number]) + qux = string + })) + + default = [ { + bar = [1] + qux = "quux" + }, + { + bar = [2] + qux = "quux_sensitive" + }] +} + +output "complex" { + value = { + root = var.sample + } +} + +output "complex_sensitive" { + sensitive = true + value = { + root = var.sample_sensitive + } +} \ No newline at end of file diff --git a/internal/command/testdata/test/complex_condition/main.tftest.hcl b/internal/command/testdata/test/complex_condition/main.tftest.hcl new file mode 100644 index 0000000000..b279319431 --- /dev/null +++ b/internal/command/testdata/test/complex_condition/main.tftest.hcl @@ -0,0 +1,70 @@ + +variables { + + foo = { + bar = "baz", + qux = "qux", + matches = "matches", + xuq = "nope" + } + + bar = { + root = [{ + bar = [1] + qux = "qux" + }, + { + bar = [2] + qux = "quux" + }] + } +} + +run "validate_diff_types" { +// the compared values are of different types, but have the same +// visual representation in the terminal. + variables { + tr1 = { + "iops" = tonumber(null) + "size" = 60 +} + tr2 = { + iops = null + size = 60 +} + } + assert { + condition = var.tr1 == var.tr2 + error_message = "expected to fail" + } +} + +run "validate_output" { + assert { + condition = output.foo == var.foo + error_message = "expected to fail due to different values" + } +} + +run "validate_complex_output" { + assert { + // just a more complex value comparison + condition = output.complex == var.bar + error_message = "expected to fail" + } +} + +run "validate_complex_output_sensitive" { + // the rhs is sensitive + assert { + condition = output.complex == output.complex_sensitive + error_message = "expected to fail" + } +} + +run "validate_complex_output_pass" { + assert { + condition = output.complex != var.foo + error_message = "should pass" + } +} diff --git a/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml b/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml index 5a3d042278..eace09d9dd 100644 --- a/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml +++ b/internal/command/testdata/test/junit-output/1pass-1fail/expected-output.xml @@ -7,7 +7,12 @@ Error: Test assertion failed on main.tftest.hcl line 7, in run "failing_assertion": 7: condition = local.number < 0 ├──────────────── - │ local.number is 10 + │ Diff: + │ --- actual + │ +++ expected + │ - 10 + │ + 0 + local variable 'number' has a value greater than zero, so assertion 2 will fail ]]> diff --git a/internal/command/testdata/test/sensitive_input_values/main.tftest.hcl b/internal/command/testdata/test/sensitive_input_values/main.tftest.hcl index ad2c16bde8..3f6f372a6e 100644 --- a/internal/command/testdata/test/sensitive_input_values/main.tftest.hcl +++ b/internal/command/testdata/test/sensitive_input_values/main.tftest.hcl @@ -13,3 +13,21 @@ run "test" { password = run.setup.password } } + +run "test_failed" { + variables { + password = run.setup.password + complex = { + foo = "bar" + baz = run.test.password + } + } + + assert { + condition = var.complex == { + foo = "bar" + baz = test_resource.resource.id + } + error_message = "expected to fail" + } +} diff --git a/internal/command/views/json/diagnostic.go b/internal/command/views/json/diagnostic.go index daef06c1b1..90f8459c23 100644 --- a/internal/command/views/json/diagnostic.go +++ b/internal/command/views/json/diagnostic.go @@ -103,6 +103,11 @@ type DiagnosticSnippet struct { // FunctionCall is information about a function call whose failure is // being reported by this diagnostic, if any. FunctionCall *DiagnosticFunctionCall `json:"function_call,omitempty"` + + // TestAssertionExpr is information derived from a diagnostic that is caused + // by a failed run assertion. This field is only populated when the assertion + // is a binary expression, i.e `a operand b``. + TestAssertionExpr *DiagnosticTestBinaryExpr `json:"test_assertion_expr,omitempty"` } // DiagnosticExpressionValue represents an HCL traversal string (e.g. @@ -129,6 +134,17 @@ type DiagnosticFunctionCall struct { Signature *Function `json:"signature,omitempty"` } +// DiagnosticTestBinaryExpr represents a failed test assertion diagnostic +// caused by a binary expression. It includes the left-hand side (LHS) and +// right-hand side (RHS) values of the binary expression, as well as a warning +// message if there is a potential issue with the values being compared. +type DiagnosticTestBinaryExpr struct { + LHS string `json:"lhs"` + RHS string `json:"rhs"` + Warning string `json:"warning"` + ShowVerbose bool `json:"show_verbose"` +} + // NewDiagnostic takes a tfdiags.Diagnostic and a map of configuration sources, // and returns a Diagnostic struct. func NewDiagnostic(diag tfdiags.Diagnostic, sources map[string][]byte) *Diagnostic { @@ -277,6 +293,7 @@ func NewDiagnostic(diag tfdiags.Diagnostic, sources map[string][]byte) *Diagnost includeUnknown := tfdiags.DiagnosticCausedByUnknown(diag) includeEphemeral := tfdiags.DiagnosticCausedByEphemeral(diag) includeSensitive := tfdiags.DiagnosticCausedBySensitive(diag) + testDiag := tfdiags.ExtraInfo[tfdiags.DiagnosticExtraCausedByTestFailure](diag) Traversals: for _, traversal := range vars { for len(traversal) > 1 { @@ -296,6 +313,22 @@ func NewDiagnostic(diag tfdiags.Diagnostic, sources map[string][]byte) *Diagnost value := DiagnosticExpressionValue{ Traversal: traversalStr, } + + // If the diagnostic is caused by a failed run assertion, + // we'll redact sensitive and ephemeral values within traversals, but format + // the values in a more human-readable way than the general case. + // If the value is unknown, we'll leave it to the general case to handle. + if testDiag != nil && val.IsKnown() { + valBuf, err := tfdiags.FormatValueStr(val) + if err != nil { + panic(err) + } + value.Statement = fmt.Sprintf("is %s", valBuf) + values = append(values, value) + seen[traversalStr] = struct{}{} + continue Traversals + } + // We'll skip any value that has a mark that we don't // know how to handle, because in that case we can't // know what that mark is intended to represent and so @@ -391,6 +424,7 @@ func NewDiagnostic(diag tfdiags.Diagnostic, sources map[string][]byte) *Diagnost sort.Slice(values, func(i, j int) bool { return values[i].Traversal < values[j].Traversal }) + diagnostic.Snippet.Values = values if callInfo := tfdiags.ExtraInfo[hclsyntax.FunctionCallDiagExtra](diag); callInfo != nil && callInfo.CalledFunctionName() != "" { @@ -408,6 +442,13 @@ func NewDiagnostic(diag tfdiags.Diagnostic, sources map[string][]byte) *Diagnost diagnostic.Snippet.FunctionCall = callInfo } + if testDiag != nil { + // If the test assertion is a binary expression, we'll include the human-readable + // formatted LHS and RHS values in the diagnostic snippet. + diagnostic.Snippet.TestAssertionExpr = formatRunBinaryDiag(ctx, fromExpr.Expression) + diagnostic.Snippet.TestAssertionExpr.ShowVerbose = testDiag.IsTestVerboseMode() + } + } } @@ -416,6 +457,36 @@ func NewDiagnostic(diag tfdiags.Diagnostic, sources map[string][]byte) *Diagnost return diagnostic } +// formatRunBinaryDiag formats the binary expression that caused the failed run diagnostic. +// The LHS and RHS values are formatted in a more human-readable way, redacting +// sensitive and ephemeral values only for the exact values that hold the mark(s). +func formatRunBinaryDiag(ctx *hcl.EvalContext, expr hcl.Expression) *DiagnosticTestBinaryExpr { + bExpr, ok := expr.(*hclsyntax.BinaryOpExpr) + if !ok { + return nil + } + // The expression has already been evaluated and failed, so we can ignore the diags here. + lhs, _ := bExpr.LHS.Value(ctx) + rhs, _ := bExpr.RHS.Value(ctx) + + lhsStr, err := tfdiags.FormatValueStr(lhs) + if err != nil { + panic(err) + } + rhsStr, err := tfdiags.FormatValueStr(rhs) + if err != nil { + panic(err) + } + + ret := &DiagnosticTestBinaryExpr{LHS: lhsStr, RHS: rhsStr} + + // The types do not match. We don't diff them. + if !lhs.Type().Equals(rhs.Type()) { + ret.Warning = "LHS and RHS values are of different types" + } + return ret +} + func parseRange(src []byte, rng hcl.Range) (*hcl.File, int) { filename := rng.Filename offset := rng.Start.Byte diff --git a/internal/lang/funcs/collection.go b/internal/lang/funcs/collection.go index e64ce85894..fd41620230 100644 --- a/internal/lang/funcs/collection.go +++ b/internal/lang/funcs/collection.go @@ -582,8 +582,14 @@ var TransposeFunc = function.New(&function.Spec{ for it := inputMap.ElementIterator(); it.Next(); { inKey, inVal := it.Element() + if inVal.IsNull() { + return cty.MapValEmpty(cty.List(cty.String)), errors.New("input must not contain null list") + } for iter := inVal.ElementIterator(); iter.Next(); { _, val := iter.Element() + if val.IsNull() { + return cty.MapValEmpty(cty.List(cty.String)), errors.New("input list must not contain null string") + } if !val.Type().Equals(cty.String) { return cty.MapValEmpty(cty.List(cty.String)), errors.New("input must be a map of lists of strings") } diff --git a/internal/lang/funcs/collection_test.go b/internal/lang/funcs/collection_test.go index 3f08edc029..e7fc9ab980 100644 --- a/internal/lang/funcs/collection_test.go +++ b/internal/lang/funcs/collection_test.go @@ -1833,6 +1833,37 @@ func TestTranspose(t *testing.T) { }).WithMarks(cty.NewValueMarks("beep", "boop", "bloop")), false, }, + { + cty.NullVal(cty.Map(cty.List(cty.String))), + cty.NilVal, + true, + }, + { + cty.MapVal(map[string]cty.Value{ + "test": cty.NullVal(cty.List(cty.String)), + }), + cty.NilVal, + true, + }, + { + cty.MapVal(map[string]cty.Value{ + "test": cty.ListVal([]cty.Value{cty.NullVal(cty.String)}), + }), + cty.NilVal, + true, + }, + { + cty.UnknownVal(cty.Map(cty.List(cty.String))), + cty.UnknownVal(cty.Map(cty.List(cty.String))).RefineNotNull(), + false, + }, + { + cty.MapVal(map[string]cty.Value{ + "test": cty.ListVal([]cty.Value{cty.UnknownVal(cty.String)}), + }), + cty.UnknownVal(cty.Map(cty.List(cty.String))).RefineNotNull(), + false, + }, } for _, test := range tests { diff --git a/internal/moduletest/graph/diagnostics.go b/internal/moduletest/graph/diagnostics.go new file mode 100644 index 0000000000..3cf3171411 --- /dev/null +++ b/internal/moduletest/graph/diagnostics.go @@ -0,0 +1,53 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package graph + +import "github.com/hashicorp/terraform/internal/tfdiags" + +// DiagnosticCausedByTestFailure implements multiple interfaces that enables it to +// be used in the "Extra" field of a diagnostic. This type should only be used as +// the Extra for diagnostics reporting assertions that fail in a run block during +// `terraform test`. +// +// DiagnosticCausedByTestFailure implements the [DiagnosticExtraCausedByTestFailure] +// interface. This allows downstream logic to identify diagnostics that are specifically +// due to assertion failures. +// +// DiagnosticCausedByTestFailure also implements the [DiagnosticExtraBecauseEphemeral], +// [DiagnosticExtraBecauseSensitive], and [DiagnosticExtraBecauseUnknown] interfaces. +// These interfaces allow the diagnostic renderer to include ephemeral, sensitive or +// unknown data if it's present. This is enabled because if a test fails then the user +// will want to know what values contributed to the failing assertion. +// +// When using this, set the Extra to DiagnosticCausedByTestFailure(true) and also +// populate the EvalContext and Expression fields of the diagnostic. + +type DiagnosticCausedByTestFailure struct { + Verbose bool +} + +var _ tfdiags.DiagnosticExtraCausedByTestFailure = DiagnosticCausedByTestFailure{false} +var _ tfdiags.DiagnosticExtraBecauseEphemeral = DiagnosticCausedByTestFailure{false} +var _ tfdiags.DiagnosticExtraBecauseSensitive = DiagnosticCausedByTestFailure{false} +var _ tfdiags.DiagnosticExtraBecauseUnknown = DiagnosticCausedByTestFailure{false} + +func (e DiagnosticCausedByTestFailure) DiagnosticCausedByTestFailure() bool { + return true +} + +func (e DiagnosticCausedByTestFailure) IsTestVerboseMode() bool { + return e.Verbose +} + +func (e DiagnosticCausedByTestFailure) DiagnosticCausedByEphemeral() bool { + return true +} + +func (e DiagnosticCausedByTestFailure) DiagnosticCausedBySensitive() bool { + return true +} + +func (e DiagnosticCausedByTestFailure) DiagnosticCausedByUnknown() bool { + return true +} diff --git a/internal/moduletest/graph/eval_context.go b/internal/moduletest/graph/eval_context.go index 1e5a4a6bbb..8ad8e1e560 100644 --- a/internal/moduletest/graph/eval_context.go +++ b/internal/moduletest/graph/eval_context.go @@ -274,8 +274,10 @@ func (ec *EvalContext) EvaluateRun(run *moduletest.Run, resultScope *lang.Scope, Subject: rule.Condition.Range().Ptr(), Expression: rule.Condition, EvalContext: hclCtx, - // Make the ephemerality visible - Extra: terraform.DiagnosticCausedByEphemeral(true), + // Diagnostic can be identified as originating from a failing test assertion. + // Also, values that are ephemeral, sensitive, or unknown are replaced with + // redacted values in renderings of the diagnostic. + Extra: DiagnosticCausedByTestFailure{Verbose: ec.verbose}, }) continue } else { diff --git a/internal/tfdiags/diagnostic_extra.go b/internal/tfdiags/diagnostic_extra.go index 944c58da9e..2bf25af830 100644 --- a/internal/tfdiags/diagnostic_extra.go +++ b/internal/tfdiags/diagnostic_extra.go @@ -233,3 +233,30 @@ func DoNotConsolidateDiagnostic(diag Diagnostic) bool { } return maybe.DoNotConsolidateDiagnostic() } + +// DiagnosticExtraCausedByTestFailure is an interface implemented by +// values in the Extra field of Diagnostic when the diagnostic is caused by a +// failing assertion in a run block during the `test` command. +// +// Just implementing this interface is not sufficient signal, though. Callers +// must also call the DiagnosticCausedByTestFailure method in order to +// confirm the result, or use the package-level function +// DiagnosticCausedByTestFailure as a convenient wrapper. +type DiagnosticExtraCausedByTestFailure interface { + // DiagnosticCausedByTestFailure returns true if the associated + // diagnostic is the result of a failed assertion in a run block. + DiagnosticCausedByTestFailure() bool + + // IsTestVerboseMode returns true if the test was executed in verbose mode. + IsTestVerboseMode() bool +} + +// DiagnosticCausedByTestFailure returns true if the given diagnostic +// is the result of a failed assertion in a run block. +func DiagnosticCausedByTestFailure(diag Diagnostic) bool { + maybe := ExtraInfo[DiagnosticExtraCausedByTestFailure](diag) + if maybe == nil { + return false + } + return maybe.DiagnosticCausedByTestFailure() +} diff --git a/internal/tfdiags/format.go b/internal/tfdiags/format.go index b696f570d0..9ac2a8e7e4 100644 --- a/internal/tfdiags/format.go +++ b/internal/tfdiags/format.go @@ -5,8 +5,11 @@ package tfdiags import ( "bytes" + "encoding/json" "fmt" + ctyjson "github.com/zclconf/go-cty/cty/json" + "github.com/hashicorp/hcl/v2" "github.com/zclconf/go-cty/cty" @@ -134,3 +137,37 @@ func TraversalStr(traversal hcl.Traversal) string { } return buf.String() } + +// FormatValueStr produces a JSON-compatible, human-readable representation of a +// cty.Value that is suitable for display in the UI. +// +// The full representation of the value is produced, but with some redaction to +// nodes within the value sensitive and ephemeral marks. +// e.g {"a": "10", "b": "password"} => {"a": "10", "b": "(sensitive value)"} +func FormatValueStr(val cty.Value) (string, error) { + var buf bytes.Buffer + + val, err := cty.Transform(val, func(path cty.Path, val cty.Value) (cty.Value, error) { + // If a value is sensitive or ephemeral or unknown, we redact it, otherwise + // we return the value as is. + if val.HasMark(marks.Sensitive) || val.HasMark(marks.Ephemeral) || !val.IsKnown() { + return cty.StringVal(CompactValueStr(val)), nil + } + return val, nil + }) + if err != nil { + return "", fmt.Errorf("unexpected error transforming value: %s", err) + } + + jsonVal, err := ctyjson.Marshal(val, val.Type()) + if err != nil { + return "", fmt.Errorf("unexpected error marshalling value: %s", err) + } + + // indent the JSON output for better readability + if err := json.Indent(&buf, jsonVal, "", " "); err != nil { + return "", fmt.Errorf("unexpected error formatting JSON: %s", err) + } + + return buf.String(), nil +} diff --git a/website/docs/language/import/index.mdx b/website/docs/language/import/index.mdx index 3088f01619..8390af81aa 100644 --- a/website/docs/language/import/index.mdx +++ b/website/docs/language/import/index.mdx @@ -96,7 +96,7 @@ locals { id = "two_1" }, { - group = "one" + group = "two" key = "bucket2" id = "two_2" },