Implement updates to testing framework based on recent feedback

go.mod

@@ -86,6 +86,7 @@ require (
 	go.opentelemetry.io/otel/sdk v1.16.0
 	go.opentelemetry.io/otel/trace v1.16.0
 	golang.org/x/crypto v0.10.0
+	golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17
 	golang.org/x/mod v0.10.0
 	golang.org/x/net v0.11.0
 	golang.org/x/oauth2 v0.8.0
@@ -220,7 +221,6 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 // indirect
 	go.opentelemetry.io/otel/metric v1.16.0 // indirect
 	go.opentelemetry.io/proto/otlp v0.20.0 // indirect
-	golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
 	golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a // indirect
 	golang.org/x/sync v0.3.0 // indirect
 	golang.org/x/time v0.3.0 // indirect

internal/command/test_test.go

@@ -420,9 +420,9 @@ func TestTest_CatchesErrorsBeforeDestroy(t *testing.T) {
 	}
 
 	expectedOut := `main.tftest.hcl... fail
-  run "test"... skip
+  run "test"... fail
 
-Failure! 0 passed, 0 failed, 1 skipped.
+Failure! 0 passed, 1 failed.
 `
 
 	expectedErr := `
@@ -552,9 +552,9 @@ variable can be declared with a variable "not_real" {} block.
 		},
 		"missing-provider": {
 			expectedOut: `main.tftest.hcl... fail
-  run "passes_validation"... skip
+  run "passes_validation"... fail
 
-Failure! 0 passed, 0 failed, 1 skipped.
+Failure! 0 passed, 1 failed.
 `,
 			expectedErr: `
 Error: Provider configuration not present
@@ -721,3 +721,243 @@ func TestTest_NestedSetupModules(t *testing.T) {
 		}
 	}
 }
+
+func TestTest_StatePropagation(t *testing.T) {
+	td := t.TempDir()
+	testCopyDir(t, testFixturePath(path.Join("test", "state_propagation")), td)
+	defer testChdir(t, td)()
+
+	provider := testing_command.NewProvider(nil)
+
+	providerSource, close := newMockProviderSource(t, map[string][]string{
+		"test": {"1.0.0"},
+	})
+	defer close()
+
+	streams, done := terminal.StreamsForTesting(t)
+	view := views.NewView(streams)
+	ui := new(cli.MockUi)
+
+	meta := Meta{
+		testingOverrides: metaOverridesForProvider(provider.Provider),
+		Ui:               ui,
+		View:             view,
+		Streams:          streams,
+		ProviderSource:   providerSource,
+	}
+
+	init := &InitCommand{
+		Meta: meta,
+	}
+
+	if code := init.Run(nil); code != 0 {
+		t.Fatalf("expected status code 0 but got %d: %s", code, ui.ErrorWriter)
+	}
+
+	c := &TestCommand{
+		Meta: meta,
+	}
+
+	code := c.Run([]string{"-verbose", "-no-color"})
+	output := done(t)
+
+	if code != 0 {
+		t.Errorf("expected status code 0 but got %d", code)
+	}
+
+	expected := `main.tftest.hcl... pass
+  run "initial_apply_example"... pass
+# test_resource.module_resource:
+resource "test_resource" "module_resource" {
+    id    = "df6h8as9"
+    value = "start"
+}
+  run "initial_apply"... pass
+# test_resource.resource:
+resource "test_resource" "resource" {
+    id    = "598318e0"
+    value = "start"
+}
+  run "plan_second_example"... pass
+
+Terraform used the selected providers to generate the following execution
+plan. Resource actions are indicated with the following symbols:
+  + create
+
+Terraform will perform the following actions:
+
+  # test_resource.second_module_resource will be created
+  + resource "test_resource" "second_module_resource" {
+      + id    = "b6a1d8cb"
+      + value = "start"
+    }
+
+Plan: 1 to add, 0 to change, 0 to destroy.
+  run "plan_update"... pass
+
+Terraform used the selected providers to generate the following execution
+plan. Resource actions are indicated with the following symbols:
+  ~ update in-place
+
+Terraform will perform the following actions:
+
+  # test_resource.resource will be updated in-place
+  ~ resource "test_resource" "resource" {
+        id    = "598318e0"
+      ~ value = "start" -> "update"
+    }
+
+Plan: 0 to add, 1 to change, 0 to destroy.
+  run "plan_update_example"... pass
+
+Terraform used the selected providers to generate the following execution
+plan. Resource actions are indicated with the following symbols:
+  ~ update in-place
+
+Terraform will perform the following actions:
+
+  # test_resource.module_resource will be updated in-place
+  ~ resource "test_resource" "module_resource" {
+        id    = "df6h8as9"
+      ~ value = "start" -> "update"
+    }
+
+Plan: 0 to add, 1 to change, 0 to destroy.
+
+Success! 5 passed, 0 failed.
+`
+
+	actual := output.All()
+
+	if diff := cmp.Diff(actual, expected); len(diff) > 0 {
+		t.Errorf("output didn't match expected:\nexpected:\n%s\nactual:\n%s\ndiff:\n%s", expected, actual, diff)
+	}
+
+	if provider.ResourceCount() > 0 {
+		t.Errorf("should have deleted all resources on completion but left %v", provider.ResourceString())
+	}
+}
+
+func TestTest_OnlyExternalModules(t *testing.T) {
+	td := t.TempDir()
+	testCopyDir(t, testFixturePath(path.Join("test", "only_modules")), td)
+	defer testChdir(t, td)()
+
+	provider := testing_command.NewProvider(nil)
+
+	providerSource, close := newMockProviderSource(t, map[string][]string{
+		"test": {"1.0.0"},
+	})
+	defer close()
+
+	streams, done := terminal.StreamsForTesting(t)
+	view := views.NewView(streams)
+	ui := new(cli.MockUi)
+
+	meta := Meta{
+		testingOverrides: metaOverridesForProvider(provider.Provider),
+		Ui:               ui,
+		View:             view,
+		Streams:          streams,
+		ProviderSource:   providerSource,
+	}
+
+	init := &InitCommand{
+		Meta: meta,
+	}
+
+	if code := init.Run(nil); code != 0 {
+		t.Fatalf("expected status code 0 but got %d: %s", code, ui.ErrorWriter)
+	}
+
+	c := &TestCommand{
+		Meta: meta,
+	}
+
+	code := c.Run([]string{"-no-color"})
+	output := done(t)
+
+	if code != 0 {
+		t.Errorf("expected status code 0 but got %d", code)
+	}
+
+	expected := `main.tftest.hcl... pass
+  run "first"... pass
+  run "second"... pass
+
+Success! 2 passed, 0 failed.
+`
+
+	actual := output.All()
+
+	if diff := cmp.Diff(actual, expected); len(diff) > 0 {
+		t.Errorf("output didn't match expected:\nexpected:\n%s\nactual:\n%s\ndiff:\n%s", expected, actual, diff)
+	}
+
+	if provider.ResourceCount() > 0 {
+		t.Errorf("should have deleted all resources on completion but left %v", provider.ResourceString())
+	}
+}
+
+func TestTest_PartialUpdates(t *testing.T) {
+	td := t.TempDir()
+	testCopyDir(t, testFixturePath(path.Join("test", "partial_updates")), td)
+	defer testChdir(t, td)()
+
+	provider := testing_command.NewProvider(nil)
+	view, done := testView(t)
+
+	c := &TestCommand{
+		Meta: Meta{
+			testingOverrides: metaOverridesForProvider(provider.Provider),
+			View:             view,
+		},
+	}
+
+	code := c.Run([]string{"-no-color"})
+	output := done(t)
+
+	if code != 0 {
+		t.Errorf("expected status code 0 but got %d", code)
+	}
+
+	expected := `main.tftest.hcl... pass
+  run "first"... pass
+
+Warning: Resource targeting is in effect
+
+You are creating a plan with the -target option, which means that the result
+of this plan may not represent all of the changes requested by the current
+configuration.
+
+The -target option is not for routine use, and is provided only for
+exceptional situations such as recovering from errors or mistakes, or when
+Terraform specifically suggests to use it as part of an error message.
+
+Warning: Applied changes may be incomplete
+
+The plan was created with the -target option in effect, so some changes
+requested in the configuration may have been ignored and the output values
+may not be fully updated. Run the following command to verify that no other
+changes are pending:
+    terraform plan
+	
+Note that the -target option is not suitable for routine use, and is provided
+only for exceptional situations such as recovering from errors or mistakes,
+or when Terraform specifically suggests to use it as part of an error
+message.
+  run "second"... pass
+
+Success! 2 passed, 0 failed.
+`
+
+	actual := output.All()
+
+	if diff := cmp.Diff(actual, expected); len(diff) > 0 {
+		t.Errorf("output didn't match expected:\nexpected:\n%s\nactual:\n%s\ndiff:\n%s", expected, actual, diff)
+	}
+
+	if provider.ResourceCount() > 0 {
+		t.Errorf("should have deleted all resources on completion but left %v", provider.ResourceString())
+	}
+}

internal/command/testdata/test/expect_failures_inputs/main.tftest.hcl

@@ -3,6 +3,8 @@ variables {
 }
 
 run "test" {
+  command = plan
+
   expect_failures = [
     var.input
   ]

internal/command/testdata/test/expect_failures_outputs/main.tftest.hcl

@@ -3,6 +3,9 @@ variables {
 }
 
 run "test" {
+
+  command = plan
+
   expect_failures = [
     output.output
   ]

internal/command/testdata/test/expect_failures_resources/main.tftest.hcl

@@ -3,6 +3,8 @@ variables {
 }
 
 run "test" {
+  command = plan
+
   expect_failures = [
     test_resource.resource
   ]

internal/command/testdata/test/invalid_default_state/main.tftest.hcl

@@ -1,8 +1,4 @@
 run "test" {
-    variables {
-        input = "Hello, world!"
-    }
-
     assert {
         condition = test_resource.resource.value == "Hello, world!"
         error_message = "wrong condition"

internal/command/testdata/test/only_modules/example/main.tf

@@ -0,0 +1,9 @@
+
+variable "input" {
+  type = string
+}
+
+resource "test_resource" "module_resource" {
+  id = "df6h8as9"
+  value = var.input
+}

internal/command/testdata/test/only_modules/main.tf

@@ -0,0 +1,9 @@
+
+variable "input" {
+  type = string
+}
+
+resource "test_resource" "resource" {
+  id = "598318e0"
+  value = var.input
+}

internal/command/testdata/test/only_modules/main.tftest.hcl

@@ -0,0 +1,23 @@
+# This is an example test file from a use case requested by a user. We only
+# refer to alternate modules and not the main configuration. This means we
+# shouldn't have to provide any data for the main configuration.
+
+run "first" {
+  module {
+    source = "./example"
+  }
+
+  variables {
+    input = "start"
+  }
+}
+
+run "second" {
+  module {
+    source = "./example"
+  }
+
+  variables {
+    input = "update"
+  }
+}

internal/command/testdata/test/partial_updates/main.tf

@@ -0,0 +1,15 @@
+
+resource "test_resource" "resource" {}
+
+locals {
+  follow = {
+    (test_resource.resource.id): "follow"
+  }
+}
+
+resource "test_resource" "follow" {
+  for_each = local.follow
+
+  id = each.key
+  value = each.value
+}

internal/command/testdata/test/partial_updates/main.tftest.hcl

@@ -0,0 +1,10 @@
+
+run "first" {
+  plan_options {
+    target = [
+      test_resource.resource,
+    ]
+  }
+}
+
+run "second" {}

internal/command/testdata/test/state_propagation/example/main.tf

@@ -0,0 +1,9 @@
+
+variable "input" {
+  type = string
+}
+
+resource "test_resource" "module_resource" {
+  id = "df6h8as9"
+  value = var.input
+}

internal/command/testdata/test/state_propagation/main.tf

@@ -0,0 +1,9 @@
+
+variable "input" {
+  type = string
+}
+
+resource "test_resource" "resource" {
+  id = "598318e0"
+  value = var.input
+}

internal/command/testdata/test/state_propagation/main.tftest.hcl

@@ -0,0 +1,54 @@
+# Our test will run this in verbose mode and we should see the plan output for
+# the second run block showing the resource being updated as the state should
+# be propagated from the first one to the second one.
+#
+# We also interweave alternate modules to test the handling of multiple states
+# within the file.
+
+run "initial_apply_example" {
+  module {
+    source = "./example"
+  }
+
+  variables {
+    input = "start"
+  }
+}
+
+run "initial_apply" {
+  variables {
+    input = "start"
+  }
+}
+
+run "plan_second_example" {
+  command = plan
+
+  module {
+    source = "./second_example"
+  }
+
+  variables {
+    input = "start"
+  }
+}
+
+run "plan_update" {
+  command = plan
+
+  variables {
+    input = "update"
+  }
+}
+
+run "plan_update_example" {
+  command = plan
+
+  module {
+    source = "./example"
+  }
+
+  variables {
+    input = "update"
+  }
+}

internal/command/testdata/test/state_propagation/second_example/main.tf

@@ -0,0 +1,9 @@
+
+variable "input" {
+  type = string
+}
+
+resource "test_resource" "second_module_resource" {
+  id = "b6a1d8cb"
+  value = var.input
+}