packer_test: add func to change assert behaviour

When a command asserts its output with checkers, by default it will
register errors through a t.Errorf.

While this works, in some cases we would want to stop execution
immediately if a function's Assert fails, as the rest of the test may
depend on the assertion being valid.

In the current state, this means either getting the result of the run to
check if an error was returned (not fully reliable as if the command was
run multiple times, and the last run succeeded, we won't get an error),
or relying on t.IsFailed() (completely reliable).

Instead, we introduce a new function on packerCommand, that lets users
change how Assert behaves, so that if an error was reported, instead of
logging the error and flagging the test as failed, we can use t.Fatalf,
so that the test immedately fails and stops execution.

packer_test/common/commands.go

@@ -11,16 +11,17 @@ import (
 )
 
 type packerCommand struct {
-	runs       int
-	packerPath string
-	args       []string
-	env        map[string]string
-	stdin      string
-	stderr     *strings.Builder
-	stdout     *strings.Builder
-	workdir    string
-	err        error
-	t          *testing.T
+	runs         int
+	packerPath   string
+	args         []string
+	env          map[string]string
+	stdin        string
+	stderr       *strings.Builder
+	stdout       *strings.Builder
+	workdir      string
+	err          error
+	t            *testing.T
+	fatalfAssert bool
 }
 
 // PackerCommand creates a skeleton of packer command with the ability to execute gadgets on the outputs of the command.
@@ -108,6 +109,16 @@ func (pc *packerCommand) Stdin(in string) *packerCommand {
 	return pc
 }
 
+// SetAssertFatal allows changing how Assert behaves when reporting an error.
+//
+// By default Assert will invoke t.Errorf with the error details, but this can be
+// changed to a t.Fatalf so that if the assertion fails, the test invoking it will
+// also immediately fail and stop execution.
+func (pc *packerCommand) SetAssertFatal() *packerCommand {
+	pc.fatalfAssert = true
+	return pc
+}
+
 // Run executes the packer command with the args/env requested and returns the
 // output streams (stdout, stderr)
 //
@@ -159,6 +170,7 @@ func (pc *packerCommand) Assert(checks ...check.Checker) {
 			checkErr := checker.Check(stdout, stderr, err)
 			if checkErr != nil {
 				checkerName := check.InferName(checker)
+
 				pc.t.Errorf("check %q failed: %s", checkerName, checkErr)
 			}
 		}
@@ -169,6 +181,10 @@ func (pc *packerCommand) Assert(checks ...check.Checker) {
 			pc.t.Logf("dumping stdout: %s", stdout)
 			pc.t.Logf("dumping stdout: %s", stderr)
 
+			if pc.fatalfAssert {
+				pc.t.Fatalf("stopping test now because of failures reported")
+			}
+
 			break
 		}
 	}

packer_test/plugin_tests/init_test.go

@@ -1,6 +1,8 @@
 package plugin_tests
 
-import "github.com/hashicorp/packer/packer_test/common/check"
+import (
+	"github.com/hashicorp/packer/packer_test/common/check"
+)
 
 func (ts *PackerPluginTestSuite) TestPackerInitForce() {
 	ts.SkipNoAcc()
@@ -29,13 +31,9 @@ func (ts *PackerPluginTestSuite) TestPackerInitUpgrade() {
 
 	cmd := ts.PackerCommand().UsePluginDir(pluginPath)
 	cmd.SetArgs("plugins", "install", "github.com/hashicorp/hashicups", "1.0.1")
+	cmd.SetAssertFatal()
 	cmd.Assert(check.MustSucceed(), check.Grep("Installed plugin github.com/hashicorp/hashicups v1.0.1", check.GrepStdout))
 
-	_, _, err := cmd.Run()
-	if err != nil {
-		ts.T().Fatalf("packer plugins install failed to install previous version of hashicups: %q", err)
-	}
-
 	ts.Run("upgrades a plugin to the latest matching version constraints", func() {
 		ts.PackerCommand().UsePluginDir(pluginPath).
 			SetArgs("init", "--upgrade", "./templates/init/hashicups.pkr.hcl").

packer_test/plugin_tests/plugins_remove_test.go

@@ -164,11 +164,9 @@ func InstalledPlugins(ts *PackerPluginTestSuite, dir string) []string {
 	ts.T().Helper()
 
 	cmd := ts.PackerCommand().UsePluginDir(dir).
-		SetArgs("plugins", "installed")
+		SetArgs("plugins", "installed").
+		SetAssertFatal()
 	cmd.Assert(check.MustSucceed())
-	if ts.T().Failed() {
-		ts.T().Fatalf("Failed to execute plugin installed for %q", dir)
-	}
 
 	out, _, _ := cmd.Run()
 	// Output will be split on '\n' after trimming all other white space