From 71169416db6671404dbe24b334692cb49f1e9d12 Mon Sep 17 00:00:00 2001
From: Lucas Bajolet <lucas.bajolet@hashicorp.com>
Date: Thu, 15 Aug 2024 10:21:35 -0400
Subject: [PATCH] packer_test: add func to change assert behaviour

When a command asserts its output with checkers, by default it will
register errors through a t.Errorf.

While this works, in some cases we would want to stop execution
immediately if a function's Assert fails, as the rest of the test may
depend on the assertion being valid.

In the current state, this means either getting the result of the run to
check if an error was returned (not fully reliable as if the command was
run multiple times, and the last run succeeded, we won't get an error),
or relying on t.IsFailed() (completely reliable).

Instead, we introduce a new function on packerCommand, that lets users
change how Assert behaves, so that if an error was reported, instead of
logging the error and flagging the test as failed, we can use t.Fatalf,
so that the test immedately fails and stops execution.
---
 packer_test/common/commands.go                | 36 +++++++++++++------
 packer_test/plugin_tests/init_test.go         | 10 +++---
 .../plugin_tests/plugins_remove_test.go       |  6 ++--
 3 files changed, 32 insertions(+), 20 deletions(-)

diff --git a/packer_test/common/commands.go b/packer_test/common/commands.go
index 4e2ebd285..9d1cf1b18 100644
--- a/packer_test/common/commands.go
+++ b/packer_test/common/commands.go
@@ -11,16 +11,17 @@ import (
 )
 
 type packerCommand struct {
-	runs       int
-	packerPath string
-	args       []string
-	env        map[string]string
-	stdin      string
-	stderr     *strings.Builder
-	stdout     *strings.Builder
-	workdir    string
-	err        error
-	t          *testing.T
+	runs         int
+	packerPath   string
+	args         []string
+	env          map[string]string
+	stdin        string
+	stderr       *strings.Builder
+	stdout       *strings.Builder
+	workdir      string
+	err          error
+	t            *testing.T
+	fatalfAssert bool
 }
 
 // PackerCommand creates a skeleton of packer command with the ability to execute gadgets on the outputs of the command.
@@ -108,6 +109,16 @@ func (pc *packerCommand) Stdin(in string) *packerCommand {
 	return pc
 }
 
+// SetAssertFatal allows changing how Assert behaves when reporting an error.
+//
+// By default Assert will invoke t.Errorf with the error details, but this can be
+// changed to a t.Fatalf so that if the assertion fails, the test invoking it will
+// also immediately fail and stop execution.
+func (pc *packerCommand) SetAssertFatal() *packerCommand {
+	pc.fatalfAssert = true
+	return pc
+}
+
 // Run executes the packer command with the args/env requested and returns the
 // output streams (stdout, stderr)
 //
@@ -159,6 +170,7 @@ func (pc *packerCommand) Assert(checks ...check.Checker) {
 			checkErr := checker.Check(stdout, stderr, err)
 			if checkErr != nil {
 				checkerName := check.InferName(checker)
+
 				pc.t.Errorf("check %q failed: %s", checkerName, checkErr)
 			}
 		}
@@ -169,6 +181,10 @@ func (pc *packerCommand) Assert(checks ...check.Checker) {
 			pc.t.Logf("dumping stdout: %s", stdout)
 			pc.t.Logf("dumping stdout: %s", stderr)
 
+			if pc.fatalfAssert {
+				pc.t.Fatalf("stopping test now because of failures reported")
+			}
+
 			break
 		}
 	}
diff --git a/packer_test/plugin_tests/init_test.go b/packer_test/plugin_tests/init_test.go
index 617449f0c..31ca8dd73 100644
--- a/packer_test/plugin_tests/init_test.go
+++ b/packer_test/plugin_tests/init_test.go
@@ -1,6 +1,8 @@
 package plugin_tests
 
-import "github.com/hashicorp/packer/packer_test/common/check"
+import (
+	"github.com/hashicorp/packer/packer_test/common/check"
+)
 
 func (ts *PackerPluginTestSuite) TestPackerInitForce() {
 	ts.SkipNoAcc()
@@ -29,13 +31,9 @@ func (ts *PackerPluginTestSuite) TestPackerInitUpgrade() {
 
 	cmd := ts.PackerCommand().UsePluginDir(pluginPath)
 	cmd.SetArgs("plugins", "install", "github.com/hashicorp/hashicups", "1.0.1")
+	cmd.SetAssertFatal()
 	cmd.Assert(check.MustSucceed(), check.Grep("Installed plugin github.com/hashicorp/hashicups v1.0.1", check.GrepStdout))
 
-	_, _, err := cmd.Run()
-	if err != nil {
-		ts.T().Fatalf("packer plugins install failed to install previous version of hashicups: %q", err)
-	}
-
 	ts.Run("upgrades a plugin to the latest matching version constraints", func() {
 		ts.PackerCommand().UsePluginDir(pluginPath).
 			SetArgs("init", "--upgrade", "./templates/init/hashicups.pkr.hcl").
diff --git a/packer_test/plugin_tests/plugins_remove_test.go b/packer_test/plugin_tests/plugins_remove_test.go
index 23e49151f..0584a8d07 100644
--- a/packer_test/plugin_tests/plugins_remove_test.go
+++ b/packer_test/plugin_tests/plugins_remove_test.go
@@ -164,11 +164,9 @@ func InstalledPlugins(ts *PackerPluginTestSuite, dir string) []string {
 	ts.T().Helper()
 
 	cmd := ts.PackerCommand().UsePluginDir(dir).
-		SetArgs("plugins", "installed")
+		SetArgs("plugins", "installed").
+		SetAssertFatal()
 	cmd.Assert(check.MustSucceed())
-	if ts.T().Failed() {
-		ts.T().Fatalf("Failed to execute plugin installed for %q", dir)
-	}
 
 	out, _, _ := cmd.Run()
 	// Output will be split on '\n' after trimming all other white space