packer_test: add func to test a cmd multiple times

When a Packer command is created for testing the tool, we generally run
it once, then the command is essentially nooping.

This change allows us to run Packer multiple times with the same
parameters, and make sure all runs conform to a specific list of checks.

This allows us to more reliably test non-deterministic behaviours.

packer_test/commands_test.go

@@ -5,12 +5,11 @@ import (
 	"os"
 	"os/exec"
 	"strings"
-	"sync"
 	"testing"
 )
 
 type packerCommand struct {
-	once       sync.Once
+	runs       int
 	packerPath string
 	args       []string
 	env        map[string]string
@@ -23,11 +22,9 @@ type packerCommand struct {
 
 // PackerCommand creates a skeleton of packer command with the ability to execute gadgets on the outputs of the command.
 func (ts *PackerTestSuite) PackerCommand() *packerCommand {
-	stderr := &strings.Builder{}
-	stdout := &strings.Builder{}
-
 	return &packerCommand{
 		packerPath: ts.packerPath,
+		runs:       1,
 		env: map[string]string{
 			"PACKER_LOG": "1",
 			// Required for Windows, otherwise since we overwrite all
@@ -41,9 +38,7 @@ func (ts *PackerTestSuite) PackerCommand() *packerCommand {
 			// are running as Administrator, but please don't).
 			"TMP": os.TempDir(),
 		},
-		stderr: stderr,
-		stdout: stdout,
-		t:      ts.T(),
+		t: ts.T(),
 	}
 }
 
@@ -77,18 +72,38 @@ func (pc *packerCommand) AddEnv(key, val string) *packerCommand {
 	return pc
 }
 
+// Runs changes the number of times the command is run.
+//
+// This is useful for testing non-deterministic bugs, which we can reasonably
+// execute multiple times and expose a dysfunctional run.
+//
+// This is not necessarily a guarantee that the code is sound, but so long as
+// we run the test enough times, we can be decently confident the problem has
+// been solved.
+func (pc *packerCommand) Runs(runs int) *packerCommand {
+	if runs <= 0 {
+		panic(fmt.Sprintf("cannot set command runs to %d", runs))
+	}
+
+	pc.runs = runs
+	return pc
+}
+
 // Run executes the packer command with the args/env requested and returns the
 // output streams (stdout, stderr)
 //
-// Note: "Run" will only execute the command once, and return the streams and
-// error from the only execution for every subsequent call
+// Note: while originally "Run" was designed to be idempotent, with the
+// introduction of multiple runs for a command, this is not the case anymore
+// and the function should not be considered thread-safe anymore.
 func (pc *packerCommand) Run() (string, string, error) {
-	pc.once.Do(pc.doRun)
+	if pc.runs <= 0 {
+		return pc.stdout.String(), pc.stderr.String(), pc.err
+	}
+	pc.runs--
 
-	return pc.stdout.String(), pc.stderr.String(), pc.err
-}
+	pc.stdout = &strings.Builder{}
+	pc.stderr = &strings.Builder{}
 
-func (pc *packerCommand) doRun() {
 	cmd := exec.Command(pc.packerPath, pc.args...)
 	for key, val := range pc.env {
 		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", key, val))
@@ -101,18 +116,29 @@ func (pc *packerCommand) doRun() {
 	}
 
 	pc.err = cmd.Run()
+
+	return pc.stdout.String(), pc.stderr.String(), pc.err
 }
 
 func (pc *packerCommand) Assert(checks ...Checker) {
-	stdout, stderr, err := pc.Run()
-
-	checks = append(checks, PanicCheck{})
+	attempt := 0
+	for pc.runs > 0 {
+		attempt++
+		stdout, stderr, err := pc.Run()
+
+		checks = append(checks, PanicCheck{})
+
+		for _, check := range checks {
+			checkErr := check.Check(stdout, stderr, err)
+			if checkErr != nil {
+				checkerName := InferName(check)
+				pc.t.Errorf("check %q failed: %s", checkerName, checkErr)
+			}
+		}
 
-	for _, check := range checks {
-		checkErr := check.Check(stdout, stderr, err)
-		if checkErr != nil {
-			checkerName := InferName(check)
-			pc.t.Errorf("check %q failed: %s", checkerName, checkErr)
+		if pc.t.Failed() {
+			pc.t.Errorf("attempt %d failed validation", attempt)
+			break
 		}
 	}
 }