diff --git a/packer_test/commands_test.go b/packer_test/commands_test.go
index 8f6beb7e5..54bdd69bf 100644
--- a/packer_test/commands_test.go
+++ b/packer_test/commands_test.go
@@ -5,12 +5,11 @@ import (
 	"os"
 	"os/exec"
 	"strings"
-	"sync"
 	"testing"
 )
 
 type packerCommand struct {
-	once       sync.Once
+	runs       int
 	packerPath string
 	args       []string
 	env        map[string]string
@@ -23,11 +22,9 @@ type packerCommand struct {
 
 // PackerCommand creates a skeleton of packer command with the ability to execute gadgets on the outputs of the command.
 func (ts *PackerTestSuite) PackerCommand() *packerCommand {
-	stderr := &strings.Builder{}
-	stdout := &strings.Builder{}
-
 	return &packerCommand{
 		packerPath: ts.packerPath,
+		runs:       1,
 		env: map[string]string{
 			"PACKER_LOG": "1",
 			// Required for Windows, otherwise since we overwrite all
@@ -41,9 +38,7 @@ func (ts *PackerTestSuite) PackerCommand() *packerCommand {
 			// are running as Administrator, but please don't).
 			"TMP": os.TempDir(),
 		},
-		stderr: stderr,
-		stdout: stdout,
-		t:      ts.T(),
+		t: ts.T(),
 	}
 }
 
@@ -77,18 +72,38 @@ func (pc *packerCommand) AddEnv(key, val string) *packerCommand {
 	return pc
 }
 
+// Runs changes the number of times the command is run.
+//
+// This is useful for testing non-deterministic bugs, which we can reasonably
+// execute multiple times and expose a dysfunctional run.
+//
+// This is not necessarily a guarantee that the code is sound, but so long as
+// we run the test enough times, we can be decently confident the problem has
+// been solved.
+func (pc *packerCommand) Runs(runs int) *packerCommand {
+	if runs <= 0 {
+		panic(fmt.Sprintf("cannot set command runs to %d", runs))
+	}
+
+	pc.runs = runs
+	return pc
+}
+
 // Run executes the packer command with the args/env requested and returns the
 // output streams (stdout, stderr)
 //
-// Note: "Run" will only execute the command once, and return the streams and
-// error from the only execution for every subsequent call
+// Note: while originally "Run" was designed to be idempotent, with the
+// introduction of multiple runs for a command, this is not the case anymore
+// and the function should not be considered thread-safe anymore.
 func (pc *packerCommand) Run() (string, string, error) {
-	pc.once.Do(pc.doRun)
+	if pc.runs <= 0 {
+		return pc.stdout.String(), pc.stderr.String(), pc.err
+	}
+	pc.runs--
 
-	return pc.stdout.String(), pc.stderr.String(), pc.err
-}
+	pc.stdout = &strings.Builder{}
+	pc.stderr = &strings.Builder{}
 
-func (pc *packerCommand) doRun() {
 	cmd := exec.Command(pc.packerPath, pc.args...)
 	for key, val := range pc.env {
 		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", key, val))
@@ -101,18 +116,29 @@ func (pc *packerCommand) doRun() {
 	}
 
 	pc.err = cmd.Run()
+
+	return pc.stdout.String(), pc.stderr.String(), pc.err
 }
 
 func (pc *packerCommand) Assert(checks ...Checker) {
-	stdout, stderr, err := pc.Run()
-
-	checks = append(checks, PanicCheck{})
+	attempt := 0
+	for pc.runs > 0 {
+		attempt++
+		stdout, stderr, err := pc.Run()
+
+		checks = append(checks, PanicCheck{})
+
+		for _, check := range checks {
+			checkErr := check.Check(stdout, stderr, err)
+			if checkErr != nil {
+				checkerName := InferName(check)
+				pc.t.Errorf("check %q failed: %s", checkerName, checkErr)
+			}
+		}
 
-	for _, check := range checks {
-		checkErr := check.Check(stdout, stderr, err)
-		if checkErr != nil {
-			checkerName := InferName(check)
-			pc.t.Errorf("check %q failed: %s", checkerName, checkErr)
+		if pc.t.Failed() {
+			pc.t.Errorf("attempt %d failed validation", attempt)
+			break
 		}
 	}
 }