test: add base suite for core acceptance testing

Acceptance testing, i.e. running Packer core commands in a controlled
environment and ensuring the behaviour is consistent to what we
expect/document, is not something we have a robust and usable framework
for at the moment.

This commit is a proposal for a base testing framework of the sort, that
is meant to be shipped with packer core, and which will eventually host
most of the tests we currently do in command where we mock an
environment.

test/base_test.go

@@ -0,0 +1,32 @@
+package test
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+)
+
+// BuildTestPacker builds a new Packer binary based on the current state of the repository.
+//
+// If for some reason the binary cannot be built, we will immediately exit with an error.
+func BuildTestPacker(t *testing.T) (string, error) {
+	testDir, err := currentDir()
+	if err != nil {
+		return "", fmt.Errorf("failed to compile packer binary: %s", err)
+	}
+
+	packerCoreDir := filepath.Dir(testDir)
+
+	outBin := filepath.Join(os.TempDir(), fmt.Sprintf("packer_core-%d", rand.Int()))
+
+	compileCommand := exec.Command("go", "build", "-C", packerCoreDir, "-o", outBin)
+	logs, err := compileCommand.CombinedOutput()
+	if err != nil {
+		t.Fatalf("failed to compile Packer core: %s\ncompilation logs: %s", err, logs)
+	}
+
+	return outBin, nil
+}

test/commands_test.go

@@ -0,0 +1,81 @@
+package test
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"sync"
+	"testing"
+)
+
+type packerCommand struct {
+	once       sync.Once
+	packerPath string
+	args       []string
+	env        map[string]string
+	stderr     *strings.Builder
+	stdout     *strings.Builder
+	err        error
+}
+
+// PackerCommand creates a skeleton of packer command with the ability to execute gadgets on the outputs of the command.
+func (ts *PackerTestSuite) PackerCommand() *packerCommand {
+	stderr := &strings.Builder{}
+	stdout := &strings.Builder{}
+
+	return &packerCommand{
+		packerPath: ts.packerPath,
+		env: map[string]string{
+			"PACKER_LOG": "1",
+		},
+		stderr: stderr,
+		stdout: stdout,
+	}
+}
+
+func (pc *packerCommand) SetArgs(args ...string) *packerCommand {
+	pc.args = args
+	return pc
+}
+
+func (pc *packerCommand) AddEnv(key, val string) *packerCommand {
+	pc.env[key] = val
+	return pc
+}
+
+// Run executes the packer command with the args/env requested and returns the
+// output streams (stdout, stderr)
+//
+// Note: "Run" will only execute the command once, and return the streams and
+// error from the only execution for every subsequent call
+func (pc *packerCommand) Run(t *testing.T) (string, string, error) {
+	pc.once.Do(pc.doRun)
+
+	if strings.Contains(pc.stdout.String(), "PACKER CRASH") || strings.Contains(pc.stderr.String(), "PACKER CRASH") {
+		t.Fatalf("Packer has crashed while running the following command: packer %#v", pc.args)
+	}
+
+	return pc.stdout.String(), pc.stderr.String(), pc.err
+}
+
+func (pc *packerCommand) doRun() {
+	cmd := exec.Command("packer", pc.args...)
+	for key, val := range pc.env {
+		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", key, val))
+	}
+	cmd.Stdout = pc.stdout
+	cmd.Stderr = pc.stderr
+
+	pc.err = cmd.Run()
+}
+
+func (pc *packerCommand) Assert(t *testing.T, checks ...Checker) {
+	stdout, stderr, err := pc.Run(t)
+
+	for _, check := range checks {
+		checkErr := check.Check(stdout, stderr, err)
+		if checkErr != nil {
+			t.Errorf("check %q failed: %s", check.Name(), checkErr)
+		}
+	}
+}

test/gadgets_test.go

@@ -0,0 +1,99 @@
+package test
+
+import (
+	"fmt"
+	"regexp"
+	"testing"
+)
+
+type Stream int
+
+const (
+	// BothStreams will use both stdout and stderr for performing a check
+	BothStreams Stream = iota
+	// OnlyStdout will only use stdout for performing a check
+	OnlyStdout
+	// OnlySterr will only use stderr for performing a check
+	OnlyStderr
+)
+
+func (s Stream) String() string {
+	switch s {
+	case BothStreams:
+		return "Both streams"
+	case OnlyStdout:
+		return "Stdout"
+	case OnlyStderr:
+		return "Stderr"
+	}
+
+	panic(fmt.Sprintf("Unknown stream value: %d", s))
+}
+
+type Checker interface {
+	Check(stdout, stderr string, err error) error
+	Name() string
+}
+
+type MustSucceed struct{}
+
+func (_ MustSucceed) Check(stdout, stderr string, err error) error {
+	return err
+}
+
+func (_ MustSucceed) Name() string {
+	return "Must succeed"
+}
+
+// Grep is essentially the equivalent to a normal grep -E on the command line.
+//
+// The `expect` string is meant to be a regexp, which will be compiled on-demand,
+// and will panic if it isn't a valid POSIX extended regexp.
+type Grep struct {
+	streams Stream
+	expect  string
+}
+
+func (g Grep) Check(stdout, stderr string, err error) error {
+	re := regexp.MustCompilePOSIX(g.expect)
+
+	streams := []string{}
+
+	switch g.streams {
+	case BothStreams:
+		streams = append(streams, stdout, stderr)
+	case OnlyStdout:
+		streams = append(streams, stdout)
+	case OnlyStderr:
+		streams = append(streams, stderr)
+	}
+
+	var found bool
+	for _, stream := range streams {
+		found = found || re.MatchString(stream)
+	}
+
+	if !found {
+		return fmt.Errorf("streams %q did not match the expected regexp %q", g.streams, g.expect)
+	}
+	return nil
+}
+
+func (g Grep) Name() string {
+	return fmt.Sprintf("command (%s) | grep -E %q", g.streams, g.expect)
+}
+
+type Dump struct {
+	t *testing.T
+}
+
+func (d Dump) Check(stdout, stderr string, err error) error {
+	d.t.Logf("Dumping command result.")
+	d.t.Logf("Stdout: %s", stdout)
+	d.t.Logf("stderr: %s", stderr)
+	return nil
+}
+
+func (_ Dump) Name() string {
+	return "dump"
+}

test/loading_test.go

@@ -0,0 +1,40 @@
+package test
+
+import (
+	"os"
+	"testing"
+)
+
+func (ts *PackerTestSuite) TestLoadingOrder() {
+	t := ts.T()
+
+	pluginDir := ts.MakePluginDir(t, "1.0.9", "1.0.10")
+	defer func() {
+		err := os.RemoveAll(pluginDir)
+		if err != nil {
+			t.Logf("failed to remove temporary plugin directory %q: %s. This may need manual intervention.", pluginDir, err)
+		}
+	}()
+
+	tests := []struct {
+		name         string
+		templatePath string
+	}{
+		{
+			"HCL2 - No required_plugins, 1.0.10 is the most recent and should load",
+			"./templates/simple.pkr.hcl",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ts.PackerCommand().
+				SetArgs("build", tt.templatePath).
+				AddEnv("PACKER_PLUGIN_PATH", pluginDir).
+				Assert(t, MustSucceed{}, Grep{
+					streams: BothStreams,
+					expect:  "packer-plugin-tester_v1\\.0\\.10[^\n]+ plugin:",
+				})
+		})
+	}
+}

test/plugin_test.go

@@ -0,0 +1,192 @@
+package test
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"sync"
+	"testing"
+
+	"github.com/hashicorp/go-version"
+)
+
+var compiledPlugins = struct {
+	pluginVersions map[string]string
+	RWMutex        sync.RWMutex
+}{
+	pluginVersions: map[string]string{},
+}
+
+func StorePluginVersion(pluginVersion, path string) {
+	compiledPlugins.RWMutex.Lock()
+	defer compiledPlugins.RWMutex.Unlock()
+	compiledPlugins.pluginVersions[pluginVersion] = path
+}
+
+func LoadPluginVersion(pluginVersion string) (string, bool) {
+	compiledPlugins.RWMutex.RLock()
+	defer compiledPlugins.RWMutex.RUnlock()
+
+	path, ok := compiledPlugins.pluginVersions[pluginVersion]
+	return path, ok
+}
+
+var tempPluginBinaryPath = struct {
+	path string
+	once sync.Once
+}{}
+
+// PluginBinaryDir returns the path to the directory where temporary binaries will be compiled
+func PluginBinaryDir() string {
+	tempPluginBinaryPath.once.Do(func() {
+		tempDir, err := os.MkdirTemp("", "packer-core-acc-test-")
+		if err != nil {
+			panic(fmt.Sprintf("failed to create temporary directory for compiled plugins: %s", err))
+		}
+
+		tempPluginBinaryPath.path = tempDir
+	})
+
+	return tempPluginBinaryPath.path
+}
+
+type PluginBuildConfig struct {
+	version *version.Version
+}
+
+func NewPluginBuildConfig(versionStr string) *PluginBuildConfig {
+	return &PluginBuildConfig{
+		version.Must(version.NewVersion(versionStr)),
+	}
+}
+
+// Version is the core version string of the test plugin.
+//
+// If the version isn't set, it'll default to 1.0.0
+func (pc PluginBuildConfig) Version() string {
+	return pc.version.Core().String()
+}
+
+func (pc PluginBuildConfig) PreRelease() string {
+	return pc.version.Prerelease()
+}
+
+func (pc PluginBuildConfig) Metadata() string {
+	return pc.version.Metadata()
+}
+
+// LDFlags compiles the ldflags for the plugin to compile based on the information provided.
+func (pc PluginBuildConfig) LDFlags() string {
+	pluginPackage := "github.com/hashicorp/packer-plugin-tester"
+
+	ldflagsArg := fmt.Sprintf("-X %s/version.Version=%s", pluginPackage, pc.Version())
+	if pc.PreRelease() != "" {
+		ldflagsArg = fmt.Sprintf("%s -X %s/version.VersionPrerelease=%s", ldflagsArg, pluginPackage, pc.PreRelease())
+	}
+	if pc.Metadata() != "" {
+		ldflagsArg = fmt.Sprintf("%s -X %s/version.VersionMetadata=%s", ldflagsArg, pluginPackage, pc.Metadata())
+	}
+
+	return ldflagsArg
+}
+
+// BinaryName is the raw name of the plugin binary to produce
+//
+// It's expected to be in the "mini-plugin_<version>[-<prerelease>][+<metadata>]" format
+func (pc PluginBuildConfig) BinaryName() string {
+	retStr := fmt.Sprintf("mini-plugin_%s", pc.Version())
+	if pc.PreRelease() != "" {
+		retStr = fmt.Sprintf("%s-%s", retStr, pc.PreRelease())
+	}
+	if pc.Metadata() != "" {
+		retStr = fmt.Sprintf("%s+%s", retStr, pc.Metadata())
+	}
+
+	return retStr
+}
+
+// BuildSimplePlugin creates a plugin that essentially does nothing.
+//
+// The plugin's code is contained in a subdirectory of this, and lets us
+// change the attributes of the plugin binary itself, like the SDK version,
+// the plugin's version, etc.
+//
+// The plugin is functional, and can be used to run builds with.
+// There won't be anything substantial created though, its goal is only
+// to validate the core functionality of Packer.
+//
+// The path to the plugin is returned, it won't be removed automatically
+// though, deletion is the caller's responsibility.
+func BuildSimplePlugin(config *PluginBuildConfig, t *testing.T) {
+	t.Logf("Building plugin in version %v", config.version)
+
+	testDir, err := currentDir()
+	if err != nil {
+		t.Fatalf("failed to compile plugin binary: %s", err)
+	}
+
+	miniPluginDir := filepath.Join(testDir, "mini_plugin")
+	outBin := filepath.Join(PluginBinaryDir(), config.BinaryName())
+
+	compileCommand := exec.Command("go", "build", "-C", miniPluginDir, "-o", outBin, "-ldflags", config.LDFlags(), ".")
+	logs, err := compileCommand.CombinedOutput()
+	if err != nil {
+		t.Fatalf("failed to compile plugin binary: %s\ncompiler logs: %s", err, logs)
+	}
+
+	StorePluginVersion(config.version.String(), outBin)
+}
+
+// currentDir returns the directory in which the current file is located.
+//
+// Since we're in tests it's reliable as they're supposed to run on the same
+// machine the binary's compiled from, but goes to say it's not meant for use
+// in distributed binaries.
+func currentDir() (string, error) {
+	// pc uintptr, file string, line int, ok bool
+	_, testDir, _, ok := runtime.Caller(0)
+	if !ok {
+		return "", fmt.Errorf("couldn't get the location of the test suite file")
+	}
+
+	return filepath.Dir(testDir), nil
+}
+
+// MakePluginDir installs a list of plugins into a temporary directory and returns its path
+//
+// This can be set in the environment for a test through a function like t.SetEnv(), so
+// packer will be able to use that directory for running its functions.
+//
+// Deletion of the directory is the caller's responsibility.
+func (ts *PackerTestSuite) MakePluginDir(t *testing.T, pluginVersions ...string) (pluginTempDir string) {
+	var err error
+
+	defer func() {
+		if err != nil {
+			if pluginTempDir != "" {
+				os.RemoveAll(pluginTempDir)
+			}
+			t.Fatalf("failed to prepare temporary plugin directory %q: %s", pluginTempDir, err)
+		}
+	}()
+
+	pluginTempDir, err = os.MkdirTemp("", "packer-plugin-dir-temp-")
+	if err != nil {
+		return
+	}
+
+	for _, pluginVersion := range pluginVersions {
+		path, _ := LoadPluginVersion(pluginVersion)
+		cmd := ts.PackerCommand().SetArgs("plugins", "install", "--path", path, "github.com/hashicorp/tester").AddEnv("PACKER_PLUGIN_PATH", pluginTempDir)
+		cmd.Assert(t, MustSucceed{})
+		out, stderr, cmdErr := cmd.Run(t)
+		if cmdErr != nil {
+			err = fmt.Errorf("failed to install tester plugin version %q: %s\nCommand stdout: %s\nCommand stderr: %s", pluginVersion, err, out, stderr)
+			return
+		}
+	}
+
+	return pluginTempDir
+}

test/suite_test.go

@@ -0,0 +1,62 @@
+package test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+)
+
+type PackerTestSuite struct {
+	suite.Suite
+	// pluginsDirectory is the directory in which plugins are compiled.
+	//
+	// Those binaries are not necessarily meant to be used as-is, but
+	// instead should be used for composing plugin installation directories.
+	pluginsDirectory string
+	// packerPath is the location in which the Packer executable is compiled
+	//
+	// Since we don't necessarily want to manually compile Packer beforehand,
+	// we compile it on demand, and use this executable for the tests.
+	packerPath string
+}
+
+func (ts *PackerTestSuite) buildPluginBinaries(t *testing.T) {
+	BuildSimplePlugin(NewPluginBuildConfig("1.0.0"), t)
+	BuildSimplePlugin(NewPluginBuildConfig("1.0.1-dev"), t)
+	BuildSimplePlugin(NewPluginBuildConfig("1.0.1"), t)
+	BuildSimplePlugin(NewPluginBuildConfig("1.0.9"), t)
+	BuildSimplePlugin(NewPluginBuildConfig("1.0.10"), t)
+}
+
+func Test_PackerCoreSuite(t *testing.T) {
+	ts := &PackerTestSuite{}
+
+	pluginsDirectory := PluginBinaryDir()
+	defer func() {
+		err := os.RemoveAll(pluginsDirectory)
+		if err != nil {
+			t.Logf("failed to cleanup directory %q: %s. This will need manual action", pluginsDirectory, err)
+		}
+	}()
+
+	ts.pluginsDirectory = pluginsDirectory
+	ts.buildPluginBinaries(t)
+
+	t.Logf("Building test packer binary...")
+	packerPath, err := BuildTestPacker(t)
+	if err != nil {
+		t.Fatalf("failed to build Packer binary: %s", err)
+	}
+	ts.packerPath = packerPath
+	t.Logf("Done")
+
+	defer func() {
+		err := os.Remove(ts.packerPath)
+		if err != nil {
+			t.Logf("failed to cleanup compiled packer binary %q: %s. This will need manual aciton", packerPath, err)
+		}
+	}()
+
+	suite.Run(t, ts)
+}

test/templates/simple.pkr.hcl

@@ -0,0 +1,14 @@
+packer {
+	required_plugins {
+		tester = {
+			source = "github.com/hashicorp/tester"
+			version = ">= 1.0.0"
+		}
+	}
+}
+
+source "tester-dynamic" "test" {}
+
+build {
+	sources = ["tester-dynamic.test"]
+}