backport of commit ef50474a9f

test/base_test.go

@@ -0,0 +1,32 @@
+package test
+
+import (
+	"fmt"
+	"math/rand"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+)
+
+// BuildTestPacker builds a new Packer binary based on the current state of the repository.
+//
+// If for some reason the binary cannot be built, we will immediately exit with an error.
+func BuildTestPacker(t *testing.T) (string, error) {
+	testDir, err := currentDir()
+	if err != nil {
+		return "", fmt.Errorf("failed to compile packer binary: %s", err)
+	}
+
+	packerCoreDir := filepath.Dir(testDir)
+
+	outBin := filepath.Join(os.TempDir(), fmt.Sprintf("packer_core-%d", rand.Int()))
+
+	compileCommand := exec.Command("go", "build", "-C", packerCoreDir, "-o", outBin)
+	logs, err := compileCommand.CombinedOutput()
+	if err != nil {
+		t.Fatalf("failed to compile Packer core: %s\ncompilation logs: %s", err, logs)
+	}
+
+	return outBin, nil
+}

test/commands_test.go

@@ -0,0 +1,81 @@
+package test
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"sync"
+	"testing"
+)
+
+type packerCommand struct {
+	once       sync.Once
+	packerPath string
+	args       []string
+	env        map[string]string
+	stderr     *strings.Builder
+	stdout     *strings.Builder
+	err        error
+}
+
+// PackerCommand creates a skeleton of packer command with the ability to execute gadgets on the outputs of the command.
+func (ts *PackerTestSuite) PackerCommand() *packerCommand {
+	stderr := &strings.Builder{}
+	stdout := &strings.Builder{}
+
+	return &packerCommand{
+		packerPath: ts.packerPath,
+		env: map[string]string{
+			"PACKER_LOG": "1",
+		},
+		stderr: stderr,
+		stdout: stdout,
+	}
+}
+
+func (pc *packerCommand) SetArgs(args ...string) *packerCommand {
+	pc.args = args
+	return pc
+}
+
+func (pc *packerCommand) AddEnv(key, val string) *packerCommand {
+	pc.env[key] = val
+	return pc
+}
+
+// Run executes the packer command with the args/env requested and returns the
+// output streams (stdout, stderr)
+//
+// Note: "Run" will only execute the command once, and return the streams and
+// error from the only execution for every subsequent call
+func (pc *packerCommand) Run(t *testing.T) (string, string, error) {
+	pc.once.Do(pc.doRun)
+
+	if strings.Contains(pc.stdout.String(), "PACKER CRASH") || strings.Contains(pc.stderr.String(), "PACKER CRASH") {
+		t.Fatalf("Packer has crashed while running the following command: packer %#v", pc.args)
+	}
+
+	return pc.stdout.String(), pc.stderr.String(), pc.err
+}
+
+func (pc *packerCommand) doRun() {
+	cmd := exec.Command("packer", pc.args...)
+	for key, val := range pc.env {
+		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", key, val))
+	}
+	cmd.Stdout = pc.stdout
+	cmd.Stderr = pc.stderr
+
+	pc.err = cmd.Run()
+}
+
+func (pc *packerCommand) Assert(t *testing.T, checks ...Checker) {
+	stdout, stderr, err := pc.Run(t)
+
+	for _, check := range checks {
+		checkErr := check.Check(stdout, stderr, err)
+		if checkErr != nil {
+			t.Errorf("check %q failed: %s", check.Name(), checkErr)
+		}
+	}
+}

test/gadgets_test.go

@@ -0,0 +1,99 @@
+package test
+
+import (
+	"fmt"
+	"regexp"
+	"testing"
+)
+
+type Stream int
+
+const (
+	// BothStreams will use both stdout and stderr for performing a check
+	BothStreams Stream = iota
+	// OnlyStdout will only use stdout for performing a check
+	OnlyStdout
+	// OnlySterr will only use stderr for performing a check
+	OnlyStderr
+)
+
+func (s Stream) String() string {
+	switch s {
+	case BothStreams:
+		return "Both streams"
+	case OnlyStdout:
+		return "Stdout"
+	case OnlyStderr:
+		return "Stderr"
+	}
+
+	panic(fmt.Sprintf("Unknown stream value: %d", s))
+}
+
+type Checker interface {
+	Check(stdout, stderr string, err error) error
+	Name() string
+}
+
+type MustSucceed struct{}
+
+func (_ MustSucceed) Check(stdout, stderr string, err error) error {
+	return err
+}
+
+func (_ MustSucceed) Name() string {
+	return "Must succeed"
+}
+
+// Grep is essentially the equivalent to a normal grep -E on the command line.
+//
+// The `expect` string is meant to be a regexp, which will be compiled on-demand,
+// and will panic if it isn't a valid POSIX extended regexp.
+type Grep struct {
+	streams Stream
+	expect  string
+}
+
+func (g Grep) Check(stdout, stderr string, err error) error {
+	re := regexp.MustCompilePOSIX(g.expect)
+
+	streams := []string{}
+
+	switch g.streams {
+	case BothStreams:
+		streams = append(streams, stdout, stderr)
+	case OnlyStdout:
+		streams = append(streams, stdout)
+	case OnlyStderr:
+		streams = append(streams, stderr)
+	}
+
+	var found bool
+	for _, stream := range streams {
+		found = found || re.MatchString(stream)
+	}
+
+	if !found {
+		return fmt.Errorf("streams %q did not match the expected regexp %q", g.streams, g.expect)
+	}
+	return nil
+}
+
+func (g Grep) Name() string {
+	return fmt.Sprintf("command (%s) | grep -E %q", g.streams, g.expect)
+}
+
+type Dump struct {
+	t *testing.T
+}
+
+func (d Dump) Check(stdout, stderr string, err error) error {
+	d.t.Logf("Dumping command result.")
+	d.t.Logf("Stdout: %s", stdout)
+	d.t.Logf("stderr: %s", stderr)
+	return nil
+}
+
+func (_ Dump) Name() string {
+	return "dump"
+}

test/loading_test.go

@@ -0,0 +1,40 @@
+package test
+
+import (
+	"os"
+	"testing"
+)
+
+func (ts *PackerTestSuite) TestLoadingOrder() {
+	t := ts.T()
+
+	pluginDir := ts.MakePluginDir(t, "1.0.9", "1.0.10")
+	defer func() {
+		err := os.RemoveAll(pluginDir)
+		if err != nil {
+			t.Logf("failed to remove temporary plugin directory %q: %s. This may need manual intervention.", pluginDir, err)
+		}
+	}()
+
+	tests := []struct {
+		name         string
+		templatePath string
+	}{
+		{
+			"HCL2 - No required_plugins, 1.0.10 is the most recent and should load",
+			"./templates/simple.pkr.hcl",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ts.PackerCommand().
+				SetArgs("build", tt.templatePath).
+				AddEnv("PACKER_PLUGIN_PATH", pluginDir).
+				Assert(t, MustSucceed{}, Grep{
+					streams: BothStreams,
+					expect:  "packer-plugin-tester_v1\\.0\\.10[^\n]+ plugin:",
+				})
+		})
+	}
+}

test/plugin_test.go

@@ -0,0 +1,192 @@
+package test
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"sync"
+	"testing"
+
+	"github.com/hashicorp/go-version"
+)
+
+var compiledPlugins = struct {
+	pluginVersions map[string]string
+	RWMutex        sync.RWMutex
+}{
+	pluginVersions: map[string]string{},
+}
+
+func StorePluginVersion(pluginVersion, path string) {
+	compiledPlugins.RWMutex.Lock()
+	defer compiledPlugins.RWMutex.Unlock()
+	compiledPlugins.pluginVersions[pluginVersion] = path
+}
+
+func LoadPluginVersion(pluginVersion string) (string, bool) {
+	compiledPlugins.RWMutex.RLock()
+	defer compiledPlugins.RWMutex.RUnlock()
+
+	path, ok := compiledPlugins.pluginVersions[pluginVersion]
+	return path, ok
+}
+
+var tempPluginBinaryPath = struct {
+	path string
+	once sync.Once
+}{}
+
+// PluginBinaryDir returns the path to the directory where temporary binaries will be compiled
+func PluginBinaryDir() string {
+	tempPluginBinaryPath.once.Do(func() {
+		tempDir, err := os.MkdirTemp("", "packer-core-acc-test-")
+		if err != nil {
+			panic(fmt.Sprintf("failed to create temporary directory for compiled plugins: %s", err))
+		}
+
+		tempPluginBinaryPath.path = tempDir
+	})
+
+	return tempPluginBinaryPath.path
+}
+
+type PluginBuildConfig struct {
+	version *version.Version
+}
+
+func NewPluginBuildConfig(versionStr string) *PluginBuildConfig {
+	return &PluginBuildConfig{
+		version.Must(version.NewVersion(versionStr)),
+	}
+}
+
+// Version is the core version string of the test plugin.
+//
+// If the version isn't set, it'll default to 1.0.0
+func (pc PluginBuildConfig) Version() string {
+	return pc.version.Core().String()
+}
+
+func (pc PluginBuildConfig) PreRelease() string {
+	return pc.version.Prerelease()
+}
+
+func (pc PluginBuildConfig) Metadata() string {
+	return pc.version.Metadata()
+}
+
+// LDFlags compiles the ldflags for the plugin to compile based on the information provided.
+func (pc PluginBuildConfig) LDFlags() string {
+	pluginPackage := "github.com/hashicorp/packer-plugin-tester"
+
+	ldflagsArg := fmt.Sprintf("-X %s/version.Version=%s", pluginPackage, pc.Version())
+	if pc.PreRelease() != "" {
+		ldflagsArg = fmt.Sprintf("%s -X %s/version.VersionPrerelease=%s", ldflagsArg, pluginPackage, pc.PreRelease())
+	}
+	if pc.Metadata() != "" {
+		ldflagsArg = fmt.Sprintf("%s -X %s/version.VersionMetadata=%s", ldflagsArg, pluginPackage, pc.Metadata())
+	}
+
+	return ldflagsArg
+}
+
+// BinaryName is the raw name of the plugin binary to produce
+//
+// It's expected to be in the "mini-plugin_<version>[-<prerelease>][+<metadata>]" format
+func (pc PluginBuildConfig) BinaryName() string {
+	retStr := fmt.Sprintf("mini-plugin_%s", pc.Version())
+	if pc.PreRelease() != "" {
+		retStr = fmt.Sprintf("%s-%s", retStr, pc.PreRelease())
+	}
+	if pc.Metadata() != "" {
+		retStr = fmt.Sprintf("%s+%s", retStr, pc.Metadata())
+	}
+
+	return retStr
+}
+
+// BuildSimplePlugin creates a plugin that essentially does nothing.
+//
+// The plugin's code is contained in a subdirectory of this, and lets us
+// change the attributes of the plugin binary itself, like the SDK version,
+// the plugin's version, etc.
+//
+// The plugin is functional, and can be used to run builds with.
+// There won't be anything substantial created though, its goal is only
+// to validate the core functionality of Packer.
+//
+// The path to the plugin is returned, it won't be removed automatically
+// though, deletion is the caller's responsibility.
+func BuildSimplePlugin(config *PluginBuildConfig, t *testing.T) {
+	t.Logf("Building plugin in version %v", config.version)
+
+	testDir, err := currentDir()
+	if err != nil {
+		t.Fatalf("failed to compile plugin binary: %s", err)
+	}
+
+	miniPluginDir := filepath.Join(testDir, "mini_plugin")
+	outBin := filepath.Join(PluginBinaryDir(), config.BinaryName())
+
+	compileCommand := exec.Command("go", "build", "-C", miniPluginDir, "-o", outBin, "-ldflags", config.LDFlags(), ".")
+	logs, err := compileCommand.CombinedOutput()
+	if err != nil {
+		t.Fatalf("failed to compile plugin binary: %s\ncompiler logs: %s", err, logs)
+	}
+
+	StorePluginVersion(config.version.String(), outBin)
+}
+
+// currentDir returns the directory in which the current file is located.
+//
+// Since we're in tests it's reliable as they're supposed to run on the same
+// machine the binary's compiled from, but goes to say it's not meant for use
+// in distributed binaries.
+func currentDir() (string, error) {
+	// pc uintptr, file string, line int, ok bool
+	_, testDir, _, ok := runtime.Caller(0)
+	if !ok {
+		return "", fmt.Errorf("couldn't get the location of the test suite file")
+	}
+
+	return filepath.Dir(testDir), nil
+}
+
+// MakePluginDir installs a list of plugins into a temporary directory and returns its path
+//
+// This can be set in the environment for a test through a function like t.SetEnv(), so
+// packer will be able to use that directory for running its functions.
+//
+// Deletion of the directory is the caller's responsibility.
+func (ts *PackerTestSuite) MakePluginDir(t *testing.T, pluginVersions ...string) (pluginTempDir string) {
+	var err error
+
+	defer func() {
+		if err != nil {
+			if pluginTempDir != "" {
+				os.RemoveAll(pluginTempDir)
+			}
+			t.Fatalf("failed to prepare temporary plugin directory %q: %s", pluginTempDir, err)
+		}
+	}()
+
+	pluginTempDir, err = os.MkdirTemp("", "packer-plugin-dir-temp-")
+	if err != nil {
+		return
+	}
+
+	for _, pluginVersion := range pluginVersions {
+		path, _ := LoadPluginVersion(pluginVersion)
+		cmd := ts.PackerCommand().SetArgs("plugins", "install", "--path", path, "github.com/hashicorp/tester").AddEnv("PACKER_PLUGIN_PATH", pluginTempDir)
+		cmd.Assert(t, MustSucceed{})
+		out, stderr, cmdErr := cmd.Run(t)
+		if cmdErr != nil {
+			err = fmt.Errorf("failed to install tester plugin version %q: %s\nCommand stdout: %s\nCommand stderr: %s", pluginVersion, err, out, stderr)
+			return
+		}
+	}
+
+	return pluginTempDir
+}

test/suite_test.go

@@ -0,0 +1,62 @@
+package test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+)
+
+type PackerTestSuite struct {
+	suite.Suite
+	// pluginsDirectory is the directory in which plugins are compiled.
+	//
+	// Those binaries are not necessarily meant to be used as-is, but
+	// instead should be used for composing plugin installation directories.
+	pluginsDirectory string
+	// packerPath is the location in which the Packer executable is compiled
+	//
+	// Since we don't necessarily want to manually compile Packer beforehand,
+	// we compile it on demand, and use this executable for the tests.
+	packerPath string
+}
+
+func (ts *PackerTestSuite) buildPluginBinaries(t *testing.T) {
+	BuildSimplePlugin(NewPluginBuildConfig("1.0.0"), t)
+	BuildSimplePlugin(NewPluginBuildConfig("1.0.1-dev"), t)
+	BuildSimplePlugin(NewPluginBuildConfig("1.0.1"), t)
+	BuildSimplePlugin(NewPluginBuildConfig("1.0.9"), t)
+	BuildSimplePlugin(NewPluginBuildConfig("1.0.10"), t)
+}
+
+func Test_PackerCoreSuite(t *testing.T) {
+	ts := &PackerTestSuite{}
+
+	pluginsDirectory := PluginBinaryDir()
+	defer func() {
+		err := os.RemoveAll(pluginsDirectory)
+		if err != nil {
+			t.Logf("failed to cleanup directory %q: %s. This will need manual action", pluginsDirectory, err)
+		}
+	}()
+
+	ts.pluginsDirectory = pluginsDirectory
+	ts.buildPluginBinaries(t)
+
+	t.Logf("Building test packer binary...")
+	packerPath, err := BuildTestPacker(t)
+	if err != nil {
+		t.Fatalf("failed to build Packer binary: %s", err)
+	}
+	ts.packerPath = packerPath
+	t.Logf("Done")
+
+	defer func() {
+		err := os.Remove(ts.packerPath)
+		if err != nil {
+			t.Logf("failed to cleanup compiled packer binary %q: %s. This will need manual aciton", packerPath, err)
+		}
+	}()
+
+	suite.Run(t, ts)
+}

test/templates/simple.pkr.hcl

@@ -0,0 +1,14 @@
+packer {
+	required_plugins {
+		tester = {
+			source = "github.com/hashicorp/tester"
+			version = ">= 1.0.0"
+		}
+	}
+}
+
+source "tester-dynamic" "test" {}
+
+build {
+	sources = ["tester-dynamic.test"]
+}