Support `packer plugins installed` plugins in JSON (and HCL without required block) (#11712)

* Parse plugins from nested installed directories in json/non-required block HCL

* Add Packer Plugins installed test

* Add checksum validation for multi-component plugins installed by the packer plugins subcommand (#11732)

This change copies a bit of the logic in hcl2template/plugin in order to
strengthen the check for multi-component plugins installed via the
`packer plugins install` commands.

* Replace ioutil with os pkg

In go1.16 the ioutil std lib package was deprecated in favor of the os
and io packages. This change just updates all Temp creation methods to
their os pkg equivalents.

Co-authored-by: Wilken Rivera <wilken@hashicorp.com>

packer/plugin-getter/plugins.go

@@ -89,7 +89,7 @@ func (pr Requirement) FilenamePrefix() string {
 	return "packer-plugin-" + pr.Identifier.Type + "_"
 }
 
-func (opts BinaryInstallationOptions) filenameSuffix() string {
+func (opts BinaryInstallationOptions) FilenameSuffix() string {
 	return "_" + opts.OS + "_" + opts.ARCH + opts.Ext
 }
 
@@ -104,7 +104,7 @@ func (opts BinaryInstallationOptions) filenameSuffix() string {
 func (pr Requirement) ListInstallations(opts ListInstallationsOptions) (InstallList, error) {
 	res := InstallList{}
 	FilenamePrefix := pr.FilenamePrefix()
-	filenameSuffix := opts.filenameSuffix()
+	filenameSuffix := opts.FilenameSuffix()
 	log.Printf("[TRACE] listing potential installations for %q that match %q. %#v", pr.Identifier, pr.VersionConstraints, opts)
 	for _, knownFolder := range opts.FromFolders {
 		glob := ""

packer/plugin.go

@@ -1,7 +1,9 @@
 package packer
 
 import (
+	"crypto/sha256"
 	"encoding/json"
+	"fmt"
 	"log"
 	"os"
 	"os/exec"
@@ -13,6 +15,7 @@ import (
 	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
 	"github.com/hashicorp/packer-plugin-sdk/pathing"
 	pluginsdk "github.com/hashicorp/packer-plugin-sdk/plugin"
+	plugingetter "github.com/hashicorp/packer/packer/plugin-getter"
 )
 
 // PluginConfig helps load and use packer plugins
@@ -200,6 +203,55 @@ func (c *PluginConfig) discoverExternalComponents(path string) error {
 		log.Printf("using external datasource %v", externallyUsed)
 	}
 
+	//Check for installed plugins using the `packer plugins install` command
+	binInstallOpts := plugingetter.BinaryInstallationOptions{
+		OS:              runtime.GOOS,
+		ARCH:            runtime.GOARCH,
+		APIVersionMajor: pluginsdk.APIVersionMajor,
+		APIVersionMinor: pluginsdk.APIVersionMinor,
+		Checksummers: []plugingetter.Checksummer{
+			{Type: "sha256", Hash: sha256.New()},
+		},
+	}
+
+	if runtime.GOOS == "windows" {
+		binInstallOpts.Ext = ".exe"
+	}
+
+	pluginPaths, err = c.discoverSingle(filepath.Join(path, "*", "*", "*", fmt.Sprintf("packer-plugin-*%s", binInstallOpts.FilenameSuffix())))
+	if err != nil {
+		return err
+	}
+
+	for pluginName, pluginPath := range pluginPaths {
+		var checksumOk bool
+		for _, checksummer := range binInstallOpts.Checksummers {
+			cs, err := checksummer.GetCacheChecksumOfFile(pluginPath)
+			if err != nil {
+				log.Printf("[TRACE] GetChecksumOfFile(%q) failed: %v", pluginPath, err)
+				continue
+			}
+
+			if err := checksummer.ChecksumFile(cs, pluginPath); err != nil {
+				log.Printf("[TRACE] ChecksumFile(%q) failed: %v", pluginPath, err)
+				continue
+			}
+			checksumOk = true
+			break
+		}
+
+		if !checksumOk {
+			log.Printf("[TRACE] No checksum found for %q ignoring possibly unsafe binary", path)
+			continue
+		}
+
+		if err := c.DiscoverMultiPlugin(pluginName, pluginPath); err != nil {
+			return err
+		}
+	}
+
+	// Manually installed plugins take precedence over all. Duplicate plugins installed
+	// prior to the packer plugins install command should be removed by user to avoid overrides.
 	pluginPaths, err = c.discoverSingle(filepath.Join(path, "packer-plugin-*"))
 	if err != nil {
 		return err
@@ -219,14 +271,14 @@ func (c *PluginConfig) discoverSingle(glob string) (map[string]string, error) {
 	if err != nil {
 		return nil, err
 	}
-
+	var prefix string
 	res := make(map[string]string)
-
-	prefix := filepath.Base(glob)
+	// Sort the matches so we add the newer version of a plugin last
+	sort.Strings(matches)
+	prefix = filepath.Base(glob)
 	prefix = prefix[:strings.Index(prefix, "*")]
 	for _, match := range matches {
 		file := filepath.Base(match)
-
 		// skip folders like packer-plugin-sdk
 		if stat, err := os.Stat(file); err == nil && stat.IsDir() {
 			continue
@@ -248,6 +300,10 @@ func (c *PluginConfig) discoverSingle(glob string) (map[string]string, error) {
 
 		// Look for foo-bar-baz. The plugin name is "baz"
 		pluginName := file[len(prefix):]
+		// multi-component plugins installed via the plugins subcommand will have a name that looks like baz_vx.y.z_x5.0_darwin_arm64.
+		// After the split the plugin name is "baz".
+		pluginName = strings.SplitN(pluginName, "_", 2)[0]
+
 		log.Printf("[DEBUG] Discovered plugin: %s = %s", pluginName, match)
 		res[pluginName] = match
 	}

packer/plugin_discover_test.go

@@ -1,8 +1,8 @@
 package packer
 
 import (
+	"crypto/sha256"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path"
@@ -14,6 +14,7 @@ import (
 	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
 	pluginsdk "github.com/hashicorp/packer-plugin-sdk/plugin"
 	"github.com/hashicorp/packer-plugin-sdk/tmp"
+	plugingetter "github.com/hashicorp/packer/packer/plugin-getter"
 )
 
 func newPluginConfig() PluginConfig {
@@ -84,7 +85,7 @@ func TestEnvVarPackerPluginPath_MultiplePaths(t *testing.T) {
 	}
 
 	// Create a second dir to look in that will be empty
-	decoyDir, err := ioutil.TempDir("", "decoy")
+	decoyDir, err := os.MkdirTemp("", "decoy")
 	if err != nil {
 		t.Fatalf("Failed to create a temporary test dir.")
 	}
@@ -127,7 +128,7 @@ func TestDiscoverDatasource(t *testing.T) {
 	}
 
 	// Create a second dir to look in that will be empty
-	decoyDir, err := ioutil.TempDir("", "decoy")
+	decoyDir, err := os.MkdirTemp("", "decoy")
 	if err != nil {
 		t.Fatalf("Failed to create a temporary test dir.")
 	}
@@ -155,7 +156,7 @@ func TestDiscoverDatasource(t *testing.T) {
 }
 
 func generateFakePlugins(dirname string, pluginNames []string) (string, []string, func(), error) {
-	dir, err := ioutil.TempDir("", dirname)
+	dir, err := os.MkdirTemp("", dirname)
 	if err != nil {
 		return "", nil, nil, fmt.Errorf("failed to create temporary test directory: %v", err)
 	}
@@ -229,10 +230,6 @@ func HasExec() bool {
 	switch runtime.GOOS {
 	case "js":
 		return false
-	case "darwin":
-		if runtime.GOARCH == "arm64" {
-			return false
-		}
 	case "windows":
 		// TODO(azr): Fix this once versioning is added and we know more
 		return false
@@ -285,12 +282,13 @@ func createMockPlugins(t *testing.T, plugins map[string]pluginsdk.Set) {
 		shPath := MustHaveCommand(t, "bash")
 		for name := range plugins {
 			plugin := path.Join(pluginDir, "packer-plugin-"+name)
+			t.Logf("creating fake plugin %s", plugin)
 			fileContent := ""
 			fileContent = fmt.Sprintf("#!%s\n", shPath)
 			fileContent += strings.Join(
 				append([]string{"PKR_WANT_TEST_PLUGINS=1"}, helperCommand(t, name, "$@")...),
 				" ")
-			if err := ioutil.WriteFile(plugin, []byte(fileContent), os.ModePerm); err != nil {
+			if err := os.WriteFile(plugin, []byte(fileContent), os.ModePerm); err != nil {
 				t.Fatalf("failed to create fake plugin binary: %v", err)
 			}
 		}
@@ -298,28 +296,141 @@ func createMockPlugins(t *testing.T, plugins map[string]pluginsdk.Set) {
 	os.Setenv("PACKER_PLUGIN_PATH", pluginDir)
 }
 
+func createMockChecksumFile(t testing.TB, filePath string) {
+	cs := plugingetter.Checksummer{
+		Type: "sha256",
+		Hash: sha256.New(),
+	}
+
+	f, err := os.Open(filePath)
+	if err != nil {
+		t.Fatalf("failed to open fake plugin binary: %v", err)
+	}
+	defer f.Close()
+
+	sum, err := cs.Sum(f)
+	if err != nil {
+		t.Fatalf("failed to checksum fake plugin binary: %v", err)
+	}
+
+	t.Logf("creating fake plugin checksum file %s with contents %x", filePath+cs.FileExt(), string(sum))
+	if err := os.WriteFile(filePath+cs.FileExt(), []byte(fmt.Sprintf("%x", sum)), os.ModePerm); err != nil {
+		t.Fatalf("failed to write checksum fake plugin binary: %v", err)
+	}
+}
+
+func createMockInstalledPlugins(t *testing.T, plugins map[string]pluginsdk.Set, opts ...func(tb testing.TB, filePath string)) {
+	pluginDir, err := tmp.Dir("pkr-multi-component-plugin-test-*")
+	{
+		// create an exectutable file with a `sh` sheebang
+		// this file will look like:
+		// #!/bin/sh
+		// PKR_WANT_TEST_PLUGINS=1 ...plugin/debug.test -test.run=TestHelperPlugins -- bird $@
+		// 'bird' is the mock plugin we want to start
+		// $@ just passes all passed arguments
+		// This will allow to run the fake plugin from go tests which in turn
+		// will run go tests callback to `TestHelperPlugins`, this one will be
+		// transparently calling our mock multi-component plugins `mockPlugins`.
+		if err != nil {
+			t.Fatal(err)
+		}
+		dir, err := os.MkdirTemp(pluginDir, "github.com")
+		if err != nil {
+			t.Fatalf("failed to create temporary test directory: %v", err)
+		}
+		dir, err = os.MkdirTemp(dir, "hashicorp")
+		if err != nil {
+			t.Fatalf("failed to create temporary test directory: %v", err)
+		}
+		dir, err = os.MkdirTemp(dir, "plugin")
+		if err != nil {
+			t.Fatalf("failed to create temporary test directory: %v", err)
+		}
+		t.Logf("putting temporary mock installed plugins in %s", dir)
+
+		shPath := MustHaveCommand(t, "bash")
+		for name := range plugins {
+			plugin := path.Join(dir, "packer-plugin-"+name)
+			t.Logf("creating fake plugin %s", plugin)
+			fileContent := ""
+			fileContent = fmt.Sprintf("#!%s\n", shPath)
+			fileContent += strings.Join(
+				append([]string{"PKR_WANT_TEST_PLUGINS=1"}, helperCommand(t, strings.Split(name, "_")[0], "$@")...),
+				" ")
+			if err := os.WriteFile(plugin, []byte(fileContent), os.ModePerm); err != nil {
+				t.Fatalf("failed to create fake plugin binary: %v", err)
+			}
+
+			for _, opt := range opts {
+				opt(t, plugin)
+			}
+		}
+	}
+	os.Setenv("PACKER_PLUGIN_PATH", pluginDir)
+}
+
+func getFormattedInstalledPluginSuffix() string {
+	return fmt.Sprintf("v1.0.0_x5.0_%s_%s", runtime.GOOS, runtime.GOARCH)
+}
+
 var (
 	mockPlugins = map[string]pluginsdk.Set{
-		"bird": pluginsdk.Set{
+		"bird": {
+			Builders: map[string]packersdk.Builder{
+				"feather":   nil,
+				"guacamole": nil,
+			},
+		},
+		"chimney": {
+			PostProcessors: map[string]packersdk.PostProcessor{
+				"smoke": nil,
+			},
+		},
+		"data": {
+			Datasources: map[string]packersdk.Datasource{
+				"source": nil,
+			},
+		},
+	}
+	mockInstalledPlugins = map[string]pluginsdk.Set{
+		fmt.Sprintf("bird_%s", getFormattedInstalledPluginSuffix()): {
 			Builders: map[string]packersdk.Builder{
 				"feather":   nil,
 				"guacamole": nil,
 			},
 		},
-		"chimney": pluginsdk.Set{
+		fmt.Sprintf("chimney_%s", getFormattedInstalledPluginSuffix()): {
 			PostProcessors: map[string]packersdk.PostProcessor{
 				"smoke": nil,
 			},
 		},
-		"data": pluginsdk.Set{
+		fmt.Sprintf("data_%s", getFormattedInstalledPluginSuffix()): {
 			Datasources: map[string]packersdk.Datasource{
 				"source": nil,
 			},
 		},
 	}
 
+	invalidInstalledPluginsMock = map[string]pluginsdk.Set{
+		"bird_v0.1.1_x5.0_wrong_architecture": {
+			Builders: map[string]packersdk.Builder{
+				"feather":   nil,
+				"guacamole": nil,
+			},
+		},
+		"chimney_cool_ranch": {
+			PostProcessors: map[string]packersdk.PostProcessor{
+				"smoke": nil,
+			},
+		},
+		"data": {
+			Datasources: map[string]packersdk.Datasource{
+				"source": nil,
+			},
+		},
+	}
 	defaultNameMock = map[string]pluginsdk.Set{
-		"foo": pluginsdk.Set{
+		"foo": {
 			Builders: map[string]packersdk.Builder{
 				"bar":                  nil,
 				"baz":                  nil,
@@ -329,7 +440,7 @@ var (
 	}
 
 	doubleDefaultMock = map[string]pluginsdk.Set{
-		"yolo": pluginsdk.Set{
+		"yolo": {
 			Builders: map[string]packersdk.Builder{
 				"bar":                  nil,
 				"baz":                  nil,
@@ -342,7 +453,7 @@ var (
 	}
 
 	badDefaultNameMock = map[string]pluginsdk.Set{
-		"foo": pluginsdk.Set{
+		"foo": {
 			Builders: map[string]packersdk.Builder{
 				"bar":                  nil,
 				"baz":                  nil,
@@ -356,7 +467,6 @@ func Test_multiplugin_describe(t *testing.T) {
 	createMockPlugins(t, mockPlugins)
 	pluginDir := os.Getenv("PACKER_PLUGIN_PATH")
 	defer os.RemoveAll(pluginDir)
-
 	c := PluginConfig{}
 	err := c.Discover()
 	if err != nil {
@@ -392,6 +502,113 @@ func Test_multiplugin_describe(t *testing.T) {
 	}
 }
 
+func Test_multiplugin_describe_installed(t *testing.T) {
+	createMockInstalledPlugins(t, mockInstalledPlugins, createMockChecksumFile)
+	pluginDir := os.Getenv("PACKER_PLUGIN_PATH")
+	defer os.RemoveAll(pluginDir)
+
+	c := PluginConfig{}
+	err := c.Discover()
+	if err != nil {
+		t.Fatalf("error discovering plugins; %s", err.Error())
+	}
+
+	for mockPluginName, plugin := range mockInstalledPlugins {
+		mockPluginName = strings.Split(mockPluginName, "_")[0]
+		for mockBuilderName := range plugin.Builders {
+			expectedBuilderName := mockPluginName + "-" + mockBuilderName
+			if !c.Builders.Has(expectedBuilderName) {
+				t.Fatalf("expected to find builder %q", expectedBuilderName)
+			}
+		}
+		for mockProvisionerName := range plugin.Provisioners {
+			expectedProvisionerName := mockPluginName + "-" + mockProvisionerName
+			if !c.Provisioners.Has(expectedProvisionerName) {
+				t.Fatalf("expected to find builder %q", expectedProvisionerName)
+			}
+		}
+		for mockPostProcessorName := range plugin.PostProcessors {
+			expectedPostProcessorName := mockPluginName + "-" + mockPostProcessorName
+			if !c.PostProcessors.Has(expectedPostProcessorName) {
+				t.Fatalf("expected to find post-processor %q", expectedPostProcessorName)
+			}
+		}
+		for mockDatasourceName := range plugin.Datasources {
+			expectedDatasourceName := mockPluginName + "-" + mockDatasourceName
+			if !c.DataSources.Has(expectedDatasourceName) {
+				t.Fatalf("expected to find datasource %q", expectedDatasourceName)
+			}
+		}
+	}
+}
+
+func Test_multiplugin_describe_installed_for_invalid(t *testing.T) {
+	tc := []struct {
+		desc                 string
+		installedPluginsMock map[string]pluginsdk.Set
+		createMockFn         func(*testing.T, map[string]pluginsdk.Set)
+	}{
+		{
+			desc:                 "Incorrectly named plugins",
+			installedPluginsMock: invalidInstalledPluginsMock,
+			createMockFn: func(t *testing.T, mocks map[string]pluginsdk.Set) {
+				createMockInstalledPlugins(t, mocks, createMockChecksumFile)
+			},
+		},
+		{
+			desc:                 "Plugins missing checksums",
+			installedPluginsMock: mockInstalledPlugins,
+			createMockFn: func(t *testing.T, mocks map[string]pluginsdk.Set) {
+				createMockInstalledPlugins(t, mocks)
+			},
+		},
+	}
+
+	for _, tt := range tc {
+		t.Run(tt.desc, func(t *testing.T) {
+			tt.createMockFn(t, tt.installedPluginsMock)
+			pluginDir := os.Getenv("PACKER_PLUGIN_PATH")
+			defer os.RemoveAll(pluginDir)
+
+			c := PluginConfig{}
+			err := c.Discover()
+			if err != nil {
+				t.Fatalf("error discovering plugins; %s", err.Error())
+			}
+			if c.Builders.Has("feather") {
+				t.Fatalf("expected to not find builder %q", "feather")
+			}
+			for mockPluginName, plugin := range tt.installedPluginsMock {
+				mockPluginName = strings.Split(mockPluginName, "_")[0]
+				for mockBuilderName := range plugin.Builders {
+					expectedBuilderName := mockPluginName + "-" + mockBuilderName
+					if c.Builders.Has(expectedBuilderName) {
+						t.Fatalf("expected to not find builder %q", expectedBuilderName)
+					}
+				}
+				for mockProvisionerName := range plugin.Provisioners {
+					expectedProvisionerName := mockPluginName + "-" + mockProvisionerName
+					if c.Provisioners.Has(expectedProvisionerName) {
+						t.Fatalf("expected to not find builder %q", expectedProvisionerName)
+					}
+				}
+				for mockPostProcessorName := range plugin.PostProcessors {
+					expectedPostProcessorName := mockPluginName + "-" + mockPostProcessorName
+					if c.PostProcessors.Has(expectedPostProcessorName) {
+						t.Fatalf("expected to not find post-processor %q", expectedPostProcessorName)
+					}
+				}
+				for mockDatasourceName := range plugin.Datasources {
+					expectedDatasourceName := mockPluginName + "-" + mockDatasourceName
+					if c.DataSources.Has(expectedDatasourceName) {
+						t.Fatalf("expected to not find datasource %q", expectedDatasourceName)
+					}
+				}
+			}
+		})
+	}
+}
+
 func Test_multiplugin_defaultName(t *testing.T) {
 	createMockPlugins(t, defaultNameMock)
 	pluginDir := os.Getenv("PACKER_PLUGIN_PATH")