allow to discover & start packer-plugin-* muliplugin binaries (#10277)

This add :
* discovery of `packer-plugin-*` binaries from the known folders and ask them to describe themselves
* tests

For testing: in go we create a bash script that in turn calls back to Go. I could not make the tests to work on windows and then would like to postpone testing this for when we know more about the finite layout of this feature. That is mainly: how things are going to work with init, versioning and such.

config.go

@@ -141,107 +141,6 @@ func (c *config) StartProvisioner(name string) (packersdk.Provisioner, error) {
 	return c.Provisioners.Start(name)
 }
 
-func (c *config) discoverExternalComponents(path string) error {
-	var err error
-
-	if !filepath.IsAbs(path) {
-		path, err = filepath.Abs(path)
-		if err != nil {
-			return err
-		}
-	}
-	externallyUsed := []string{}
-
-	pluginPaths, err := c.discoverSingle(filepath.Join(path, "packer-builder-*"))
-	if err != nil {
-		return err
-	}
-	for pluginName, pluginPath := range pluginPaths {
-		newPath := pluginPath // this needs to be stored in a new variable for the func below
-		c.Builders[pluginName] = func() (packersdk.Builder, error) {
-			return c.Plugins.Client(newPath).Builder()
-		}
-		externallyUsed = append(externallyUsed, pluginName)
-	}
-	if len(externallyUsed) > 0 {
-		sort.Strings(externallyUsed)
-		log.Printf("using external builders %v", externallyUsed)
-		externallyUsed = nil
-	}
-
-	pluginPaths, err = c.discoverSingle(filepath.Join(path, "packer-post-processor-*"))
-	if err != nil {
-		return err
-	}
-	for pluginName, pluginPath := range pluginPaths {
-		newPath := pluginPath // this needs to be stored in a new variable for the func below
-		c.PostProcessors[pluginName] = func() (packersdk.PostProcessor, error) {
-			return c.Plugins.Client(newPath).PostProcessor()
-		}
-		externallyUsed = append(externallyUsed, pluginName)
-	}
-	if len(externallyUsed) > 0 {
-		sort.Strings(externallyUsed)
-		log.Printf("using external post-processors %v", externallyUsed)
-		externallyUsed = nil
-	}
-
-	pluginPaths, err = c.discoverSingle(filepath.Join(path, "packer-provisioner-*"))
-	if err != nil {
-		return err
-	}
-	for pluginName, pluginPath := range pluginPaths {
-		newPath := pluginPath // this needs to be stored in a new variable for the func below
-		c.Provisioners[pluginName] = func() (packersdk.Provisioner, error) {
-			return c.Plugins.Client(newPath).Provisioner()
-		}
-		externallyUsed = append(externallyUsed, pluginName)
-	}
-	if len(externallyUsed) > 0 {
-		sort.Strings(externallyUsed)
-		log.Printf("using external provisioners %v", externallyUsed)
-		externallyUsed = nil
-	}
-
-	return nil
-}
-
-func (c *config) discoverSingle(glob string) (map[string]string, error) {
-	matches, err := filepath.Glob(glob)
-	if err != nil {
-		return nil, err
-	}
-
-	res := make(map[string]string)
-
-	prefix := filepath.Base(glob)
-	prefix = prefix[:strings.Index(prefix, "*")]
-	for _, match := range matches {
-		file := filepath.Base(match)
-
-		// On Windows, ignore any plugins that don't end in .exe.
-		// We could do a full PATHEXT parse, but this is probably good enough.
-		if runtime.GOOS == "windows" && strings.ToLower(filepath.Ext(file)) != ".exe" {
-			log.Printf(
-				"[DEBUG] Ignoring plugin match %s, no exe extension",
-				match)
-			continue
-		}
-
-		// If the filename has a ".", trim up to there
-		if idx := strings.Index(file, ".exe"); idx >= 0 {
-			file = file[:idx]
-		}
-
-		// Look for foo-bar-baz. The plugin name is "baz"
-		pluginName := file[len(prefix):]
-		log.Printf("[DEBUG] Discovered plugin: %s = %s", pluginName, match)
-		res[pluginName] = match
-	}
-
-	return res, nil
-}
-
 func (c *config) discoverInternalComponents() error {
 	// Get the packer binary path
 	packerPath, err := os.Executable()

packer-plugin-sdk/plugin/set.go

@@ -78,10 +78,10 @@ func (i *Set) RegisterProvisioner(name string, provisioner packersdk.Provisioner
 //  * "start post-processor example" starts the post-processor "example"
 func (i *Set) Run() error {
 	args := os.Args[1:]
-	return i.run(args...)
+	return i.RunCommand(args...)
 }
 
-func (i *Set) run(args ...string) error {
+func (i *Set) RunCommand(args ...string) error {
 	if len(args) < 1 {
 		return fmt.Errorf("needs at least one argument")
 	}

packer-plugin-sdk/plugin/set_test.go

@@ -50,7 +50,7 @@ func TestSet(t *testing.T) {
 		t.Fatalf("Unexpected description: %s", diff)
 	}
 
-	err := set.run("start", "builder", "example")
+	err := set.RunCommand("start", "builder", "example")
 	if diff := cmp.Diff(err.Error(), ErrManuallyStartedPlugin.Error()); diff != "" {
 		t.Fatalf("Unexpected error: %s", diff)
 	}

packer/plugin/discover.go

@@ -1,6 +1,7 @@
 package plugin
 
 import (
+	"encoding/json"
 	"log"
 	"os"
 	"os/exec"
@@ -159,6 +160,17 @@ func (c *Config) discoverExternalComponents(path string) error {
 		log.Printf("using external provisioners %v", externallyUsed)
 	}
 
+	pluginPaths, err = c.discoverSingle(filepath.Join(path, "packer-plugin-*"))
+	if err != nil {
+		return err
+	}
+
+	for pluginName, pluginPath := range pluginPaths {
+		if err := c.discoverMultiPlugin(pluginName, pluginPath); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -175,6 +187,11 @@ func (c *Config) discoverSingle(glob string) (map[string]string, error) {
 	for _, match := range matches {
 		file := filepath.Base(match)
 
+		// skip folders like packer-plugin-sdk
+		if stat, err := os.Stat(file); err == nil && stat.IsDir() {
+			continue
+		}
+
 		// On Windows, ignore any plugins that don't end in .exe.
 		// We could do a full PATHEXT parse, but this is probably good enough.
 		if runtime.GOOS == "windows" && strings.ToLower(filepath.Ext(file)) != ".exe" {
@@ -198,11 +215,62 @@ func (c *Config) discoverSingle(glob string) (map[string]string, error) {
 	return res, nil
 }
 
-func (c *Config) Client(path string) *Client {
+// discoverMultiPlugin takes the description from a multiplugin binary and
+// makes the plugins available to use in Packer. Each plugin found in the
+// binary will be addressable using `${pluginName}-${builderName}` for example.
+// pluginName could be manually set. It usually is a cloud name like amazon.
+// pluginName can be extrapolated from the filename of the binary; so
+// if the "packer-plugin-amazon" binary had an "ebs" builder one could use
+// the "amazon-ebs" builder.
+func (c *Config) discoverMultiPlugin(pluginName, pluginPath string) error {
+	out, err := exec.Command(pluginPath, "describe").Output()
+	if err != nil {
+		return err
+	}
+	var desc pluginsdk.SetDescription
+	if err := json.Unmarshal(out, &desc); err != nil {
+		return err
+	}
+
+	pluginPrefix := pluginName + "-"
+
+	for _, builderName := range desc.Builders {
+		builderName := builderName // copy to avoid pointer overwrite issue
+		c.builders[pluginPrefix+builderName] = func() (packersdk.Builder, error) {
+			return c.Client(pluginPath, "start", "builder", builderName).Builder()
+		}
+	}
+	if len(desc.Builders) > 0 {
+		log.Printf("found external %v builders from %s plugin", desc.Builders, pluginName)
+	}
+
+	for _, postProcessorName := range desc.PostProcessors {
+		postProcessorName := postProcessorName // copy to avoid pointer overwrite issue
+		c.postProcessors[pluginPrefix+postProcessorName] = func() (packersdk.PostProcessor, error) {
+			return c.Client(pluginPath, "start", "post-processor", postProcessorName).PostProcessor()
+		}
+	}
+	if len(desc.PostProcessors) > 0 {
+		log.Printf("found external %v post-processors from %s plugin", desc.PostProcessors, pluginName)
+	}
+
+	for _, provisionerName := range desc.Provisioners {
+		provisionerName := provisionerName // copy to avoid pointer overwrite issue
+		c.provisioners[pluginPrefix+provisionerName] = func() (packersdk.Provisioner, error) {
+			return c.Client(pluginPath, "start", "provisioner", provisionerName).Provisioner()
+		}
+	}
+	if len(desc.Provisioners) > 0 {
+		log.Printf("found external %v provisioner from %s plugin", desc.Provisioners, pluginName)
+	}
+
+	return nil
+}
+
+func (c *Config) Client(path string, args ...string) *Client {
 	originalPath := path
 
 	// Check for special case using `packer plugin PLUGIN`
-	args := []string{}
 	if strings.Contains(path, PACKERSPACE) {
 		parts := strings.Split(path, PACKERSPACE)
 		path = parts[0]

packer/plugin/discover_test.go

@@ -4,11 +4,16 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"os/exec"
+	"path"
 	"path/filepath"
 	"runtime"
+	"strings"
 	"testing"
 
+	"github.com/hashicorp/packer/packer-plugin-sdk/packer"
 	pluginsdk "github.com/hashicorp/packer/packer-plugin-sdk/plugin"
+	"github.com/hashicorp/packer/packer-plugin-sdk/tmp"
 )
 
 func newConfig() Config {
@@ -134,3 +139,159 @@ func generateFakePlugins(dirname string, pluginNames []string) (string, []string
 
 	return dir, plugins, cleanUpFunc, nil
 }
+
+// TestHelperProcess isn't a real test. It's used as a helper process
+// for multiplugin-binary tests.
+func TestHelperPlugins(*testing.T) {
+	if os.Getenv("PKR_WANT_TEST_PLUGINS") != "1" {
+		return
+	}
+	defer os.Exit(0)
+
+	args := os.Args
+	for len(args) > 0 {
+		if args[0] == "--" {
+			args = args[1:]
+			break
+		}
+		args = args[1:]
+	}
+	if len(args) == 0 {
+		fmt.Fprintf(os.Stderr, "No command\n")
+		os.Exit(2)
+	}
+
+	pluginName, args := args[0], args[1:]
+	plugin, found := mockPlugins[pluginName]
+	if !found {
+		fmt.Fprintf(os.Stderr, "No %q plugin found\n", pluginName)
+		os.Exit(2)
+	}
+
+	err := plugin.RunCommand(args...)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}
+
+// HasExec reports whether the current system can start new processes
+// using os.StartProcess or (more commonly) exec.Command.
+func HasExec() bool {
+	switch runtime.GOOS {
+	case "js":
+		return false
+	case "darwin":
+		if runtime.GOARCH == "arm64" {
+			return false
+		}
+	case "windows":
+		// TODO(azr): Fix this once versioning is added and we know more
+		return false
+	}
+	return true
+}
+
+// MustHaveExec checks that the current system can start new processes
+// using os.StartProcess or (more commonly) exec.Command.
+// If not, MustHaveExec calls t.Skip with an explanation.
+func MustHaveExec(t testing.TB) {
+	if !HasExec() {
+		t.Skipf("skipping test: cannot exec subprocess on %s/%s", runtime.GOOS, runtime.GOARCH)
+	}
+}
+
+func MustHaveCommand(t testing.TB, cmd string) string {
+	path, err := exec.LookPath(cmd)
+	if err != nil {
+		t.Skipf("skipping test: cannot find the %q command: %v", cmd, err)
+	}
+	return path
+}
+
+func helperCommand(t *testing.T, s ...string) []string {
+	MustHaveExec(t)
+
+	cmd := []string{os.Args[0], "-test.run=TestHelperPlugins", "--"}
+	return append(cmd, s...)
+}
+
+var (
+	mockPlugins = map[string]pluginsdk.Set{
+		"bird": pluginsdk.Set{
+			Builders: map[string]packer.Builder{
+				"feather":   nil,
+				"guacamole": nil,
+			},
+		},
+		"chimney": pluginsdk.Set{
+			PostProcessors: map[string]packer.PostProcessor{
+				"smoke": nil,
+			},
+		},
+	}
+)
+
+func Test_multiplugin_describe(t *testing.T) {
+
+	pluginDir, err := tmp.Dir("pkr-multiplugin-test-*")
+	{
+		// create an exectutable file with a `sh` sheebang
+		// this file will look like:
+		// #!/bin/sh
+		// PKR_WANT_TEST_PLUGINS=1 ...plugin/debug.test -test.run=TestHelperPlugins -- bird $@
+		// 'bird' is the mock plugin we want to start
+		// $@ just passes all passed arguments
+		// This will allow to run the fake plugin from go tests which in turn
+		// will run go tests callback to `TestHelperPlugins`, this one will be
+		// transparently calling our mock multiplugins `mockPlugins`.
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer os.RemoveAll(pluginDir)
+
+		t.Logf("putting temporary mock plugins in %s", pluginDir)
+		defer os.RemoveAll(pluginDir)
+
+		shPath := MustHaveCommand(t, "bash")
+		for name := range mockPlugins {
+			plugin := path.Join(pluginDir, "packer-plugin-"+name)
+			fileContent := ""
+			fileContent = fmt.Sprintf("#!%s\n", shPath)
+			fileContent += strings.Join(
+				append([]string{"PKR_WANT_TEST_PLUGINS=1"}, helperCommand(t, name, "$@")...),
+				" ")
+			if err := ioutil.WriteFile(plugin, []byte(fileContent), os.ModePerm); err != nil {
+				t.Fatalf("failed to create fake plugin binary: %v", err)
+			}
+		}
+	}
+	os.Setenv("PACKER_PLUGIN_PATH", pluginDir)
+
+	c := Config{}
+	err = c.Discover()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for mockPluginName, plugin := range mockPlugins {
+		for mockBuilderName := range plugin.Builders {
+			expectedBuilderName := mockPluginName + "-" + mockBuilderName
+			if _, found := c.builders[expectedBuilderName]; !found {
+				t.Fatalf("expected to find builder %q", expectedBuilderName)
+			}
+		}
+		for mockProvisionerName := range plugin.Provisioners {
+			expectedProvisionerName := mockPluginName + "-" + mockProvisionerName
+			if _, found := c.provisioners[expectedProvisionerName]; !found {
+				t.Fatalf("expected to find builder %q", expectedProvisionerName)
+			}
+		}
+		for mockPostProcessorName := range plugin.PostProcessors {
+			expectedPostProcessorName := mockPluginName + "-" + mockPostProcessorName
+			if _, found := c.postProcessors[expectedPostProcessorName]; !found {
+				t.Fatalf("expected to find post-processor %q", expectedPostProcessorName)
+			}
+		}
+	}
+}