add aws multiplugin binary canary code (#10272)

This creates a new plumbing to allow to have binaries that can define any plugin on any type.

cmd/packer-plugin-amazon/main.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/hashicorp/packer/builder/amazon/ebs"
+	"github.com/hashicorp/packer/builder/amazon/ebssurrogate"
+	"github.com/hashicorp/packer/builder/amazon/ebsvolume"
+	"github.com/hashicorp/packer/builder/osc/chroot"
+	"github.com/hashicorp/packer/packer-plugin-sdk/plugin"
+	amazonimport "github.com/hashicorp/packer/post-processor/amazon-import"
+)
+
+func main() {
+	pps := plugin.NewSet()
+	pps.RegisterBuilder("ebs", new(ebs.Builder))
+	pps.RegisterBuilder("chroot", new(chroot.Builder))
+	pps.RegisterBuilder("ebssurrogate", new(ebssurrogate.Builder))
+	pps.RegisterBuilder("ebsvolume", new(ebsvolume.Builder))
+	pps.RegisterPostProcessor("import", new(amazonimport.PostProcessor))
+	err := pps.Run()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
+	}
+}

packer-plugin-sdk/plugin/server.go

@@ -38,12 +38,14 @@ const MagicCookieValue = "d602bf8f470bc67ca7faa0386276bbdd4330efaf76d1a219cb4d69
 // know how to speak it.
 const APIVersion = "5"
 
+var ErrManuallyStartedPlugin = errors.New(
+	"Please do not execute plugins directly. Packer will execute these for you.")
+
 // Server waits for a connection to this plugin and returns a Packer
 // RPC server that you can use to register components and serve them.
 func Server() (*packrpc.Server, error) {
 	if os.Getenv(MagicCookieKey) != MagicCookieValue {
-		return nil, errors.New(
-			"Please do not execute plugins directly. Packer will execute these for you.")
+		return nil, ErrManuallyStartedPlugin
 	}
 
 	// If there is no explicit number of Go threads to use, then set it

packer-plugin-sdk/plugin/set.go

@@ -0,0 +1,166 @@
+package plugin
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"sort"
+
+	packersdk "github.com/hashicorp/packer/packer-plugin-sdk/packer"
+	"github.com/hashicorp/packer/version"
+)
+
+// Set is a plugin set. It's API is meant to be very close to what is returned
+// by plugin.Server
+// It can describe itself or run a single plugin using the CLI arguments.
+type Set struct {
+	version        string
+	sdkVersion     string
+	Builders       map[string]packersdk.Builder
+	PostProcessors map[string]packersdk.PostProcessor
+	Provisioners   map[string]packersdk.Provisioner
+}
+
+// SetDescription describes a Set.
+type SetDescription struct {
+	Version        string   `json:"version"`
+	SDKVersion     string   `json:"sdk_version"`
+	Builders       []string `json:"builders"`
+	PostProcessors []string `json:"post_processors"`
+	Provisioners   []string `json:"provisioners"`
+}
+
+////
+// Setup
+////
+
+func NewSet() *Set {
+	return &Set{
+		version:        version.String(),
+		sdkVersion:     version.String(), // TODO: Set me after the split
+		Builders:       map[string]packersdk.Builder{},
+		PostProcessors: map[string]packersdk.PostProcessor{},
+		Provisioners:   map[string]packersdk.Provisioner{},
+	}
+}
+
+func (i *Set) RegisterBuilder(name string, builder packersdk.Builder) {
+	if _, found := i.Builders[name]; found {
+		panic(fmt.Errorf("registering duplicate %s builder", name))
+	}
+	i.Builders[name] = builder
+}
+
+func (i *Set) RegisterPostProcessor(name string, postProcessor packersdk.PostProcessor) {
+	if _, found := i.PostProcessors[name]; found {
+		panic(fmt.Errorf("registering duplicate %s post-processor", name))
+	}
+	i.PostProcessors[name] = postProcessor
+}
+
+func (i *Set) RegisterProvisioner(name string, provisioner packersdk.Provisioner) {
+	if _, found := i.Provisioners[name]; found {
+		panic(fmt.Errorf("registering duplicate %s provisioner", name))
+	}
+	i.Provisioners[name] = provisioner
+}
+
+// Run takes the os Args and runs a packer plugin command from it.
+//  * "describe" command makes the plugin set describe itself.
+//  * "start builder builder-name" starts the builder "builder-name"
+//  * "start post-processor example" starts the post-processor "example"
+func (i *Set) Run() error {
+	args := os.Args[1:]
+	return i.run(args...)
+}
+
+func (i *Set) run(args ...string) error {
+	if len(args) < 1 {
+		return fmt.Errorf("needs at least one argument")
+	}
+
+	switch args[0] {
+	case "describe":
+		return i.jsonDescribe(os.Stdout)
+	case "start":
+		args = args[1:]
+		if len(args) != 2 {
+			return fmt.Errorf("start takes two arguments, for example 'start builder example-builder'. Found: %v", args)
+		}
+		return i.start(args[0], args[1])
+	default:
+		return fmt.Errorf("Unknown command: %q", args[0])
+	}
+}
+
+func (i *Set) start(kind, name string) error {
+	server, err := Server()
+	if err != nil {
+		return err
+	}
+
+	log.Printf("[TRACE] starting %s %s", kind, name)
+
+	switch kind {
+	case "builder":
+		err = server.RegisterBuilder(i.Builders[name])
+	case "post-processor":
+		err = server.RegisterPostProcessor(i.PostProcessors[name])
+	case "provisioners":
+		err = server.RegisterProvisioner(i.Provisioners[name])
+	default:
+		err = fmt.Errorf("Unknown plugin type: %s", kind)
+	}
+	if err != nil {
+		return err
+	}
+	server.Serve()
+	return nil
+}
+
+////
+// Describe
+////
+
+func (i *Set) description() SetDescription {
+	return SetDescription{
+		Version:        i.version,
+		SDKVersion:     i.sdkVersion,
+		Builders:       i.buildersDescription(),
+		PostProcessors: i.postProcessorsDescription(),
+		Provisioners:   i.provisionersDescription(),
+	}
+}
+
+func (i *Set) jsonDescribe(out io.Writer) error {
+	return json.NewEncoder(out).Encode(i.description())
+}
+
+func (i *Set) buildersDescription() []string {
+	out := []string{}
+	for key := range i.Builders {
+		out = append(out, key)
+	}
+	sort.Strings(out)
+	return out
+}
+
+func (i *Set) postProcessorsDescription() []string {
+	out := []string{}
+	for key := range i.PostProcessors {
+		out = append(out, key)
+	}
+	sort.Strings(out)
+	return out
+}
+
+func (i *Set) provisionersDescription() []string {
+	out := []string{}
+	for key := range i.Provisioners {
+		out = append(out, key)
+	}
+	sort.Strings(out)
+	return out
+}

packer-plugin-sdk/plugin/set_test.go

@@ -0,0 +1,54 @@
+package plugin
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	packersdk "github.com/hashicorp/packer/packer-plugin-sdk/packer"
+	"github.com/hashicorp/packer/version"
+)
+
+type MockBuilder struct {
+	packersdk.Builder
+}
+
+var _ packersdk.Builder = new(MockBuilder)
+
+type MockProvisioner struct {
+	packersdk.Provisioner
+}
+
+var _ packersdk.Provisioner = new(MockProvisioner)
+
+type MockPostProcessor struct {
+	packersdk.PostProcessor
+}
+
+var _ packersdk.PostProcessor = new(MockPostProcessor)
+
+func TestSet(t *testing.T) {
+	set := NewSet()
+	set.RegisterBuilder("example-2", new(MockBuilder))
+	set.RegisterBuilder("example", new(MockBuilder))
+	set.RegisterPostProcessor("example", new(MockPostProcessor))
+	set.RegisterPostProcessor("example-2", new(MockPostProcessor))
+	set.RegisterProvisioner("example", new(MockProvisioner))
+	set.RegisterProvisioner("example-2", new(MockProvisioner))
+
+	outputDesc := set.description()
+
+	if diff := cmp.Diff(SetDescription{
+		Version:        version.String(),
+		SDKVersion:     version.String(),
+		Builders:       []string{"example", "example-2"},
+		PostProcessors: []string{"example", "example-2"},
+		Provisioners:   []string{"example", "example-2"},
+	}, outputDesc); diff != "" {
+		t.Fatalf("Unexpected description: %s", diff)
+	}
+
+	err := set.run("start", "builder", "example")
+	if diff := cmp.Diff(err.Error(), ErrManuallyStartedPlugin.Error()); diff != "" {
+		t.Fatalf("Unexpected error: %s", diff)
+	}
+}

website/pages/docs/extending/plugins.mdx

@@ -34,8 +34,8 @@ of Packer starts and communicates with.
 
 These plugin applications aren't meant to be run manually. Instead, Packer core
 executes them as a sub-process, run as a sub-command (`packer plugin`) and
-communicates with them. For example, the VMware builder is actually run as
-`packer plugin packer-builder-vmware`. The next time you run a Packer build,
+communicates with them. For example, the Shell provisioner is actually run as
+`packer plugin packer-provisioner-shell`. The next time you run a Packer build,
 look at your process list and you should see a handful of `packer-` prefixed
 applications running.
 
@@ -43,9 +43,9 @@ applications running.
 
 The easiest way to install a plugin is to name it correctly, then place it in
 the proper directory. To name a plugin correctly, make sure the binary is named
-`packer-TYPE-NAME`. For example, `packer-builder-amazon-ebs` for a "builder"
-type plugin named "amazon-ebs". Valid types for plugins are down this page
-more.
+`packer-plugin-NAME`. For example, `packer-plugin-amazon` for a "plugin"
+binary named "amazon". This binary will make one or more plugins
+available to use. Valid types for plugins are down this page.
 
 Once the plugin is named properly, Packer automatically discovers plugins in
 the following directories in the given order. If a conflicting plugin is found
@@ -72,6 +72,9 @@ later, it will take precedence over one found earlier.
 
 The valid types for plugins are:
 
+- `plugin` - A plugin binary that can contain one or more of each Packer plugin
+  type.
+
 - `builder` - Plugins responsible for building images for a specific
   platform.
 
@@ -81,6 +84,13 @@ The valid types for plugins are:
 - `provisioner` - A provisioner to install software on images created by a
   builder.
 
+~> **Note**: Only _multi-plugin binaries_ -- that is plugins named
+packer-plugin-*, like the `packer-plugin-amazon` described before -- are
+expected to work with Packer's plugin manager. The legacy `builder`,
+`post-processor` and `provisioner` plugin types will keep on being detected but
+Packer cannot install them automatically.
+
+
 ## Developing Plugins
 
 This page will document how you can develop your own Packer plugins. Prior to
@@ -141,8 +151,12 @@ There are two steps involved in creating a plugin:
 2.  Serve the interface by calling the appropriate plugin serving method in
     your main method.
 
-A basic example is shown below. In this example, assume the `Builder` struct
-implements the `packer.Builder` interface:
+Basic examples are shown below. Note that if you can define a multi-plugin
+binary as it will allow to add more that one plugin per binary.
+
+
+<Tabs>
+<Tab heading="Multi-plugin Binary">
 
 ```go
 import (
@@ -150,22 +164,67 @@ import (
 )
 
 // Assume this implements packer.Builder
-type Builder struct{}
+type ExampleBuilder struct{}
+
+// Assume this implements packer.PostProcessor
+type FooPostProcessor struct{}
+
+// Assume this implements packer.Provisioner
+type BarProvisioner struct{}
 
 func main() {
- 	server, err := plugin.Server()
+	pps := plugin.NewSet()
+	pps.RegisterBuilder("example", new(ExampleBuilder))
+	pps.RegisterPostProcessor("foo", new(FooPostProcessor))
+	pps.RegisterProvisioner("bar", new(BarProvisioner))
+	err := pps.Run()
 	if err != nil {
-		panic(err)
+		fmt.Fprintln(os.Stderr, err.Error())
+		os.Exit(1)
 	}
-	server.RegisterBuilder(new(Builder))
-	server.Serve()
 }
 ```
 
-**That's it!** `plugin.Server` handles all the nitty gritty of
+**That's it!** `plugin.NewSet` handles all the nitty gritty of
 communicating with Packer core and serving your builder over RPC. It can't get
 much easier than that.
 
+Here the name of the plugin will be used to use each plugin, so if your plugin
+is named `packer-plugin-my`, this would make the following parts available:
+
+* the `my-example` builder
+* the `my-foo` post-processor
+* the `my-bar` provisioner
+
+</Tab>
+<Tab heading="Single Plugin Binary">
+
+```go
+import (
+  "github.com/hashicorp/packer/packer/plugin"
+)
+
+// Assume this implements packer.Builder
+type Builder struct{}
+
+func main() {
+    server, err := plugin.Server()
+    if err != nil {
+        panic(err)
+    }
+    server.RegisterBuilder(new(Builder))
+    server.Serve()
+}
+```
+
+**That's it!** `plugin.Server` handles all the nitty gritty of communicating with
+Packer core and serving your builder over RPC. It can't get much easier than
+that.
+
+</Tab>
+</Tabs>
+
+
 Next, just build your plugin like a normal Go application, using `go build` or
 however you please. The resulting binary is the plugin that can be installed
 using standard installation procedures.
@@ -207,10 +266,10 @@ concerns.
 #### Naming Conventions
 
 It is standard practice to name the resulting plugin application in the format
-of `packer-TYPE-NAME`. For example, if you're building a new builder for
+of `packer-plugin-NAME`. For example, if you're building a new builder for
 CustomCloud, it would be standard practice to name the resulting plugin
-`packer-builder-custom-cloud`. This naming convention helps users identify the
-purpose of a plugin.
+`packer-plugin-custom-cloud`. This naming convention helps users identify the
+scope of a plugin.
 
 #### Testing Plugins