packer: use protobuf when plugins support it

With the next minor version of the SDK, we'll introduce experimental
protobuf support for serialising data between Packer and plugins.
This is exposed to plugins under the `PACKER_PLUGIN_PB` environment
variable, which is automatically set when all the plugins able to be
loaded by Packer (i.e. the highest version compatible with Packer's
loading process) are capable to communicate with Protobuf.

If any plugin uses version 5.0 of the API, we default to using Gob,
thereby maintaining retro-compatibility will all the existing plugins.

hcl2template/plugin.go

@@ -7,12 +7,14 @@ import (
 	"crypto/sha256"
 	"fmt"
 	"log"
+	"os"
 	"runtime"
 	"strings"
 
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/packer-plugin-sdk/didyoumean"
 	pluginsdk "github.com/hashicorp/packer-plugin-sdk/plugin"
+	"github.com/hashicorp/packer/packer"
 	plugingetter "github.com/hashicorp/packer/packer/plugin-getter"
 )
 
@@ -132,6 +134,15 @@ func (cfg *PackerConfig) DetectPluginBinaries(releaseOnly bool) hcl.Diagnostics
 		})
 	}
 
+	// If no installed plugin is incompatible with Protobuf, we setup the
+	// environment before invoking any plugin so that they are setup to
+	// use protobuf.
+	if packer.UseProtobuf {
+		os.Setenv("PACKER_RPC_PB", "1")
+	} else {
+		os.Setenv("PACKER_RPC_PB", "0")
+	}
+
 	return diags
 }
 

main.go

@@ -194,6 +194,11 @@ func wrappedMain() int {
 	// the arguments...
 	args, machineReadable := extractMachineReadable(os.Args[1:])
 
+	// Set potential protobuf usage off if requested
+	if !packer.MayUseProtobuf() {
+		packer.UseProtobuf = false
+	}
+
 	defer packer.CleanupClients()
 
 	var ui packersdk.Ui

packer/core.go

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"os"
 	"regexp"
 	"sort"
 	"strconv"
@@ -147,6 +148,15 @@ func (c *Core) DetectPluginBinaries(releaseOnly bool) hcl.Diagnostics {
 		})
 	}
 
+	// If no installed plugin is incompatible with Protobuf, we setup the
+	// environment before invoking any plugin so that they are setup to
+	// use protobuf.
+	if UseProtobuf {
+		os.Setenv("PACKER_RPC_PB", "1")
+	} else {
+		os.Setenv("PACKER_RPC_PB", "0")
+	}
+
 	return diags
 }
 

packer/env.go

@@ -0,0 +1,25 @@
+package packer
+
+import (
+	"os"
+
+	"github.com/hashicorp/packer-plugin-sdk/rpc"
+)
+
+// MayUseProtobuf is meant to look into the environment to prohibit protobuf
+//
+// If the PACKER_USE_PB environment variable is unset or set to a non-empty
+// string that is neither "0", "no", or "false", Packer will choose dynamically
+// which protocol to use when communicating with plugins.
+//
+// If however it is explicitly set to one of the false values, Packer will not
+// attempt to detect which protocol to use, and instead will forcibly use gob.
+func MayUseProtobuf() bool {
+	usePB := os.Getenv(rpc.PackerUsePBEnvVar)
+	switch usePB {
+	case "0", "no", "false":
+		return false
+	}
+
+	return true
+}

packer/plugin.go

@@ -6,6 +6,7 @@ package packer
 import (
 	"crypto/sha256"
 	"encoding/json"
+	"fmt"
 	"log"
 	"os"
 	"os/exec"
@@ -13,6 +14,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"strconv"
 	"strings"
 
 	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
@@ -42,6 +44,16 @@ const PACKERSPACE = "-PACKERSPACE-"
 
 var extractPluginBasename = regexp.MustCompile("^packer-plugin-([^_]+)")
 
+// UseProtobuf is updated when plugins are discovered.
+//
+// If any plugin is discovered that doesn't support protobuf encoding for
+// their HCL specs, Packer will only use Gob, and avertise as such to the
+// plugins it uses.
+//
+// TODO: check if we can dynamically switch serialisation formats without
+// being too intrusive for plugins.
+var UseProtobuf bool = true
+
 // Discover discovers the latest installed version of each installed plugin.
 //
 // Search the directory of the executable, then the plugins directory, and
@@ -114,6 +126,8 @@ func (c *PluginConfig) Discover(releasesOnly bool) error {
 	return nil
 }
 
+var apiVersionMinorRe = regexp.MustCompile("[0-9]+$")
+
 // DiscoverMultiPlugin takes the description from a multi-component plugin
 // binary and makes the plugins available to use in Packer. Each plugin found in the
 // binary will be addressable using `${pluginName}-${builderName}` for example.
@@ -131,6 +145,23 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error
 		return err
 	}
 
+	// If the major version isn't a match with what Packer expects, the
+	// plugin won't be considered for loading, so we can safely ignore that
+	// check here, and only check that the version is at least 5.1 for
+	// protobuf support.
+	// So by only looking at the minor version number, we'll be good here,
+	// at least until we officially remove gob support in v6.0, in which
+	// case, this check will be removed.
+	minVersion := apiVersionMinorRe.FindString(desc.APIVersion)
+	if minVersion == "" {
+		return fmt.Errorf("plugin API minor number could not be extracted, this is a Packer bug, please open an issue on the project tracker.")
+	}
+	minNb, _ := strconv.Atoi(minVersion)
+	if minNb == 0 {
+		log.Printf("[DEBUG] - plugin %q uses an older SDK, won't attempt to use protobuf for serialisation", pluginName)
+		UseProtobuf = false
+	}
+
 	pluginPrefix := pluginName + "-"
 
 	for _, builderName := range desc.Builders {