From bda59b4d0a4a235abcaba6ba9dc63933756eff49 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Mon, 19 Feb 2024 12:02:43 -0500 Subject: [PATCH] packer: use protobuf when plugins support it With the next minor version of the SDK, we'll introduce experimental protobuf support for serialising data between Packer and plugins. This is exposed to plugins under the `PACKER_PLUGIN_PB` environment variable, which is automatically set when all the plugins able to be loaded by Packer (i.e. the highest version compatible with Packer's loading process) are capable to communicate with Protobuf. If any plugin uses version 5.0 of the API, we default to using Gob, thereby maintaining retro-compatibility will all the existing plugins. --- hcl2template/plugin.go | 11 +++++++++++ main.go | 5 +++++ packer/core.go | 10 ++++++++++ packer/env.go | 25 +++++++++++++++++++++++++ packer/plugin.go | 31 +++++++++++++++++++++++++++++++ 5 files changed, 82 insertions(+) create mode 100644 packer/env.go diff --git a/hcl2template/plugin.go b/hcl2template/plugin.go index c0f94e3f9..d0cd211fb 100644 --- a/hcl2template/plugin.go +++ b/hcl2template/plugin.go @@ -7,12 +7,14 @@ import ( "crypto/sha256" "fmt" "log" + "os" "runtime" "strings" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/packer-plugin-sdk/didyoumean" pluginsdk "github.com/hashicorp/packer-plugin-sdk/plugin" + "github.com/hashicorp/packer/packer" plugingetter "github.com/hashicorp/packer/packer/plugin-getter" ) @@ -132,6 +134,15 @@ func (cfg *PackerConfig) DetectPluginBinaries(releaseOnly bool) hcl.Diagnostics }) } + // If no installed plugin is incompatible with Protobuf, we setup the + // environment before invoking any plugin so that they are setup to + // use protobuf. + if packer.UseProtobuf { + os.Setenv("PACKER_RPC_PB", "1") + } else { + os.Setenv("PACKER_RPC_PB", "0") + } + return diags } diff --git a/main.go b/main.go index 553c173f4..4ee91088e 100644 --- a/main.go +++ b/main.go @@ -194,6 +194,11 @@ func wrappedMain() int { // the arguments... args, machineReadable := extractMachineReadable(os.Args[1:]) + // Set potential protobuf usage off if requested + if !packer.MayUseProtobuf() { + packer.UseProtobuf = false + } + defer packer.CleanupClients() var ui packersdk.Ui diff --git a/packer/core.go b/packer/core.go index 369ead37f..efca9f305 100644 --- a/packer/core.go +++ b/packer/core.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "log" + "os" "regexp" "sort" "strconv" @@ -147,6 +148,15 @@ func (c *Core) DetectPluginBinaries(releaseOnly bool) hcl.Diagnostics { }) } + // If no installed plugin is incompatible with Protobuf, we setup the + // environment before invoking any plugin so that they are setup to + // use protobuf. + if UseProtobuf { + os.Setenv("PACKER_RPC_PB", "1") + } else { + os.Setenv("PACKER_RPC_PB", "0") + } + return diags } diff --git a/packer/env.go b/packer/env.go new file mode 100644 index 000000000..0cc28eb21 --- /dev/null +++ b/packer/env.go @@ -0,0 +1,25 @@ +package packer + +import ( + "os" + + "github.com/hashicorp/packer-plugin-sdk/rpc" +) + +// MayUseProtobuf is meant to look into the environment to prohibit protobuf +// +// If the PACKER_USE_PB environment variable is unset or set to a non-empty +// string that is neither "0", "no", or "false", Packer will choose dynamically +// which protocol to use when communicating with plugins. +// +// If however it is explicitly set to one of the false values, Packer will not +// attempt to detect which protocol to use, and instead will forcibly use gob. +func MayUseProtobuf() bool { + usePB := os.Getenv(rpc.PackerUsePBEnvVar) + switch usePB { + case "0", "no", "false": + return false + } + + return true +} diff --git a/packer/plugin.go b/packer/plugin.go index 96ff96734..570e4036f 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -6,6 +6,7 @@ package packer import ( "crypto/sha256" "encoding/json" + "fmt" "log" "os" "os/exec" @@ -13,6 +14,7 @@ import ( "path/filepath" "regexp" "runtime" + "strconv" "strings" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" @@ -42,6 +44,16 @@ const PACKERSPACE = "-PACKERSPACE-" var extractPluginBasename = regexp.MustCompile("^packer-plugin-([^_]+)") +// UseProtobuf is updated when plugins are discovered. +// +// If any plugin is discovered that doesn't support protobuf encoding for +// their HCL specs, Packer will only use Gob, and avertise as such to the +// plugins it uses. +// +// TODO: check if we can dynamically switch serialisation formats without +// being too intrusive for plugins. +var UseProtobuf bool = true + // Discover discovers the latest installed version of each installed plugin. // // Search the directory of the executable, then the plugins directory, and @@ -114,6 +126,8 @@ func (c *PluginConfig) Discover(releasesOnly bool) error { return nil } +var apiVersionMinorRe = regexp.MustCompile("[0-9]+$") + // DiscoverMultiPlugin takes the description from a multi-component plugin // binary and makes the plugins available to use in Packer. Each plugin found in the // binary will be addressable using `${pluginName}-${builderName}` for example. @@ -131,6 +145,23 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error return err } + // If the major version isn't a match with what Packer expects, the + // plugin won't be considered for loading, so we can safely ignore that + // check here, and only check that the version is at least 5.1 for + // protobuf support. + // So by only looking at the minor version number, we'll be good here, + // at least until we officially remove gob support in v6.0, in which + // case, this check will be removed. + minVersion := apiVersionMinorRe.FindString(desc.APIVersion) + if minVersion == "" { + return fmt.Errorf("plugin API minor number could not be extracted, this is a Packer bug, please open an issue on the project tracker.") + } + minNb, _ := strconv.Atoi(minVersion) + if minNb == 0 { + log.Printf("[DEBUG] - plugin %q uses an older SDK, won't attempt to use protobuf for serialisation", pluginName) + UseProtobuf = false + } + pluginPrefix := pluginName + "-" for _, builderName := range desc.Builders {