From fd5f668ee9f0b0260d4a07acd0d5e1aaa0559d5d Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Wed, 17 Jan 2024 15:52:59 -0500 Subject: [PATCH] plugin: consolidate loading logic Right now we had two paths for discovering installed plugins, i.e. through plugin-getter's `ListInstallations' function, or through the `Discover' call, which relied on a glob to list the installations. This was required since we allowed plugins to be installed in multiple locations, and with different constraints. Now that we force a certain convention, we can consolidate the logic into ListInstallations, and rely on that logic in `Discover' to load a plugin into the PluginConfig for the current Packer run. --- hcl2template/plugin.go | 2 +- packer/plugin-getter/plugins.go | 7 + packer/plugin-getter/plugins_test.go | 226 --------------------------- packer/plugin.go | 141 +++++------------ packer/plugin_discover_test.go | 2 + 5 files changed, 48 insertions(+), 330 deletions(-) diff --git a/hcl2template/plugin.go b/hcl2template/plugin.go index 15358573d..279f534e0 100644 --- a/hcl2template/plugin.go +++ b/hcl2template/plugin.go @@ -94,7 +94,7 @@ func (cfg *PackerConfig) DetectPluginBinaries() hcl.Diagnostics { continue } log.Printf("[TRACE] Found the following %q installations: %v", pluginRequirement.Identifier, sortedInstalls) - install := sortedInstalls[len(sortedInstalls)-1] + install := sortedInstalls[0] err = cfg.parser.PluginConfig.DiscoverMultiPlugin(pluginRequirement.Accessor, install.BinaryPath) if err != nil { diags = append(diags, &hcl.Diagnostic{ diff --git a/packer/plugin-getter/plugins.go b/packer/plugin-getter/plugins.go index 85ba3ba32..d9379351c 100644 --- a/packer/plugin-getter/plugins.go +++ b/packer/plugin-getter/plugins.go @@ -11,7 +11,10 @@ import ( "io" "log" "os" + "os/exec" + "path" "path/filepath" + "regexp" "sort" "strconv" "strings" @@ -19,8 +22,10 @@ import ( "github.com/hashicorp/go-multierror" "github.com/hashicorp/go-version" + pluginsdk "github.com/hashicorp/packer-plugin-sdk/plugin" "github.com/hashicorp/packer-plugin-sdk/tmp" "github.com/hashicorp/packer/hcl2template/addrs" + "golang.org/x/mod/semver" ) type Requirements []*Requirement @@ -182,6 +187,8 @@ func (pr Requirement) ListInstallations(opts ListInstallationsOptions) (InstallL }) } + sort.Sort(res) + return res, nil } diff --git a/packer/plugin-getter/plugins_test.go b/packer/plugin-getter/plugins_test.go index 1f58e3496..4cc78acc3 100644 --- a/packer/plugin-getter/plugins_test.go +++ b/packer/plugin-getter/plugins_test.go @@ -18,7 +18,6 @@ import ( "github.com/google/go-cmp/cmp" "github.com/hashicorp/go-version" - "github.com/hashicorp/hcl/v2" "github.com/hashicorp/packer/hcl2template/addrs" ) @@ -58,231 +57,6 @@ func TestChecksumFileEntry_init(t *testing.T) { } } -func TestPlugin_ListInstallations(t *testing.T) { - - type fields struct { - Identifier string - VersionConstraints version.Constraints - } - tests := []struct { - name string - fields fields - opts ListInstallationsOptions - wantErr bool - want InstallList - }{ - - { - "windows_all_plugins", - fields{ - // empty - }, - ListInstallationsOptions{ - pluginFolderOne, - BinaryInstallationOptions{ - OS: "windows", ARCH: "amd64", - Ext: ".exe", - Checksummers: []Checksummer{ - { - Type: "sha256", - Hash: sha256.New(), - }, - }, - }, - }, - false, - []*Installation{ - { - Version: "v1.2.3", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.3_x5.0_windows_amd64.exe"), - }, - { - Version: "v1.2.4", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.4_x5.0_windows_amd64.exe"), - }, - { - Version: "v1.2.5", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.5_x5.0_windows_amd64.exe"), - }, - { - Version: "v4.5.6", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "google", "packer-plugin-google_v4.5.6_x5.0_windows_amd64.exe"), - }, - { - Version: "v4.5.7", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "google", "packer-plugin-google_v4.5.7_x5.0_windows_amd64.exe"), - }, - { - Version: "v4.5.8", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "google", "packer-plugin-google_v4.5.8_x5.0_windows_amd64.exe"), - }, - }, - }, - - { - "darwin_amazon_prot_5.0", - fields{ - Identifier: "github.com/hashicorp/amazon", - }, - ListInstallationsOptions{ - pluginFolderOne, - BinaryInstallationOptions{ - APIVersionMajor: "5", APIVersionMinor: "0", - OS: "darwin", ARCH: "amd64", - Checksummers: []Checksummer{ - { - Type: "sha256", - Hash: sha256.New(), - }, - }, - }, - }, - false, - []*Installation{ - { - Version: "v1.2.3", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.3_x5.0_darwin_amd64"), - }, - { - Version: "v1.2.4", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.4_x5.0_darwin_amd64"), - }, - { - Version: "v1.2.5", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.5_x5.0_darwin_amd64"), - }, - }, - }, - { - "darwin_amazon_prot_5.1", - fields{ - Identifier: "github.com/hashicorp/amazon", - }, - ListInstallationsOptions{ - pluginFolderOne, - BinaryInstallationOptions{ - APIVersionMajor: "5", APIVersionMinor: "1", - OS: "darwin", ARCH: "amd64", - Checksummers: []Checksummer{ - { - Type: "sha256", - Hash: sha256.New(), - }, - }, - }, - }, - false, - []*Installation{ - { - Version: "v1.2.3", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.3_x5.0_darwin_amd64"), - }, - { - Version: "v1.2.3", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.3_x5.1_darwin_amd64"), - }, - { - Version: "v1.2.4", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.4_x5.0_darwin_amd64"), - }, - { - Version: "v1.2.5", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.5_x5.0_darwin_amd64"), - }, - }, - }, - { - "windows_amazon", - fields{ - Identifier: "github.com/hashicorp/amazon", - }, - ListInstallationsOptions{ - pluginFolderOne, - BinaryInstallationOptions{ - APIVersionMajor: "5", APIVersionMinor: "0", - OS: "windows", ARCH: "amd64", - Ext: ".exe", - Checksummers: []Checksummer{ - { - Type: "sha256", - Hash: sha256.New(), - }, - }, - }, - }, - false, - []*Installation{ - { - Version: "v1.2.3", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.3_x5.0_windows_amd64.exe"), - }, - { - Version: "v1.2.4", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.4_x5.0_windows_amd64.exe"), - }, - { - Version: "v1.2.5", - BinaryPath: filepath.Join(pluginFolderOne, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.5_x5.0_windows_amd64.exe"), - }, - }, - }, - { - "test nil identifier - multiple plugins with same version", - fields{ - Identifier: "", - }, - ListInstallationsOptions{ - pluginFolderThree, - BinaryInstallationOptions{ - APIVersionMajor: "5", APIVersionMinor: "0", - OS: "linux", ARCH: "amd64", - Checksummers: []Checksummer{ - { - Type: "sha256", - Hash: sha256.New(), - }, - }, - }, - }, - false, - []*Installation{ - { - Version: "v1.2.5", - BinaryPath: filepath.Join(pluginFolderThree, "github.com", "hashicorp", "alazon", "packer-plugin-alazon_v1.2.5_x5.0_linux_amd64"), - }, - { - Version: "v1.2.5", - BinaryPath: filepath.Join(pluginFolderThree, "github.com", "hashicorp", "amazon", "packer-plugin-amazon_v1.2.5_x5.0_linux_amd64"), - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var identifier *addrs.Plugin - if tt.fields.Identifier != "" { - var diags hcl.Diagnostics - identifier, diags = addrs.ParsePluginSourceString(tt.fields.Identifier) - if diags.HasErrors() { - t.Fatalf("%v", diags) - } - } - p := Requirement{ - Identifier: identifier, - VersionConstraints: tt.fields.VersionConstraints, - } - got, err := p.ListInstallations(tt.opts) - if (err != nil) != tt.wantErr { - t.Errorf("Plugin.ListInstallations() error = %v, wantErr %v", err, tt.wantErr) - return - } - if diff := cmp.Diff(tt.want, got); diff != "" { - t.Errorf("Plugin.ListInstallations() unexpected output: %s", diff) - } - }) - } -} - func TestRequirement_InstallLatest(t *testing.T) { type fields struct { Identifier string diff --git a/packer/plugin.go b/packer/plugin.go index 4d5a67397..5fcef8954 100644 --- a/packer/plugin.go +++ b/packer/plugin.go @@ -6,13 +6,13 @@ package packer import ( "crypto/sha256" "encoding/json" - "fmt" "log" "os" "os/exec" + "path" "path/filepath" + "regexp" "runtime" - "sort" "strings" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" @@ -40,7 +40,9 @@ type PluginConfig struct { // without being confused with spaces in the path to the command itself. const PACKERSPACE = "-PACKERSPACE-" -// Discover discovers plugins. +var extractPluginBasename = regexp.MustCompile("^packer-plugin-([^_]+)") + +// Discover discovers the latest installed version of each installed plugin. // // Search the directory of the executable, then the plugins directory, and // finally the CWD, in that order. Any conflicts will overwrite previously @@ -71,67 +73,54 @@ func (c *PluginConfig) Discover() error { c.PluginDirectory, _ = PluginFolder() } - if err := c.discoverInstalledComponents(); err != nil { + installations, err := plugingetter.Requirement{}.ListInstallations(plugingetter.ListInstallationsOptions{ + PluginDirectory: c.PluginDirectory, + BinaryInstallationOptions: plugingetter.BinaryInstallationOptions{ + OS: runtime.GOOS, + ARCH: runtime.GOARCH, + Checksummers: []plugingetter.Checksummer{ + {Type: "sha256", Hash: sha256.New()}, + }, + }, + }) + if err != nil { return err } - return nil -} - -func (c *PluginConfig) discoverSingle(glob string) (map[string]string, error) { - matches, err := filepath.Glob(glob) - if err != nil { - return nil, err - } - var prefix string - res := make(map[string]string) - // Sort the matches so we add the newer version of a plugin last - sort.Strings(matches) - prefix = filepath.Base(glob) - prefix = prefix[:strings.Index(prefix, "*")] - for _, match := range matches { - file := filepath.Base(match) - // skip folders like packer-plugin-sdk - if stat, err := os.Stat(file); err == nil && stat.IsDir() { + // Map of plugin basename to executable + // + // We'll use that later to register the components for each plugin + pluginMap := map[string]string{} + for _, install := range installations { + pluginBasename := path.Base(install.BinaryPath) + matches := extractPluginBasename.FindStringSubmatch(pluginBasename) + if len(matches) != 2 { + log.Printf("[INFO] - plugin %q could not have its name matched, ignoring", pluginBasename) continue } - // On Windows, ignore any plugins that don't end in .exe. - // We could do a full PATHEXT parse, but this is probably good enough. - if runtime.GOOS == "windows" && strings.ToLower(filepath.Ext(file)) != ".exe" { - log.Printf( - "[TRACE] Ignoring plugin match %s, no exe extension", - match) - continue - } + pluginName := matches[1] - if strings.Contains(strings.ToUpper(file), defaultChecksummer.FileExt()) { - log.Printf( - "[TRACE] Ignoring plugin match %s, which looks to be a checksum file", - match) + // If the plugin is already registered in the plugin map, we + // can ignore the current executable, as they're sorted by + // version in descending order, so if it's already in the map, + // a more recent version was already discovered. + _, ok := pluginMap[pluginName] + if ok { continue - - } - - // If the filename has a ".", trim up to there - if idx := strings.Index(file, ".exe"); idx >= 0 { - file = file[:idx] } - // Look for foo-bar-baz. The plugin name is "baz" - pluginName := file[len(prefix):] - // multi-component plugins installed via the plugins subcommand will have a name that looks like baz_vx.y.z_x5.0_darwin_arm64. - // After the split the plugin name is "baz". - pluginName = strings.SplitN(pluginName, "_", 2)[0] + pluginMap[pluginName] = install.BinaryPath + } - pluginPath, err := filepath.Abs(match) + for name, path := range pluginMap { + err := c.DiscoverMultiPlugin(name, path) if err != nil { - pluginPath = match + return err } - res[pluginName] = pluginPath } - return res, nil + return nil } // DiscoverMultiPlugin takes the description from a multi-component plugin @@ -257,57 +246,3 @@ func (c *PluginConfig) Client(path string, args ...string) *PluginClient { config.MaxPort = c.PluginMaxPort return NewClient(&config) } - -// discoverInstalledComponents scans the provided path for plugins installed by running packer plugins install or packer init. -// Valid plugins contain a matching system binary and valid checksum file. -func (c *PluginConfig) discoverInstalledComponents() error { - //Check for installed plugins using the `packer plugins install` command - binInstallOpts := plugingetter.BinaryInstallationOptions{ - OS: runtime.GOOS, - ARCH: runtime.GOARCH, - APIVersionMajor: pluginsdk.APIVersionMajor, - APIVersionMinor: pluginsdk.APIVersionMinor, - Checksummers: []plugingetter.Checksummer{ - defaultChecksummer, - }, - } - - if runtime.GOOS == "windows" { - binInstallOpts.Ext = ".exe" - } - - pluginPath := filepath.Join(c.PluginDirectory, "*", "*", "*", fmt.Sprintf("packer-plugin-*%s", binInstallOpts.FilenameSuffix())) - pluginPaths, err := c.discoverSingle(pluginPath) - if err != nil { - return err - } - - for pluginName, pluginPath := range pluginPaths { - var checksumOk bool - for _, checksummer := range binInstallOpts.Checksummers { - cs, err := checksummer.GetCacheChecksumOfFile(pluginPath) - if err != nil { - log.Printf("[TRACE] GetChecksumOfFile(%q) failed: %v\n", pluginPath, err) - continue - } - - if err := checksummer.ChecksumFile(cs, pluginPath); err != nil { - log.Printf("[TRACE] ChecksumFile(%q) failed: %v\n", pluginPath, err) - continue - } - checksumOk = true - break - } - - if !checksumOk { - log.Printf("[WARN] No checksum found for %q ignoring possibly unsafe binary", pluginPath) - continue - } - - if err := c.DiscoverMultiPlugin(pluginName, pluginPath); err != nil { - return err - } - } - - return nil -} diff --git a/packer/plugin_discover_test.go b/packer/plugin_discover_test.go index 472f9d6e0..3eca150d8 100644 --- a/packer/plugin_discover_test.go +++ b/packer/plugin_discover_test.go @@ -17,6 +17,7 @@ import ( packersdk "github.com/hashicorp/packer-plugin-sdk/packer" pluginsdk "github.com/hashicorp/packer-plugin-sdk/plugin" "github.com/hashicorp/packer-plugin-sdk/tmp" + "github.com/hashicorp/packer-plugin-sdk/version" plugingetter "github.com/hashicorp/packer/packer/plugin-getter" ) @@ -305,6 +306,7 @@ func TestHelperPlugins(t *testing.T) { for _, mock := range allMocks { plugin, found := mock[pluginName] if found { + plugin.SetVersion(version.InitializePluginVersion("1.0.0", "")) err := plugin.RunCommand(args...) if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err)