From a23412674732aba2f25d65f6d5357c504b993d45 Mon Sep 17 00:00:00 2001 From: anshulSharma Date: Tue, 29 Jul 2025 08:50:23 +0530 Subject: [PATCH] changes for pulling binary from releases.hashicorp.com (#13431) * changes for pulling binary from releases.hashicorp.com * cahnges for getting release from release official site * cahnges for getting release from release official site * unit test cases * unit test cases * unit test coverage * changes for getter releases.hashicorp.com * lint fix * lint fix * lint fix * manifest.json related changes * manifest.json related changes * manifest.json related changes * manifest.json related changes * github getter test cases * added test cases for getters * added test cases for getters * added test cases for getters * added test cases for getters * added test cases for getters * added test cases for plugins getter * unit test cases for getting release from official site * description to the methods --- command/init.go | 7 + command/plugins_install.go | 6 + packer/plugin-getter/github/getter.go | 62 ++- packer/plugin-getter/github/getter_test.go | 192 ++++++++ packer/plugin-getter/plugins.go | 417 ++++++++--------- packer/plugin-getter/plugins_test.go | 495 +++++++++++++++++++- packer/plugin-getter/release/getter.go | 211 +++++++++ packer/plugin-getter/release/getter_test.go | 153 ++++++ 8 files changed, 1305 insertions(+), 238 deletions(-) create mode 100644 packer/plugin-getter/github/getter_test.go create mode 100644 packer/plugin-getter/release/getter.go create mode 100644 packer/plugin-getter/release/getter_test.go diff --git a/command/init.go b/command/init.go index a9a993426..f1a28993e 100644 --- a/command/init.go +++ b/command/init.go @@ -11,6 +11,8 @@ import ( "runtime" "strings" + "github.com/hashicorp/packer/packer/plugin-getter/release" + gversion "github.com/hashicorp/go-version" pluginsdk "github.com/hashicorp/packer-plugin-sdk/plugin" "github.com/hashicorp/packer/packer" @@ -95,7 +97,11 @@ for more info.`) log.Printf("[TRACE] init: %#v", opts) + // the ordering of the getters is important here, place the getter on top which you want to try first getters := []plugingetter.Getter{ + &release.Getter{ + Name: "releases.hashicorp.com", + }, &github.Getter{ // In the past some terraform plugins downloads were blocked from a // specific aws region by s3. Changing the user agent unblocked the @@ -105,6 +111,7 @@ for more info.`) // TODO: allow to set this from the config file or an environment // variable. UserAgent: "packer-getter-github-" + version.String(), + Name: "github.com", }, } diff --git a/command/plugins_install.go b/command/plugins_install.go index b1d6c3613..78f909f56 100644 --- a/command/plugins_install.go +++ b/command/plugins_install.go @@ -17,6 +17,8 @@ import ( "runtime" "strings" + "github.com/hashicorp/packer/packer/plugin-getter/release" + "github.com/hashicorp/go-version" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/packer-plugin-sdk/plugin" @@ -176,6 +178,9 @@ func (c *PluginsInstallCommand) RunContext(buildCtx context.Context, args *Plugi } getters := []plugingetter.Getter{ + &release.Getter{ + Name: "releases.hashicorp.com", + }, &github.Getter{ // In the past some terraform plugins downloads were blocked from a // specific aws region by s3. Changing the user agent unblocked the @@ -185,6 +190,7 @@ func (c *PluginsInstallCommand) RunContext(buildCtx context.Context, args *Plugi // TODO: allow to set this from the config file or an environment // variable. UserAgent: "packer-getter-github-" + pkrversion.String(), + Name: "github.com", }, } diff --git a/packer/plugin-getter/github/getter.go b/packer/plugin-getter/github/getter.go index e4cd4f603..df4fc0fe1 100644 --- a/packer/plugin-getter/github/getter.go +++ b/packer/plugin-getter/github/getter.go @@ -18,9 +18,10 @@ import ( "path/filepath" "strings" + plugingetter "github.com/hashicorp/packer/packer/plugin-getter" + "github.com/google/go-github/v33/github" "github.com/hashicorp/packer/hcl2template/addrs" - plugingetter "github.com/hashicorp/packer/packer/plugin-getter" "golang.org/x/oauth2" ) @@ -33,11 +34,21 @@ const ( type Getter struct { Client *github.Client UserAgent string + Name string } var _ plugingetter.Getter = &Getter{} -func transformChecksumStream() func(in io.ReadCloser) (io.ReadCloser, error) { +type PluginMetadata struct { + Versions map[string]PluginVersion `json:"versions"` +} + +type PluginVersion struct { + Name string `json:"name"` + Version string `json:"version"` +} + +func TransformChecksumStream() func(in io.ReadCloser) (io.ReadCloser, error) { return func(in io.ReadCloser) (io.ReadCloser, error) { defer in.Close() rd := bufio.NewReader(in) @@ -188,7 +199,12 @@ func (gp GithubPlugin) RealRelativePath() string { ) } +func (gp GithubPlugin) PluginType() string { + return fmt.Sprintf("packer-plugin-%s", gp.Type) +} + func (g *Getter) Get(what string, opts plugingetter.GetOptions) (io.ReadCloser, error) { + log.Printf("[TRACE] Getting %s of %s plugin from %s", what, opts.PluginRequirement.Identifier, g.Name) ghURI, err := NewGithubPlugin(opts.PluginRequirement.Identifier) if err != nil { return nil, err @@ -237,7 +253,7 @@ func (g *Getter) Get(what string, opts plugingetter.GetOptions) (io.ReadCloser, u, nil, ) - transform = transformChecksumStream() + transform = TransformChecksumStream() case "zip": u := filepath.ToSlash("https://github.com/" + ghURI.RealRelativePath() + "/releases/download/" + opts.Version() + "/" + opts.ExpectedZipFilename()) req, err = g.Client.NewRequest( @@ -277,3 +293,43 @@ func (g *Getter) Get(what string, opts plugingetter.GetOptions) (io.ReadCloser, return transform(resp.Body) } + +// Init method: a file inside will look like so: +// +// packer-plugin-comment_v0.2.12_x5.0_freebsd_amd64.zip +func (g *Getter) Init(req *plugingetter.Requirement, entry *plugingetter.ChecksumFileEntry) error { + filename := entry.Filename + res := strings.TrimPrefix(filename, req.FilenamePrefix()) + // res now looks like v0.2.12_x5.0_freebsd_amd64.zip + + entry.Ext = filepath.Ext(res) + + res = strings.TrimSuffix(res, entry.Ext) + // res now looks like v0.2.12_x5.0_freebsd_amd64 + + parts := strings.Split(res, "_") + // ["v0.2.12", "x5.0", "freebsd", "amd64"] + if len(parts) < 4 { + return fmt.Errorf("malformed filename expected %s{version}_x{protocol-version}_{os}_{arch}", req.FilenamePrefix()) + } + + entry.BinVersion, entry.ProtVersion, entry.Os, entry.Arch = parts[0], parts[1], parts[2], parts[3] + + return nil +} + +func (g *Getter) Validate(opt plugingetter.GetOptions, expectedVersion string, installOpts plugingetter.BinaryInstallationOptions, entry *plugingetter.ChecksumFileEntry) error { + expectedBinVersion := "v" + expectedVersion + if entry.BinVersion != expectedBinVersion { + return fmt.Errorf("wrong version: %s does not match expected %s", entry.BinVersion, expectedBinVersion) + } + if entry.Os != installOpts.OS || entry.Arch != installOpts.ARCH { + return fmt.Errorf("wrong system, expected %s_%s", installOpts.OS, installOpts.ARCH) + } + + return installOpts.CheckProtocolVersion(entry.ProtVersion) +} + +func (g *Getter) ExpectedFileName(pr *plugingetter.Requirement, version string, entry *plugingetter.ChecksumFileEntry, zipFileName string) string { + return zipFileName +} diff --git a/packer/plugin-getter/github/getter_test.go b/packer/plugin-getter/github/getter_test.go new file mode 100644 index 000000000..055c30b5d --- /dev/null +++ b/packer/plugin-getter/github/getter_test.go @@ -0,0 +1,192 @@ +package github + +import ( + "testing" + + plugingetter "github.com/hashicorp/packer/packer/plugin-getter" + "github.com/stretchr/testify/assert" +) + +func TestInit(t *testing.T) { + + tests := []struct { + name string + entry *plugingetter.ChecksumFileEntry + binVersion string + protocolVersion string + os string + arch string + wantErr bool + }{ + { + name: "valid format parses", + entry: &plugingetter.ChecksumFileEntry{ + Filename: "packer-plugin-v0.2.12_x5.0_freebsd_amd64.zip", + }, + binVersion: "v0.2.12", + protocolVersion: "x5.0", + os: "freebsd", + arch: "amd64", + wantErr: false, + }, + { + name: "malformed filename returns error", + entry: &plugingetter.ChecksumFileEntry{ + Filename: "packer-plugin-v0.2.12_x5.0.zip", + }, + binVersion: "v0.2.12", + protocolVersion: "x5.0", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := &plugingetter.Requirement{} + + getter := &Getter{} + err := getter.Init(req, tt.entry) + + if err != nil && !tt.wantErr { + t.Fatalf("unexpected error: %s", err) + } + + if err == nil && tt.wantErr { + t.Fatal("expected error but got nil") + } + + if !tt.wantErr && (tt.entry.BinVersion != "v0.2.12" || tt.entry.ProtVersion != "x5.0" || tt.entry.Os != "freebsd" || tt.entry.Arch != "amd64") { + t.Fatalf("unexpected parsed values: %+v", tt.entry) + } + }) + } +} + +func TestValidate(t *testing.T) { + tests := []struct { + name string + option plugingetter.GetOptions + installOpts plugingetter.BinaryInstallationOptions + entry *plugingetter.ChecksumFileEntry + version string + wantErr bool + }{ + { + name: "valid entry", + installOpts: plugingetter.BinaryInstallationOptions{ + OS: "linux", + ARCH: "amd64", + }, + entry: &plugingetter.ChecksumFileEntry{ + BinVersion: "v1.2.3", + Os: "linux", + Arch: "amd64", + ProtVersion: "x5.0", + }, + version: "1.2.3", + wantErr: false, + }, + { + name: "invalid bin version", + installOpts: plugingetter.BinaryInstallationOptions{ + OS: "linux", + ARCH: "amd64", + }, + entry: &plugingetter.ChecksumFileEntry{ + BinVersion: "v1.2.3", + Os: "linux", + Arch: "amd64", + ProtVersion: "x5.0", + }, + version: "1.2.4", + wantErr: true, + }, + { + name: "wrong OS", + installOpts: plugingetter.BinaryInstallationOptions{ + OS: "linux", + ARCH: "amd64", + }, + entry: &plugingetter.ChecksumFileEntry{ + BinVersion: "v1.2.3", + Os: "darwin", + Arch: "amd64", + ProtVersion: "x5.0", + }, + version: "1.2.3", + wantErr: true, + }, + { + name: "wrong Arch", + installOpts: plugingetter.BinaryInstallationOptions{ + OS: "linux", + ARCH: "amd64", + }, + entry: &plugingetter.ChecksumFileEntry{ + BinVersion: "v1.2.3", + Os: "linux", + Arch: "arm64", + ProtVersion: "x5.0", + }, + version: "1.2.3", + wantErr: true, + }, + { + name: "invalid API major version", + installOpts: plugingetter.BinaryInstallationOptions{ + OS: "linux", + ARCH: "amd64", + APIVersionMajor: "5", + APIVersionMinor: "0", + }, + entry: &plugingetter.ChecksumFileEntry{ + BinVersion: "v1.2.3", + Os: "linux", + Arch: "amd64", + ProtVersion: "x4.0", + }, + version: "1.2.3", + wantErr: true, + }, + { + name: "invalid API minor version", + installOpts: plugingetter.BinaryInstallationOptions{ + OS: "linux", + ARCH: "amd64", + APIVersionMajor: "5", + APIVersionMinor: "0", + }, + entry: &plugingetter.ChecksumFileEntry{ + BinVersion: "v1.2.3", + Os: "linux", + Arch: "amd64", + ProtVersion: "x5.4", + }, + version: "1.2.3", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + getter := &Getter{} + err := getter.Validate(plugingetter.GetOptions{}, tt.version, tt.installOpts, tt.entry) + + if err != nil && !tt.wantErr { + t.Fatalf("unexpected error: %s", err) + } + + if err == nil && tt.wantErr { + t.Fatal("expected error but got nil") + } + }) + } +} + +func TestExpectedFileName(t *testing.T) { + getter := &Getter{} + pr := plugingetter.Requirement{} + fileName := getter.ExpectedFileName(&pr, "1.2.3", &plugingetter.ChecksumFileEntry{}, "packer-plugin-docker_v1.2.3_x5.0_linux_amd64.zip") + assert.Equal(t, "packer-plugin-docker_v1.2.3_x5.0_linux_amd64.zip", fileName) +} diff --git a/packer/plugin-getter/plugins.go b/packer/plugin-getter/plugins.go index 314eff022..c1fe0a6f8 100644 --- a/packer/plugin-getter/plugins.go +++ b/packer/plugin-getter/plugins.go @@ -31,6 +31,17 @@ import ( "golang.org/x/mod/semver" ) +const JSONExtension = ".json" + +var HTTPFailure = errors.New("http call failed to releases.hashicorp.com failed") + +type ManifestMeta struct { + Metadata Metadata `json:"metadata"` +} +type Metadata struct { + ProtocolVersion string `json:"protocol_version"` +} + type Requirements []*Requirement // Requirement describes a required plugin and how it is installed. Usually a list @@ -521,13 +532,13 @@ func (binOpts *BinaryInstallationOptions) CheckProtocolVersion(remoteProt string } if localVersion.Major != remoteVersion.Major { - return fmt.Errorf("Unsupported remote protocol MAJOR version %d. The current MAJOR protocol version is %d."+ - " This version of Packer can only communicate with plugins using that version.", remoteVersion.Major, localVersion.Major) + return fmt.Errorf("unsupported remote protocol MAJOR version %d. The current MAJOR protocol version is %d."+ + " This version of Packer can only communicate with plugins using that version", remoteVersion.Major, localVersion.Major) } if remoteVersion.Minor > localVersion.Minor { - return fmt.Errorf("Unsupported remote protocol MINOR version %d. The supported MINOR protocol versions are version %d and below. "+ - "Please upgrade Packer or use an older version of the plugin if possible.", remoteVersion.Minor, localVersion.Minor) + return fmt.Errorf("unsupported remote protocol MINOR version %d. The supported MINOR protocol versions are version %d and below. "+ + "Please upgrade Packer or use an older version of the plugin if possible", remoteVersion.Minor, localVersion.Minor) } return nil @@ -537,6 +548,10 @@ func (gp *GetOptions) Version() string { return "v" + gp.version.String() } +func (gp *GetOptions) VersionString() string { + return gp.version.String() +} + // A Getter helps get the appropriate files to download a binary. type Getter interface { // Get allows Packer to know more information about releases of a plugin in @@ -586,6 +601,15 @@ type Getter interface { // packer-plugin-amazon_v1.0.0_x5.0_linux_amd64 file that will be checksum // verified then copied to the correct plugin location. Get(what string, opts GetOptions) (io.ReadCloser, error) + + // Init this method parses the checksum file and initializes the entry + Init(req *Requirement, entry *ChecksumFileEntry) error + + // Validate checks if OS, ARCH, and protocol version matches with the system and local version + Validate(opt GetOptions, expectedVersion string, installOpts BinaryInstallationOptions, entry *ChecksumFileEntry) error + + // ExpectedFileName returns the expected file name for the binary, which needs to be installed + ExpectedFileName(pr *Requirement, version string, entry *ChecksumFileEntry, zipFileName string) string } type Release struct { @@ -601,49 +625,8 @@ func ParseReleases(f io.ReadCloser) ([]Release, error) { type ChecksumFileEntry struct { Filename string `json:"filename"` Checksum string `json:"checksum"` - ext, binVersion, os, arch string - protVersion string -} - -func (e ChecksumFileEntry) Ext() string { return e.ext } -func (e ChecksumFileEntry) BinVersion() string { return e.binVersion } -func (e ChecksumFileEntry) ProtVersion() string { return e.protVersion } -func (e ChecksumFileEntry) Os() string { return e.os } -func (e ChecksumFileEntry) Arch() string { return e.arch } - -// a file inside will look like so: -// -// packer-plugin-comment_v0.2.12_x5.0_freebsd_amd64.zip -func (e *ChecksumFileEntry) init(req *Requirement) (err error) { - filename := e.Filename - res := strings.TrimPrefix(filename, req.FilenamePrefix()) - // res now looks like v0.2.12_x5.0_freebsd_amd64.zip - - e.ext = filepath.Ext(res) - - res = strings.TrimSuffix(res, e.ext) - // res now looks like v0.2.12_x5.0_freebsd_amd64 - - parts := strings.Split(res, "_") - // ["v0.2.12", "x5.0", "freebsd", "amd64"] - if len(parts) < 4 { - return fmt.Errorf("malformed filename expected %s{version}_x{protocol-version}_{os}_{arch}", req.FilenamePrefix()) - } - - e.binVersion, e.protVersion, e.os, e.arch = parts[0], parts[1], parts[2], parts[3] - - return err -} - -func (e *ChecksumFileEntry) validate(expectedVersion string, installOpts BinaryInstallationOptions) error { - if e.binVersion != expectedVersion { - return fmt.Errorf("wrong version: '%s' does not match expected %s ", e.binVersion, expectedVersion) - } - if e.os != installOpts.OS || e.arch != installOpts.ARCH { - return fmt.Errorf("wrong system, expected %s_%s ", installOpts.OS, installOpts.ARCH) - } - - return installOpts.CheckProtocolVersion(e.protVersion) + Ext, BinVersion, Os, Arch string + ProtVersion string } func ParseChecksumFileEntries(f io.Reader) ([]ChecksumFileEntry, error) { @@ -655,7 +638,6 @@ func (pr *Requirement) InstallLatest(opts InstallOptions) (*Installation, error) getters := opts.Getters - log.Printf("[TRACE] getting available versions for the %s plugin", pr.Identifier) versions := goversion.Collection{} var errs *multierror.Error for _, getter := range getters { @@ -665,9 +647,12 @@ func (pr *Requirement) InstallLatest(opts InstallOptions) (*Installation, error) BinaryInstallationOptions: opts.BinaryInstallationOptions, }) if err != nil { + if errors.Is(err, HTTPFailure) { + continue + } errs = multierror.Append(errs, err) log.Printf("[TRACE] %s", err.Error()) - continue + return nil, errs } releases, err := ParseReleases(releasesFile) @@ -702,40 +687,25 @@ func (pr *Requirement) InstallLatest(opts InstallOptions) (*Installation, error) continue } - break - } + // Here we want to try every release in order, starting from the highest one + // that matches the requirements. The system and protocol version need to + // match too. + sort.Sort(sort.Reverse(versions)) + log.Printf("[DEBUG] will try to install: %s", versions) - if len(versions) == 0 { - if errs.Len() == 0 { - err := fmt.Errorf("no release version found for constraints: %q", pr.VersionConstraints.String()) - errs = multierror.Append(errs, err) - } - return nil, errs - } - - // Here we want to try every release in order, starting from the highest one - // that matches the requirements. The system and protocol version need to - // match too. - sort.Sort(sort.Reverse(versions)) - log.Printf("[DEBUG] will try to install: %s", versions) + for _, version := range versions { + //TODO(azr): split in its own InstallVersion(version, opts) function - for _, version := range versions { - //TODO(azr): split in its own InstallVersion(version, opts) function + outputFolder := filepath.Join( + // Pick last folder as it's the one with the highest priority + opts.PluginDirectory, + // add expected full path + filepath.Join(pr.Identifier.Parts()...), + ) - outputFolder := filepath.Join( - // Pick last folder as it's the one with the highest priority - opts.PluginDirectory, - // add expected full path - filepath.Join(pr.Identifier.Parts()...), - ) + log.Printf("[TRACE] fetching checksums file for the %q version of the %s plugin in %q...", version, pr.Identifier, outputFolder) - log.Printf("[TRACE] fetching checksums file for the %q version of the %s plugin in %q...", version, pr.Identifier, outputFolder) - - var checksum *FileChecksum - for _, getter := range getters { - if checksum != nil { - break - } + var checksum *FileChecksum for _, checksummer := range opts.Checksummers { if checksum != nil { break @@ -746,11 +716,15 @@ func (pr *Requirement) InstallLatest(opts InstallOptions) (*Installation, error) version: version, }) if err != nil { + if errors.Is(err, HTTPFailure) { + return nil, err + } err := fmt.Errorf("could not get %s checksum file for %s version %s. Is the file present on the release and correctly named ? %w", checksummer.Type, pr.Identifier, version, err) errs = multierror.Append(errs, err) log.Printf("[TRACE] %s", err) continue } + entries, err := ParseChecksumFileEntries(checksumFile) _ = checksumFile.Close() if err != nil { @@ -761,13 +735,22 @@ func (pr *Requirement) InstallLatest(opts InstallOptions) (*Installation, error) } for _, entry := range entries { - if err := entry.init(pr); err != nil { + if filepath.Ext(entry.Filename) == JSONExtension { + continue + } + if err := getter.Init(pr, &entry); err != nil { err := fmt.Errorf("could not parse checksum filename %s. Is it correctly formatted ? %s", entry.Filename, err) errs = multierror.Append(errs, err) log.Printf("[TRACE] %s", err) continue } - if err := entry.validate("v"+version.String(), opts.BinaryInstallationOptions); err != nil { + + metaOpts := GetOptions{ + PluginRequirement: pr, + BinaryInstallationOptions: opts.BinaryInstallationOptions, + version: version, + } + if err := getter.Validate(metaOpts, version.String(), opts.BinaryInstallationOptions, &entry); err != nil { continue } @@ -786,8 +769,10 @@ func (pr *Requirement) InstallLatest(opts InstallOptions) (*Installation, error) Expected: cs, Checksummer: checksummer, } + expectedZipFilename := checksum.Filename - expectedBinaryFilename := strings.TrimSuffix(expectedZipFilename, filepath.Ext(expectedZipFilename)) + opts.BinaryInstallationOptions.Ext + expectedBinFilename := getter.ExpectedFileName(pr, version.String(), &entry, expectedZipFilename) + expectedBinaryFilename := strings.TrimSuffix(expectedBinFilename, filepath.Ext(expectedBinFilename)) + opts.BinaryInstallationOptions.Ext outputFileName := filepath.Join(outputFolder, expectedBinaryFilename) for _, potentialChecksumer := range opts.Checksummers { @@ -811,153 +796,165 @@ func (pr *Requirement) InstallLatest(opts InstallOptions) (*Installation, error) } } - for _, getter := range getters { - // start fetching binary - remoteZipFile, err := getter.Get("zip", GetOptions{ - PluginRequirement: pr, - BinaryInstallationOptions: opts.BinaryInstallationOptions, - version: version, - expectedZipFilename: expectedZipFilename, - }) - if err != nil { - errs = multierror.Append(errs, - fmt.Errorf("could not get binary for %s version %s. Is the file present on the release and correctly named ? %s", - pr.Identifier, version, err)) - continue - } - // create temporary file that will receive a temporary binary.zip - tmpFile, err := tmp.File("packer-plugin-*.zip") - if err != nil { - err = fmt.Errorf("could not create temporary file to download plugin: %w", err) - errs = multierror.Append(errs, err) - return nil, errs - } - defer func() { - tmpFilePath := tmpFile.Name() - tmpFile.Close() - os.Remove(tmpFilePath) - }() - // write binary to tmp file - _, err = io.Copy(tmpFile, remoteZipFile) - _ = remoteZipFile.Close() - if err != nil { - err := fmt.Errorf("Error getting plugin, trying another getter: %w", err) - errs = multierror.Append(errs, err) - continue - } - if _, err := tmpFile.Seek(0, 0); err != nil { - err := fmt.Errorf("Error seeking beginning of temporary file for checksumming, continuing: %w", err) - errs = multierror.Append(errs, err) - continue - } - // verify that the checksum for the zip is what we expect. - if err := checksum.Checksummer.Checksum(checksum.Expected, tmpFile); err != nil { - err := fmt.Errorf("%w. Is the checksum file correct ? Is the binary file correct ?", err) - errs = multierror.Append(errs, err) - continue - } - zr, err := zip.OpenReader(tmpFile.Name()) - if err != nil { - errs = multierror.Append(errs, fmt.Errorf("zip : %v", err)) - return nil, errs + // start fetching binary + remoteZipFile, err := getter.Get("zip", GetOptions{ + PluginRequirement: pr, + BinaryInstallationOptions: opts.BinaryInstallationOptions, + version: version, + expectedZipFilename: expectedZipFilename, + }) + if err != nil { + if errors.Is(err, HTTPFailure) { + return nil, err } + errs = multierror.Append(errs, + fmt.Errorf("could not get binary for %s version %s. Is the file present on the release and correctly named ? %s", + pr.Identifier, version, err)) + continue + } + // create temporary file that will receive a temporary binary.zip + tmpFile, err := tmp.File("packer-plugin-*.zip") + if err != nil { + err = fmt.Errorf("could not create temporary file to download plugin: %w", err) + errs = multierror.Append(errs, err) + return nil, errs + } + defer func() { + tmpFilePath := tmpFile.Name() + tmpFile.Close() + os.Remove(tmpFilePath) + }() + + // write binary to tmp file + _, err = io.Copy(tmpFile, remoteZipFile) + _ = remoteZipFile.Close() + if err != nil { + err := fmt.Errorf("Error getting plugin, trying another getter: %w", err) + errs = multierror.Append(errs, err) + continue + } + if _, err := tmpFile.Seek(0, 0); err != nil { + err := fmt.Errorf("Error seeking beginning of temporary file for checksumming, continuing: %w", err) + errs = multierror.Append(errs, err) + continue + } - var copyFrom io.ReadCloser - for _, f := range zr.File { - if f.Name != expectedBinaryFilename { - continue - } - copyFrom, err = f.Open() - if err != nil { - errs = multierror.Append(errs, fmt.Errorf("failed to open temp file: %w", err)) - return nil, errs - } - break - } - if copyFrom == nil { - err := fmt.Errorf("could not find a %q file in zipfile", expectedBinaryFilename) - errs = multierror.Append(errs, err) - return nil, errs - } + // verify that the checksum for the zip is what we expect. + if err := checksum.Checksummer.Checksum(checksum.Expected, tmpFile); err != nil { + err := fmt.Errorf("%w. Is the checksum file correct ? Is the binary file correct ?", err) + errs = multierror.Append(errs, err) + continue + } + zr, err := zip.OpenReader(tmpFile.Name()) + if err != nil { + errs = multierror.Append(errs, fmt.Errorf("zip : %v", err)) + return nil, errs + } - var outputFileData bytes.Buffer - if _, err := io.Copy(&outputFileData, copyFrom); err != nil { - err := fmt.Errorf("extract file: %w", err) - errs = multierror.Append(errs, err) - return nil, errs + var copyFrom io.ReadCloser + for _, f := range zr.File { + if f.Name != expectedBinaryFilename { + continue } - tmpBinFileName := filepath.Join(os.TempDir(), expectedBinaryFilename) - tmpOutputFile, err := os.OpenFile(tmpBinFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) + copyFrom, err = f.Open() if err != nil { - err = fmt.Errorf("could not create temporary file to download plugin: %w", err) - errs = multierror.Append(errs, err) + errs = multierror.Append(errs, fmt.Errorf("failed to open temp file: %w", err)) return nil, errs } - defer func() { - os.Remove(tmpBinFileName) - }() - - if _, err := tmpOutputFile.Write(outputFileData.Bytes()); err != nil { - err := fmt.Errorf("extract file: %w", err) - errs = multierror.Append(errs, err) - return nil, errs - } - tmpOutputFile.Close() + break + } + if copyFrom == nil { + err := fmt.Errorf("could not find a %q file in zipfile", expectedBinaryFilename) + errs = multierror.Append(errs, err) + return nil, errs + } - if err := checkVersion(tmpBinFileName, pr.Identifier.String(), version); err != nil { - errs = multierror.Append(errs, err) - var continuableError *ContinuableInstallError - if errors.As(err, &continuableError) { - continue - } - return nil, errs - } + var outputFileData bytes.Buffer + if _, err := io.Copy(&outputFileData, copyFrom); err != nil { + err := fmt.Errorf("extract file: %w", err) + errs = multierror.Append(errs, err) + return nil, errs + } + tmpBinFileName := filepath.Join(os.TempDir(), expectedBinaryFilename) + tmpOutputFile, err := os.OpenFile(tmpBinFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + err = fmt.Errorf("could not create temporary file to download plugin: %w", err) + errs = multierror.Append(errs, err) + return nil, errs + } + defer func() { + os.Remove(tmpBinFileName) + }() - // create directories if need be - if err := os.MkdirAll(outputFolder, 0755); err != nil { - err := fmt.Errorf("could not create plugin folder %q: %w", outputFolder, err) - errs = multierror.Append(errs, err) - log.Printf("[TRACE] %s", err.Error()) - return nil, errs - } - outputFile, err := os.OpenFile(outputFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) - if err != nil { - err = fmt.Errorf("could not create final plugin binary file: %w", err) - errs = multierror.Append(errs, err) - return nil, errs - } - if _, err := outputFile.Write(outputFileData.Bytes()); err != nil { - err = fmt.Errorf("could not write final plugin binary file: %w", err) - errs = multierror.Append(errs, err) - return nil, errs - } - outputFile.Close() + if _, err := tmpOutputFile.Write(outputFileData.Bytes()); err != nil { + err := fmt.Errorf("extract file: %w", err) + errs = multierror.Append(errs, err) + return nil, errs + } + tmpOutputFile.Close() - cs, err := checksum.Checksummer.Sum(&outputFileData) - if err != nil { - err := fmt.Errorf("failed to checksum binary file: %s", err) - errs = multierror.Append(errs, err) - log.Printf("[WARNING] %v, ignoring", err) - } - if err := os.WriteFile(outputFileName+checksum.Checksummer.FileExt(), []byte(hex.EncodeToString(cs)), 0644); err != nil { - err := fmt.Errorf("failed to write local binary checksum file: %s", err) - errs = multierror.Append(errs, err) - log.Printf("[WARNING] %v, ignoring", err) - os.Remove(outputFileName) + if err := checkVersion(tmpBinFileName, pr.Identifier.String(), version); err != nil { + errs = multierror.Append(errs, err) + var continuableError *ContinuableInstallError + if errors.As(err, &continuableError) { continue } + return nil, errs + } + + // create directories if need be + if err := os.MkdirAll(outputFolder, 0755); err != nil { + err := fmt.Errorf("could not create plugin folder %q: %w", outputFolder, err) + errs = multierror.Append(errs, err) + log.Printf("[TRACE] %s", err.Error()) + return nil, errs + } + outputFile, err := os.OpenFile(outputFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + err = fmt.Errorf("could not create final plugin binary file: %w", err) + errs = multierror.Append(errs, err) + return nil, errs + } + if _, err := outputFile.Write(outputFileData.Bytes()); err != nil { + err = fmt.Errorf("could not write final plugin binary file: %w", err) + errs = multierror.Append(errs, err) + return nil, errs + } + outputFile.Close() - // Success !! - return &Installation{ - BinaryPath: strings.ReplaceAll(outputFileName, "\\", "/"), - Version: "v" + version.String(), - }, nil + cs, err = checksum.Checksummer.Sum(&outputFileData) + if err != nil { + err := fmt.Errorf("failed to checksum binary file: %s", err) + errs = multierror.Append(errs, err) + log.Printf("[WARNING] %v, ignoring", err) } + if err := os.WriteFile(outputFileName+checksum.Checksummer.FileExt(), []byte(hex.EncodeToString(cs)), 0644); err != nil { + err := fmt.Errorf("failed to write local binary checksum file: %s", err) + errs = multierror.Append(errs, err) + log.Printf("[WARNING] %v, ignoring", err) + os.Remove(outputFileName) + continue + } + + // Success !! + return &Installation{ + BinaryPath: strings.ReplaceAll(outputFileName, "\\", "/"), + Version: "v" + version.String(), + }, nil } + } } } + if len(versions) == 0 { + if errs.Len() == 0 { + err := fmt.Errorf("no release version found for constraints: %q", pr.VersionConstraints.String()) + errs = multierror.Append(errs, err) + } + return nil, errs + } + if errs.ErrorOrNil() == nil { err := fmt.Errorf("could not find a local nor a remote checksum for plugin %q %q", pr.Identifier, pr.VersionConstraints) errs = multierror.Append(errs, err) diff --git a/packer/plugin-getter/plugins_test.go b/packer/plugin-getter/plugins_test.go index ca53500b1..881663796 100644 --- a/packer/plugin-getter/plugins_test.go +++ b/packer/plugin-getter/plugins_test.go @@ -28,31 +28,7 @@ var ( pluginFolderTwo = filepath.Join("testdata", "plugins_2") ) -func TestChecksumFileEntry_init(t *testing.T) { - expectedVersion := "v0.3.0" - req := &Requirement{ - Identifier: &addrs.Plugin{ - Source: "github.com/ddelnano/xenserver", - }, - } - - checkSum := &ChecksumFileEntry{ - Filename: fmt.Sprintf("packer-plugin-xenserver_%s_x5.0_darwin_amd64.zip", expectedVersion), - Checksum: "0f5969b069b9c0a58f2d5786c422341c70dfe17bd68f896fcbd46677e8c913f1", - } - - err := checkSum.init(req) - - if err != nil { - t.Fatalf("ChecksumFileEntry.init failure: %v", err) - } - - if checkSum.binVersion != expectedVersion { - t.Errorf("failed to parse ChecksumFileEntry properly expected version '%s' but found '%s'", expectedVersion, checkSum.binVersion) - } -} - -func TestRequirement_InstallLatest(t *testing.T) { +func TestRequirement_InstallLatestFromGithub(t *testing.T) { type fields struct { Identifier string VersionConstraints string @@ -72,6 +48,7 @@ func TestRequirement_InstallLatest(t *testing.T) { args{InstallOptions{ []Getter{ &mockPluginGetter{ + Name: "github.com", Releases: []Release{ {Version: "v1.2.3"}, }, @@ -109,6 +86,7 @@ func TestRequirement_InstallLatest(t *testing.T) { args{InstallOptions{ []Getter{ &mockPluginGetter{ + Name: "github.com", Releases: []Release{ {Version: "v1.2.3"}, }, @@ -144,6 +122,7 @@ func TestRequirement_InstallLatest(t *testing.T) { args{InstallOptions{ []Getter{ &mockPluginGetter{ + Name: "github.com", Releases: []Release{ {Version: "v1.2.3"}, {Version: "v1.2.4"}, @@ -185,6 +164,7 @@ func TestRequirement_InstallLatest(t *testing.T) { args{InstallOptions{ []Getter{ &mockPluginGetter{ + Name: "github.com", Releases: []Release{ {Version: "v1.2.3"}, {Version: "v1.2.4"}, @@ -236,6 +216,7 @@ echo '{"version":"v2.10.0","api_version":"x6.0"}'`, args{InstallOptions{ []Getter{ &mockPluginGetter{ + Name: "github.com", Releases: []Release{ {Version: "v1.2.3"}, {Version: "v1.2.4"}, @@ -288,6 +269,7 @@ echo '{"version":"v2.10.1","api_version":"x6.1"}'`, args{InstallOptions{ []Getter{ &mockPluginGetter{ + Name: "github.com", Releases: []Release{ {Version: "v1.2.3"}, {Version: "v1.2.4"}, @@ -340,6 +322,7 @@ echo '{"version":"v2.10.0","api_version":"x6.1"}'`, args{InstallOptions{ []Getter{ &mockPluginGetter{ + Name: "github.com", Releases: []Release{ {Version: "v2.10.0"}, }, @@ -380,6 +363,7 @@ echo '{"version":"v2.10.0","api_version":"x6.1"}'`, args{InstallOptions{ []Getter{ &mockPluginGetter{ + Name: "github.com", Releases: []Release{ {Version: "v2.10.0"}, }, @@ -461,10 +445,450 @@ echo '{"version":"v2.10.0","api_version":"x6.1"}'`, } } +func TestRequirement_InstallLatestFromOfficialRelease(t *testing.T) { + type fields struct { + Identifier string + VersionConstraints string + } + type args struct { + opts InstallOptions + } + tests := []struct { + name string + fields fields + args args + want *Installation + wantErr bool + }{ + {"already-installed-same-api-version", + fields{"amazon", "v1.2.3"}, + args{InstallOptions{ + []Getter{ + &mockPluginGetter{ + Name: "releases.hashicorp.com", + Releases: []Release{ + {Version: "v1.2.3"}, + }, + ChecksumFileEntries: map[string][]ChecksumFileEntry{ + "1.2.3": {{ + // here the checksum file tells us what zipfiles + // to expect. maybe we could cache the zip file + // ? but then the plugin is present on the drive + // twice. + Filename: "packer-plugin-amazon_1.2.3_darwin_amd64.zip", + Checksum: "1337c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }}, + }, + Manifest: map[string]io.ReadCloser{ + "github.com/hashicorp/packer-plugin-amazon/packer-plugin-amazon_1.2.3_darwin_amd64_manifest.json": manifestFile(map[string]map[string]string{ + "metadata": {"protocol_version": "5.0"}, + }), + }, + }, + }, + pluginFolderOne, + false, + BinaryInstallationOptions{ + APIVersionMajor: "5", APIVersionMinor: "0", + OS: "darwin", ARCH: "amd64", + Checksummers: []Checksummer{ + { + Type: "sha256", + Hash: sha256.New(), + }, + }, + }, + }}, + nil, false}, + + {"already-installed-compatible-api-minor-version", + // here 'packer' uses the procol version 5.1 which is compatible + // with the 5.0 one of an already installed plugin. + fields{"amazon", "v1.2.3"}, + args{InstallOptions{ + []Getter{ + &mockPluginGetter{ + Name: "releases.hashicorp.com", + Releases: []Release{ + {Version: "v1.2.3"}, + }, + ChecksumFileEntries: map[string][]ChecksumFileEntry{ + "1.2.3": {{ + Filename: "packer-plugin-amazon_1.2.3_darwin_amd64.zip", + Checksum: "1337c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }}, + }, + Manifest: map[string]io.ReadCloser{ + "github.com/hashicorp/packer-plugin-amazon/packer-plugin-amazon_1.2.3_darwin_amd64_manifest.json": manifestFile(map[string]map[string]string{ + "metadata": {"protocol_version": "5.0"}, + }), + }, + }, + }, + pluginFolderOne, + false, + BinaryInstallationOptions{ + APIVersionMajor: "5", APIVersionMinor: "1", + OS: "darwin", ARCH: "amd64", + Checksummers: []Checksummer{ + { + Type: "sha256", + Hash: sha256.New(), + }, + }, + }, + }}, + nil, false}, + + {"ignore-incompatible-higher-protocol-version", + // here 'packer' needs a binary with protocol version 5.0, and a + // working plugin is already installed; but a plugin with version + // 6.0 is available locally and remotely. It simply needs to be + // ignored. + fields{"amazon", ">= v1"}, + args{InstallOptions{ + []Getter{ + &mockPluginGetter{ + Name: "releases.hashicorp.com", + Releases: []Release{ + {Version: "v1.2.3"}, + {Version: "v1.2.4"}, + {Version: "v1.2.5"}, + {Version: "v2.0.0"}, + }, + ChecksumFileEntries: map[string][]ChecksumFileEntry{ + "1.2.5": {{ + Filename: "packer-plugin-amazon_1.2.5_darwin_amd64.zip", + Checksum: "1337c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }}, + }, + Manifest: map[string]io.ReadCloser{ + "github.com/hashicorp/packer-plugin-amazon/packer-plugin-amazon_1.2.5_darwin_amd64_manifest.json": manifestFile(map[string]map[string]string{ + "metadata": {"protocol_version": "5.0"}, + }), + }, + }, + }, + pluginFolderOne, + false, + BinaryInstallationOptions{ + APIVersionMajor: "5", APIVersionMinor: "0", + OS: "darwin", ARCH: "amd64", + Checksummers: []Checksummer{ + { + Type: "sha256", + Hash: sha256.New(), + }, + }, + }, + }}, + nil, false}, + + {"upgrade-with-diff-protocol-version", + // here we have something locally and test that a newer version will + // be installed, the newer version has a lower minor protocol + // version than the one we support. + fields{"amazon", ">= v2"}, + args{InstallOptions{ + []Getter{ + &mockPluginGetter{ + Name: "releases.hashicorp.com", + Releases: []Release{ + {Version: "v1.2.3"}, + {Version: "v1.2.4"}, + {Version: "v1.2.5"}, + {Version: "v2.0.0"}, + {Version: "v2.1.0"}, + {Version: "v2.10.0"}, + }, + ChecksumFileEntries: map[string][]ChecksumFileEntry{ + "2.10.0": {{ + Filename: "packer-plugin-amazon_2.10.0_darwin_amd64.zip", + Checksum: "5763f8b5b5ed248894e8511a089cf399b96c7ef92d784fb30ee6242a7cb35bce", + }}, + }, + Zips: map[string]io.ReadCloser{ + "github.com/hashicorp/packer-plugin-amazon/packer-plugin-amazon_2.10.0_darwin_amd64.zip": zipFile(map[string]string{ + // Make the false plugin echo an output that matches a subset of `describe` for install to work + // + // Note: this won't work on Windows as they don't have bin/sh, but this will + // eventually be replaced by acceptance tests. + "packer-plugin-amazon_v2.10.0_x6.0_darwin_amd64": `#!/bin/sh +echo '{"version":"v2.10.0","api_version":"x6.0"}'`, + }), + }, + Manifest: map[string]io.ReadCloser{ + "github.com/hashicorp/packer-plugin-amazon/packer-plugin-amazon_2.10.0_darwin_amd64_manifest.json": manifestFile(map[string]map[string]string{ + "metadata": {"protocol_version": "6.0"}, + }), + }, + }, + }, + pluginFolderTwo, + false, + BinaryInstallationOptions{ + APIVersionMajor: "6", APIVersionMinor: "1", + OS: "darwin", ARCH: "amd64", + Checksummers: []Checksummer{ + { + Type: "sha256", + Hash: sha256.New(), + }, + }, + }, + }}, + &Installation{ + BinaryPath: "testdata/plugins_2/github.com/hashicorp/amazon/packer-plugin-amazon_v2.10.0_x6.0_darwin_amd64", + Version: "v2.10.0", + }, false}, + + {"wrong-zip-checksum", + // here we have something locally and test that a newer version with + // a wrong checksum will not be installed and error. + fields{"amazon", ">= v2"}, + args{InstallOptions{ + []Getter{ + &mockPluginGetter{ + Name: "releases.hashicorp.com", + Releases: []Release{ + {Version: "v2.10.0"}, + }, + ChecksumFileEntries: map[string][]ChecksumFileEntry{ + "2.10.0": {{ + Filename: "packer-plugin-amazon_2.10.0_darwin_amd64.zip", + Checksum: "133713371337133713371337c4a152edd277366a7f71ff3812583e4a35dd0d4a", + }}, + }, + Zips: map[string]io.ReadCloser{ + "github.com/hashicorp/packer-plugin-amazon/packer-plugin-amazon_2.10.0_darwin_amd64.zip": zipFile(map[string]string{ + "packer-plugin-amazon_v2.10.0_x6.0_darwin_amd64": "h4xx", + }), + }, + Manifest: map[string]io.ReadCloser{ + "github.com/hashicorp/packer-plugin-amazon/packer-plugin-amazon_2.10.0_darwin_amd64_manifest.json": manifestFile(map[string]map[string]string{ + "metadata": {"protocol_version": "6.0"}, + }), + }, + }, + }, + pluginFolderTwo, + false, + BinaryInstallationOptions{ + APIVersionMajor: "6", APIVersionMinor: "1", + OS: "darwin", ARCH: "amd64", + Checksummers: []Checksummer{ + { + Type: "sha256", + Hash: sha256.New(), + }, + }, + }, + }}, + + nil, true}, + + {"wrong-local-checksum", + // here we have something wrong locally and test that a newer + // version with a wrong checksum will not be installed + // this should totally error. + fields{"amazon", ">= v1"}, + args{InstallOptions{ + []Getter{ + &mockPluginGetter{ + Name: "releases.hashicorp.com", + Releases: []Release{ + {Version: "v2.10.0"}, + }, + ChecksumFileEntries: map[string][]ChecksumFileEntry{ + "2.10.0": {{ + Filename: "packer-plugin-amazon_2.10.0_darwin_amd64.zip", + Checksum: "133713371337133713371337c4a152edd277366a7f71ff3812583e4a35dd0d4a", + }}, + }, + Zips: map[string]io.ReadCloser{ + "github.com/hashicorp/packer-plugin-amazon/packer-plugin-amazon_2.10.0_darwin_amd64.zip": zipFile(map[string]string{ + "packer-plugin-amazon_v2.10.0_x6.0_darwin_amd64": "h4xx", + }), + }, + Manifest: map[string]io.ReadCloser{ + "github.com/hashicorp/packer-plugin-amazon/packer-plugin-amazon_2.10.0_darwin_amd64_manifest.json": manifestFile(map[string]map[string]string{ + "metadata": {"protocol_version": "6.0"}, + }), + }, + }, + }, + pluginFolderTwo, + false, + BinaryInstallationOptions{ + APIVersionMajor: "6", APIVersionMinor: "1", + OS: "darwin", ARCH: "amd64", + Checksummers: []Checksummer{ + { + Type: "sha256", + Hash: sha256.New(), + }, + }, + }, + }}, + + nil, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + switch tt.name { + case "upgrade-with-diff-protocol-version", + "upgrade-with-same-protocol-version", + "upgrade-with-one-missing-checksum-file": + if runtime.GOOS != "windows" { + break + } + t.Skipf("Test %q cannot run on Windows because of a shell script being invoked, skipping.", tt.name) + } + + log.Printf("starting %s test", tt.name) + + identifier, err := addrs.ParsePluginSourceString("github.com/hashicorp/" + tt.fields.Identifier) + if err != nil { + t.Fatalf("ParsePluginSourceString(%q): %v", tt.fields.Identifier, err) + } + cts, err := version.NewConstraint(tt.fields.VersionConstraints) + if err != nil { + t.Fatalf("version.NewConstraint(%q): %v", tt.fields.Identifier, err) + } + pr := &Requirement{ + Identifier: identifier, + VersionConstraints: cts, + } + got, err := pr.InstallLatest(tt.args.opts) + if (err != nil) != tt.wantErr { + t.Errorf("Requirement.InstallLatest() error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Errorf("Requirement.InstallLatest() %s", diff) + } + if tt.want != nil && tt.want.BinaryPath != "" { + // Cleanup. + // These two files should be here by now and os.Remove will fail if + // they aren't. + if err := os.Remove(filepath.Clean(tt.want.BinaryPath)); err != nil { + t.Fatal(err) + } + if err := os.Remove(filepath.Clean(tt.want.BinaryPath + "_SHA256SUM")); err != nil { + t.Fatal(err) + } + } + }) + } +} + type mockPluginGetter struct { Releases []Release ChecksumFileEntries map[string][]ChecksumFileEntry Zips map[string]io.ReadCloser + Name string + APIMajor string + APIMinor string + Manifest map[string]io.ReadCloser +} + +func (g *mockPluginGetter) Init(req *Requirement, entry *ChecksumFileEntry) error { + filename := entry.Filename + res := strings.TrimPrefix(filename, req.FilenamePrefix()) + // res now looks like v0.2.12_x5.0_freebsd_amd64.zip + + entry.Ext = filepath.Ext(res) + + res = strings.TrimSuffix(res, entry.Ext) + // res now looks like v0.2.12_x5.0_freebsd_amd64 + + parts := strings.Split(res, "_") + // ["v0.2.12", "x5.0", "freebsd", "amd64"] + + if g.Name == "github.com" { + if len(parts) < 4 { + return fmt.Errorf("malformed filename expected %s{version}_x{protocol-version}_{os}_{arch}", req.FilenamePrefix()) + } + + entry.BinVersion, entry.ProtVersion, entry.Os, entry.Arch = parts[0], parts[1], parts[2], parts[3] + } else { + if len(parts) < 3 { + return fmt.Errorf("malformed filename expected %s{version}_{os}_{arch}", req.FilenamePrefix()) + } + + entry.BinVersion, entry.Os, entry.Arch = parts[0], parts[1], parts[2] + entry.BinVersion = strings.TrimPrefix(entry.BinVersion, "v") + } + + return nil +} + +func (g *mockPluginGetter) Validate(opt GetOptions, expectedVersion string, installOpts BinaryInstallationOptions, entry *ChecksumFileEntry) error { + if g.Name == "github.com" { + expectedBinVersion := "v" + expectedVersion + + if entry.BinVersion != expectedBinVersion { + return fmt.Errorf("wrong version: %s does not match expected %s", entry.BinVersion, expectedBinVersion) + } + if entry.Os != installOpts.OS || entry.Arch != installOpts.ARCH { + return fmt.Errorf("wrong system, expected %s_%s", installOpts.OS, installOpts.ARCH) + } + + return installOpts.CheckProtocolVersion(entry.ProtVersion) + } else { + if entry.BinVersion != expectedVersion { + return fmt.Errorf("wrong version: %s does not match expected %s", entry.BinVersion, expectedVersion) + } + if entry.Os != installOpts.OS || entry.Arch != installOpts.ARCH { + return fmt.Errorf("wrong system, expected %s_%s got %s_%s", installOpts.OS, installOpts.ARCH, entry.Os, entry.Arch) + } + + manifest, err := g.Get("meta", opt) + if err != nil { + return err + } + + var data ManifestMeta + body, err := io.ReadAll(manifest) + if err != nil { + log.Printf("Failed to unmarshal manifest json: %s", err) + return err + } + + err = json.Unmarshal(body, &data) + if err != nil { + log.Printf("Failed to unmarshal manifest json: %s", err) + return err + } + + err = installOpts.CheckProtocolVersion("x" + data.Metadata.ProtocolVersion) + if err != nil { + return err + } + + g.APIMajor = strings.Split(data.Metadata.ProtocolVersion, ".")[0] + g.APIMinor = strings.Split(data.Metadata.ProtocolVersion, ".")[1] + + log.Printf("#### versions API %s.%s, entry %s.%s", g.APIMajor, g.APIMinor, entry.ProtVersion, entry.BinVersion) + + return nil + } +} + +func (g *mockPluginGetter) ExpectedFileName(pr *Requirement, version string, entry *ChecksumFileEntry, zipFileName string) string { + if g.Name == "github.com" { + return zipFileName + } else { + pluginSourceParts := strings.Split(pr.Identifier.Source, "/") + + // We need to verify that the plugin source is in the expected format + return strings.Join([]string{fmt.Sprintf("packer-plugin-%s", pluginSourceParts[2]), + "v" + version, + "x" + g.APIMajor + "." + g.APIMinor, + entry.Os, + entry.Arch + ".zip", + }, "_") + } } func (g *mockPluginGetter) Get(what string, options GetOptions) (io.ReadCloser, error) { @@ -490,6 +914,17 @@ func (g *mockPluginGetter) Get(what string, options GetOptions) (io.ReadCloser, panic(fmt.Sprintf("could not find zipfile %s. %v", acc, g.Zips)) } return zip, nil + case "meta": + // Note: we'll act as if the plugin sources would always be github sources for now. + // This test will need to be updated if/when we move on to support other sources. + parts := options.PluginRequirement.Identifier.Parts() + acc := fmt.Sprintf("%s/%s/packer-plugin-%s/packer-plugin-%s_%s_%s_%s_manifest.json", parts[0], parts[1], parts[2], parts[2], options.version, options.BinaryInstallationOptions.OS, options.BinaryInstallationOptions.ARCH) + + manifest, found := g.Manifest[acc] + if found == false { + panic(fmt.Sprintf("could not find manifest file %s. %v", acc, g.Zips)) + } + return manifest, nil default: panic("Don't know how to get " + what) } @@ -527,6 +962,16 @@ func zipFile(content map[string]string) io.ReadCloser { return io.NopCloser(buff) } +func manifestFile(content map[string]map[string]string) io.ReadCloser { + jsonBytes, err := json.Marshal(content) + if err != nil { + fmt.Println("Error marshaling JSON:", err) + } + + buffer := bytes.NewBuffer(jsonBytes) + return io.NopCloser(buffer) +} + var _ Getter = &mockPluginGetter{} func Test_LessInstallList(t *testing.T) { diff --git a/packer/plugin-getter/release/getter.go b/packer/plugin-getter/release/getter.go new file mode 100644 index 000000000..ca7083e40 --- /dev/null +++ b/packer/plugin-getter/release/getter.go @@ -0,0 +1,211 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package release + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "path/filepath" + "strings" + + plugingetter "github.com/hashicorp/packer/packer/plugin-getter" + gh "github.com/hashicorp/packer/packer/plugin-getter/github" +) + +const officialReleaseURL = "https://releases.hashicorp.com/" + +type Getter struct { + APIMajor string + APIMinor string + HttpClient *http.Client + Name string +} + +var _ plugingetter.Getter = &Getter{} + +func transformZipStream() func(in io.ReadCloser) (io.ReadCloser, error) { + return func(in io.ReadCloser) (io.ReadCloser, error) { + defer in.Close() + buf := new(bytes.Buffer) + _, err := io.Copy(buf, in) + if err != nil { + panic(err) + } + return io.NopCloser(buf), nil + } +} + +// transformReleasesVersionStream get a stream from github tags and transforms it into +// something Packer wants, namely a json list of Release. +func transformReleasesVersionStream(in io.ReadCloser) (io.ReadCloser, error) { + if in == nil { + return nil, fmt.Errorf("transformReleasesVersionStream got nil body") + } + defer in.Close() + dec := json.NewDecoder(in) + + var m gh.PluginMetadata + if err := dec.Decode(&m); err != nil { + return nil, err + } + + var out []plugingetter.Release + for _, m := range m.Versions { + out = append(out, plugingetter.Release{ + Version: "v" + m.Version, + }) + } + + buf := &bytes.Buffer{} + if err := json.NewEncoder(buf).Encode(out); err != nil { + return nil, err + } + + return io.NopCloser(buf), nil +} + +func (g *Getter) Get(what string, opts plugingetter.GetOptions) (io.ReadCloser, error) { + log.Printf("[TRACE] Getting %s of %s plugin from %s", what, opts.PluginRequirement.Identifier, g.Name) + // The gitHub plugin we are using because we are not changing the plugin source string, if we decide to change that, + // then we need to write this method for release getter as well, but that will change the packer init and install command as well + ghURI, err := gh.NewGithubPlugin(opts.PluginRequirement.Identifier) + if err != nil { + return nil, err + } + + if g.HttpClient == nil { + g.HttpClient = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + } + + var req *http.Request + transform := transformZipStream() + + switch what { + case "releases": + // https://releases.hashicorp.com/packer-plugin-docker/index.json + url := filepath.ToSlash(officialReleaseURL + ghURI.PluginType() + "/index.json") + req, err = http.NewRequest("GET", url, nil) + transform = transformReleasesVersionStream + case "sha256": + // https://releases.hashicorp.com/packer-plugin-docker/8.0.0/packer-plugin-docker_1.1.1_SHA256SUMS + url := filepath.ToSlash(officialReleaseURL + ghURI.PluginType() + "/" + opts.VersionString() + "/" + ghURI.PluginType() + "_" + opts.VersionString() + "_SHA256SUMS") + transform = gh.TransformChecksumStream() + req, err = http.NewRequest("GET", url, nil) + case "meta": + // https://releases.hashicorp.com/packer-plugin-docker/8.0.0/packer-plugin-docker_1.1.1_manifest.json + url := filepath.ToSlash(officialReleaseURL + ghURI.PluginType() + "/" + opts.VersionString() + "/" + ghURI.PluginType() + "_" + opts.VersionString() + "_manifest.json") + req, err = http.NewRequest("GET", url, nil) + case "zip": + // https://releases.hashicorp.com/packer-plugin-docker/1.1.1/packer-plugin-docker_1.1.1_darwin_arm64.zip + url := filepath.ToSlash(officialReleaseURL + ghURI.PluginType() + "/" + opts.VersionString() + "/" + opts.ExpectedZipFilename()) + req, err = http.NewRequest("GET", url, nil) + default: + return nil, fmt.Errorf("%q not implemented", what) + } + + if err != nil { + log.Printf("[ERROR] http-getter: error creating request for %q: %s", what, err) + return nil, err + } + + resp, err := g.HttpClient.Do(req) + if err != nil || resp.StatusCode >= 400 { + log.Printf("[ERROR] Got error while trying getting data from releases.hashicorp.com, %v", err) + return nil, plugingetter.HTTPFailure + } + + defer func(Body io.ReadCloser) { + err = Body.Close() + if err != nil { + log.Printf("[ERROR] http-getter: error closing response body: %s", err) + } + }(resp.Body) + + return transform(resp.Body) +} + +// Init method : a file inside will look like so: +// +// packer-plugin-comment_0.2.12_freebsd_amd64.zip +func (g *Getter) Init(req *plugingetter.Requirement, entry *plugingetter.ChecksumFileEntry) error { + filename := entry.Filename + //remove the test line below where hardcoded prefix being used + res := strings.TrimPrefix(filename, req.FilenamePrefix()) + // res now looks like v0.2.12_freebsd_amd64.zip + + entry.Ext = filepath.Ext(res) + + res = strings.TrimSuffix(res, entry.Ext) + // res now looks like 0.2.12_freebsd_amd64 + + parts := strings.Split(res, "_") + // ["0.2.12", "freebsd", "amd64"] + if len(parts) < 3 { + return fmt.Errorf("malformed filename expected %s{version}_{os}_{arch}", req.FilenamePrefix()) + } + + entry.BinVersion, entry.Os, entry.Arch = parts[0], parts[1], parts[2] + entry.BinVersion = strings.TrimPrefix(entry.BinVersion, "v") + + return nil +} + +func (g *Getter) Validate(opt plugingetter.GetOptions, expectedVersion string, installOpts plugingetter.BinaryInstallationOptions, entry *plugingetter.ChecksumFileEntry) error { + + if entry.BinVersion != expectedVersion { + return fmt.Errorf("wrong version: %s does not match expected %s", entry.BinVersion, expectedVersion) + } + if entry.Os != installOpts.OS || entry.Arch != installOpts.ARCH { + return fmt.Errorf("wrong system, expected %s_%s got %s_%s", installOpts.OS, installOpts.ARCH, entry.Os, entry.Arch) + } + + manifest, err := g.Get("meta", opt) + if err != nil { + return err + } + + var data plugingetter.ManifestMeta + body, err := io.ReadAll(manifest) + if err != nil { + log.Printf("Failed to unmarshal manifest json: %s", err) + return err + } + + err = json.Unmarshal(body, &data) + if err != nil { + log.Printf("Failed to unmarshal manifest json: %s", err) + return err + } + + err = installOpts.CheckProtocolVersion("x" + data.Metadata.ProtocolVersion) + if err != nil { + return err + } + + g.APIMajor = strings.Split(data.Metadata.ProtocolVersion, ".")[0] + g.APIMinor = strings.Split(data.Metadata.ProtocolVersion, ".")[1] + + return nil +} + +func (g *Getter) ExpectedFileName(pr *plugingetter.Requirement, version string, entry *plugingetter.ChecksumFileEntry, zipFileName string) string { + pluginSourceParts := strings.Split(pr.Identifier.Source, "/") + + // We need to verify that the plugin source is in the expected format + return strings.Join([]string{fmt.Sprintf("packer-plugin-%s", pluginSourceParts[2]), + "v" + version, + "x" + g.APIMajor + "." + g.APIMinor, + entry.Os, + entry.Arch + ".zip", + }, "_") +} diff --git a/packer/plugin-getter/release/getter_test.go b/packer/plugin-getter/release/getter_test.go new file mode 100644 index 000000000..7b5f647bb --- /dev/null +++ b/packer/plugin-getter/release/getter_test.go @@ -0,0 +1,153 @@ +package release + +import ( + "testing" + + "github.com/hashicorp/packer/hcl2template/addrs" + plugingetter "github.com/hashicorp/packer/packer/plugin-getter" + "github.com/stretchr/testify/assert" +) + +func TestInit(t *testing.T) { + + tests := []struct { + name string + entry *plugingetter.ChecksumFileEntry + binVersion string + protocolVersion string + os string + arch string + wantErr bool + }{ + { + name: "valid format parses", + entry: &plugingetter.ChecksumFileEntry{ + Filename: "packer-plugin-v0.2.12_freebsd_amd64.zip", + }, + binVersion: "v0.2.12", + protocolVersion: "x5.0", + os: "freebsd", + arch: "amd64", + wantErr: false, + }, + { + name: "malformed filename returns error", + entry: &plugingetter.ChecksumFileEntry{ + Filename: "packer-plugin-v0.2.12.zip", + }, + binVersion: "v0.2.12", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := &plugingetter.Requirement{} + + getter := &Getter{} + err := getter.Init(req, tt.entry) + + if err != nil && !tt.wantErr { + t.Fatalf("unexpected error: %s", err) + } + + if err == nil && tt.wantErr { + t.Fatal("expected error but got nil") + } + + if !tt.wantErr && (tt.entry.BinVersion != "0.2.12" || tt.entry.Os != "freebsd" || tt.entry.Arch != "amd64") { + t.Fatalf("unexpected parsed values: %+v", tt.entry) + } + }) + } +} + +func TestValidate(t *testing.T) { + tests := []struct { + name string + option plugingetter.GetOptions + installOpts plugingetter.BinaryInstallationOptions + entry *plugingetter.ChecksumFileEntry + version string + wantErr bool + }{ + { + name: "invalid bin version", + installOpts: plugingetter.BinaryInstallationOptions{ + OS: "linux", + ARCH: "amd64", + }, + entry: &plugingetter.ChecksumFileEntry{ + BinVersion: "1.2.3", + Os: "linux", + Arch: "amd64", + }, + version: "1.2.4", + wantErr: true, + }, + { + name: "wrong OS", + installOpts: plugingetter.BinaryInstallationOptions{ + OS: "linux", + ARCH: "amd64", + }, + entry: &plugingetter.ChecksumFileEntry{ + BinVersion: "1.2.3", + Os: "darwin", + Arch: "amd64", + }, + version: "1.2.3", + wantErr: true, + }, + { + name: "wrong Arch", + installOpts: plugingetter.BinaryInstallationOptions{ + OS: "linux", + ARCH: "amd64", + }, + entry: &plugingetter.ChecksumFileEntry{ + BinVersion: "1.2.3", + Os: "linux", + Arch: "arm64", + ProtVersion: "x5.0", + }, + version: "1.2.3", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + getter := &Getter{} + err := getter.Validate(plugingetter.GetOptions{}, tt.version, tt.installOpts, tt.entry) + + if err != nil && !tt.wantErr { + t.Fatalf("unexpected error: %s", err) + } + + if err == nil && tt.wantErr { + t.Fatal("expected error but got nil") + } + }) + } +} + +func TestExpectedFileName(t *testing.T) { + getter := &Getter{ + APIMajor: "5", + APIMinor: "0", + } + pr := plugingetter.Requirement{ + Identifier: &addrs.Plugin{ + Source: "github.com/hashicorp/docker", + }, + } + + entry := &plugingetter.ChecksumFileEntry{ + Os: "linux", + Arch: "amd64", + } + fileName := getter.ExpectedFileName(&pr, "1.2.3", entry, "packer-plugin-docker_v1.2.3_x5.0_linux_amd64.zip") + assert.Equal(t, "packer-plugin-docker_v1.2.3_x5.0_linux_amd64.zip", fileName) +}