diff --git a/packer/plugin-getter/github/getter.go b/packer/plugin-getter/github/getter.go index e4cd4f603..ef3f9534e 100644 --- a/packer/plugin-getter/github/getter.go +++ b/packer/plugin-getter/github/getter.go @@ -7,10 +7,13 @@ import ( "bufio" "bytes" "context" + "crypto/tls" + "crypto/x509" "encoding/json" "errors" "fmt" "io" + "io/ioutil" "log" "net/http" "os" @@ -31,12 +34,22 @@ const ( ) type Getter struct { - Client *github.Client - UserAgent string + Client *github.Client + HttpClient *http.Client + UserAgent string } var _ plugingetter.Getter = &Getter{} +type PluginMetadata struct { + Versions map[string]PluginVersion `json:"versions"` +} + +type PluginVersion struct { + Name string `json:"name"` + Version string `json:"version"` +} + func transformChecksumStream() func(in io.ReadCloser) (io.ReadCloser, error) { return func(in io.ReadCloser) (io.ReadCloser, error) { defer in.Close() @@ -107,6 +120,35 @@ func transformVersionStream(in io.ReadCloser) (io.ReadCloser, error) { return io.NopCloser(buf), nil } +// transformReleasesVersionStream get a stream from github tags and transforms it into +// something Packer wants, namely a json list of Release. +func transformReleasesVersionStream(in io.ReadCloser) (io.ReadCloser, error) { + if in == nil { + return nil, fmt.Errorf("transformReleasesVersionStream got nil body") + } + defer in.Close() + dec := json.NewDecoder(in) + + var m PluginMetadata + if err := dec.Decode(&m); err != nil { + return nil, err + } + + var out []plugingetter.Release + for _, m := range m.Versions { + out = append(out, plugingetter.Release{ + Version: "v" + m.Version, + }) + } + + buf := &bytes.Buffer{} + if err := json.NewEncoder(buf).Encode(out); err != nil { + return nil, err + } + + return io.NopCloser(buf), nil +} + // HostSpecificTokenAuthTransport makes sure the http roundtripper only sets an // auth token for requests aimed at a specific host. // @@ -188,6 +230,10 @@ func (gp GithubPlugin) RealRelativePath() string { ) } +func (gp GithubPlugin) PluginType() string { + return fmt.Sprintf("packer-plugin-%s", gp.Type) +} + func (g *Getter) Get(what string, opts plugingetter.GetOptions) (io.ReadCloser, error) { ghURI, err := NewGithubPlugin(opts.PluginRequirement.Identifier) if err != nil { @@ -277,3 +323,89 @@ func (g *Getter) Get(what string, opts plugingetter.GetOptions) (io.ReadCloser, return transform(resp.Body) } + +func (g *Getter) GetOfficialRelease(what string, opts plugingetter.GetOptions) (io.ReadCloser, error) { + ghURI, err := NewGithubPlugin(opts.PluginRequirement.Identifier) + if err != nil { + return nil, err + } + + if g.HttpClient == nil { + caCert, err := ioutil.ReadFile("/Users/anshulsharma/Documents/test_official/cert.pem") + if err != nil { + panic(err) + } + + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + + // Create custom TLS config + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + } + + g.HttpClient = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsConfig, + }, + } + //g.HttpClient = &http.Client{} + } + + var req *http.Request + transform := func(in io.ReadCloser) (io.ReadCloser, error) { + return in, nil + } + + switch what { + case "releases": + url := filepath.ToSlash("https://releases.hashicorp.com/" + ghURI.PluginType() + "/index.json") + req, err = http.NewRequest("GET", url, nil) + transform = transformReleasesVersionStream + case "sha256": + // something like https://github.com/sylviamoss/packer-plugin-comment/releases/download/v0.2.11/packer-plugin-comment_v0.2.11_x5_SHA256SUMS + url := filepath.ToSlash("https://releases.hashicorp.com/" + ghURI.PluginType() + "/" + opts.VersionString() + "/" + ghURI.PluginType() + "_" + opts.VersionString() + "_SHA256SUMS") + transform = transformChecksumStream() + log.Printf("[DEBUG] github-getter: getting %q", url) + req, err = http.NewRequest("GET", url, nil) + case "zip": + // https://releases.hashicorp.com/terraform-provider-akamai/8.0.0/terraform-provider-akamai_8.0.0_darwin_arm64.zip + url := filepath.ToSlash("https://releases.hashicorp.com/" + ghURI.PluginType() + "/" + opts.VersionString() + "/" + opts.ExpectedZipFilename()) + req, err = http.NewRequest("GET", url, nil) + transform = transformZipStream() + default: + return nil, fmt.Errorf("%q not implemented", what) + } + + if err != nil { + log.Printf("[ERROR] http-getter: error creating request for %q: %s", what, err) + return nil, err + } + + resp, err := g.HttpClient.Do(req) + if err != nil || resp.StatusCode >= 400 { + log.Printf("[ERROR] Got error while getting data from releases.hashicorp.com, %v", err) + return nil, plugingetter.HTTPFailure + } + + defer func(Body io.ReadCloser) { + err = Body.Close() + if err != nil { + log.Printf("[ERROR] http-getter: error closing response body: %s", err) + } + }(resp.Body) + + return transform(resp.Body) +} + +func transformZipStream() func(in io.ReadCloser) (io.ReadCloser, error) { + return func(in io.ReadCloser) (io.ReadCloser, error) { + defer in.Close() + buf := new(bytes.Buffer) + _, err := io.Copy(buf, in) + if err != nil { + panic(err) + } + return io.NopCloser(buf), nil + } +} diff --git a/packer/plugin-getter/plugins.go b/packer/plugin-getter/plugins.go index 314eff022..e8f5b1905 100644 --- a/packer/plugin-getter/plugins.go +++ b/packer/plugin-getter/plugins.go @@ -31,6 +31,10 @@ import ( "golang.org/x/mod/semver" ) +const JSONExtension = ".json" + +var HTTPFailure = errors.New("http call failed to releases.hashicorp.com failed") + type Requirements []*Requirement // Requirement describes a required plugin and how it is installed. Usually a list @@ -537,6 +541,10 @@ func (gp *GetOptions) Version() string { return "v" + gp.version.String() } +func (gp *GetOptions) VersionString() string { + return gp.version.String() +} + // A Getter helps get the appropriate files to download a binary. type Getter interface { // Get allows Packer to know more information about releases of a plugin in @@ -586,6 +594,8 @@ type Getter interface { // packer-plugin-amazon_v1.0.0_x5.0_linux_amd64 file that will be checksum // verified then copied to the correct plugin location. Get(what string, opts GetOptions) (io.ReadCloser, error) + + GetOfficialRelease(what string, opts GetOptions) (io.ReadCloser, error) } type Release struct { @@ -635,6 +645,32 @@ func (e *ChecksumFileEntry) init(req *Requirement) (err error) { return err } +// a file inside will look like so: +// +// packer-plugin-comment_v0.2.12_freebsd_amd64.zip +func (e *ChecksumFileEntry) initRelease(req *Requirement) (err error) { + filename := e.Filename + //remove the test line below where hardcoded prefix being used + res := strings.TrimPrefix(filename, req.FilenamePrefix()) + // res now looks like v0.2.12_freebsd_amd64.zip + + e.ext = filepath.Ext(res) + + res = strings.TrimSuffix(res, e.ext) + // res now looks like v0.2.12_freebsd_amd64 + + parts := strings.Split(res, "_") + // ["v0.2.12", "freebsd", "amd64"] + if len(parts) < 3 { + return fmt.Errorf("malformed filename expected %s{version}_{os}_{arch}", req.FilenamePrefix()) + } + + e.binVersion, e.os, e.arch = parts[0], parts[1], parts[2] + e.binVersion = strings.TrimPrefix(e.binVersion, "v") + + return err +} + func (e *ChecksumFileEntry) validate(expectedVersion string, installOpts BinaryInstallationOptions) error { if e.binVersion != expectedVersion { return fmt.Errorf("wrong version: '%s' does not match expected %s ", e.binVersion, expectedVersion) @@ -646,16 +682,44 @@ func (e *ChecksumFileEntry) validate(expectedVersion string, installOpts BinaryI return installOpts.CheckProtocolVersion(e.protVersion) } +func (e *ChecksumFileEntry) validateRelease(expectedVersion string, installOpts BinaryInstallationOptions) error { + if e.binVersion != expectedVersion { + return fmt.Errorf("wrong version: '%s' does not match expected %s ", e.binVersion, expectedVersion) + } + if e.os != installOpts.OS || e.arch != installOpts.ARCH { + return fmt.Errorf("wrong system, expected %s_%s got %s_%s", installOpts.OS, installOpts.ARCH, e.os, e.arch) + } + + return nil +} + func ParseChecksumFileEntries(f io.Reader) ([]ChecksumFileEntry, error) { var entries []ChecksumFileEntry return entries, json.NewDecoder(f).Decode(&entries) } func (pr *Requirement) InstallLatest(opts InstallOptions) (*Installation, error) { + newInstall, err := pr.InstallOfficial(opts) + if err != nil { + if errors.Is(err, HTTPFailure) { + log.Printf("[TRACE] Failure from release site, trying to install from gitHub.com %s", pr.Identifier) + newInstall, err = pr.InstallFromGitHub(opts) + if err != nil { + return nil, err + } + } else { + return nil, err + } + } + + return newInstall, nil +} + +func (pr *Requirement) InstallFromGitHub(opts InstallOptions) (*Installation, error) { getters := opts.Getters - log.Printf("[TRACE] getting available versions for the %s plugin", pr.Identifier) + log.Printf("[TRACE] getting available versions for the %s plugin from github.com", pr.Identifier) versions := goversion.Collection{} var errs *multierror.Error for _, getter := range getters { @@ -966,9 +1030,347 @@ func (pr *Requirement) InstallLatest(opts InstallOptions) (*Installation, error) return nil, errs } +func (pr *Requirement) InstallOfficial(opts InstallOptions) (*Installation, error) { + + getters := opts.Getters + + log.Printf("[TRACE] getting available versions for the %s plugin from releases.hasicorp.com", pr.Identifier) + versions := goversion.Collection{} + var errs *multierror.Error + for _, getter := range getters { + + releasesFile, err := getter.GetOfficialRelease("releases", GetOptions{ + PluginRequirement: pr, + BinaryInstallationOptions: opts.BinaryInstallationOptions, + }) + if err != nil { + if errors.Is(err, HTTPFailure) { + return nil, err + } + errs = multierror.Append(errs, err) + log.Printf("[TRACE] %s", err.Error()) + continue + } + + releases, err := ParseReleases(releasesFile) + if err != nil { + err := fmt.Errorf("could not parse release: %w", err) + errs = multierror.Append(errs, err) + log.Printf("[TRACE] %s", err.Error()) + continue + } + if len(releases) == 0 { + err := fmt.Errorf("no release found") + errs = multierror.Append(errs, err) + log.Printf("[TRACE] %s", err.Error()) + continue + } + for _, release := range releases { + v, err := goversion.NewVersion(release.Version) + if err != nil { + err := fmt.Errorf("could not parse release version %s. %w", release.Version, err) + errs = multierror.Append(errs, err) + log.Printf("[TRACE] %s, ignoring it", err.Error()) + continue + } + if pr.VersionConstraints.Check(v) { + versions = append(versions, v) + } + } + if len(versions) == 0 { + err := fmt.Errorf("no matching version found in releases. In %v", releases) + errs = multierror.Append(errs, err) + log.Printf("[TRACE] %s", err.Error()) + continue + } + + break + } + + if len(versions) == 0 { + if errs.Len() == 0 { + err := fmt.Errorf("no release version found for constraints: %q", pr.VersionConstraints.String()) + errs = multierror.Append(errs, err) + } + return nil, errs + } + + // Here we want to try every release in order, starting from the highest one + // that matches the requirements. The system and protocol version need to + // match too. + sort.Sort(sort.Reverse(versions)) + log.Printf("[DEBUG] will try to install: %s", versions) + + for _, version := range versions { + //TODO(azr): split in its own InstallVersion(version, opts) function + + outputFolder := filepath.Join( + // Pick last folder as it's the one with the highest priority + opts.PluginDirectory, + // add expected full path + filepath.Join(pr.Identifier.Parts()...), + ) + + log.Printf("[TRACE] fetching checksums file for the %q version of the %s plugin in %q...", version, pr.Identifier, outputFolder) + + var checksum *FileChecksum + for _, getter := range getters { + if checksum != nil { + break + } + for _, checksummer := range opts.Checksummers { + if checksum != nil { + break + } + + checksumFile, err := getter.GetOfficialRelease(checksummer.Type, GetOptions{ + PluginRequirement: pr, + BinaryInstallationOptions: opts.BinaryInstallationOptions, + version: version, + }) + if err != nil { + if errors.Is(err, HTTPFailure) { + return nil, err + } + err := fmt.Errorf("could not get %s checksum file for %s version %s. Is the file present on the release and correctly named ? %w", checksummer.Type, pr.Identifier, version, err) + errs = multierror.Append(errs, err) + log.Printf("[TRACE] %s", err) + continue + } + + entries, err := ParseChecksumFileEntries(checksumFile) + _ = checksumFile.Close() + if err != nil { + err := fmt.Errorf("could not parse %s checksumfile: %v. Make sure the checksum file contains a checksum and a binary filename per line", checksummer.Type, err) + errs = multierror.Append(errs, err) + log.Printf("[TRACE] %s", err) + continue + } + + for _, entry := range entries { + if filepath.Ext(entry.Filename) == JSONExtension { + continue + } + if err := entry.initRelease(pr); err != nil { + err := fmt.Errorf("could not parse checksum filename %s. Is it correctly formatted ? %s", entry.Filename, err) + errs = multierror.Append(errs, err) + log.Printf("[TRACE] %s", err) + continue + } + + if err := entry.validateRelease(version.String(), opts.BinaryInstallationOptions); err != nil { + log.Printf("#### Got error %v", err) + continue + } + + log.Printf("[TRACE] About to get: %s", entry.Filename) + + cs, err := checksummer.ParseChecksum(strings.NewReader(entry.Checksum)) + if err != nil { + err := fmt.Errorf("could not parse %s checksum: %s. Make sure the checksum file contains the checksum and only the checksum", checksummer.Type, err) + errs = multierror.Append(errs, err) + log.Printf("[TRACE] %s", err) + continue + } + + checksum = &FileChecksum{ + Filename: entry.Filename, + Expected: cs, + Checksummer: checksummer, + } + + log.Printf("#### getChecksum filename: %s", checksum.Filename) + expectedZipFilename := checksum.Filename + expectedBinaryFilename := strings.TrimSuffix(expectedZipFilename, filepath.Ext(expectedZipFilename)) + opts.BinaryInstallationOptions.Ext + outputFileName := filepath.Join(outputFolder, expectedBinaryFilename) + + log.Printf("#### output filename: %s", outputFileName) + + for _, potentialChecksumer := range opts.Checksummers { + // First check if a local checksum file is already here in the expected + // download folder. Here we want to download a binary so we only check + // for an existing checksum file from the folder we want to download + // into. + cs, err := potentialChecksumer.GetCacheChecksumOfFile(outputFileName) + if err == nil && len(cs) > 0 { + localChecksum := &FileChecksum{ + Expected: cs, + Checksummer: potentialChecksumer, + } + + log.Printf("[TRACE] found a pre-existing %q checksum file", potentialChecksumer.Type) + // if outputFile is there and matches the checksum: do nothing more. + if err := localChecksum.ChecksumFile(localChecksum.Expected, outputFileName); err == nil && !opts.Force { + log.Printf("[INFO] %s v%s plugin is already correctly installed in %q", pr.Identifier, version, outputFileName) + return nil, nil // success + } + } + } + + for _, getter := range getters { + // start fetching binary + remoteZipFile, err := getter.GetOfficialRelease("zip", GetOptions{ + PluginRequirement: pr, + BinaryInstallationOptions: opts.BinaryInstallationOptions, + version: version, + expectedZipFilename: expectedZipFilename, + }) + if err != nil { + if errors.Is(err, HTTPFailure) { + return nil, err + } + errs = multierror.Append(errs, + fmt.Errorf("could not get binary for %s version %s. Is the file present on the release and correctly named ? %s", + pr.Identifier, version, err)) + continue + } + // create temporary file that will receive a temporary binary.zip + tmpFile, err := tmp.File("packer-plugin-*.zip") + if err != nil { + err = fmt.Errorf("could not create temporary file to download plugin: %w", err) + errs = multierror.Append(errs, err) + return nil, errs + } + defer func() { + tmpFilePath := tmpFile.Name() + tmpFile.Close() + os.Remove(tmpFilePath) + }() + // write binary to tmp file + _, err = io.Copy(tmpFile, remoteZipFile) + _ = remoteZipFile.Close() + if err != nil { + err := fmt.Errorf("Error getting plugin, trying another getter: %w", err) + errs = multierror.Append(errs, err) + continue + } + if _, err := tmpFile.Seek(0, 0); err != nil { + err := fmt.Errorf("Error seeking beginning of temporary file for checksumming, continuing: %w", err) + errs = multierror.Append(errs, err) + continue + } + // verify that the checksum for the zip is what we expect. + if err := checksum.Checksummer.Checksum(checksum.Expected, tmpFile); err != nil { + err := fmt.Errorf("%w. Is the checksum file correct ? Is the binary file correct ?", err) + errs = multierror.Append(errs, err) + continue + } + zr, err := zip.OpenReader(tmpFile.Name()) + if err != nil { + errs = multierror.Append(errs, fmt.Errorf("zip : %v", err)) + return nil, errs + } + + var copyFrom io.ReadCloser + for _, f := range zr.File { + if f.Name != "terraform-provider-akamai_v8.0.0" { + continue + } + copyFrom, err = f.Open() + if err != nil { + errs = multierror.Append(errs, fmt.Errorf("failed to open temp file: %w", err)) + return nil, errs + } + break + } + if copyFrom == nil { + err := fmt.Errorf("could not find a %q file in zipfile", expectedBinaryFilename) + errs = multierror.Append(errs, err) + return nil, errs + } + + var outputFileData bytes.Buffer + if _, err := io.Copy(&outputFileData, copyFrom); err != nil { + err := fmt.Errorf("extract file: %w", err) + errs = multierror.Append(errs, err) + return nil, errs + } + tmpBinFileName := filepath.Join(os.TempDir(), expectedBinaryFilename) + tmpOutputFile, err := os.OpenFile(tmpBinFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + err = fmt.Errorf("could not create temporary file to download plugin: %w", err) + errs = multierror.Append(errs, err) + return nil, errs + } + defer func() { + os.Remove(tmpBinFileName) + }() + + if _, err := tmpOutputFile.Write(outputFileData.Bytes()); err != nil { + err := fmt.Errorf("extract file: %w", err) + errs = multierror.Append(errs, err) + return nil, errs + } + tmpOutputFile.Close() + + if err := checkVersion(tmpBinFileName, pr.Identifier.String(), version); err != nil { + errs = multierror.Append(errs, err) + var continuableError *ContinuableInstallError + if errors.As(err, &continuableError) { + continue + } + return nil, errs + } + + // create directories if need be + if err := os.MkdirAll(outputFolder, 0755); err != nil { + err := fmt.Errorf("could not create plugin folder %q: %w", outputFolder, err) + errs = multierror.Append(errs, err) + log.Printf("[TRACE] %s", err.Error()) + return nil, errs + } + outputFile, err := os.OpenFile(outputFileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + err = fmt.Errorf("could not create final plugin binary file: %w", err) + errs = multierror.Append(errs, err) + return nil, errs + } + if _, err := outputFile.Write(outputFileData.Bytes()); err != nil { + err = fmt.Errorf("could not write final plugin binary file: %w", err) + errs = multierror.Append(errs, err) + return nil, errs + } + outputFile.Close() + + cs, err := checksum.Checksummer.Sum(&outputFileData) + if err != nil { + err := fmt.Errorf("failed to checksum binary file: %s", err) + errs = multierror.Append(errs, err) + log.Printf("[WARNING] %v, ignoring", err) + } + if err := os.WriteFile(outputFileName+checksum.Checksummer.FileExt(), []byte(hex.EncodeToString(cs)), 0644); err != nil { + err := fmt.Errorf("failed to write local binary checksum file: %s", err) + errs = multierror.Append(errs, err) + log.Printf("[WARNING] %v, ignoring", err) + os.Remove(outputFileName) + continue + } + + // Success !! + return &Installation{ + BinaryPath: strings.ReplaceAll(outputFileName, "\\", "/"), + Version: "v" + version.String(), + }, nil + } + } + } + } + } + + if errs.ErrorOrNil() == nil { + err := fmt.Errorf("could not find a local nor a remote checksum for plugin %q %q", pr.Identifier, pr.VersionConstraints) + errs = multierror.Append(errs, err) + } + errs = multierror.Append(errs, fmt.Errorf("could not install any compatible version of plugin %q", pr.Identifier)) + return nil, errs + + return nil, errs +} + func GetPluginDescription(pluginPath string) (pluginsdk.SetDescription, error) { out, err := exec.Command(pluginPath, "describe").Output() if err != nil { + log.Printf("#### could not get plugin description for %q %q", pluginPath, err) return pluginsdk.SetDescription{}, err }