Merge pull request #7804 from hashicorp/esx5_checksum_files

parse checksum file so that we can use the checksums inside in the vm…

common/iso_config.go

@@ -1,10 +1,15 @@
 package common
 
 import (
+	"encoding/hex"
 	"errors"
 	"fmt"
+	"log"
+	"net/url"
+	"os"
 	"strings"
 
+	getter "github.com/hashicorp/go-getter"
 	"github.com/hashicorp/packer/template/interpolate"
 )
 
@@ -53,18 +58,47 @@ func (c *ISOConfig) Prepare(ctx *interpolate.Context) (warnings []string, errs [
 	}
 
 	if c.ISOChecksumURL != "" {
-		if strings.HasSuffix(strings.ToLower(c.ISOChecksumURL), ".iso") {
-			errs = append(errs, fmt.Errorf("Error parsing checksum:"+
-				" .iso is not a valid checksum extension"))
+		if c.ISOChecksum != "" {
+			warnings = append(warnings, "You have provided both an "+
+				"iso_checksum and an iso_checksum_url. Discarding the "+
+				"iso_checksum_url and using the checksum.")
+		} else {
+			if strings.HasSuffix(strings.ToLower(c.ISOChecksumURL), ".iso") {
+				errs = append(errs, fmt.Errorf("Error parsing checksum:"+
+					" .iso is not a valid checksum extension"))
+			}
+			// go-getter auto-parses checksum files
+			c.ISOChecksumType = "file"
+			c.ISOChecksum = c.ISOChecksumURL
 		}
-		// go-getter auto-parses checksum files
-		c.ISOChecksumType = "file"
-		c.ISOChecksum = c.ISOChecksumURL
 	}
 
 	if c.ISOChecksum == "" {
 		errs = append(errs, fmt.Errorf("A checksum must be specified"))
 	}
+	if c.ISOChecksumType == "file" {
+		u, err := url.Parse(c.ISOUrls[0])
+		wd, err := os.Getwd()
+		if err != nil {
+			log.Printf("get working directory: %v", err)
+			// here we ignore the error in case the
+			// working directory is not needed.
+		}
+		gc := getter.Client{
+			Dst:     "no-op",
+			Src:     u.String(),
+			Pwd:     wd,
+			Dir:     false,
+			Getters: getter.Getters,
+		}
+		cksum, err := gc.ChecksumFromFile(c.ISOChecksumURL, u)
+		if cksum == nil || err != nil {
+			errs = append(errs, fmt.Errorf("Couldn't extract checksum from checksum file"))
+		} else {
+			c.ISOChecksumType = cksum.Type
+			c.ISOChecksum = hex.EncodeToString(cksum.Value)
+		}
+	}
 
 	return warnings, errs
 }

common/iso_config_test.go

@@ -75,11 +75,16 @@ func TestISOConfigPrepare_ISOChecksum(t *testing.T) {
 func TestISOConfigPrepare_ISOChecksumURLBad(t *testing.T) {
 	i := testISOConfig()
 	i.ISOChecksumURL = "file:///not_read"
+	i.ISOChecksum = "shouldoverride"
 
 	// Test ISOChecksum overrides url
 	warns, err := i.Prepare(nil)
-	if len(warns) > 0 && len(err) > 0 {
-		t.Fatalf("bad: %#v, %#v", warns, err)
+	if len(warns) != 1 {
+		t.Fatalf("Bad: should have warned because both checksum and " +
+			"checksumURL are set.")
+	}
+	if len(err) > 0 {
+		t.Fatalf("Bad; should have warned but not errored.")
 	}
 
 	// Test that we won't try to read an iso into memory because of a user

go.mod

@@ -44,7 +44,7 @@ require (
 	github.com/hashicorp/errwrap v1.0.0
 	github.com/hashicorp/go-checkpoint v0.0.0-20171009173528-1545e56e46de
 	github.com/hashicorp/go-cleanhttp v0.5.0
-	github.com/hashicorp/go-getter v1.2.0
+	github.com/hashicorp/go-getter v1.3.1-0.20190627223108-da0323b9545e
 	github.com/hashicorp/go-multierror v1.0.0
 	github.com/hashicorp/go-oracle-terraform v0.0.0-20181016190316-007121241b79
 	github.com/hashicorp/go-retryablehttp v0.5.2 // indirect

go.sum

@@ -177,6 +177,12 @@ github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6K
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-getter v1.2.0 h1:E05bVPilzyh2yXgT6srn7WEkfMZaH+LuX9tDJw/4kaE=
 github.com/hashicorp/go-getter v1.2.0/go.mod h1:/O1k/AizTN0QmfEKknCYGvICeyKUDqCYA8vvWtGWDeQ=
+github.com/hashicorp/go-getter v1.3.1-0.20190627194702-0c4919d4eb92 h1:Tck3az71eOyEcxueXyZfM4YO5oy2Hq8AbTpvc9HrQN0=
+github.com/hashicorp/go-getter v1.3.1-0.20190627194702-0c4919d4eb92/go.mod h1:/O1k/AizTN0QmfEKknCYGvICeyKUDqCYA8vvWtGWDeQ=
+github.com/hashicorp/go-getter v1.3.1-0.20190627194820-3fc511e2f341 h1:v+hsBJMakfQLeYkPuv/Y/T86jKvDQrOU1TyzximcF2Y=
+github.com/hashicorp/go-getter v1.3.1-0.20190627194820-3fc511e2f341/go.mod h1:/O1k/AizTN0QmfEKknCYGvICeyKUDqCYA8vvWtGWDeQ=
+github.com/hashicorp/go-getter v1.3.1-0.20190627223108-da0323b9545e h1:6krcdHPiS+aIP9XKzJzSahfjD7jG7Z+4+opm0z39V1M=
+github.com/hashicorp/go-getter v1.3.1-0.20190627223108-da0323b9545e/go.mod h1:/O1k/AizTN0QmfEKknCYGvICeyKUDqCYA8vvWtGWDeQ=
 github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4=

vendor/github.com/hashicorp/go-getter/README.md

@@ -280,6 +280,16 @@ None
   * `depth` - The Git clone depth. The provided number specifies the last `n`
     revisions to clone from the repository.
 
+
+The `git` getter accepts both URL-style SSH addresses like
+`git::ssh://git@example.com/foo/bar`, and "scp-style" addresses like
+`git::git@example.com/foo/bar`. In the latter case, omitting the `git::`
+force prefix is allowed if the username prefix is exactly `git@`.
+
+The "scp-style" addresses _cannot_ be used in conjunction with the `ssh://`
+scheme prefix, because in that case the colon is used to mark an optional
+port number to connect on, rather than to delimit the path from the host.
+
 ### Mercurial (`hg`)
 
   * `rev` - The Mercurial revision to checkout.

vendor/github.com/hashicorp/go-getter/checksum.go

@@ -19,8 +19,8 @@ import (
 	urlhelper "github.com/hashicorp/go-getter/helper/url"
 )
 
-// fileChecksum helps verifying the checksum for a file.
-type fileChecksum struct {
+// FileChecksum helps verifying the checksum for a file.
+type FileChecksum struct {
 	Type     string
 	Hash     hash.Hash
 	Value    []byte
@@ -50,7 +50,7 @@ func (cerr *ChecksumError) Error() string {
 
 // checksum is a simple method to compute the checksum of a source file
 // and compare it to the given expected value.
-func (c *fileChecksum) checksum(source string) error {
+func (c *FileChecksum) checksum(source string) error {
 	f, err := os.Open(source)
 	if err != nil {
 		return fmt.Errorf("Failed to open file for checksum: %s", err)
@@ -74,7 +74,7 @@ func (c *fileChecksum) checksum(source string) error {
 	return nil
 }
 
-// extractChecksum will return a fileChecksum based on the 'checksum'
+// extractChecksum will return a FileChecksum based on the 'checksum'
 // parameter of u.
 // ex:
 //  http://hashicorp.com/terraform?checksum=<checksumValue>
@@ -93,7 +93,7 @@ func (c *fileChecksum) checksum(source string) error {
 //  <checksum> *file2
 //
 // see parseChecksumLine for more detail on checksum file parsing
-func (c *Client) extractChecksum(u *url.URL) (*fileChecksum, error) {
+func (c *Client) extractChecksum(u *url.URL) (*FileChecksum, error) {
 	q := u.Query()
 	v := q.Get("checksum")
 
@@ -115,14 +115,14 @@ func (c *Client) extractChecksum(u *url.URL) (*fileChecksum, error) {
 
 	switch checksumType {
 	case "file":
-		return c.checksumFromFile(checksumValue, u)
+		return c.ChecksumFromFile(checksumValue, u)
 	default:
 		return newChecksumFromType(checksumType, checksumValue, filepath.Base(u.EscapedPath()))
 	}
 }
 
-func newChecksum(checksumValue, filename string) (*fileChecksum, error) {
-	c := &fileChecksum{
+func newChecksum(checksumValue, filename string) (*FileChecksum, error) {
+	c := &FileChecksum{
 		Filename: filename,
 	}
 	var err error
@@ -133,7 +133,7 @@ func newChecksum(checksumValue, filename string) (*fileChecksum, error) {
 	return c, nil
 }
 
-func newChecksumFromType(checksumType, checksumValue, filename string) (*fileChecksum, error) {
+func newChecksumFromType(checksumType, checksumValue, filename string) (*FileChecksum, error) {
 	c, err := newChecksum(checksumValue, filename)
 	if err != nil {
 		return nil, err
@@ -157,7 +157,7 @@ func newChecksumFromType(checksumType, checksumValue, filename string) (*fileChe
 	return c, nil
 }
 
-func newChecksumFromValue(checksumValue, filename string) (*fileChecksum, error) {
+func newChecksumFromValue(checksumValue, filename string) (*FileChecksum, error) {
 	c, err := newChecksum(checksumValue, filename)
 	if err != nil {
 		return nil, err
@@ -183,14 +183,14 @@ func newChecksumFromValue(checksumValue, filename string) (*fileChecksum, error)
 	return c, nil
 }
 
-// checksumsFromFile will return all the fileChecksums found in file
+// ChecksumFromFile will return all the FileChecksums found in file
 //
-// checksumsFromFile will try to guess the hashing algorithm based on content
+// ChecksumFromFile will try to guess the hashing algorithm based on content
 // of checksum file
 //
-// checksumsFromFile will only return checksums for files that match file
+// ChecksumFromFile will only return checksums for files that match file
 // behind src
-func (c *Client) checksumFromFile(checksumFile string, src *url.URL) (*fileChecksum, error) {
+func (c *Client) ChecksumFromFile(checksumFile string, src *url.URL) (*FileChecksum, error) {
 	checksumFileURL, err := urlhelper.Parse(checksumFile)
 	if err != nil {
 		return nil, err
@@ -286,7 +286,7 @@ func (c *Client) checksumFromFile(checksumFile string, src *url.URL) (*fileCheck
 // of a line.
 // for BSD type sums parseChecksumLine guesses the hashing algorithm
 // by checking the length of the checksum.
-func parseChecksumLine(line string) (*fileChecksum, error) {
+func parseChecksumLine(line string) (*FileChecksum, error) {
 	parts := strings.Fields(line)
 
 	switch len(parts) {

vendor/github.com/hashicorp/go-getter/decompress_zip.go

@@ -9,7 +9,7 @@ import (
 )
 
 // ZipDecompressor is an implementation of Decompressor that can
-// decompress tar.gzip files.
+// decompress zip files.
 type ZipDecompressor struct{}
 
 func (d *ZipDecompressor) Decompress(dst, src string, dir bool) error {

vendor/github.com/hashicorp/go-getter/detect_bitbucket.go

@@ -35,7 +35,7 @@ func (d *BitBucketDetector) detectHTTP(src string) (string, bool, error) {
 	var info struct {
 		SCM string `json:"scm"`
 	}
-	infoUrl := "https://api.bitbucket.org/1.0/repositories" + u.Path
+	infoUrl := "https://api.bitbucket.org/2.0/repositories" + u.Path
 	resp, err := http.Get(infoUrl)
 	if err != nil {
 		return "", true, fmt.Errorf("error looking up BitBucket URL: %s", err)

vendor/github.com/hashicorp/go-getter/get_git.go

@@ -34,6 +34,15 @@ func (g *GitGetter) Get(dst string, u *url.URL) error {
 		return fmt.Errorf("git must be available and on the PATH")
 	}
 
+	// The port number must be parseable as an integer. If not, the user
+	// was probably trying to use a scp-style address, in which case the
+	// ssh:// prefix must be removed to indicate that.
+	if portStr := u.Port(); portStr != "" {
+		if _, err := strconv.ParseUint(portStr, 10, 16); err != nil {
+			return fmt.Errorf("invalid port number %q; if using the \"scp-like\" git address scheme where a colon introduces the path instead, remove the ssh:// portion and use just the git:: prefix", portStr)
+		}
+	}
+
 	// Extract some query parameters we use
 	var ref, sshKey string
 	var depth int
@@ -90,26 +99,6 @@ func (g *GitGetter) Get(dst string, u *url.URL) error {
 		}
 	}
 
-	// For SSH-style URLs, if they use the SCP syntax of host:path, then
-	// the URL will be mangled. We detect that here and correct the path.
-	// Example: host:path/bar will turn into host/path/bar
-	if u.Scheme == "ssh" {
-		if idx := strings.Index(u.Host, ":"); idx > -1 {
-			// Copy the URL so we don't modify the input
-			var newU url.URL = *u
-			u = &newU
-
-			// Path includes the part after the ':'.
-			u.Path = u.Host[idx+1:] + u.Path
-			if u.Path[0] != '/' {
-				u.Path = "/" + u.Path
-			}
-
-			// Host trims up to the :
-			u.Host = u.Host[:idx]
-		}
-	}
-
 	// Clone or update the repository
 	_, err := os.Stat(dst)
 	if err != nil && !os.IsNotExist(err) {

vendor/modules.txt

@@ -235,7 +235,7 @@ github.com/hashicorp/errwrap
 github.com/hashicorp/go-checkpoint
 # github.com/hashicorp/go-cleanhttp v0.5.0
 github.com/hashicorp/go-cleanhttp
-# github.com/hashicorp/go-getter v1.2.0
+# github.com/hashicorp/go-getter v1.3.1-0.20190627223108-da0323b9545e
 github.com/hashicorp/go-getter
 github.com/hashicorp/go-getter/helper/url
 # github.com/hashicorp/go-immutable-radix v1.0.0