internal/providercache: Installer records its selections in a file

Just as with the old installer mechanism, our goal is that explicit
provider installation is the only way that new provider versions can be
selected.

To achieve that, we conclude each call to EnsureProviderVersions by
writing a selections lock file into the target directory. A later caller
can then recall the selections from that file by calling SelectedPackages,
which both ensures that it selects the same set of versions and also
verifies that the checksums recorded by the installer still match.

This new selections.json file has a different layout than our old
plugins.json lock file. Not only does it use a different hashing algorithm
than before, we also record explicitly which version of each provider
was selected. In the old model, we'd repeat normal discovery when
reloading the lock file and then fail with a confusing error message if
discovery happened to select a different version, but now we'll be able
to distinguish between a package that's gone missing since installation
(which could previously have then selected a different available version)
from a package that has been modified.

internal/providercache/installer.go

@@ -3,6 +3,7 @@ package providercache
 import (
 	"context"
 	"fmt"
+	"path/filepath"
 	"sort"
 	"strings"
 
@@ -267,6 +268,7 @@ NeedProvider:
 		new := installTo.ProviderVersion(provider, version)
 		if new == nil {
 			err := fmt.Errorf("after installing %s it is still not detected in the target directory; this is a bug in Terraform", provider)
+			errs[provider] = err
 			if cb := evts.FetchPackageFailure; cb != nil {
 				cb(provider, version, err)
 			}
@@ -292,6 +294,41 @@ NeedProvider:
 		}
 	}
 
+	// We'll remember our selections in a lock file inside the target directory,
+	// so callers can recover those exact selections later by calling
+	// SelectedPackages on the same installer.
+	lockEntries := map[addrs.Provider]lockFileEntry{}
+	for provider, version := range selected {
+		cached := i.targetDir.ProviderVersion(provider, version)
+		if cached == nil {
+			err := fmt.Errorf("selected package for %s is no longer present in the target directory; this is a bug in Terraform", provider)
+			errs[provider] = err
+			if cb := evts.HashPackageFailure; cb != nil {
+				cb(provider, version, err)
+			}
+			continue
+		}
+		hash, err := cached.Hash()
+		if err != nil {
+			errs[provider] = fmt.Errorf("failed to calculate checksum for installed provider %s package: %s", provider, err)
+			if cb := evts.HashPackageFailure; cb != nil {
+				cb(provider, version, err)
+			}
+			continue
+		}
+		lockEntries[provider] = lockFileEntry{
+			SelectedVersion: version,
+			PackageHash:     hash,
+		}
+	}
+	err := i.lockFile().Write(lockEntries)
+	if err != nil {
+		// This is one of few cases where this function does _not_ return an
+		// InstallerError, because failure to write the lock file is a more
+		// general problem, not specific to a certain provider.
+		return selected, fmt.Errorf("failed to record a manifest of selected providers: %s", err)
+	}
+
 	if len(errs) > 0 {
 		return selected, InstallerError{
 			ProviderErrors: errs,
@@ -300,6 +337,60 @@ NeedProvider:
 	return selected, nil
 }
 
+func (i *Installer) lockFile() *lockFile {
+	return &lockFile{
+		filename: filepath.Join(i.targetDir.baseDir, "selections.json"),
+	}
+}
+
+// SelectedPackages returns the metadata about the packages chosen by the
+// most recent call to EnsureProviderVersions, which are recorded in a lock
+// file in the installer's target directory.
+//
+// If EnsureProviderVersions has never been run against the current target
+// directory, the result is a successful empty response indicating that nothing
+// is selected.
+//
+// SelectedPackages also verifies that the package contents are consistent
+// with the checksums that were recorded at installation time, reporting an
+// error if not.
+func (i *Installer) SelectedPackages() (map[addrs.Provider]*CachedProvider, error) {
+	entries, err := i.lockFile().Read()
+	if err != nil {
+		// Read does not return an error for "file not found", so this should
+		// always be some other error.
+		return nil, fmt.Errorf("failed to read selections file: %s", err)
+	}
+
+	ret := make(map[addrs.Provider]*CachedProvider, len(entries))
+	errs := make(map[addrs.Provider]error)
+	for provider, entry := range entries {
+		cached := i.targetDir.ProviderVersion(provider, entry.SelectedVersion)
+		if cached == nil {
+			errs[provider] = fmt.Errorf("package for selected version %s is no longer available in the local cache directory", entry.SelectedVersion)
+			continue
+		}
+
+		ok, err := cached.MatchesHash(entry.PackageHash)
+		if err != nil {
+			errs[provider] = fmt.Errorf("failed to verify checksum for v%s package: %s", entry.SelectedVersion, err)
+			continue
+		}
+		if !ok {
+			errs[provider] = fmt.Errorf("checksum mismatch for v%s package", entry.SelectedVersion)
+			continue
+		}
+		ret[provider] = cached
+	}
+
+	if len(errs) > 0 {
+		return ret, InstallerError{
+			ProviderErrors: errs,
+		}
+	}
+	return ret, nil
+}
+
 // InstallMode customizes the details of how an install operation treats
 // providers that have versions already cached in the target directory.
 type InstallMode rune

internal/providercache/installer_events.go

@@ -93,6 +93,12 @@ type InstallerEvents struct {
 	FetchPackageSuccess func(provider addrs.Provider, version getproviders.Version, localDir string)
 	FetchPackageRetry   func(provider addrs.Provider, version getproviders.Version, err error)
 	FetchPackageFailure func(provider addrs.Provider, version getproviders.Version, err error)
+
+	// HashPackageFailure is called if the installer is unable to determine
+	// the hash of the contents of an installed package after installation.
+	// In that case, the selection will not be recorded in the target cache
+	// directory's lock file.
+	HashPackageFailure func(provider addrs.Provider, version getproviders.Version, err error)
 }
 
 // OnContext produces a context with all of the same behaviors as the given

internal/providercache/lock_file.go

@@ -0,0 +1,114 @@
+package providercache
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/internal/getproviders"
+)
+
+// lockFile represents a file on disk that captures selected versions and
+// their associated package checksums resulting from an install process, so
+// that later consumers of that install process can be sure they are reading
+// an identical set of providers to what the install process intended.
+//
+// This is an internal type used to encapsulate the reading, parsing,
+// serializing, and writing of lock files. Its public interface is via methods
+// on type Installer.
+type lockFile struct {
+	filename string
+}
+
+// LockFileEntry represents an entry for a specific provider in a LockFile.
+type lockFileEntry struct {
+	SelectedVersion getproviders.Version
+	PackageHash     string
+}
+
+var _ json.Marshaler = (*lockFileEntry)(nil)
+var _ json.Unmarshaler = (*lockFileEntry)(nil)
+
+// Read returns the current locks captured in the lock file.
+//
+// If the file does not exist, the result is successful but empty to indicate
+// that no providers at all are available for use.
+func (lf *lockFile) Read() (map[addrs.Provider]lockFileEntry, error) {
+	buf, err := ioutil.ReadFile(lf.filename)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil // no file means no locks yet
+		}
+		return nil, err
+	}
+
+	var rawEntries map[string]*lockFileEntry
+	err = json.Unmarshal(buf, &rawEntries)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing %s: %s", lf.filename, err)
+	}
+
+	ret := make(map[addrs.Provider]lockFileEntry, len(rawEntries))
+	for providerStr, entry := range rawEntries {
+		provider, diags := addrs.ParseProviderSourceString(providerStr)
+		if diags.HasErrors() {
+			// This file is both generated and consumed by Terraform, so we
+			// don't use super-detailed error messages for problems in it.
+			// If we get here without someone tampering with the file then
+			// it's presumably a bug in either our serializer or our parser.
+			return nil, fmt.Errorf("error parsing %s: invalid provider address %q", lf.filename, providerStr)
+		}
+		ret[provider] = *entry
+	}
+
+	return ret, nil
+}
+
+// Write stores a new set of entries in the lock file, disarding any
+// selections previously stored there.
+func (lf *lockFile) Write(new map[addrs.Provider]lockFileEntry) error {
+	toStore := make(map[string]*lockFileEntry, len(new))
+	for provider := range new {
+		entry := new[provider] // so that each reference below is to a different object
+		toStore[provider.String()] = &entry
+	}
+
+	buf, err := json.MarshalIndent(toStore, "", "  ")
+	if err != nil {
+		return fmt.Errorf("error writing %s: %s", lf.filename, err)
+	}
+
+	os.MkdirAll(
+		filepath.Dir(lf.filename), 0660,
+	) // ignore error since WriteFile below will generate a better one anyway
+	return ioutil.WriteFile(lf.filename, buf, 0660)
+}
+
+func (lfe *lockFileEntry) UnmarshalJSON(src []byte) error {
+	type Raw struct {
+		VersionStr string `json:"version"`
+		Hash       string `json:"hash"`
+	}
+	var raw Raw
+	err := json.Unmarshal(src, &raw)
+	if err != nil {
+		return err
+	}
+	version, err := getproviders.ParseVersion(raw.VersionStr)
+	if err != nil {
+		return fmt.Errorf("invalid version number: %s", err)
+	}
+	lfe.SelectedVersion = version
+	lfe.PackageHash = raw.Hash
+	return nil
+}
+
+func (lfe *lockFileEntry) MarshalJSON() ([]byte, error) {
+	return json.Marshal(map[string]interface{}{
+		"version": lfe.SelectedVersion.String(),
+		"hash":    lfe.PackageHash,
+	})
+}