internal/providercache: verify that the provider protocol version is compatible (#24737)

* internal/providercache: verify that the provider protocol version is
compatible

The public registry includes a list of supported provider protocol
versions for each provider version. This change adds verification of
support and adds a specific error message pointing users to the closest
matching version.

command/init_test.go

@@ -1556,7 +1556,7 @@ func newMockProviderSource(t *testing.T, availableProviderVersions map[string][]
 				close()
 				t.Fatalf("failed to parse %q as a version number for %q: %s", versionStr, name, err)
 			}
-			meta, close, err := getproviders.FakeInstallablePackageMeta(addr, version, getproviders.CurrentPlatform)
+			meta, close, err := getproviders.FakeInstallablePackageMeta(addr, version, getproviders.VersionList{getproviders.MustParseVersion("5.0")}, getproviders.CurrentPlatform)
 			if err != nil {
 				close()
 				t.Fatalf("failed to prepare fake package for %s %s: %s", name, versionStr, err)
@@ -1616,7 +1616,7 @@ func installFakeProviderPackagesElsewhere(t *testing.T, cacheDir *providercache.
 			if err != nil {
 				t.Fatalf("failed to parse %q as a version number for %q: %s", versionStr, name, err)
 			}
-			meta, close, err := getproviders.FakeInstallablePackageMeta(addr, version, getproviders.CurrentPlatform)
+			meta, close, err := getproviders.FakeInstallablePackageMeta(addr, version, getproviders.VersionList{getproviders.MustParseVersion("5.0")}, getproviders.CurrentPlatform)
 			// We're going to install all these fake packages before we return,
 			// so we don't need to preserve them afterwards.
 			defer close()

internal/getproviders/memoize_source_test.go

@@ -10,8 +10,9 @@ import (
 func TestMemoizeSource(t *testing.T) {
 	provider := addrs.NewDefaultProvider("foo")
 	version := MustParseVersion("1.0.0")
+	protocols := VersionList{MustParseVersion("5.0")}
 	platform := Platform{OS: "gameboy", Arch: "lr35902"}
-	meta := FakePackageMeta(provider, version, platform)
+	meta := FakePackageMeta(provider, version, protocols, platform)
 	nonexistProvider := addrs.NewDefaultProvider("nonexist")
 	nonexistPlatform := Platform{OS: "gamegear", Arch: "z80"}
 

internal/getproviders/mock_source.go

@@ -117,11 +117,12 @@ func (s *MockSource) CallLog() [][]interface{} {
 // FakePackageMeta constructs and returns a PackageMeta that carries the given
 // metadata but has fake location information that is likely to fail if
 // attempting to install from it.
-func FakePackageMeta(provider addrs.Provider, version Version, target Platform) PackageMeta {
+func FakePackageMeta(provider addrs.Provider, version Version, protocols VersionList, target Platform) PackageMeta {
 	return PackageMeta{
-		Provider:       provider,
-		Version:        version,
-		TargetPlatform: target,
+		Provider:         provider,
+		Version:          version,
+		ProtocolVersions: protocols,
+		TargetPlatform:   target,
 
 		// Some fake but somewhat-realistic-looking other metadata. This
 		// points nowhere, so will fail if attempting to actually use it.
@@ -140,7 +141,7 @@ func FakePackageMeta(provider addrs.Provider, version Version, target Platform)
 // alongside the result in order to clean up the temporary file. The caller
 // should call the callback even if this function returns an error, because
 // some error conditions leave a partially-created file on disk.
-func FakeInstallablePackageMeta(provider addrs.Provider, version Version, target Platform) (PackageMeta, func(), error) {
+func FakeInstallablePackageMeta(provider addrs.Provider, version Version, protocols VersionList, target Platform) (PackageMeta, func(), error) {
 	f, err := ioutil.TempFile("", "terraform-getproviders-fake-package-")
 	if err != nil {
 		return PackageMeta{}, func() {}, err
@@ -179,9 +180,10 @@ func FakeInstallablePackageMeta(provider addrs.Provider, version Version, target
 	h.Sum(checksum[:0])
 
 	meta := PackageMeta{
-		Provider:       provider,
-		Version:        version,
-		TargetPlatform: target,
+		Provider:         provider,
+		Version:          version,
+		ProtocolVersions: protocols,
+		TargetPlatform:   target,
 
 		Location: PackageLocalArchive(f.Name()),
 

internal/getproviders/multi_source_test.go

@@ -16,16 +16,19 @@ func TestMultiSourceAvailableVersions(t *testing.T) {
 			FakePackageMeta(
 				addrs.NewDefaultProvider("foo"),
 				MustParseVersion("1.0.0"),
+				VersionList{MustParseVersion("5.0")},
 				platform1,
 			),
 			FakePackageMeta(
 				addrs.NewDefaultProvider("foo"),
 				MustParseVersion("1.0.0"),
+				VersionList{MustParseVersion("5.0")},
 				platform2,
 			),
 			FakePackageMeta(
 				addrs.NewDefaultProvider("bar"),
 				MustParseVersion("1.0.0"),
+				VersionList{MustParseVersion("5.0")},
 				platform2,
 			),
 		})
@@ -33,16 +36,19 @@ func TestMultiSourceAvailableVersions(t *testing.T) {
 			FakePackageMeta(
 				addrs.NewDefaultProvider("foo"),
 				MustParseVersion("1.0.0"),
+				VersionList{MustParseVersion("5.0")},
 				platform1,
 			),
 			FakePackageMeta(
 				addrs.NewDefaultProvider("foo"),
 				MustParseVersion("1.2.0"),
+				VersionList{MustParseVersion("5.0")},
 				platform1,
 			),
 			FakePackageMeta(
 				addrs.NewDefaultProvider("bar"),
 				MustParseVersion("1.0.0"),
+				VersionList{MustParseVersion("5.0")},
 				platform1,
 			),
 		})
@@ -81,11 +87,13 @@ func TestMultiSourceAvailableVersions(t *testing.T) {
 			FakePackageMeta(
 				addrs.NewDefaultProvider("foo"),
 				MustParseVersion("1.0.0"),
+				VersionList{MustParseVersion("5.0")},
 				platform1,
 			),
 			FakePackageMeta(
 				addrs.NewDefaultProvider("bar"),
 				MustParseVersion("1.0.0"),
+				VersionList{MustParseVersion("5.0")},
 				platform1,
 			),
 		})
@@ -93,11 +101,13 @@ func TestMultiSourceAvailableVersions(t *testing.T) {
 			FakePackageMeta(
 				addrs.NewDefaultProvider("foo"),
 				MustParseVersion("1.2.0"),
+				VersionList{MustParseVersion("5.0")},
 				platform1,
 			),
 			FakePackageMeta(
 				addrs.NewDefaultProvider("bar"),
 				MustParseVersion("1.2.0"),
+				VersionList{MustParseVersion("5.0")},
 				platform1,
 			),
 		})
@@ -158,16 +168,19 @@ func TestMultiSourcePackageMeta(t *testing.T) {
 	onlyInS1 := fakeFilename("s1", FakePackageMeta(
 		addrs.NewDefaultProvider("foo"),
 		MustParseVersion("1.0.0"),
+		VersionList{MustParseVersion("5.0")},
 		platform2,
 	))
 	onlyInS2 := fakeFilename("s2", FakePackageMeta(
 		addrs.NewDefaultProvider("foo"),
 		MustParseVersion("1.2.0"),
+		VersionList{MustParseVersion("5.0")},
 		platform1,
 	))
 	inBothS1 := fakeFilename("s1", FakePackageMeta(
 		addrs.NewDefaultProvider("foo"),
 		MustParseVersion("1.0.0"),
+		VersionList{MustParseVersion("5.0")},
 		platform1,
 	))
 	inBothS2 := fakeFilename("s2", inBothS1)
@@ -177,6 +190,7 @@ func TestMultiSourcePackageMeta(t *testing.T) {
 		fakeFilename("s1", FakePackageMeta(
 			addrs.NewDefaultProvider("bar"),
 			MustParseVersion("1.0.0"),
+			VersionList{MustParseVersion("5.0")},
 			platform2,
 		)),
 	})
@@ -186,6 +200,7 @@ func TestMultiSourcePackageMeta(t *testing.T) {
 		fakeFilename("s2", FakePackageMeta(
 			addrs.NewDefaultProvider("bar"),
 			MustParseVersion("1.0.0"),
+			VersionList{MustParseVersion("5.0")},
 			platform1,
 		)),
 	})

internal/providercache/installer.go

@@ -12,6 +12,7 @@ import (
 	"github.com/hashicorp/terraform/addrs"
 	"github.com/hashicorp/terraform/internal/copydir"
 	"github.com/hashicorp/terraform/internal/getproviders"
+	tfversion "github.com/hashicorp/terraform/version"
 )
 
 // Installer is the main type in this package, representing a provider installer
@@ -41,8 +42,17 @@ type Installer struct {
 	// namespace, which we use for providers that are built in to Terraform
 	// and thus do not need any separate installation step.
 	builtInProviderTypes []string
+
+	// pluginProtocolVersion is the protocol version terrafrom core supports to
+	// communicate with servers, and is used to resolve plugin discovery with
+	// terraform registry, in addition to any specified plugin version
+	// constraints.
+	pluginProtocolVersion getproviders.VersionConstraints
 }
 
+// The currently-supported plugin protocol version.
+var SupportedPluginProtocols = getproviders.MustParseVersionConstraints("~> 5")
+
 // NewInstaller constructs and returns a new installer with the given target
 // directory and provider source.
 //
@@ -54,8 +64,9 @@ type Installer struct {
 // or the result is undefined.
 func NewInstaller(targetDir *Dir, source getproviders.Source) *Installer {
 	return &Installer{
-		targetDir: targetDir,
-		source:    source,
+		targetDir:             targetDir,
+		source:                source,
+		pluginProtocolVersion: SupportedPluginProtocols,
 	}
 }
 
@@ -301,6 +312,45 @@ NeedProvider:
 			continue
 		}
 
+		// if the package meta includes provider protocol versions, verify that terraform supports it.
+		if len(meta.ProtocolVersions) > 0 {
+			protoVersions := versions.MeetingConstraints(i.pluginProtocolVersion)
+			match := false
+			for _, version := range meta.ProtocolVersions {
+				if protoVersions.Has(version) {
+					match = true
+				}
+			}
+			if match == false {
+				// Find the closest matching version
+				closestAvailable := i.findClosestProtocolCompatibleVersion(provider, version)
+				if closestAvailable == versions.Unspecified {
+					err := fmt.Errorf(errProviderVersionIncompatible, provider)
+					errs[provider] = err
+					if cb := evts.FetchPackageFailure; cb != nil {
+						cb(provider, version, err)
+					}
+					continue
+				}
+
+				// Determine if the closest matching provider is newer or older
+				// than the requirement in order to send the appropriate error
+				// message.
+				var protoErr string
+				if version.GreaterThan(closestAvailable) {
+					protoErr = providerProtocolTooNew
+				} else {
+					protoErr = providerProtocolTooOld
+				}
+
+				errs[provider] = fmt.Errorf(protoErr, provider, version, tfversion.String(), closestAvailable.String(), closestAvailable.String(), getproviders.VersionConstraintsString(reqs[provider]))
+				if cb := evts.FetchPackageFailure; cb != nil {
+					cb(provider, version, err)
+				}
+				continue
+			}
+		}
+
 		// Step 3c: Retrieve the package indicated by the metadata we received,
 		// either directly into our target directory or via the global cache
 		// directory.
@@ -498,3 +548,56 @@ func (err InstallerError) Error() string {
 	}
 	return b.String()
 }
+
+// findClosestProtocolCompatibleVersion searches for the provider version with the closest protocol match.
+func (i *Installer) findClosestProtocolCompatibleVersion(provider addrs.Provider, version versions.Version) versions.Version {
+	var match versions.Version
+	available, _ := i.source.AvailableVersions(provider)
+	available.Sort()                                       // put the versions in increasing order of precedence
+	for index := len(available) - 1; index >= 0; index-- { // walk backwards to consider newer versions first
+		meta, _ := i.source.PackageMeta(provider, available[index], i.targetDir.targetPlatform)
+		if len(meta.ProtocolVersions) > 0 {
+			protoVersions := versions.MeetingConstraints(i.pluginProtocolVersion)
+			for _, version := range meta.ProtocolVersions {
+				if protoVersions.Has(version) {
+					match = available[index]
+					break // we will only consider the newest matching version
+				}
+			}
+		}
+	}
+	return match
+}
+
+// providerProtocolTooOld is a message sent to the CLI UI if the provider's
+// supported protocol versions are too old for the user's version of terraform,
+// but an older version of the provider is compatible.
+const providerProtocolTooOld = `
+Provider %q v%s is not compatible with Terraform %s.
+Provider version %s is the earliest compatible version. Select it with 
+the following version constraint:
+	version = %q
+Terraform checked all of the plugin versions matching the given constraint:
+	%s
+Consult the documentation for this provider for more information on
+compatibility between provider and Terraform versions.
+`
+
+// providerProtocolTooNew is a message sent to the CLI UI if the provider's
+// supported protocol versions are too new for the user's version of terraform,
+// and the user could either upgrade terraform or choose an older version of the
+// provider
+const providerProtocolTooNew = `
+Provider %q v%s is not compatible with Terraform %s.
+Provider version %s is the latest compatible version. Select it with 
+the following constraint:
+	version = %q
+Terraform checked all of the plugin versions matching the given constraint:
+	%s
+Consult the documentation for this provider for more information on
+compatibility between provider and Terraform versions.
+Alternatively, upgrade to the latest version of Terraform for compatibility with newer provider releases.
+`
+
+// there does exist a version outside of the constaints that is compatible.
+const errProviderVersionIncompatible = `No compatible versions of provider %s were found.`

internal/providercache/installer_test.go

@@ -0,0 +1,100 @@
+package providercache
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/hashicorp/terraform/addrs"
+	"github.com/hashicorp/terraform/internal/getproviders"
+)
+
+func TestEnsureProviderVersions(t *testing.T) {
+	// Set up a test provider "foo" with two versions which support different protocols
+	// used by both package metas
+	provider := addrs.NewDefaultProvider("foo")
+	platform := getproviders.Platform{OS: "gameboy", Arch: "lr35902"}
+
+	// foo version 1.0 supports protocol 4
+	version1 := getproviders.MustParseVersion("1.0.0")
+	protocols1 := getproviders.VersionList{getproviders.MustParseVersion("4.0")}
+	meta1, close1, _ := getproviders.FakeInstallablePackageMeta(provider, version1, protocols1, platform)
+	defer close1()
+
+	// foo version 2.0 supports protocols 4 and 5.2
+	version2 := getproviders.MustParseVersion("2.0.0")
+	protocols2 := getproviders.VersionList{getproviders.MustParseVersion("4.0"), getproviders.MustParseVersion("5.2")}
+	meta2, close2, _ := getproviders.FakeInstallablePackageMeta(provider, version2, protocols2, platform)
+	defer close2()
+
+	// foo version 3.0 supports protocol 6
+	version3 := getproviders.MustParseVersion("3.0.0")
+	protocols3 := getproviders.VersionList{getproviders.MustParseVersion("6.0")}
+	meta3, close3, _ := getproviders.FakeInstallablePackageMeta(provider, version3, protocols3, platform)
+	defer close3()
+
+	// set up the mock source
+	source := getproviders.NewMockSource(
+		[]getproviders.PackageMeta{meta1, meta2, meta3},
+	)
+
+	// create a temporary workdir
+	tmpDirPath, err := ioutil.TempDir("", "terraform-test-providercache")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDirPath)
+
+	// set up the installer using the temporary directory and mock source
+	dir := newDirWithPlatform(tmpDirPath, platform)
+	installer := NewInstaller(dir, source)
+
+	// First test: easy case. The requested version supports the current plugin protocol version
+	reqs := getproviders.Requirements{
+		provider: getproviders.MustParseVersionConstraints("2.0"),
+	}
+	ctx := context.TODO()
+	selections, err := installer.EnsureProviderVersions(ctx, reqs, InstallNewProvidersOnly)
+	if err != nil {
+		t.Fatalf("expected sucess, got error: %s", err)
+	}
+	if len(selections) != 1 {
+		t.Fatalf("wrong number of results. Got %d, expected 1", len(selections))
+	}
+	got := selections[provider]
+	if !got.Same(version2) {
+		t.Fatalf("wrong result. Expected provider version %s, got %s", version2, got)
+	}
+
+	// For the second test, set the requirement to something later than the
+	// version that supports the current plugin protocol version 5.0
+	reqs[provider] = getproviders.MustParseVersionConstraints("3.0")
+
+	selections, err = installer.EnsureProviderVersions(ctx, reqs, InstallNewProvidersOnly)
+	if err == nil {
+		t.Fatalf("expected error, got success")
+	}
+	if len(selections) != 0 {
+		t.Errorf("wrong number of results. Got %d, expected 0", len(selections))
+	}
+	if !strings.Contains(err.Error(), "Provider version 2.0.0 is the latest compatible version.") {
+		t.Fatalf("wrong error: %s", err)
+	}
+
+	// For the third test, set the requirement to something earlier than the
+	// version that supports the current plugin protocol version 5.0
+	reqs[provider] = getproviders.MustParseVersionConstraints("1.0")
+
+	selections, err = installer.EnsureProviderVersions(ctx, reqs, InstallNewProvidersOnly)
+	if err == nil {
+		t.Fatalf("expected error, got success")
+	}
+	if len(selections) != 0 {
+		t.Errorf("wrong number of results. Got %d, expected 0", len(selections))
+	}
+	if !strings.Contains(err.Error(), "Provider version 2.0.0 is the earliest compatible version.") {
+		t.Fatalf("wrong error: %s", err)
+	}
+}