main: skip direct provider installation for providers available locally

This more closely replicates the 0.12-and-earlier behavior, where having
at least one version of a provider installed locally would totally disable
any attempt to look for newer versions remotely.

This is just for the implicit default behavior. Assumption is that later
we'll have an explicit configuration mechanism that will allow the user
to specify exactly where to look for what, and thus avoid tricky
heuristics like this.

command/e2etest/init_test.go

@@ -130,6 +130,49 @@ func TestInitProvidersVendored(t *testing.T) {
 
 }
 
+func TestInitProvidersLocalOnly(t *testing.T) {
+	t.Parallel()
+
+	// This test should not reach out to the network if it is behaving as
+	// intended. If it _does_ try to access an upstream registry and encounter
+	// an error doing so then that's a legitimate test failure that should be
+	// fixed. (If it incorrectly reaches out anywhere then it's likely to be
+	// to the host "example.com", which is the placeholder domain we use in
+	// the test fixture.)
+
+	fixturePath := filepath.Join("testdata", "local-only-provider")
+	tf := e2e.NewBinary(terraformBin, fixturePath)
+	defer tf.Close()
+
+	// Our fixture dir has a generic os_arch dir, which we need to customize
+	// to the actual OS/arch where this test is running in order to get the
+	// desired result.
+	fixtMachineDir := tf.Path("terraform.d/plugins/example.com/awesomecorp/happycloud/1.2.0/os_arch")
+	wantMachineDir := tf.Path("terraform.d/plugins/example.com/awesomecorp/happycloud/1.2.0/", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH))
+	err := os.Rename(fixtMachineDir, wantMachineDir)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	stdout, stderr, err := tf.Run("init")
+	if err != nil {
+		t.Errorf("unexpected error: %s", err)
+	}
+
+	if stderr != "" {
+		t.Errorf("unexpected stderr output:\n%s", stderr)
+	}
+
+	if !strings.Contains(stdout, "Terraform has been successfully initialized!") {
+		t.Errorf("success message is missing from output:\n%s", stdout)
+	}
+
+	if !strings.Contains(stdout, "- Installing example.com/awesomecorp/happycloud v1.2.0") {
+		t.Errorf("provider download message is missing from output:\n%s", stdout)
+		t.Logf("(this can happen if you have a conflicting copy of the plugin in one of the global plugin search dirs)")
+	}
+}
+
 func TestInitProviders_pluginCache(t *testing.T) {
 	t.Parallel()
 

command/e2etest/testdata/local-only-provider/main.tf

@@ -0,0 +1,21 @@
+# The purpose of this test is to refer to a provider whose address contains
+# a hostname that is only used for namespacing purposes and doesn't actually
+# have a provider registry deployed at it.
+#
+# A user can install such a provider in one of the implied local filesystem
+# directories and Terraform should accept that as the selection for that
+# provider without producing any errors about the fact that example.com
+# does not have a provider registry.
+#
+# For this test in particular we're using the "vendor" directory that is
+# the documented way to include provider plugins directly inside a
+# configuration uploaded to Terraform Cloud, but this functionality applies
+# to all of the implicit local filesystem search directories.
+
+terraform {
+  required_providers {
+    happycloud = {
+      source = "example.com/awesomecorp/happycloud"
+    }
+  }
+}

command/e2etest/testdata/local-only-provider/terraform.d/plugins/example.com/awesomecorp/happycloud/1.2.0/os_arch/terraform-provider-happycloud_v1.2.0

@@ -0,0 +1,2 @@
+This is not a real plugin executable. It's just here to be discovered by the
+provider installation process.

provider_source.go

@@ -6,8 +6,9 @@ import (
 	"path/filepath"
 
 	"github.com/apparentlymart/go-userdirs/userdirs"
-
 	"github.com/hashicorp/terraform-svchost/disco"
+
+	"github.com/hashicorp/terraform/addrs"
 	"github.com/hashicorp/terraform/command/cliconfig"
 	"github.com/hashicorp/terraform/internal/getproviders"
 )
@@ -19,8 +20,21 @@ import (
 func providerSource(services *disco.Disco) getproviders.Source {
 	// We're not yet using the CLI config here because we've not implemented
 	// yet the new configuration constructs to customize provider search
-	// locations. That'll come later.
-	// For now, we have a fixed set of search directories:
+	// locations. That'll come later. For now, we just always use the
+	// implicit default provider source.
+	return implicitProviderSource(services)
+}
+
+// implicitProviderSource builds a default provider source to use if there's
+// no explicit provider installation configuration in the CLI config.
+//
+// This implicit source looks in a number of local filesystem directories and
+// directly in a provider's upstream registry. Any providers that have at least
+// one version available in a local directory are implicitly excluded from
+// direct installation, as if the user had listed them explicitly in the
+// "exclude" argument in the direct provider source in the CLI config.
+func implicitProviderSource(services *disco.Disco) getproviders.Source {
+	// The local search directories we use for implicit configuration are:
 	// - The "terraform.d/plugins" directory in the current working directory,
 	//   which we've historically documented as a place to put plugins as a
 	//   way to include them in bundles uploaded to Terraform Cloud, where
@@ -31,10 +45,17 @@ func providerSource(services *disco.Disco) getproviders.Source {
 	//   following e.g. the XDG base directory specification on Unix systems,
 	//   Apple's guidelines on OS X, and "known folders" on Windows.
 	//
-	// Those directories are checked in addition to the direct upstream
-	// registry specified in the provider's address.
+	// Any provider we find in one of those implicit directories will be
+	// automatically excluded from direct installation from an upstream
+	// registry. Anything not available locally will query its primary
+	// upstream registry.
 	var searchRules []getproviders.MultiSourceSelector
 
+	// We'll track any providers we can find in the local search directories
+	// along the way, and then exclude them from the registry source we'll
+	// finally add at the end.
+	foundLocally := map[addrs.Provider]struct{}{}
+
 	addLocalDir := func(dir string) {
 		// We'll make sure the directory actually exists before we add it,
 		// because otherwise installation would always fail trying to look
@@ -44,9 +65,23 @@ func providerSource(services *disco.Disco) getproviders.Source {
 		// don't exist to help users get their configurations right.)
 		if info, err := os.Stat(dir); err == nil && info.IsDir() {
 			log.Printf("[DEBUG] will search for provider plugins in %s", dir)
+			fsSource := getproviders.NewFilesystemMirrorSource(dir)
+
+			// We'll peep into the source to find out what providers it seems
+			// to be providing, so that we can exclude those from direct
+			// install. This might fail, in which case we'll just silently
+			// ignore it and assume it would fail during installation later too
+			// and therefore effectively doesn't provide _any_ packages.
+			if available, err := fsSource.AllAvailablePackages(); err == nil {
+				for found := range available {
+					foundLocally[found] = struct{}{}
+				}
+			}
+
 			searchRules = append(searchRules, getproviders.MultiSourceSelector{
-				Source: getproviders.NewFilesystemMirrorSource(dir),
+				Source: fsSource,
 			})
+
 		} else {
 			log.Printf("[DEBUG] ignoring non-existing provider search directory %s", dir)
 		}
@@ -72,6 +107,13 @@ func providerSource(services *disco.Disco) getproviders.Source {
 		addLocalDir(dir)
 	}
 
+	// Anything we found in local directories above is excluded from being
+	// looked up via the registry source we're about to construct.
+	var directExcluded getproviders.MultiSourceMatchingPatterns
+	for addr := range foundLocally {
+		directExcluded = append(directExcluded, addr)
+	}
+
 	// Last but not least, the main registry source! We'll wrap a caching
 	// layer around this one to help optimize the several network requests
 	// we'll end up making to it while treating it as one of several sources
@@ -83,6 +125,7 @@ func providerSource(services *disco.Disco) getproviders.Source {
 		Source: getproviders.NewMemoizeSource(
 			getproviders.NewRegistrySource(services),
 		),
+		Exclude: directExcluded,
 	})
 
 	return getproviders.MultiSource(searchRules)