test: Enable mocking http provider sources in tests (#38206)

* test: Update`MockSource` to be able to present providers as being installed via HTTP, add `newMockProviderSourceViaHTTP` for direct use in tests.

Compare newMockProviderSourceViaHTTP to the existing newMockProviderSource. newMockProviderSource uses FakeInstallablePackageMeta, which creates metadata for providers that says it's being sourced from a local archive. In newMockProviderSourceViaHTTP the metadata reports the provider is downloaded via HTTP. The URL is created using a base URL that should be obtained from a test http server created in the test.

* test: Add `newMockProviderSourceUsingTestHttpServer` helper, to abstract away making the http test server. Add test showing it in use.

* refactor: Update test helpers to call t.Cleanup internally

internal/command/init_test.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"log"
 	"maps"
 	"net/http"
@@ -3450,6 +3451,59 @@ func TestInit_testsWithModule(t *testing.T) {
 
 // Testing init's behaviors with `state_store` when run in an empty working directory
 func TestInit_stateStore_newWorkingDir(t *testing.T) {
+	t.Run("temporary: test showing use of HTTP server in mock provider source", func(t *testing.T) {
+		// Create a temporary, uninitialized working directory with configuration including a state store
+		td := t.TempDir()
+		testCopyDir(t, testFixturePath("init-with-state-store"), td)
+		t.Chdir(td)
+
+		// Mock provider still needs to be supplied via testingOverrides despite the mock HTTP source
+		mockProvider := mockPluggableStateStorageProvider()
+		mockProviderVersion := getproviders.MustParseVersion("1.2.3")
+		mockProviderAddress := addrs.NewDefaultProvider("test")
+
+		// Set up mock provider source that mocks out downloading hashicorp/test v1.2.3 via HTTP.
+		// This stops Terraform auto-approving the provider installation.
+		source := newMockProviderSourceUsingTestHttpServer(t, mockProviderAddress, mockProviderVersion)
+
+		ui := new(cli.MockUi)
+		view, done := testView(t)
+		meta := Meta{
+			Ui:                        ui,
+			View:                      view,
+			AllowExperimentalFeatures: true,
+			testingOverrides: &testingOverrides{
+				Providers: map[addrs.Provider]providers.Factory{
+					mockProviderAddress: providers.FactoryFixed(mockProvider),
+				},
+			},
+			ProviderSource: source,
+		}
+		c := &InitCommand{
+			Meta: meta,
+		}
+
+		args := []string{"-enable-pluggable-state-storage-experiment=true"}
+		code := c.Run(args)
+		testOutput := done(t)
+		if code != 0 {
+			t.Fatalf("expected code 0 exit code, got %d, output: \n%s", code, testOutput.All())
+		}
+
+		// Check output
+		output := testOutput.All()
+		expectedOutputs := []string{
+			"Initializing the state store...",
+			"Terraform created an empty state file for the default workspace",
+			"Terraform has been successfully initialized!",
+		}
+		for _, expected := range expectedOutputs {
+			if !strings.Contains(output, expected) {
+				t.Fatalf("expected output to include %q, but got':\n %s", expected, output)
+			}
+		}
+	})
+
 	t.Run("the init command creates a backend state file, and creates the default workspace by default", func(t *testing.T) {
 		// Create a temporary, uninitialized working directory with configuration including a state store
 		td := t.TempDir()
@@ -5827,6 +5881,120 @@ func newMockProviderSource(t *testing.T, availableProviderVersions map[string][]
 	return getproviders.NewMockSource(packages, nil), close
 }
 
+// newMockProviderSourceViaHTTP is similar to newMockProviderSource except that the metadata (PackageMeta) for each provider
+// reports that the provider is going to be accessed via HTTP
+//
+// Provider binaries are not available via the mock HTTP provider source. This source is sufficient only to allow Terraform
+// to complete the provider installation process while believing it's installing providers over HTTP.
+// This method is not sufficient to enable Terraform to use providers with those names.
+//
+// When using `newMockProviderSourceViaHTTP` to set a value for `(Meta).ProviderSource` in a test, also set up `testOverrides`
+// in the same Meta. That way the provider source will allow the download process to complete, and when Terraform attempts to use
+// those binaries it will instead use the testOverride providers.
+func newMockProviderSourceViaHTTP(t *testing.T, availableProviderVersions map[string][]string, address string) (source *getproviders.MockSource) {
+	t.Helper()
+	var packages []getproviders.PackageMeta
+	var closes []func()
+	close := func() {
+		for _, f := range closes {
+			f()
+		}
+	}
+	for source, versions := range availableProviderVersions {
+		addr := addrs.MustParseProviderSourceString(source)
+		for _, versionStr := range versions {
+			version, err := getproviders.ParseVersion(versionStr)
+			if err != nil {
+				close()
+				t.Fatalf("failed to parse %q as a version number for %q: %s", versionStr, addr.ForDisplay(), err)
+			}
+			meta, close, err := getproviders.FakePackageMetaViaHTTP(addr, version, getproviders.VersionList{getproviders.MustParseVersion("5.0")}, getproviders.CurrentPlatform, address, "")
+			if err != nil {
+				close()
+				t.Fatalf("failed to prepare fake package for %s %s: %s", addr.ForDisplay(), versionStr, err)
+			}
+			closes = append(closes, close)
+			packages = append(packages, meta)
+		}
+	}
+
+	t.Cleanup(close)
+	return getproviders.NewMockSource(packages, nil)
+}
+
+// newMockProviderSourceUsingTestHttpServer is a helper that makes it easier to use newMockProviderSourceViaHTTP.
+// This helper sets up a test HTTP server for use with newMockProviderSourceViaHTTP, and configures a handler that will respond when
+// Terraform attempts to download provider binaries during installation. The mock source is returned ready to use and all cleanup is
+// handled internally to this helper.
+//
+// This source is not sufficient for providers to be available to use during a test; when using this helper, also set up testOverrides in
+// the same Meta to provide the actual provider implementations for use during the test.
+//
+// Currently this helper only allows one provider/version to be mocked. In future we could extend it to allow multiple providers/versions.
+func newMockProviderSourceUsingTestHttpServer(t *testing.T, p addrs.Provider, v getproviders.Version) *getproviders.MockSource {
+	// Get un-started server so we can obtain the port it'll run on.
+	server := httptest.NewUnstartedServer(nil)
+
+	// Set up mock provider source that mocks installation via HTTP.
+	source := newMockProviderSourceViaHTTP(
+		t,
+		map[string][]string{
+			fmt.Sprintf("%s/%s", p.Namespace, p.Type): {v.String()},
+		},
+		server.Listener.Addr().String(),
+	)
+
+	// Supply a download location so that the installation completes ok
+	// while Terraform still believes it's downloading a provider via HTTP.
+	providerMetadata, err := source.PackageMeta(
+		context.Background(),
+		p,
+		v,
+		getproviders.CurrentPlatform,
+	)
+	if err != nil {
+		t.Fatalf("failed to get provider metadata: %s", err)
+	}
+
+	// Make Terraform believe it's downloading the provider.
+	// Any requests to the test server that aren't for that purpose will cause the test to fail.
+	server.Config = &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		providerLocationPath := strings.ReplaceAll(
+			providerMetadata.Location.String(),
+			"http://"+server.Listener.Addr().String(),
+			"",
+		)
+		// This is the URL that the init command will hit to download the provider, so we return a valid provider archive.
+		if r.URL.Path == providerLocationPath {
+			// This code returns data in the temporary file that's created by the mock provider source.
+			// This 'downloaded' is not used when Terraform uses the provider after the mock installation completes;
+			// Terraform will look for will use testOverrides in the Meta set up for this test.
+			//
+			// Although it's not used later we need to use this file (versus empty or made-up bytes) to enable installation
+			// logic to receive data with the correct checksum.
+			f, err := os.Open(providerMetadata.Filename)
+			if err != nil {
+				t.Fatalf("failed to open mock source file: %s", err)
+			}
+			defer f.Close()
+			archiveBytes, err := io.ReadAll(f)
+			if err != nil {
+				t.Fatalf("failed to read mock source file: %s", err)
+			}
+			w.WriteHeader(http.StatusOK)
+			w.Write(archiveBytes)
+			return
+		} else {
+			t.Fatalf("unexpected URL path: %s", r.URL.Path)
+		}
+	})}
+
+	server.Start()
+	t.Cleanup(server.Close)
+
+	return source
+}
+
 // installFakeProviderPackages installs a fake package for the given provider
 // names (interpreted as a "default" provider address) and versions into the
 // local plugin cache for the given "meta".

internal/getproviders/memoize_source_test.go

@@ -88,7 +88,6 @@ func TestMemoizeSource(t *testing.T) {
 		if warns[0] != "WARNING!" {
 			t.Fatalf("wrong result! Got %s, expected \"WARNING!\"", warns[0])
 		}
-
 	})
 	t.Run("PackageMeta for existing provider", func(t *testing.T) {
 		mock := NewMockSource([]PackageMeta{meta}, nil)

internal/getproviders/mock_source.go

@@ -6,7 +6,6 @@ import (
 	"crypto/sha256"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 
 	"github.com/hashicorp/terraform/internal/addrs"
@@ -153,7 +152,7 @@ func FakePackageMeta(provider addrs.Provider, version Version, protocols Version
 // should call the callback even if this function returns an error, because
 // some error conditions leave a partially-created file on disk.
 func FakeInstallablePackageMeta(provider addrs.Provider, version Version, protocols VersionList, target Platform, execFilename string) (PackageMeta, func(), error) {
-	f, err := ioutil.TempFile("", "terraform-getproviders-fake-package-")
+	f, err := os.CreateTemp("", "terraform-getproviders-fake-package-")
 	if err != nil {
 		return PackageMeta{}, func() {}, err
 	}
@@ -212,6 +211,79 @@ func FakeInstallablePackageMeta(provider addrs.Provider, version Version, protoc
 	return meta, close, nil
 }
 
+// This is basically the same as FakePackageMeta, except that we'll use a PackageHTTPURL instead of a PackageLocalArchive when creating metadata for the provider.
+// By doing so, we create a mock source that makes Terraform believe it's downloading the provider via HTTP, instead of from a local archive.
+//
+// The caller is responsible for calling the close callback to clean up the temporary file.
+// The temporary file is only used to calculate checksums and isn't actually used to install the provider in the test.
+func FakePackageMetaViaHTTP(provider addrs.Provider, version Version, protocols VersionList, target Platform, locationBaseUrl string, execFilename string) (PackageMeta, func(), error) {
+	f, err := os.CreateTemp("", "terraform-getproviders-fake-package-")
+	if err != nil {
+		return PackageMeta{}, func() {}, err
+	}
+
+	// After this point, all of our return paths should include this as the
+	// close callback.
+	close := func() {
+		f.Close()
+		os.Remove(f.Name())
+	}
+
+	if execFilename == "" {
+		execFilename = fmt.Sprintf("terraform-provider-%s_%s", provider.Type, version.String())
+		if target.OS == "windows" {
+			// For a little more (technically unnecessary) realism...
+			execFilename += ".exe"
+		}
+	}
+
+	zw := zip.NewWriter(f)
+	fw, err := zw.Create(execFilename)
+	if err != nil {
+		return PackageMeta{}, close, fmt.Errorf("failed to add %s to mock zip file: %s", execFilename, err)
+	}
+	fmt.Fprintf(fw, "This is a fake provider package for %s %s, not a real provider.\n", provider, version)
+	err = zw.Close()
+	if err != nil {
+		return PackageMeta{}, close, fmt.Errorf("failed to close the mock zip file: %s", err)
+	}
+
+	// Compute the SHA256 checksum of the generated file, to allow package
+	// authentication code to be exercised.
+	f.Seek(0, io.SeekStart)
+	h := sha256.New()
+	io.Copy(h, f)
+	checksum := [32]byte{}
+	h.Sum(checksum[:0])
+
+	meta := PackageMeta{
+		Provider:         provider,
+		Version:          version,
+		ProtocolVersions: protocols,
+		TargetPlatform:   target,
+
+		Location: PackageHTTPURL(
+			fmt.Sprintf(
+				"http://%[1]s/terraform-provider-%[2]s/%[3]s/terraform-provider-%[2]s_%[3]s_%[4]s.zip",
+				locationBaseUrl,
+				provider.Type,
+				version.String(),
+				target.String(),
+			),
+		),
+
+		// This is a fake filename that mimics what a real registry might
+		// indicate as a good filename for this package, in case some caller
+		// intends to use it to name a local copy of the temporary file.
+		// (At the time of writing, no caller actually does that, but who
+		// knows what the future holds?)
+		Filename: f.Name(),
+
+		Authentication: NewArchiveChecksumAuthentication(target, checksum),
+	}
+	return meta, close, nil
+}
+
 func (s *MockSource) ForDisplay(provider addrs.Provider) string {
 	return "mock source"
 }

internal/getproviders/multi_source_test.go

@@ -182,7 +182,6 @@ func TestMultiSourceAvailableVersions(t *testing.T) {
 		if err.Error() != wantErr {
 			t.Fatalf("wrong error.\ngot:  %s\nwant: %s\n", err, wantErr)
 		}
-
 	})
 
 	t.Run("merging with warnings", func(t *testing.T) {

internal/providercache/installer_test.go

@@ -1816,8 +1816,7 @@ func TestEnsureProviderVersions(t *testing.T) {
 		"failed install of a non-existing built-in provider": {
 			Source: getproviders.NewMockSource(
 				[]getproviders.PackageMeta{},
-				nil,
-			),
+				nil),
 			Prepare: func(t *testing.T, inst *Installer, dir *Dir) {
 				// NOTE: We're intentionally not calling
 				// inst.SetBuiltInProviderTypes to make the "terraform"

internal/providercache/package_install.go

@@ -30,6 +30,7 @@ func installFromHTTPURL(ctx context.Context, meta getproviders.PackageMeta, targ
 	// When we're installing from an HTTP URL we expect the URL to refer to
 	// a zip file. We'll fetch that into a temporary file here and then
 	// delegate to installFromLocalArchive below to actually extract it.
+
 	httpGetter := getter.HttpGetter{
 		Client:                httpclient.New(),
 		Netrc:                 true,

internal/rpcapi/packages.go

@@ -87,7 +87,6 @@ func (p *packagesServer) ProviderPackageVersions(ctx context.Context, request *p
 }
 
 func (p *packagesServer) FetchProviderPackage(ctx context.Context, request *packages.FetchProviderPackage_Request) (*packages.FetchProviderPackage_Response, error) {
-
 	response := new(packages.FetchProviderPackage_Response)
 
 	version, err := versions.ParseVersion(request.Version)

internal/rpcapi/packages_test.go

@@ -23,7 +23,6 @@ import (
 )
 
 func TestPackagesServer_ProviderPackageVersions(t *testing.T) {
-
 	tcs := map[string]struct {
 		source           string
 		expectedVersions []string
@@ -126,7 +125,6 @@ func TestPackagesServer_ProviderPackageVersions(t *testing.T) {
 			}
 		})
 	}
-
 }
 
 func TestPackagesServer_FetchProviderPackage(t *testing.T) {