aherman
/
terraform
mirror of https://github.com/hashicorp/terraform
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

test: Add test helper for mocking out use of a network mirror as a provider source (#38447)

Browse Source 
* refactor: Add `NewMockHTTPMirrorSource` test helper. This required refactoring `NewMockHTTPMirrorSource` to pull out reused logic.

* refactor: Move structs defining expected HTTP responses from a network mirror into `getproviders` package

* feat: Add `newHTTPMirrorProviderSourceUsingTestHttpServer` test helper, for tests that need to include use of a network mirror when downloading providers.
pull/38448/head
Sarah French 3 weeks ago committed by GitHub
parent 85be5e4634
commit 1edd518394
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 353 additions and 28 deletions
Show Stats Download Patch File Download Diff File

321
internal/command/init_test.go
Unescape View File

@ -3897,6 +3897,59 @@ func TestInit_stateStore_newWorkingDir(t *testing.T) {
}
})
t.Run("temporary: test showing use of network mirror in mock provider source", func(t *testing.T) {
// Create a temporary, uninitialized working directory with configuration including a state store
td := t.TempDir()
testCopyDir(t, testFixturePath("init-with-state-store"), td)
t.Chdir(td)
// Mock provider still needs to be supplied via testingOverrides despite the mock network mirror
mockProvider := mockPluggableStateStorageProvider()
mockProviderAddress := addrs.NewDefaultProvider("test")
// Set up mock provider source that mocks out downloading hashicorp/test v1.2.3 from a network mirror.
source := newHTTPMirrorProviderSourceUsingTestHttpServer(t, map[string][]string{
"hashicorp/test": {"1.0.0", "1.2.3"},
}, true)
ui := new(cli.MockUi)
view, done := testView(t)
meta := Meta{
Ui: ui,
View: view,
AllowExperimentalFeatures: true,
testingOverrides: &testingOverrides{
Providers: map[addrs.Provider]providers.Factory{
mockProviderAddress: providers.FactoryFixed(mockProvider),
},
},
ProviderSource: source,
}
c := &InitCommand{
Meta: meta,
}
args := []string{"-enable-pluggable-state-storage-experiment=true"}
code := c.Run(args)
testOutput := done(t)
if code != 0 {
t.Fatalf("expected code 0 exit code, got %d, output: \n%s", code, testOutput.All())
}
// Check output
output := testOutput.All()
expectedOutputs := []string{
"Initializing the state store...",
" Installed hashicorp/test v1.2.3 (verified checksum)", // verified checksum message due to hashes matching those described by the network mirror.
"Terraform has been successfully initialized!",
}
for _, expected := range expectedOutputs {
if !strings.Contains(output, expected) {
t.Fatalf("expected output to include %q, but got':\n %s", expected, output)
}
}
})
t.Run("the init command creates a backend state file, and the default workspace is not made by default", func(t *testing.T) {
// Create a temporary, uninitialized working directory with configuration including a state store
td := t.TempDir()
@ -5978,16 +6031,19 @@ func newMockProviderSource(t *testing.T, availableProviderVersions map[string][]
return getproviders.NewMockSource(packages, nil)
}
// newMockProviderSourceViaHTTP is similar to newMockProviderSource except that the metadata (PackageMeta) for each provider
// reports that the provider is going to be accessed via HTTP
// newMockProviderSourceViaHTTP returns a mock provider source that will return
// metadata for providers defined by the calling test code. That metadata will
// report that the provider package is available for download via HTTP from a given
// address. That address is built using the address parameter.
//
// Provider binaries are not available via the mock HTTP provider source. This source is sufficient only to allow Terraform
// to complete the provider installation process while believing it's installing providers over HTTP.
// This method is not sufficient to enable Terraform to use providers with those names.
// The mock HTTP provider source returned by newMockProviderSourceViaHTTP is not
// sufficient for Terraform to complete a provider installation process successfully;
// the provider source will supply Terraform with metadata describing where packages
// can be downloaded from, only. Without an HTTP server that serves files matching the
// metadata returned from this source, Terraform will fail during provider download.
//
// When using `newMockProviderSourceViaHTTP` to set a value for `(Meta).ProviderSource` in a test, also set up `testOverrides`
// in the same Meta. That way the provider source will allow the download process to complete, and when Terraform attempts to use
// those binaries it will instead use the testOverride providers.
// Use newMockProviderSourceUsingTestHttpServer, a helper that sets up a test HTTP server
// to use in combination with this source.
func newMockProviderSourceViaHTTP(t *testing.T, availableProviderVersions map[string][]string, address string) (source *getproviders.MockSource) {
t.Helper()
var packages []getproviders.PackageMeta
@ -6015,13 +6071,15 @@ func newMockProviderSourceViaHTTP(t *testing.T, availableProviderVersions map[st
return getproviders.NewMockSource(packages, nil)
}
// newMockProviderSourceUsingTestHttpServer is a helper that makes it easier to use newMockProviderSourceViaHTTP.
// This helper sets up a test HTTP server for use with newMockProviderSourceViaHTTP, and configures a handler that will respond when
// Terraform attempts to download provider binaries during installation. The mock source is returned ready to use and all cleanup is
// handled internally to this helper.
// newMockProviderSourceUsingTestHttpServer is a helper that returns a mock provider
// source that is paired with a test HTTP server. The provider source will tell Terraform
// that a given provider can be downloaded via HTTP from a given URL, and the test HTTP
// server will enable Terraform to perform that download successfully.
//
// This source is not sufficient for providers to be available to use during a test; when using this helper, also set up testOverrides in
// the same Meta to provide the actual provider implementations for use during the test.
// This source is not sufficient for providers to be available to _use_ during a test,
// it is only sufficient to enable a provider installation process to complete successfully.
// If your test expects a provider to be installed and then used, ensure that testOverrides
// provides the actual provider implementations for use during the test.
func newMockProviderSourceUsingTestHttpServer(t *testing.T, availableProviderVersions map[string][]string) *getproviders.MockSource {
t.Helper()
@ -6076,7 +6134,7 @@ func newMockProviderSourceUsingTestHttpServer(t *testing.T, availableProviderVer
t.Fatalf("unexpected URL path, test doesn't define a matching provider version: %s", r.URL.Path)
}
// This code returns data in the temporary file that's created by the mock provider source.
// This code returns data in the temporary file that's created by this test helper.
// This 'download' is not used when Terraform uses the provider after the mock installation completes;
// Terraform will look for will use testOverrides in the Meta set up for this test.
//
@ -6101,6 +6159,239 @@ func newMockProviderSourceUsingTestHttpServer(t *testing.T, availableProviderVer
return source
}
// newHTTPMirrorProviderSourceUsingTestHttpServer returns an HTTPMirrorSource that is backed
// by a test HTTPS server that acts as a network mirror. The test HTTP server will serve the
// providers and versions defined in the input map using the Provider Network Mirror Protocol.
//
// Calling code has the option of allowing the mirror to report hashes for each provider version,
// or to force the mirror to not report hashes.
//
// All cleanup is handled internally using t.Cleanup.
//
// This source is not sufficient for providers to be available to _use_ during a test,
// it is only sufficient to enable a provider installation process to complete successfully.
// If your test expects a provider to be installed and then used, ensure that testOverrides
// provides the actual provider implementations for use during the test.
func newHTTPMirrorProviderSourceUsingTestHttpServer(t *testing.T, input map[string][]string, allowReturnHashes bool) *getproviders.HTTPMirrorSource {
t.Helper()
// Get un-started server so we can obtain the port it'll run on.
server := httptest.NewUnstartedServer(nil)
address := server.Listener.Addr().String()
// Parse input map for convenience
availableProviderVersions := make(map[addrs.Provider][]getproviders.Version)
for pSource, versions := range input {
addr := addrs.MustParseProviderSourceString(pSource)
var parsedVersions []getproviders.Version
for _, versionStr := range versions {
version := getproviders.MustParseVersion(versionStr)
parsedVersions = append(parsedVersions, version)
}
availableProviderVersions[addr] = parsedVersions
}
// Create tmp files for each provider version defined in the availableProviderVersions map.
// These files are later served by the mock network mirror HTTP server.
tmpFileLocations := map[addrs.Provider]map[getproviders.Version]string{}
for addr, versions := range availableProviderVersions {
for _, version := range versions {
f, _, err := getproviders.CreateFakeFileWithChecksumForProvider(t, addr, version, getproviders.CurrentPlatform, "") // file cleanup already handled
if err != nil {
t.Fatalf("failed to create fake package for provider %s %s: %s", addr, version, err)
}
if _, ok := tmpFileLocations[addr]; !ok {
tmpFileLocations[addr] = make(map[getproviders.Version]string)
}
tmpFileLocations[addr][version] = f.Name()
}
}
// Implement a network mirror
//
// First we define handlers for different endpoints, and then we set up the
// server to use those handlers.
// Handler for endpoints that list available versions of a provider
// GET :hostname/:namespace/:type/index.json
handleListEndpoint := func(addr addrs.Provider, w http.ResponseWriter, r *http.Request) {
// Create response body with the versions available for this provider.
response := getproviders.ListVersionsResponseBody{
Versions: make(map[string]struct{}),
}
versions := availableProviderVersions[addr]
for _, v := range versions {
response.Versions[v.String()] = struct{}{}
}
b, err := json.Marshal(response)
if err != nil {
t.Fatalf("failed to marshal versions response for provider %s: %s", addr, err)
}
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write(b)
}
// Handler for endpoints that list data about a specific versions of a provider
// GET :hostname/:namespace/:type/:version.json
handleVersionEndpoint := func(addr addrs.Provider, v getproviders.Version, w http.ResponseWriter, r *http.Request) {
// Create response body with metadata for single package that matches current platform
response := getproviders.ListInstallationPackagesResponseBody{
Archives: make(map[string]*getproviders.ListInstallationPackagesArchiveMeta),
}
// E.g. terraform-provider-foobar_1.0.0_darwin_amd64.zip
path := fmt.Sprintf(
"terraform-provider-%s_%s_%s.zip",
addr.Type,
v.String(),
getproviders.CurrentPlatform.String(),
)
// Make hashes
if _, ok := tmpFileLocations[addr]; !ok {
t.Fatalf("no package metadata found in the mock network mirror for provider source %q", addr)
}
if _, ok := tmpFileLocations[addr][v]; !ok {
t.Fatalf("no package metadata found in the mock network mirror for provider source %q and version %q", addr, v)
}
fileLocation := tmpFileLocations[addr][v]
zHash, err := getproviders.PackageHashLegacyZipSHA(getproviders.PackageLocalArchive(fileLocation))
if err != nil {
t.Fatalf("failed to compute hash for provider source %q and version %q: %s", addr, v, err)
}
h1Hash, err := getproviders.PackageHashV1(getproviders.PackageLocalArchive(fileLocation))
if err != nil {
t.Fatalf("failed to compute hash for provider source %q and version %q: %s", addr, v, err)
}
m := &getproviders.ListInstallationPackagesArchiveMeta{
RelativeURL: path,
}
if allowReturnHashes {
// Test may want no hashes to be returned, to test the behaviour when no authentication data
// is available for a provider package.
m.Hashes = []string{
zHash.String(),
h1Hash.String(),
}
}
response.Archives[getproviders.CurrentPlatform.String()] = m
b, err := json.Marshal(response)
if err != nil {
t.Fatalf("failed to marshal versions response for provider %s: %s", addr, err)
}
w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write(b)
}
// Handler for endpoints that allow downloading a provider archive.
// GET hostname/:namespace/:type/:filename , where filename always ends in .zip
handleZipDownloadEndpoint := func(addr addrs.Provider, v getproviders.Version, w http.ResponseWriter, r *http.Request) {
// This code returns data in the temporary file that's created by this test helper.
// This 'download' is not used when Terraform uses the provider after the mock installation completes;
// Terraform will look for will use testOverrides in the Meta set up for this test.
//
// Although it's not used later we need to use this file (versus empty or made-up bytes) to enable installation
// logic to receive data with the correct checksum.
fileLocation, ok := tmpFileLocations[addr][v]
if !ok {
t.Fatalf("no package metadata found in the mock network mirror for provider source %q and version %q", addr, v)
}
f, err := os.Open(fileLocation)
if err != nil {
t.Fatalf("failed to open mock source file: %s", err)
}
defer f.Close()
archiveBytes, err := io.ReadAll(f)
if err != nil {
t.Fatalf("failed to read mock source file: %s", err)
}
w.Header().Add("Content-Type", "application/zip")
w.WriteHeader(http.StatusOK)
w.Write(archiveBytes)
}
// Set up how the server handles requests.
// This includes validating that the request matches a provider version that was specified in the input map
// from the calling test code.
server.Config = &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
parts := strings.Split(r.URL.Path, "/")
if len(parts) < 5 {
t.Fatalf("unexpected URL path in request to test network mirror: %s", r.URL.Path)
}
// Parse provider source; avoid repeated logic in handlers below
providerSource := strings.Join(parts[1:len(parts)-1], "/")
providerAddr, diag := addrs.ParseProviderSourceString(providerSource)
if diag.HasErrors() {
t.Fatalf("failed to parse provider source from URL path %q: %s", r.URL.Path, diag.Err())
}
// Is the request for a provider source and version combo specified by the test?
if _, ok := availableProviderVersions[providerAddr]; !ok {
t.Fatalf("provider source %q is not available in the mock network mirror", providerAddr)
}
var v getproviders.Version
if !strings.HasSuffix(r.URL.Path, "/index.json") {
// Only parse and assert version for requests that include a version in the path
versionRegex := regexp.MustCompile(`([0-9]+\.[0-9]+\.[0-9]+)`)
versionStr := versionRegex.FindString(r.URL.Path)
if versionStr == "" {
t.Fatalf("expected to be able to parse version from URL path %q", r.URL.Path)
}
v = getproviders.MustParseVersion(versionStr)
found := false
for _, pv := range availableProviderVersions[providerAddr] {
if pv.Same(v) {
found = true
break
}
}
if !found {
t.Fatalf("provider source %q and version %q is not available in the mock network mirror", providerAddr, v)
}
}
switch {
case strings.HasSuffix(r.URL.Path, "/index.json"):
// List versions for a provider
// GET :hostname/:namespace/:type/index.json
handleListEndpoint(providerAddr, w, r)
return
case strings.HasSuffix(r.URL.Path, ".json"):
// Show archives for a specific version
// GET :hostname/:namespace/:type/:version.json
handleVersionEndpoint(providerAddr, v, w, r)
return
case strings.HasSuffix(r.URL.Path, ".zip"):
// Handle provider binary download requests.
// GET :hostname/:namespace/:type/:filename.zip
handleZipDownloadEndpoint(providerAddr, v, w, r)
return
default:
t.Fatalf("unhandled request to mock network mirror HTTP server:\npath: %s\n request: %#v", r.URL.Path, r)
}
})}
server.Start() // Mock HTTP Mirror source doesn't enforce TLS
t.Cleanup(server.Close)
// Set up mock provider source that mocks installation via HTTP.
url := &url.URL{
Scheme: "http", // No TLS again
Host: address,
}
source := getproviders.NewMockHTTPMirrorSource(t, url)
return source
}
// installFakeProviderPackages installs a fake package for the given provider
// names (interpreted as a "default" provider address) and versions into the
// local plugin cache for the given "meta".

60
internal/getproviders/http_mirror_source.go
Unescape View File

@ -14,6 +14,7 @@ import (
"net/url"
"path"
"strings"
"testing"
"github.com/hashicorp/go-retryablehttp"
svchost "github.com/hashicorp/terraform-svchost"
@ -55,14 +56,35 @@ func NewHTTPMirrorSource(baseURL *url.URL, creds svcauth.CredentialsSource) *HTT
}
return nil
}
// Enforce TLS
return newHTTPMirrorSourceWithHTTPClientTLS(baseURL, creds, httpClient)
}
func NewMockHTTPMirrorSource(t *testing.T, baseURL *url.URL) *HTTPMirrorSource {
httpClient := httpclient.New()
httpClient.Timeout = requestTimeout
httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
// If we get redirected more than five times we'll assume we're
// in a redirect loop and bail out, rather than hanging forever.
if len(via) > 5 {
return fmt.Errorf("too many redirects")
}
return nil
}
// In tests we don't enforce TLS and also there are no creds
var creds svcauth.CredentialsSource // nil
return newHTTPMirrorSourceWithHTTPClient(baseURL, creds, httpClient)
}
func newHTTPMirrorSourceWithHTTPClient(baseURL *url.URL, creds svcauth.CredentialsSource, httpClient *http.Client) *HTTPMirrorSource {
func newHTTPMirrorSourceWithHTTPClientTLS(baseURL *url.URL, creds svcauth.CredentialsSource, httpClient *http.Client) *HTTPMirrorSource {
if baseURL.Scheme != "https" {
panic("non-https URL for HTTP mirror")
}
return newHTTPMirrorSourceWithHTTPClient(baseURL, creds, httpClient)
}
func newHTTPMirrorSourceWithHTTPClient(baseURL *url.URL, creds svcauth.CredentialsSource, httpClient *http.Client) *HTTPMirrorSource {
// We borrow the retry settings and behaviors from the registry client,
// because our needs here are very similar to those of the registry client.
retryableClient := retryablehttp.NewClient()
@ -117,10 +139,7 @@ func (s *HTTPMirrorSource) AvailableVersions(ctx context.Context, provider addrs
// If we got here then the response had status OK and so our body
// will be non-nil and should contain some JSON for us to parse.
type ResponseBody struct {
Versions map[string]struct{} `json:"versions"`
}
var bodyContent ResponseBody
var bodyContent ListVersionsResponseBody
dec := json.NewDecoder(body)
if err := dec.Decode(&bodyContent); err != nil {
@ -181,14 +200,7 @@ func (s *HTTPMirrorSource) PackageMeta(ctx context.Context, provider addrs.Provi
// If we got here then the response had status OK and so our body
// will be non-nil and should contain some JSON for us to parse.
type ResponseArchiveMeta struct {
RelativeURL string `json:"url"`
Hashes []string
}
type ResponseBody struct {
Archives map[string]*ResponseArchiveMeta `json:"archives"`
}
var bodyContent ResponseBody
var bodyContent ListInstallationPackagesResponseBody
dec := json.NewDecoder(body)
if err := dec.Decode(&bodyContent); err != nil {
@ -247,6 +259,28 @@ func (s *HTTPMirrorSource) ForDisplay(provider addrs.Provider) string {
return "provider mirror at " + s.baseURL.String()
}
// ListVersionsResponseBody is the JSON structure of a response when a user queries the available versions
// for a provider in the network mirror, i.e. a GET to path :hostname/:namespace/:type/index.json
// See: https://developer.hashicorp.com/terraform/internals/provider-network-mirror-protocol#list-available-versions
type ListVersionsResponseBody struct {
Versions map[string]struct{} `json:"versions"`
}
// ListInstallationPackagesResponseBody is the structure of the JSON response when a user queries the
// available packages for a given provider version in the network mirror.
// i.e. at the path :hostname/:namespace/:type/:version.json
// See: https://developer.hashicorp.com/terraform/internals/provider-network-mirror-protocol#list-available-installation-packages
type ListInstallationPackagesResponseBody struct {
Archives map[string]*ListInstallationPackagesArchiveMeta `json:"archives"`
}
// ListInstallationPackagesArchiveMeta is the JSON structure of each archive entry for a specific provider
// version in the network mirror. See ListInstallationPackagesResponseBody.
type ListInstallationPackagesArchiveMeta struct {
RelativeURL string `json:"url"`
Hashes []string
}
// mirrorHost extracts the hostname portion of the configured base URL and
// returns it as a svchost.Hostname, normalized in the usual ways.
//

Write Preview
Loading…
Cancel
Save