diff --git a/config/module/get_test.go b/config/module/get_test.go index de2a13bab5..6b94ac3725 100644 --- a/config/module/get_test.go +++ b/config/module/get_test.go @@ -1,7 +1,9 @@ package module import ( + "encoding/json" "fmt" + "io" "net/http" "net/http/httptest" "net/url" @@ -14,6 +16,7 @@ import ( getter "github.com/hashicorp/go-getter" version "github.com/hashicorp/go-version" + "github.com/hashicorp/terraform/registry/response" ) // Map of module names and location of test modules. @@ -26,22 +29,28 @@ type testMod struct { // All the locationes from the mockRegistry start with a file:// scheme. If // the the location string here doesn't have a scheme, the mockRegistry will // find the absolute path and return a complete URL. -var testMods = map[string]testMod{ - "registry/foo/bar": { +var testMods = map[string][]testMod{ + "registry/foo/bar": {{ location: "file:///download/registry/foo/bar/0.2.3//*?archive=tar.gz", version: "0.2.3", - }, - "registry/foo/baz": { + }}, + "registry/foo/baz": {{ location: "file:///download/registry/foo/baz/1.10.0//*?archive=tar.gz", version: "1.10.0", - }, - "registry/local/sub": { + }}, + "registry/local/sub": {{ location: "test-fixtures/registry-tar-subdir/foo.tgz//*?archive=tar.gz", version: "0.1.2", - }, - "exists-in-registry/identifier/provider": { + }}, + "exists-in-registry/identifier/provider": {{ location: "file:///registry/exists", version: "0.2.0", + }}, + "test-versions/name/provider": { + {version: "2.2.0"}, + {version: "2.1.1"}, + {version: "1.2.2"}, + {version: "1.2.1"}, }, } @@ -59,45 +68,114 @@ func latestVersion(versions []string) string { return col[len(col)-1].String() } -// Just enough like a registry to exercise our code. -// Returns the location of the latest version -func mockRegistry() *httptest.Server { +func mockRegHandler() http.Handler { mux := http.NewServeMux() - server := httptest.NewServer(mux) + + download := func(w http.ResponseWriter, r *http.Request) { + p := strings.TrimLeft(r.URL.Path, "/") + // handle download request + re := regexp.MustCompile(`^([-a-z]+/\w+/\w+)/download$`) + // download lookup + matches := re.FindStringSubmatch(p) + if len(matches) != 2 { + w.WriteHeader(http.StatusBadRequest) + return + } + + versions, ok := testMods[matches[1]] + if !ok { + http.NotFound(w, r) + return + } + mod := versions[0] + + location := mod.location + if !strings.HasPrefix(location, "file:///") { + // we can't use filepath.Abs because it will clean `//` + wd, _ := os.Getwd() + location = fmt.Sprintf("file://%s/%s", wd, location) + } + + w.Header().Set("X-Terraform-Get", location) + w.WriteHeader(http.StatusNoContent) + // no body + return + } + + versions := func(w http.ResponseWriter, r *http.Request) { + p := strings.TrimLeft(r.URL.Path, "/") + re := regexp.MustCompile(`^([-a-z]+/\w+/\w+)/versions$`) + matches := re.FindStringSubmatch(p) + if len(matches) != 2 { + w.WriteHeader(http.StatusBadRequest) + return + } + + name := matches[1] + versions, ok := testMods[name] + if !ok { + http.NotFound(w, r) + return + } + + // only adding the single requested module for now + // this is the minimal that any regisry is epected to support + mpvs := &response.ModuleProviderVersions{ + Source: name, + } + + for _, v := range versions { + mv := &response.ModuleVersion{ + Version: v.version, + } + mpvs.Versions = append(mpvs.Versions, mv) + } + + resp := response.ModuleVersions{ + Modules: []*response.ModuleProviderVersions{mpvs}, + } + + js, err := json.Marshal(resp) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.Write(js) + } mux.Handle("/v1/modules/", http.StripPrefix("/v1/modules/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - p := strings.TrimLeft(r.URL.Path, "/") - // handle download request - download := regexp.MustCompile(`^([-a-z]+/\w+/\w+)/download$`) - - // download lookup - matches := download.FindStringSubmatch(p) - if len(matches) != 2 { - w.WriteHeader(http.StatusBadRequest) + if strings.HasSuffix(r.URL.Path, "/download") { + download(w, r) return } - mod, ok := testMods[matches[1]] - if !ok { - w.WriteHeader(http.StatusNotFound) + if strings.HasSuffix(r.URL.Path, "/versions") { + versions(w, r) return } - location := mod.location - if !strings.HasPrefix(location, "file:///") { - // we can't use filepath.Abs because it will clean `//` - wd, _ := os.Getwd() - location = fmt.Sprintf("file://%s/%s", wd, location) - } - - w.Header().Set("X-Terraform-Get", location) - w.WriteHeader(http.StatusNoContent) - // no body - return + http.NotFound(w, r) })), ) + mux.HandleFunc("/.well-known/terraform.json", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"modules.v1":"/v1/modules/"}`) + }) + return mux +} + +// Just enough like a registry to exercise our code. +// Returns the location of the latest version +func mockRegistry() *httptest.Server { + server := httptest.NewServer(mockRegHandler()) + return server +} + +func mockTLSRegistry() *httptest.Server { + server := httptest.NewTLSServer(mockRegHandler()) return server } @@ -138,12 +216,12 @@ func TestDetectRegistry(t *testing.T) { }{ { source: "registry/foo/bar", - location: testMods["registry/foo/bar"].location, + location: testMods["registry/foo/bar"][0].location, found: true, }, { source: "registry/foo/baz", - location: testMods["registry/foo/baz"].location, + location: testMods["registry/foo/baz"][0].location, found: true, }, // this should not be found, and is no longer valid as a local source diff --git a/config/module/registry.go b/config/module/registry.go index 2753871944..a1fef61619 100644 --- a/config/module/registry.go +++ b/config/module/registry.go @@ -30,7 +30,6 @@ const ( var ( client *http.Client tfVersion = version.String() - regDisco = disco.NewDisco() ) func init() { @@ -45,21 +44,27 @@ func (e errModuleNotFound) Error() string { } // Lookup module versions in the registry. -func lookupModuleVersions(module *regsrc.Module) (*response.ModuleVersions, error) { +func lookupModuleVersions(regDisco *disco.Disco, module *regsrc.Module) (*response.ModuleVersions, error) { if module.RawHost == nil { module.RawHost = regsrc.NewFriendlyHost(defaultRegistry) } - regUrl := regDisco.DiscoverServiceURL(svchost.Hostname(module.RawHost.Normalized()), serviceID) - if regUrl == nil { - regUrl = &url.URL{ + regURL := regDisco.DiscoverServiceURL(svchost.Hostname(module.RawHost.Normalized()), serviceID) + if regURL == nil { + regURL = &url.URL{ Scheme: "https", Host: module.RawHost.String(), Path: defaultApiPath, } } - location := fmt.Sprintf("%s/%s/%s/%s/versions", regUrl, module.RawNamespace, module.RawName, module.RawProvider) + service := regURL.String() + + if service[len(service)-1] != '/' { + service += "/" + } + + location := fmt.Sprintf("%s%s/%s/%s/versions", service, module.RawNamespace, module.RawName, module.RawProvider) log.Printf("[DEBUG] fetching module versions from %q", location) req, err := http.NewRequest("GET", location, nil) @@ -69,6 +74,15 @@ func lookupModuleVersions(module *regsrc.Module) (*response.ModuleVersions, erro req.Header.Set(xTerraformVersion, tfVersion) + // if discovery required a custom transport, then we should use that too + client := client + if regDisco.Transport != nil { + client = &http.Client{ + Transport: regDisco.Transport, + Timeout: requestTimeout, + } + } + resp, err := client.Do(req) if err != nil { return nil, err diff --git a/config/module/registry_test.go b/config/module/registry_test.go new file mode 100644 index 0000000000..2c784ae5de --- /dev/null +++ b/config/module/registry_test.go @@ -0,0 +1,152 @@ +package module + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + cleanhttp "github.com/hashicorp/go-cleanhttp" + version "github.com/hashicorp/go-version" + "github.com/hashicorp/terraform/registry/regsrc" + "github.com/hashicorp/terraform/svchost/disco" +) + +// Return a transport to use for this test server. +// This not only loads the tls.Config from the test server for proper cert +// validation, but also inserts a Dialer that resolves localhost and +// example.com to 127.0.0.1 with the correct port, since 127.0.0.1 on its own +// isn't a valid registry hostname. +// TODO: cert validation not working here, so we use don't verify for now. +func mockTransport(server *httptest.Server) *http.Transport { + u, _ := url.Parse(server.URL) + _, port, _ := net.SplitHostPort(u.Host) + + transport := cleanhttp.DefaultTransport() + transport.TLSClientConfig = server.TLS + transport.TLSClientConfig.InsecureSkipVerify = true + transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + host, _, _ := net.SplitHostPort(addr) + switch host { + case "example.com", "localhost", "localhost.localdomain", "registry.terraform.io": + addr = "127.0.0.1" + if port != "" { + addr += ":" + port + } + } + return (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext(ctx, network, addr) + } + return transport +} + +func TestMockDiscovery(t *testing.T) { + server := mockTLSRegistry() + defer server.Close() + + regDisco := disco.NewDisco() + regDisco.Transport = mockTransport(server) + + regURL := regDisco.DiscoverServiceURL("example.com", serviceID) + + if regURL == nil { + t.Fatal("no registry service discovered") + } + + if regURL.Host != "example.com" { + t.Fatal("expected registry host example.com, got:", regURL.Host) + } +} + +func TestLookupModuleVersions(t *testing.T) { + server := mockTLSRegistry() + defer server.Close() + regDisco := disco.NewDisco() + regDisco.Transport = mockTransport(server) + + // test with and without a hostname + for _, src := range []string{ + "example.com/test-versions/name/provider", + "test-versions/name/provider", + } { + modsrc, err := regsrc.ParseModuleSource(src) + if err != nil { + t.Fatal(err) + } + + resp, err := lookupModuleVersions(regDisco, modsrc) + if err != nil { + t.Fatal(err) + } + + if len(resp.Modules) != 1 { + t.Fatal("expected 1 module, got", len(resp.Modules)) + } + + mod := resp.Modules[0] + name := "test-versions/name/provider" + if mod.Source != name { + t.Fatalf("expected module name %q, got %q", name, mod.Source) + } + + if len(mod.Versions) != 4 { + t.Fatal("expected 4 versions, got", len(mod.Versions)) + } + + for _, v := range mod.Versions { + _, err := version.NewVersion(v.Version) + if err != nil { + t.Fatalf("invalid version %q: %s", v.Version, err) + } + } + } +} + +func TestACCLookupModuleVersions(t *testing.T) { + server := mockTLSRegistry() + defer server.Close() + regDisco := disco.NewDisco() + + // test with and without a hostname + for _, src := range []string{ + "terraform-aws-modules/vpc/aws", + defaultRegistry + "/terraform-aws-modules/vpc/aws", + } { + modsrc, err := regsrc.ParseModuleSource(src) + if err != nil { + t.Fatal(err) + } + + resp, err := lookupModuleVersions(regDisco, modsrc) + if err != nil { + t.Fatal(err) + } + + if len(resp.Modules) != 1 { + t.Fatal("expected 1 module, got", len(resp.Modules)) + } + + mod := resp.Modules[0] + name := "terraform-aws-modules/vpc/aws" + if mod.Source != name { + t.Fatalf("expected module name %q, got %q", name, mod.Source) + } + + if len(mod.Versions) == 0 { + t.Fatal("expected multiple versions, got 0") + } + + for _, v := range mod.Versions { + _, err := version.NewVersion(v.Version) + if err != nil { + t.Fatalf("invalid version %q: %s", v.Version, err) + } + } + } +}