handle go-getter subdirs in Tree.Load

In order to remain backward compatible with some modules, we need to
handle subdirs during Load. This means duplicating part of the go-getter
code path for subDir handling so we can resolve any subDirs and globs
internally, while keeping the entire remote directory structure within
the file storage.

config/module/get_test.go

@@ -315,7 +315,6 @@ func TestRegistryGitHubArchive(t *testing.T) {
 	if actual != expected {
 		t.Fatalf("got: \n\n%s\nexpected: \n\n%s", actual, expected)
 	}
-
 }
 
 func TestAccRegistryDiscover(t *testing.T) {
@@ -342,3 +341,31 @@ func TestAccRegistryDiscover(t *testing.T) {
 		t.Fatalf("url doesn't contain 'consul': %s", u.String())
 	}
 }
+
+func TestAccRegistryLoad(t *testing.T) {
+	if os.Getenv("TF_ACC") == "" {
+		t.Skip("skipping ACC test")
+	}
+
+	storage := testStorage(t)
+	tree := NewTree("", testConfig(t, "registry-load"))
+
+	if err := tree.Load(storage, GetModeGet); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	if !tree.Loaded() {
+		t.Fatal("should be loaded")
+	}
+
+	// This should no longer error
+	if err := tree.Load(storage, GetModeNone); err != nil {
+		t.Fatalf("err: %s", err)
+	}
+
+	// TODO expand this further by fetching some metadata from the registry
+	actual := strings.TrimSpace(tree.String())
+	if !strings.Contains(actual, "(path: vault)") {
+		t.Fatal("missing vault module, got:\n", actual)
+	}
+}

config/module/test-fixtures/registry-load/main.tf

@@ -0,0 +1,3 @@
+module "vault" {
+    source = "hashicorp/vault/aws"
+}

config/module/tree.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"fmt"
+	"log"
 	"strings"
 	"sync"
 
@@ -176,13 +177,43 @@ func (t *Tree) Load(s getter.Storage, mode GetMode) error {
 		copy(path, t.path)
 		path = append(path, m.Name)
 
-		source, err := getter.Detect(m.Source, t.config.Dir, detectors)
+		log.Printf("[TRACE] module source %q", m.Source)
+		// Split out the subdir if we have one.
+		// Terraform keeps the entire request tree for now, so that modules can
+		// reference sibling modules from the same archive or repo.
+		source, subDir := getter.SourceDirSubdir(m.Source)
+
+		log.Printf("[TRACE] module source: %q", source)
+
+		source, err := getter.Detect(source, t.config.Dir, detectors)
 		if err != nil {
 			return fmt.Errorf("module %s: %s", m.Name, err)
 		}
+
+		log.Printf("[TRACE] detected module source %q", source)
+
+		// Check if the detector introduced something new.
+		// For example, the registry always adds a subdir of `//*`,
+		// indicating that we need to strip off the first component from the
+		// tar archive, though we may not yet know what it is called.
+		//
+		// TODO: This can cause us to lose the previously detected subdir. It
+		// was never an issue before, since none of the supported detectors
+		// previously had this behavior, but we may want to add this ability to
+		// registry modules.
+		source, subDir2 := getter.SourceDirSubdir(source)
+		if subDir2 != "" {
+			subDir = subDir2
+		}
+
+		log.Printf("[TRACE] getting module source %q", source)
+
 		// Get the directory where this module is so we can load it
 		key := strings.Join(path, ".")
-		key = fmt.Sprintf("module.%s-%s", key, m.Source)
+
+		// The key is the string being hashed to uniquely id the Source.  The
+		// leading digit can be incremented to re-fetch all existing modules.
+		key = fmt.Sprintf("0.root.%s-%s", key, m.Source)
 
 		dir, ok, err := getStorage(s, key, source, mode)
 		if err != nil {
@@ -193,6 +224,13 @@ func (t *Tree) Load(s getter.Storage, mode GetMode) error {
 				"module %s: not found, may need to be downloaded using 'terraform get'", m.Name)
 		}
 
+		// Expand the subDir if required.
+		dir, err = getter.SubdirGlob(dir, subDir)
+		if err != nil {
+			return err
+		}
+
+		// Load the configurations.Dir(source)
 		children[m.Name], err = NewTreeModule(m.Name, dir)
 		if err != nil {
 			return fmt.Errorf(