diff --git a/configs/configload/loader_install.go b/configs/configload/loader_install.go new file mode 100644 index 0000000000..2ab6529d33 --- /dev/null +++ b/configs/configload/loader_install.go @@ -0,0 +1,237 @@ +package configload + +import ( + "fmt" + "log" + "os" + "path/filepath" + "strings" + + version "github.com/hashicorp/go-version" + "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/registry/regsrc" +) + +// InstallModules analyses the root module in the given directory and installs +// all of its direct and transitive dependencies into the loader's modules +// directory, which must already exist. +// +// Since InstallModules makes possibly-time-consuming calls to remote services, +// a hook interface is supported to allow the caller to be notified when +// each module is installed and, for remote modules, when downloading begins. +// LoadConfig guarantees that two hook calls will not happen concurrently but +// it does not guarantee any particular ordering of hook calls. This mechanism +// is for UI feedback only and does not give the caller any control over the +// process. +// +// If modules are already installed in the target directory, they will be +// skipped unless their source address or version have changed or unless +// the upgrade flag is set. +// +// InstallModules never deletes any directory, except in the case where it +// needs to replace a directory that is already present with a newly-extracted +// package. +// +// If the returned diagnostics contains errors then the module installation +// may have wholly or partially completed. Modules must be loaded in order +// to find their dependencies, so this function does many of the same checks +// as LoadConfig as a side-effect. +func (l *Loader) InstallModules(rootDir string, upgrade bool, hooks InstallHooks) hcl.Diagnostics { + rootMod, diags := l.parser.LoadConfigDir(rootDir) + if rootMod == nil { + return diags + } + + if hooks == nil { + // Use our no-op implementation as a placeholder + hooks = InstallHooksImpl{} + } + + // Create a manifest record for the root module. This will be used if + // there are any relative-pathed modules in the root. + l.modules.manifest[""] = moduleRecord{ + Key: "", + Dir: rootDir, + } + + _, cDiags := configs.BuildConfig(rootMod, configs.ModuleWalkerFunc( + func(req *configs.ModuleRequest) (*configs.Module, *version.Version, hcl.Diagnostics) { + + key := manifestKey(req.Path) + instPath := l.packageInstallPath(req.Path) + + log.Printf("[DEBUG] Module installer: begin %s", key) + + // First we'll check if we need to upgrade/replace an existing + // installed module, and delete it out of the way if so. + replace := upgrade + if !replace { + record, recorded := l.modules.manifest[key] + switch { + case !recorded: + log.Printf("[TRACE] %s is not yet installed", key) + replace = true + case record.SourceAddr != req.SourceAddr: + log.Printf("[TRACE] %s source address has changed from %q to %q", key, record.SourceAddr, req.SourceAddr) + replace = true + case record.Version != nil && !req.VersionConstraint.Required.Check(record.Version): + log.Printf("[TRACE] %s version %s no longer compatible with constraints %s", key, record.Version, req.VersionConstraint.Required) + replace = true + } + } + + // If we _are_ planning to replace this module, then we'll remove + // it now so our installation code below won't conflict with any + // existing remnants. + if replace { + if _, recorded := l.modules.manifest[key]; recorded { + log.Printf("[TRACE] discarding previous record of %s prior to reinstall", key) + } + delete(l.modules.manifest, key) + // Deleting a module invalidates all of its descendent modules too. + keyPrefix := key + "." + for subKey := range l.modules.manifest { + if strings.HasPrefix(subKey, keyPrefix) { + if _, recorded := l.modules.manifest[subKey]; recorded { + log.Printf("[TRACE] also discarding downstream %s", subKey) + } + delete(l.modules.manifest, subKey) + } + } + } + + record, recorded := l.modules.manifest[key] + if !recorded { + // Clean up any stale cache directory that might be present. + // If this is a local (relative) source then the dir will + // not exist, but we'll ignore that. + log.Printf("[TRACE] cleaning directory %s prior to install of %s", instPath, key) + err := l.modules.FS.RemoveAll(instPath) + if err != nil && !os.IsNotExist(err) { + log.Printf("[TRACE] failed to remove %s: %s", key, err) + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to remove local module cache", + Detail: fmt.Sprintf( + "Terraform tried to remove %s in order to reinstall this module, but encountered an error: %s", + instPath, err, + ), + Subject: &req.CallRange, + }) + return nil, nil, diags + } + } else { + // If this module is already recorded and its root directory + // exists then we will just load what's already there and + // keep our existing record. + info, err := l.modules.FS.Stat(record.Dir) + if err == nil && info.IsDir() { + mod, mDiags := l.parser.LoadConfigDir(record.Dir) + diags = append(diags, mDiags...) + + log.Printf("[TRACE] Module installer: %s %s already installed in %s", key, record.Version, record.Dir) + return mod, record.Version, diags + } + } + + // If we get down here then it's finally time to actually install + // the module. There are some variants to this process depending + // on what type of module source address we have. + switch { + + case isLocalSourceAddr(req.SourceAddr): + log.Printf("[TRACE] %s has local path %q", key, req.SourceAddr) + mod, mDiags := l.installLocalModule(req, key, hooks) + diags = append(diags, mDiags...) + return mod, nil, diags + + case isRegistrySourceAddr(req.SourceAddr): + addr, err := regsrc.ParseModuleSource(req.SourceAddr) + if err != nil { + // Should never happen because isRegistrySourceAddr already validated + panic(err) + } + log.Printf("[TRACE] %s is a registry module at %s", key, addr) + + // TODO: Implement + panic("registry source installation not yet implemented") + + default: + log.Printf("[TRACE] %s address %q will be interpreted with go-getter", key, req.SourceAddr) + + // TODO: Implement + panic("fallback source installation not yet implemented") + + } + + }, + )) + diags = append(diags, cDiags...) + + err := l.modules.writeModuleManifestSnapshot() + if err != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Failed to update module manifest", + Detail: fmt.Sprintf("Unable to write the module manifest file: %s", err), + }) + } + + return diags +} + +func (l *Loader) installLocalModule(req *configs.ModuleRequest, key string, hooks InstallHooks) (*configs.Module, hcl.Diagnostics) { + var diags hcl.Diagnostics + + parentKey := manifestKey(req.Parent.Path) + parentRecord, recorded := l.modules.manifest[parentKey] + if !recorded { + // This is indicative of a bug rather than a user-actionable error + panic(fmt.Errorf("missing manifest record for parent module %s", parentKey)) + } + + if len(req.VersionConstraint.Required) != 0 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid version constraint", + Detail: "A version constraint cannot be applied to a module at a relative local path.", + Subject: &req.VersionConstraint.DeclRange, + }) + } + + // For local sources we don't actually need to modify the + // filesystem at all because the parent already wrote + // the files we need, and so we just load up what's already here. + newDir := filepath.Join(parentRecord.Dir, req.SourceAddr) + log.Printf("[TRACE] %s uses directory from parent: %s", key, newDir) + mod, mDiags := l.parser.LoadConfigDir(newDir) + if mod == nil { + // nil indicates missing or unreadable directory, so we'll + // discard the returned diags and return a more specific + // error message here. + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Unreadable module directory", + Detail: fmt.Sprintf("The directory %s could not be read.", newDir), + Subject: &req.SourceAddrRange, + }) + } else { + diags = append(diags, mDiags...) + } + + // Note the local location in our manifest. + l.modules.manifest[key] = moduleRecord{ + Key: key, + Dir: newDir, + SourceAddr: req.SourceAddr, + } + log.Printf("[TRACE] Module installer: %s installed at %s", key, newDir) + hooks.Install(key, nil, newDir) + + return mod, diags +} + +func (l *Loader) packageInstallPath(modulePath []string) string { + return filepath.Join(l.modules.Dir, strings.Join(modulePath, ".")) +} diff --git a/configs/configload/loader_install_hooks.go b/configs/configload/loader_install_hooks.go new file mode 100644 index 0000000000..058369414d --- /dev/null +++ b/configs/configload/loader_install_hooks.go @@ -0,0 +1,34 @@ +package configload + +import version "github.com/hashicorp/go-version" + +// InstallHooks is an interface used to provide notifications about the +// installation process being orchestrated by InstallModules. +// +// This interface may have new methods added in future, so implementers should +// embed InstallHooksImpl to get no-op implementations of any unimplemented +// methods. +type InstallHooks interface { + // Download is called for modules that are retrieved from a remote source + // before that download begins, to allow a caller to give feedback + // on progress through a possibly-long sequence of downloads. + Download(moduleAddr, packageAddr string, version *version.Version) + + // Install is called for each module that is installed, even if it did + // not need to be downloaded from a remote source. + Install(moduleAddr string, version *version.Version, localPath string) +} + +// InstallHooksImpl is a do-nothing implementation of InstallHooks that +// can be embedded in another implementation struct to allow only partial +// implementation of the interface. +type InstallHooksImpl struct { +} + +func (h InstallHooksImpl) Download(moduleAddr, packageAddr string, version *version.Version) { +} + +func (h InstallHooksImpl) Install(moduleAddr string, version *version.Version, localPath string) { +} + +var _ InstallHooks = InstallHooksImpl{} diff --git a/configs/configload/loader_install_test.go b/configs/configload/loader_install_test.go new file mode 100644 index 0000000000..608b08a009 --- /dev/null +++ b/configs/configload/loader_install_test.go @@ -0,0 +1,65 @@ +package configload + +import ( + "path/filepath" + "testing" + + version "github.com/hashicorp/go-version" +) + +func TestLoaderInstallModules_local(t *testing.T) { + fixtureDir := filepath.Clean("test-fixtures/local-modules") + loader := newTestLoader(filepath.Join(fixtureDir, ".terraform/modules")) + + hooks := &testInstallHooks{} + + diags := loader.InstallModules(fixtureDir, false, hooks) + assertNoDiagnostics(t, diags) + + wantCalls := []testInstallHookCall{ + { + Name: "Install", + ModuleAddr: "child_a", + PackageAddr: "", + LocalPath: "test-fixtures/local-modules/child_a", + }, + { + Name: "Install", + ModuleAddr: "child_a.child_b", + PackageAddr: "", + LocalPath: "test-fixtures/local-modules/child_a/child_b", + }, + } + + assertResultDeepEqual(t, hooks.Calls, wantCalls) +} + +type testInstallHooks struct { + Calls []testInstallHookCall +} + +type testInstallHookCall struct { + Name string + ModuleAddr string + PackageAddr string + Version *version.Version + LocalPath string +} + +func (h *testInstallHooks) Download(moduleAddr, packageAddr string, version *version.Version) { + h.Calls = append(h.Calls, testInstallHookCall{ + Name: "Download", + ModuleAddr: moduleAddr, + PackageAddr: packageAddr, + Version: version, + }) +} + +func (h *testInstallHooks) Install(moduleAddr string, version *version.Version, localPath string) { + h.Calls = append(h.Calls, testInstallHookCall{ + Name: "Install", + ModuleAddr: moduleAddr, + Version: version, + LocalPath: localPath, + }) +} diff --git a/configs/configload/loader_test.go b/configs/configload/loader_test.go index 512b99f3fa..6ab1fc8c55 100644 --- a/configs/configload/loader_test.go +++ b/configs/configload/loader_test.go @@ -4,11 +4,42 @@ import ( "reflect" "testing" + "github.com/spf13/afero" + "github.com/davecgh/go-spew/spew" "github.com/hashicorp/hcl2/hcl" + "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/registry" "github.com/zclconf/go-cty/cty" ) +// newTestLoader is like NewLoader but it uses a copy-on-write overlay filesystem +// over the real filesystem so that any files that are created cannot persist +// between test runs. +// +// It will also panic if there are any errors creating the loader, since +// these should never happen in a testing scenario. +func newTestLoader(dir string) *Loader { + realFS := afero.NewOsFs() + overlayFS := afero.NewMemMapFs() + fs := afero.NewCopyOnWriteFs(realFS, overlayFS) + parser := configs.NewParser(fs) + reg := registry.NewClient(nil, nil, nil) + ret := &Loader{ + parser: parser, + modules: moduleMgr{ + FS: afero.Afero{fs}, + Dir: dir, + Registry: reg, + }, + } + err := ret.modules.readModuleManifestSnapshot() + if err != nil { + panic(err) + } + return ret +} + func assertNoDiagnostics(t *testing.T, diags hcl.Diagnostics) bool { t.Helper() return assertDiagnosticCount(t, diags, 0) diff --git a/configs/configload/module_manifest.go b/configs/configload/module_manifest.go index 8ecc720a74..777f3a282f 100644 --- a/configs/configload/module_manifest.go +++ b/configs/configload/module_manifest.go @@ -30,7 +30,7 @@ type moduleRecord struct { // VersionStr is the version specifier string. This is used only for // serialization in snapshots and should not be accessed or updated // by any other codepaths; use "Version" instead. - VersionStr string `json:"Version"` + VersionStr string `json:"Version,omitempty"` // Dir is the path to the local directory where the module is installed. Dir string `json:"Dir"` @@ -115,7 +115,11 @@ func (m *moduleMgr) writeModuleManifestSnapshot() error { for _, record := range m.manifest { // Make sure VersionStr is in sync with Version, since we encourage // callers to manipulate Version and ignore VersionStr. - record.VersionStr = record.Version.String() + if record.Version != nil { + record.VersionStr = record.Version.String() + } else { + record.VersionStr = "" + } write.Records = append(write.Records, record) } diff --git a/configs/configload/source_addr.go b/configs/configload/source_addr.go new file mode 100644 index 0000000000..7767859c57 --- /dev/null +++ b/configs/configload/source_addr.go @@ -0,0 +1,28 @@ +package configload + +import ( + "strings" + + "github.com/hashicorp/terraform/registry/regsrc" +) + +var localSourcePrefixes = []string{ + "./", + "../", + ".\\", + "..\\", +} + +func isLocalSourceAddr(addr string) bool { + for _, prefix := range localSourcePrefixes { + if strings.HasPrefix(addr, prefix) { + return true + } + } + return false +} + +func isRegistrySourceAddr(addr string) bool { + _, err := regsrc.ParseModuleSource(addr) + return err == nil +} diff --git a/configs/configload/test-fixtures/local-modules/child_a/child_a.tf b/configs/configload/test-fixtures/local-modules/child_a/child_a.tf new file mode 100644 index 0000000000..eb2c0044f2 --- /dev/null +++ b/configs/configload/test-fixtures/local-modules/child_a/child_a.tf @@ -0,0 +1,4 @@ + +module "child_b" { + source = "./child_b" +} diff --git a/configs/configload/test-fixtures/local-modules/child_a/child_b/child_b.tf b/configs/configload/test-fixtures/local-modules/child_a/child_b/child_b.tf new file mode 100644 index 0000000000..4900e9796c --- /dev/null +++ b/configs/configload/test-fixtures/local-modules/child_a/child_b/child_b.tf @@ -0,0 +1,4 @@ + +output "hello" { + value = "Hello from child_b!" +} diff --git a/configs/configload/test-fixtures/local-modules/root.tf b/configs/configload/test-fixtures/local-modules/root.tf new file mode 100644 index 0000000000..1a6721b5df --- /dev/null +++ b/configs/configload/test-fixtures/local-modules/root.tf @@ -0,0 +1,4 @@ + +module "child_a" { + source = "./child_a" +}