Update init to use new graph-based workflow

The init command and the module installer are now using the new
graph-based workflow to install modules instead of using the recursive
BuildConfig.
pull/38232/head
Daniel Banck 3 months ago committed by Daniel Schmidt
parent 11d819048a
commit fb60c15670

@ -49,11 +49,24 @@ func (c *InitCommand) Run(args []string) int {
var diags tfdiags.Diagnostics
args = c.Meta.process(args)
initArgs, initDiags := arguments.ParseInit(args, c.Meta.AllowExperimentalFeatures)
diags = diags.Append(initDiags)
view := views.NewInit(initArgs.ViewType, c.View)
if initDiags.HasErrors() {
diags = diags.Append(initDiags)
loader, err := c.initConfigLoader()
if err != nil {
diags = diags.Append(err)
view.Diagnostics(diags)
return 1
}
var varDiags tfdiags.Diagnostics
c.VariableValues, varDiags = initArgs.Vars.CollectValues(func(filename string, src []byte) {
loader.Parser().ForceFileSource(filename, src)
})
diags = diags.Append(varDiags)
if diags.HasErrors() {
view.Diagnostics(diags)
return 1
}

@ -186,6 +186,8 @@ type Meta struct {
// flag is set, to reinforce that experiments are not for production use.
AllowExperimentalFeatures bool
VariableValues map[string]arguments.UnparsedVariableValue
//----------------------------------------------------------
// Protected: commands can set these
//----------------------------------------------------------
@ -835,7 +837,7 @@ func (m *Meta) checkRequiredVersion() tfdiags.Diagnostics {
return diags
}
config, configDiags := loader.LoadConfig(pwd)
config, configDiags := m.loadConfig(pwd)
if configDiags.HasErrors() {
diags = diags.Append(configDiags)
return diags

@ -201,7 +201,21 @@ func (m *Meta) installModules(ctx context.Context, rootDir, testsDir string, upg
return true, diags
}
inst := initwd.NewModuleInstaller(m.modulesDir(), loader, m.registryClient())
initializer := func(rootMod *configs.Module, walker configs.ModuleWalker) (*configs.Config, tfdiags.Diagnostics) {
variables, diags := backendrun.ParseConstVariableValues(m.VariableValues, rootMod.Variables)
ctx, ctxDiags := terraform.NewContext(&terraform.ContextOpts{
Parallelism: 1,
})
diags = diags.Append(ctxDiags)
if diags.HasErrors() {
return nil, diags
}
return ctx.Init(rootMod, terraform.InitOpts{
Walker: walker,
SetVariables: variables,
})
}
inst := initwd.NewModuleInstaller(m.modulesDir(), loader, m.registryClient(), initializer)
_, moreDiags := inst.InstallModules(ctx, rootDir, testsDir, upgrade, installErrsOnly, hooks)
diags = diags.Append(moreDiags)

@ -6,7 +6,6 @@ package initwd
import (
"context"
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
@ -19,6 +18,7 @@ import (
"github.com/hashicorp/terraform/internal/copy"
"github.com/hashicorp/terraform/internal/getmodules"
"github.com/hashicorp/terraform/internal/getmodules/moduleaddrs"
"github.com/zclconf/go-cty/cty"
version "github.com/hashicorp/go-version"
"github.com/hashicorp/terraform/internal/modsdir"
@ -59,7 +59,7 @@ func DirFromModule(ctx context.Context, loader *configload.Loader, rootDir, modu
// The target directory must exist but be empty.
{
entries, err := ioutil.ReadDir(rootDir)
entries, err := os.ReadDir(rootDir)
if err != nil {
if os.IsNotExist(err) {
diags = diags.Append(tfdiags.Sourceless(
@ -94,7 +94,7 @@ func DirFromModule(ctx context.Context, loader *configload.Loader, rootDir, modu
}
instDir := filepath.Join(rootDir, ".terraform/init-from-module")
inst := NewModuleInstaller(instDir, loader, reg)
inst := NewModuleInstaller(instDir, loader, reg, nil)
log.Printf("[DEBUG] installing modules in %s to initialize working directory from %q", instDir, sourceAddrStr)
os.RemoveAll(instDir) // if this fails then we'll fail on MkdirAll below too
err := os.MkdirAll(instDir, os.ModePerm)
@ -129,24 +129,17 @@ func DirFromModule(ctx context.Context, loader *configload.Loader, rootDir, modu
// Now we need to create an artificial root module that will seed our
// installation process.
sourceAddr, err := moduleaddrs.ParseModuleSource(sourceAddrStr)
if err != nil {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Invalid module source address",
fmt.Sprintf("Failed to parse module source address: %s", err),
))
fakeRange := hcl.Range{
Filename: initFromModuleRootFilename,
Start: hcl.InitialPos,
End: hcl.InitialPos,
}
fakeRootModule := &configs.Module{
ModuleCalls: map[string]*configs.ModuleCall{
initFromModuleRootCallName: {
Name: initFromModuleRootCallName,
SourceAddr: sourceAddr,
DeclRange: hcl.Range{
Filename: initFromModuleRootFilename,
Start: hcl.InitialPos,
End: hcl.InitialPos,
},
SourceExpr: hcl.StaticExpr(cty.StringVal(sourceAddrStr), fakeRange),
DeclRange: fakeRange,
},
},
ProviderRequirements: &configs.RequiredProviders{},
@ -167,11 +160,20 @@ func DirFromModule(ctx context.Context, loader *configload.Loader, rootDir, modu
fetcher := getmodules.NewPackageFetcher()
walker := inst.moduleInstallWalker(ctx, instManifest, true, wrapHooks, fetcher)
_, cDiags := inst.installDescendantModules(fakeRootModule, instManifest, walker, true)
_, cDiags := inst.installDescendantModules(fakeRootModule, walker, true)
if cDiags.HasErrors() {
return diags.Append(cDiags)
}
err = instManifest.WriteSnapshotToDir(instDir)
if err != nil {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Failed to update module manifest",
fmt.Sprintf("Unable to write the module manifest file: %s", err),
))
}
// If all of that succeeded then we'll now migrate what was installed
// into the final directory structure.
err = os.MkdirAll(modulesDir, os.ModePerm)
@ -215,9 +217,12 @@ func DirFromModule(ctx context.Context, loader *configload.Loader, rootDir, modu
mod, _ := loader.Parser().LoadConfigDir(rootDir) // ignore diagnostics since we're just doing value-add here anyway
if mod != nil {
for _, mc := range mod.ModuleCalls {
if pathTraversesUp(mc.SourceAddrRaw) {
// TODO improve this
sourceVal, _ := mc.SourceExpr.Value(nil)
sourceRaw := sourceVal.AsString()
if pathTraversesUp(sourceRaw) {
packageAddr, givenSubdir := moduleaddrs.SplitPackageSubdir(sourceAddrStr)
newSubdir := filepath.Join(givenSubdir, mc.SourceAddrRaw)
newSubdir := filepath.Join(givenSubdir, sourceRaw)
if pathTraversesUp(newSubdir) {
// This should never happen in any reasonable
// configuration since this suggests a path that

@ -107,7 +107,7 @@ func TestDirFromModule_registry(t *testing.T) {
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
config, loadDiags := loader.LoadConfig(".")
config, loadDiags := loader.LoadStaticConfig(".")
tfdiags.AssertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags))
wantTraces := map[string]string{
@ -187,7 +187,7 @@ func TestDirFromModule_submodules(t *testing.T) {
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
config, loadDiags := loader.LoadConfig(".")
config, loadDiags := loader.LoadStaticConfig(".")
tfdiags.AssertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags))
wantTraces := map[string]string{
@ -312,7 +312,7 @@ func TestDirFromModule_rel_submodules(t *testing.T) {
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
config, loadDiags := loader.LoadConfig(".")
config, loadDiags := loader.LoadStaticConfig(".")
tfdiags.AssertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags))
wantTraces := map[string]string{

@ -30,6 +30,8 @@ import (
"github.com/hashicorp/terraform/internal/tfdiags"
)
type Initializer func(rootMod *configs.Module, walker configs.ModuleWalker) (*configs.Config, tfdiags.Diagnostics)
type ModuleInstaller struct {
modsDir string
loader *configload.Loader
@ -42,6 +44,8 @@ type ModuleInstaller struct {
// The keys in moduleVersionsUrl are the moduleVersion struct below and
// addresses and the values are underlying remote source addresses.
registryPackageSources map[moduleVersion]addrs.ModuleSourceRemote
initializer Initializer
}
type moduleVersion struct {
@ -49,13 +53,14 @@ type moduleVersion struct {
version string
}
func NewModuleInstaller(modsDir string, loader *configload.Loader, reg *registry.Client) *ModuleInstaller {
func NewModuleInstaller(modsDir string, loader *configload.Loader, reg *registry.Client, initializer Initializer) *ModuleInstaller {
return &ModuleInstaller{
modsDir: modsDir,
loader: loader,
reg: reg,
registryPackageVersions: make(map[addrs.ModuleRegistryPackage]*response.ModuleVersions),
registryPackageSources: make(map[moduleVersion]addrs.ModuleSourceRemote),
initializer: initializer,
}
}
@ -137,8 +142,31 @@ func (i *ModuleInstaller) InstallModules(ctx context.Context, rootDir, testsDir
}
walker := i.moduleInstallWalker(ctx, manifest, upgrade, hooks, fetcher)
cfg, instDiags := i.installDescendantModules(rootMod, manifest, walker, installErrsOnly)
diags = append(diags, instDiags...)
var cfg *configs.Config
var instDiags tfdiags.Diagnostics
if i.initializer != nil {
cfg, instDiags = i.initializer(rootMod, walker)
diags = diags.Append(instDiags)
} else {
cfg, instDiags = i.installDescendantModules(rootMod, walker, installErrsOnly)
diags = diags.Append(instDiags)
}
finalDiags := configs.FinalizeConfig(cfg, walker, configs.MockDataLoaderFunc(i.loader.LoadExternalMockData))
diags = diags.Append(finalDiags)
if diags.HasErrors() {
return nil, diags
}
err = manifest.WriteSnapshotToDir(i.modsDir)
if err != nil {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Failed to update module manifest",
fmt.Sprintf("Unable to write the module manifest file: %s", err),
))
}
return cfg, diags
}
@ -292,7 +320,7 @@ func (i *ModuleInstaller) moduleInstallWalker(ctx context.Context, manifest mods
)
}
func (i *ModuleInstaller) installDescendantModules(rootMod *configs.Module, manifest modsdir.Manifest, installWalker configs.ModuleWalker, installErrsOnly bool) (*configs.Config, tfdiags.Diagnostics) {
func (i *ModuleInstaller) installDescendantModules(rootMod *configs.Module, installWalker configs.ModuleWalker, installErrsOnly bool) (*configs.Config, tfdiags.Diagnostics) {
var diags tfdiags.Diagnostics
// When attempting to initialize the current directory with a module
@ -332,15 +360,6 @@ func (i *ModuleInstaller) installDescendantModules(rootMod *configs.Module, mani
diags = tfdiags.OverrideAll(diags, tfdiags.Warning, nil)
}
err := manifest.WriteSnapshotToDir(i.modsDir)
if err != nil {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Failed to update module manifest",
fmt.Sprintf("Unable to write the module manifest file: %s", err),
))
}
return cfg, diags
}

@ -45,7 +45,7 @@ func TestModuleInstaller(t *testing.T) {
modulesDir := filepath.Join(dir, ".terraform/modules")
loader, close := configload.NewLoaderForTests(t)
defer close()
inst := NewModuleInstaller(modulesDir, loader, nil)
inst := NewModuleInstaller(modulesDir, loader, nil, nil)
_, diags := inst.InstallModules(context.Background(), ".", "tests", false, false, hooks)
tfdiags.AssertNoDiagnostics(t, diags)
@ -77,7 +77,7 @@ func TestModuleInstaller(t *testing.T) {
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
config, loadDiags := loader.LoadConfig(".")
config, loadDiags := loader.LoadStaticConfig(".")
tfdiags.AssertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags))
wantTraces := map[string]string{
@ -109,7 +109,7 @@ func TestModuleInstaller_error(t *testing.T) {
loader, close := configload.NewLoaderForTests(t)
defer close()
inst := NewModuleInstaller(modulesDir, loader, nil)
inst := NewModuleInstaller(modulesDir, loader, nil, nil)
_, diags := inst.InstallModules(context.Background(), ".", "tests", false, false, hooks)
if !diags.HasErrors() {
@ -130,7 +130,7 @@ func TestModuleInstaller_emptyModuleName(t *testing.T) {
loader, close := configload.NewLoaderForTests(t)
defer close()
inst := NewModuleInstaller(modulesDir, loader, nil)
inst := NewModuleInstaller(modulesDir, loader, nil, nil)
_, diags := inst.InstallModules(context.Background(), ".", "tests", false, false, hooks)
if !diags.HasErrors() {
@ -151,7 +151,7 @@ func TestModuleInstaller_invalidModuleName(t *testing.T) {
loader, close := configload.NewLoaderForTests(t)
defer close()
inst := NewModuleInstaller(modulesDir, loader, registry.NewClient(nil, nil))
inst := NewModuleInstaller(modulesDir, loader, registry.NewClient(nil, nil), nil)
_, diags := inst.InstallModules(context.Background(), dir, "tests", false, false, hooks)
if !diags.HasErrors() {
@ -189,7 +189,7 @@ func TestModuleInstaller_packageEscapeError(t *testing.T) {
loader, close := configload.NewLoaderForTests(t)
defer close()
inst := NewModuleInstaller(modulesDir, loader, nil)
inst := NewModuleInstaller(modulesDir, loader, nil, nil)
_, diags := inst.InstallModules(context.Background(), ".", "tests", false, false, hooks)
if !diags.HasErrors() {
@ -227,7 +227,7 @@ func TestModuleInstaller_explicitPackageBoundary(t *testing.T) {
loader, close := configload.NewLoaderForTests(t)
defer close()
inst := NewModuleInstaller(modulesDir, loader, nil)
inst := NewModuleInstaller(modulesDir, loader, nil, nil)
_, diags := inst.InstallModules(context.Background(), ".", "tests", false, false, hooks)
if diags.HasErrors() {
@ -250,7 +250,7 @@ func TestModuleInstaller_ExactMatchPrerelease(t *testing.T) {
loader, close := configload.NewLoaderForTests(t)
defer close()
inst := NewModuleInstaller(modulesDir, loader, registry.NewClient(nil, nil))
inst := NewModuleInstaller(modulesDir, loader, registry.NewClient(nil, nil), nil)
cfg, diags := inst.InstallModules(context.Background(), ".", "tests", false, false, hooks)
if diags.HasErrors() {
@ -277,7 +277,7 @@ func TestModuleInstaller_PartialMatchPrerelease(t *testing.T) {
loader, close := configload.NewLoaderForTests(t)
defer close()
inst := NewModuleInstaller(modulesDir, loader, registry.NewClient(nil, nil))
inst := NewModuleInstaller(modulesDir, loader, registry.NewClient(nil, nil), nil)
cfg, diags := inst.InstallModules(context.Background(), ".", "tests", false, false, hooks)
if diags.HasErrors() {
@ -300,7 +300,7 @@ func TestModuleInstaller_invalid_version_constraint_error(t *testing.T) {
loader, close := configload.NewLoaderForTests(t)
defer close()
inst := NewModuleInstaller(modulesDir, loader, nil)
inst := NewModuleInstaller(modulesDir, loader, nil, nil)
_, diags := inst.InstallModules(context.Background(), ".", "tests", false, false, hooks)
if !diags.HasErrors() {
@ -326,7 +326,7 @@ func TestModuleInstaller_invalidVersionConstraintGetter(t *testing.T) {
loader, close := configload.NewLoaderForTests(t)
defer close()
inst := NewModuleInstaller(modulesDir, loader, nil)
inst := NewModuleInstaller(modulesDir, loader, nil, nil)
_, diags := inst.InstallModules(context.Background(), ".", "tests", false, false, hooks)
if !diags.HasErrors() {
@ -352,7 +352,7 @@ func TestModuleInstaller_invalidVersionConstraintLocal(t *testing.T) {
loader, close := configload.NewLoaderForTests(t)
defer close()
inst := NewModuleInstaller(modulesDir, loader, nil)
inst := NewModuleInstaller(modulesDir, loader, nil, nil)
_, diags := inst.InstallModules(context.Background(), ".", "tests", false, false, hooks)
if !diags.HasErrors() {
@ -378,7 +378,7 @@ func TestModuleInstaller_symlink(t *testing.T) {
loader, close := configload.NewLoaderForTests(t)
defer close()
inst := NewModuleInstaller(modulesDir, loader, nil)
inst := NewModuleInstaller(modulesDir, loader, nil, nil)
_, diags := inst.InstallModules(context.Background(), ".", "tests", false, false, hooks)
tfdiags.AssertNoDiagnostics(t, diags)
@ -410,7 +410,7 @@ func TestModuleInstaller_symlink(t *testing.T) {
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
config, loadDiags := loader.LoadConfig(".")
config, loadDiags := loader.LoadStaticConfig(".")
tfdiags.AssertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags))
wantTraces := map[string]string{
@ -454,7 +454,7 @@ func TestLoaderInstallModules_invalidRegistry(t *testing.T) {
loader, close := configload.NewLoaderForTests(t)
defer close()
inst := NewModuleInstaller(modulesDir, loader, registry.NewClient(nil, nil))
inst := NewModuleInstaller(modulesDir, loader, registry.NewClient(nil, nil), nil)
_, diags := inst.InstallModules(context.Background(), dir, "tests", false, false, hooks)
if !diags.HasErrors() {
@ -493,7 +493,7 @@ func TestLoaderInstallModules_registry(t *testing.T) {
loader, close := configload.NewLoaderForTests(t)
defer close()
inst := NewModuleInstaller(modulesDir, loader, registry.NewClient(nil, nil))
inst := NewModuleInstaller(modulesDir, loader, registry.NewClient(nil, nil), nil)
_, diags := inst.InstallModules(context.Background(), dir, "tests", false, false, hooks)
tfdiags.AssertNoDiagnostics(t, diags)
@ -608,7 +608,7 @@ func TestLoaderInstallModules_registry(t *testing.T) {
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
config, loadDiags := loader.LoadConfig(".")
config, loadDiags := loader.LoadStaticConfig(".")
tfdiags.AssertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags))
wantTraces := map[string]string{
@ -656,7 +656,7 @@ func TestLoaderInstallModules_goGetter(t *testing.T) {
loader, close := configload.NewLoaderForTests(t)
defer close()
inst := NewModuleInstaller(modulesDir, loader, registry.NewClient(nil, nil))
inst := NewModuleInstaller(modulesDir, loader, registry.NewClient(nil, nil), nil)
_, diags := inst.InstallModules(context.Background(), dir, "tests", false, false, hooks)
tfdiags.AssertNoDiagnostics(t, diags)
@ -738,7 +738,7 @@ func TestLoaderInstallModules_goGetter(t *testing.T) {
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
config, loadDiags := loader.LoadConfig(".")
config, loadDiags := loader.LoadStaticConfig(".")
tfdiags.AssertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags))
wantTraces := map[string]string{
@ -774,7 +774,7 @@ func TestModuleInstaller_fromTests(t *testing.T) {
modulesDir := filepath.Join(dir, ".terraform/modules")
loader, close := configload.NewLoaderForTests(t)
defer close()
inst := NewModuleInstaller(modulesDir, loader, nil)
inst := NewModuleInstaller(modulesDir, loader, nil, nil)
_, diags := inst.InstallModules(context.Background(), ".", "tests", false, false, hooks)
tfdiags.AssertNoDiagnostics(t, diags)
@ -800,7 +800,7 @@ func TestModuleInstaller_fromTests(t *testing.T) {
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
config, loadDiags := loader.LoadConfigWithTests(".", "tests")
config, loadDiags := loader.LoadStaticConfigWithTests(".", "tests")
tfdiags.AssertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags))
if config.Module.Tests["tests/main.tftest.hcl"].Runs[0].ConfigUnderTest == nil {
@ -831,7 +831,7 @@ func TestLoadInstallModules_registryFromTest(t *testing.T) {
loader, close := configload.NewLoaderForTests(t)
defer close()
inst := NewModuleInstaller(modulesDir, loader, registry.NewClient(nil, nil))
inst := NewModuleInstaller(modulesDir, loader, registry.NewClient(nil, nil), nil)
_, diags := inst.InstallModules(context.Background(), dir, "tests", false, false, hooks)
tfdiags.AssertNoDiagnostics(t, diags)
@ -909,7 +909,7 @@ func TestLoadInstallModules_registryFromTest(t *testing.T) {
// Make sure the configuration is loadable now.
// (This ensures that correct information is recorded in the manifest.)
config, loadDiags := loader.LoadConfigWithTests(".", "tests")
config, loadDiags := loader.LoadStaticConfigWithTests(".", "tests")
tfdiags.AssertNoDiagnostics(t, tfdiags.Diagnostics{}.Append(loadDiags))
if config.Module.Tests["main.tftest.hcl"].Runs[0].ConfigUnderTest == nil {

@ -37,7 +37,7 @@ func LoadConfigForTests(t *testing.T, rootDir string, testsDir string) (*configs
var diags tfdiags.Diagnostics
loader, cleanup := configload.NewLoaderForTests(t)
inst := NewModuleInstaller(loader.ModulesDir(), loader, registry.NewClient(nil, nil))
inst := NewModuleInstaller(loader.ModulesDir(), loader, registry.NewClient(nil, nil), nil)
_, moreDiags := inst.InstallModules(context.Background(), rootDir, testsDir, true, false, ModuleInstallHooksImpl{})
diags = diags.Append(moreDiags)
@ -53,7 +53,7 @@ func LoadConfigForTests(t *testing.T, rootDir string, testsDir string) (*configs
t.Fatalf("failed to refresh modules after installation: %s", err)
}
config, hclDiags := loader.LoadConfigWithTests(rootDir, testsDir)
config, hclDiags := loader.LoadStaticConfigWithTests(rootDir, testsDir)
diags = diags.Append(hclDiags)
return config, loader, cleanup, diags
}

Loading…
Cancel
Save