From 902be52976e2851c777797b37a15a5f8e5515d3e Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 27 Mar 2018 17:22:51 -0700 Subject: [PATCH] terraform: HCL2-flavored module dependency resolver For the moment this is just a lightly-adapted copy of ModuleTreeDependencies named ConfigTreeDependencies, with the goal that the two can live concurrently for the moment while not all callers are yet updated and then we can drop ModuleTreeDependencies and its helper functions altogether in a later commit. This can then be used to make "terraform init" and "terraform providers" work properly with the HCL2-powered configuration loader. --- command/init.go | 62 +++++---- command/providers.go | 8 +- configs/resource.go | 43 +++++++ plugin/discovery/version_set.go | 5 + terraform/module_dependencies.go | 212 ++++++++++++++++++++++++++----- 5 files changed, 268 insertions(+), 62 deletions(-) diff --git a/command/init.go b/command/init.go index 2228849a6c..6718a5884c 100644 --- a/command/init.go +++ b/command/init.go @@ -7,7 +7,6 @@ import ( "sort" "strings" - multierror "github.com/hashicorp/go-multierror" "github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/terraform/backend" backendinit "github.com/hashicorp/terraform/backend/init" @@ -145,6 +144,8 @@ func (c *InitCommand) Run(args []string) int { c.showDiagnostics(diags) return 1 } + + c.Ui.Output("") } // If our directory is empty, then we're done. We can't get or setup @@ -290,10 +291,10 @@ func (c *InitCommand) Run(args []string) int { } // Now that we have loaded all modules, check the module tree for missing providers. - err = c.getProviders(path, state, flagUpgrade) - if err != nil { - // this function provides its own output - log.Printf("[ERROR] %s", err) + providerDiags := c.getProviders(path, state, flagUpgrade) + diags = diags.Append(providerDiags) + if providerDiags.HasErrors() { + c.showDiagnostics(diags) return 1 } @@ -303,6 +304,11 @@ func (c *InitCommand) Run(args []string) int { c.Ui.Output("") } + // If we accumulated any warnings along the way that weren't accompanied + // by errors then we'll output them here so that the success message is + // still the final thing shown. + c.showDiagnostics(diags) + c.Ui.Output(c.Colorize().Color(strings.TrimSpace(outputInitSuccess))) if !c.RunningInAutomation { // If we're not running in an automation wrapper, give the user @@ -385,24 +391,25 @@ func (c *InitCommand) backendConfigOverrideBody(flags rawFlags, schema *configsc // Load the complete module tree, and fetch any missing providers. // This method outputs its own Ui. -func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade bool) error { - mod, diags := c.Module(path) +func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade bool) tfdiags.Diagnostics { + config, diags := c.loadConfig(path) if diags.HasErrors() { - c.showDiagnostics(diags) - return diags.Err() + return diags } if err := terraform.CheckStateVersion(state); err != nil { diags = diags.Append(err) - c.showDiagnostics(diags) - return err + return diags } - if err := terraform.CheckRequiredVersion(mod); err != nil { - diags = diags.Append(err) - c.showDiagnostics(diags) - return err - } + // FIXME: Restore this once terraform.CheckRequiredVersion is updated to + // work with a configs.Config instead of a legacy module.Tree. + /* + if err := terraform.CheckRequiredVersion(mod); err != nil { + diags = diags.Append(err) + return diags + } + */ var available discovery.PluginMetaSet if upgrade { @@ -413,7 +420,7 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade available = c.providerPluginSet() } - requirements := terraform.ModuleTreeDependencies(mod, state).AllPluginRequirements() + requirements := terraform.ConfigTreeDependencies(config, state).AllPluginRequirements() if len(requirements) == 0 { // nothing to initialize return nil @@ -425,7 +432,6 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade missing := c.missingPlugins(available, requirements) - var errs error if c.getPlugins { if len(missing) > 0 { c.Ui.Output(fmt.Sprintf("- Checking for available provider plugins on %s...", @@ -462,12 +468,12 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade c.Ui.Error(fmt.Sprintf(errProviderInstallError, provider, err.Error(), DefaultPluginVendorDir)) } - errs = multierror.Append(errs, err) + diags = diags.Append(err) } } - if errs != nil { - return errs + if diags.HasErrors() { + return diags } } else if len(missing) > 0 { // we have missing providers, but aren't going to try and download them @@ -478,11 +484,11 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade } else { lines = append(lines, fmt.Sprintf("* %s (%s)\n", provider, reqd.Versions)) } - errs = multierror.Append(errs, fmt.Errorf("missing provider %q", provider)) + diags = diags.Append(fmt.Errorf("missing provider %q", provider)) } sort.Strings(lines) c.Ui.Error(fmt.Sprintf(errMissingProvidersNoInstall, strings.Join(lines, ""), DefaultPluginVendorDir)) - return errs + return diags } // With all the providers downloaded, we'll generate our lock file @@ -498,8 +504,8 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade for name, meta := range chosen { digest, err := meta.SHA256() if err != nil { - c.Ui.Error(fmt.Sprintf("failed to read provider plugin %s: %s", meta.Path, err)) - return err + diags = diags.Append(fmt.Errorf("Failed to read provider plugin %s: %s", meta.Path, err)) + return diags } digests[name] = digest if c.ignorePluginChecksum { @@ -508,8 +514,8 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade } err := c.providerPluginsLock().Write(digests) if err != nil { - c.Ui.Error(fmt.Sprintf("failed to save provider manifest: %s", err)) - return err + diags = diags.Append(fmt.Errorf("failed to save provider manifest: %s", err)) + return diags } { @@ -557,7 +563,7 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade } } - return nil + return diags } func (c *InitCommand) AutocompleteArgs() complete.Predictor { diff --git a/command/providers.go b/command/providers.go index 83341bccdd..97d628715d 100644 --- a/command/providers.go +++ b/command/providers.go @@ -5,6 +5,7 @@ import ( "sort" "github.com/hashicorp/terraform/moduledeps" + "github.com/hashicorp/terraform/terraform" "github.com/hashicorp/terraform/tfdiags" "github.com/xlab/treeprint" ) @@ -69,11 +70,8 @@ func (c *ProvidersCommand) Run(args []string) int { return 1 } - // FIXME: Restore this once the "terraform" package is updated to deal - // with HCL2 config types. - //s := state.State() - //depTree := terraform.ModuleTreeDependencies(config, s) - var depTree *moduledeps.Module + s := state.State() + depTree := terraform.ConfigTreeDependencies(config, s) depTree.SortDescendents() printRoot := treeprint.New() diff --git a/configs/resource.go b/configs/resource.go index c310dbb54e..f9534762e5 100644 --- a/configs/resource.go +++ b/configs/resource.go @@ -2,6 +2,7 @@ package configs import ( "fmt" + "strings" "github.com/hashicorp/hcl2/gohcl" "github.com/hashicorp/hcl2/hcl" @@ -39,6 +40,22 @@ func (r *ManagedResource) moduleUniqueKey() string { return fmt.Sprintf("%s.%s", r.Name, r.Type) } +// ProviderConfigKey returns a string key for the provider configuration +// that should be used for this resource. This function implements the +// default behavior of extracting the type from the resource type name if +// an explicit "provider" argument was not provided. +func (r *ManagedResource) ProviderConfigKey() string { + if r.ProviderConfigRef == nil { + typeName := r.Type + if under := strings.Index(typeName, "_"); under != -1 { + return typeName[:under] + } + return typeName + } + + return r.ProviderConfigRef.String() +} + func decodeResourceBlock(block *hcl.Block) (*ManagedResource, hcl.Diagnostics) { r := &ManagedResource{ Type: block.Labels[0], @@ -234,6 +251,22 @@ func (r *DataResource) moduleUniqueKey() string { return fmt.Sprintf("data.%s.%s", r.Name, r.Type) } +// ProviderConfigKey returns a string key for the provider configuration +// that should be used for this resource. This function implements the +// default behavior of extracting the type from the resource type name if +// an explicit "provider" argument was not provided. +func (r *DataResource) ProviderConfigKey() string { + if r.ProviderConfigRef == nil { + typeName := r.Type + if under := strings.Index(typeName, "_"); under != -1 { + return typeName[:under] + } + return typeName + } + + return r.ProviderConfigRef.String() +} + func decodeDataBlock(block *hcl.Block) (*DataResource, hcl.Diagnostics) { r := &DataResource{ Type: block.Labels[0], @@ -370,6 +403,16 @@ func decodeProviderConfigRef(attr *hcl.Attribute) (*ProviderConfigRef, hcl.Diagn return ret, diags } +func (r *ProviderConfigRef) String() string { + if r == nil { + return "" + } + if r.Alias != "" { + return fmt.Sprintf("%s.%s", r.Name, r.Alias) + } + return r.Name +} + var commonResourceAttributes = []hcl.AttributeSchema{ { Name: "count", diff --git a/plugin/discovery/version_set.go b/plugin/discovery/version_set.go index 0aefd759fd..de02f5ec5b 100644 --- a/plugin/discovery/version_set.go +++ b/plugin/discovery/version_set.go @@ -36,6 +36,11 @@ type Constraints struct { raw version.Constraints } +// NewConstraints creates a Constraints based on a version.Constraints. +func NewConstraints(c version.Constraints) Constraints { + return Constraints{c} +} + // AllVersions is a Constraints containing all versions var AllVersions Constraints diff --git a/terraform/module_dependencies.go b/terraform/module_dependencies.go index 4594cb6033..98fcfd1c56 100644 --- a/terraform/module_dependencies.go +++ b/terraform/module_dependencies.go @@ -1,36 +1,34 @@ package terraform import ( + version "github.com/hashicorp/go-version" "github.com/hashicorp/terraform/config" "github.com/hashicorp/terraform/config/module" + "github.com/hashicorp/terraform/configs" "github.com/hashicorp/terraform/moduledeps" "github.com/hashicorp/terraform/plugin/discovery" ) -// ModuleTreeDependencies returns the dependencies of the tree of modules -// described by the given configuration tree and state. +// ConfigTreeDependencies returns the dependencies of the tree of modules +// described by the given configuration and state. // // Both configuration and state are required because there can be resources // implied by instances in the state that no longer exist in config. -// -// This function will panic if any invalid version constraint strings are -// present in the configuration. This is guaranteed not to happen for any -// configuration that has passed a call to Config.Validate(). -func ModuleTreeDependencies(root *module.Tree, state *State) *moduledeps.Module { +func ConfigTreeDependencies(root *configs.Config, state *State) *moduledeps.Module { // First we walk the configuration tree to build the overall structure // and capture the explicit/implicit/inherited provider dependencies. - deps := moduleTreeConfigDependencies(root, nil) + deps := configTreeConfigDependencies(root, nil) // Next we walk over the resources in the state to catch any additional // dependencies created by existing resources that are no longer in config. // Most things we find in state will already be present in 'deps', but // we're interested in the rare thing that isn't. - moduleTreeMergeStateDependencies(deps, state) + configTreeMergeStateDependencies(deps, state) return deps } -func moduleTreeConfigDependencies(root *module.Tree, inheritProviders map[string]*config.ProviderConfig) *moduledeps.Module { +func configTreeConfigDependencies(root *configs.Config, inheritProviders map[string]*configs.Provider) *moduledeps.Module { if root == nil { // If no config is provided, we'll make a synthetic root. // This isn't necessarily correct if we're called with a nil that @@ -40,37 +38,88 @@ func moduleTreeConfigDependencies(root *module.Tree, inheritProviders map[string } } + name := "root" + if len(root.Path) != 0 { + name = root.Path[len(root.Path)-1] + } + ret := &moduledeps.Module{ - Name: root.Name(), + Name: name, } - cfg := root.Config() - providerConfigs := cfg.ProviderConfigsByFullName() + module := root.Module // Provider dependencies { - providers := make(moduledeps.Providers, len(providerConfigs)) + providers := make(moduledeps.Providers) - // Any providerConfigs elements are *explicit* provider dependencies, - // which is the only situation where the user might provide an actual - // version constraint. We'll take care of these first. - for fullName, pCfg := range providerConfigs { + // The main way to declare a provider dependency is explicitly inside + // the "terraform" block, which allows declaring a requirement without + // also creating a configuration. + for fullName, constraints := range module.ProviderRequirements { inst := moduledeps.ProviderInstance(fullName) - versionSet := discovery.AllVersions - if pCfg.Version != "" { - versionSet = discovery.ConstraintStr(pCfg.Version).MustParse() + + // The handling here is a bit fiddly because the moduledeps package + // was designed around the legacy (pre-0.12) configuration model + // and hasn't yet been revised to handle the new model. As a result, + // we need to do some translation here. + // FIXME: Eventually we should adjust the underlying model so we + // can also retain the source location of each constraint, for + // more informative output from the "terraform providers" command. + var rawConstraints version.Constraints + for _, constraint := range constraints { + rawConstraints = append(rawConstraints, constraint.Required...) } + discoConstraints := discovery.NewConstraints(rawConstraints) + providers[inst] = moduledeps.ProviderDependency{ - Constraints: versionSet, + Constraints: discoConstraints, Reason: moduledeps.ProviderDependencyExplicit, } } + // Provider configurations can also include version constraints, + // allowing for more terse declaration in situations where both a + // configuration and a constraint are defined in the same module. + for fullName, pCfg := range module.ProviderConfigs { + inst := moduledeps.ProviderInstance(fullName) + discoConstraints := discovery.AllVersions + if pCfg.Version.Required != nil { + discoConstraints = discovery.NewConstraints(pCfg.Version.Required) + } + if existing, exists := providers[inst]; exists { + existing.Constraints = existing.Constraints.Append(discoConstraints) + } else { + providers[inst] = moduledeps.ProviderDependency{ + Constraints: discoConstraints, + Reason: moduledeps.ProviderDependencyExplicit, + } + } + } + // Each resource in the configuration creates an *implicit* provider // dependency, though we'll only record it if there isn't already // an explicit dependency on the same provider. - for _, rc := range cfg.Resources { - fullName := rc.ProviderFullName() + for _, rc := range module.ManagedResources { + fullName := rc.ProviderConfigKey() + inst := moduledeps.ProviderInstance(fullName) + if _, exists := providers[inst]; exists { + // Explicit dependency already present + continue + } + + reason := moduledeps.ProviderDependencyImplicit + if _, inherited := inheritProviders[fullName]; inherited { + reason = moduledeps.ProviderDependencyInherited + } + + providers[inst] = moduledeps.ProviderDependency{ + Constraints: discovery.AllVersions, + Reason: reason, + } + } + for _, rc := range module.DataResources { + fullName := rc.ProviderConfigKey() inst := moduledeps.ProviderInstance(fullName) if _, exists := providers[inst]; exists { // Explicit dependency already present @@ -91,21 +140,21 @@ func moduleTreeConfigDependencies(root *module.Tree, inheritProviders map[string ret.Providers = providers } - childInherit := make(map[string]*config.ProviderConfig) + childInherit := make(map[string]*configs.Provider) for k, v := range inheritProviders { childInherit[k] = v } - for k, v := range providerConfigs { + for k, v := range module.ProviderConfigs { childInherit[k] = v } - for _, c := range root.Children() { - ret.Children = append(ret.Children, moduleTreeConfigDependencies(c, childInherit)) + for _, c := range root.Children { + ret.Children = append(ret.Children, configTreeConfigDependencies(c, childInherit)) } return ret } -func moduleTreeMergeStateDependencies(root *moduledeps.Module, state *State) { +func configTreeMergeStateDependencies(root *moduledeps.Module, state *State) { if state == nil { return } @@ -151,5 +200,110 @@ func moduleTreeMergeStateDependencies(root *moduledeps.Module, state *State) { } } } +} +// ModuleTreeDependencies returns the dependencies of the tree of modules +// described by the given configuration tree and state. +// +// Both configuration and state are required because there can be resources +// implied by instances in the state that no longer exist in config. +// +// This function will panic if any invalid version constraint strings are +// present in the configuration. This is guaranteed not to happen for any +// configuration that has passed a call to Config.Validate(). +func ModuleTreeDependencies(root *module.Tree, state *State) *moduledeps.Module { + // First we walk the configuration tree to build the overall structure + // and capture the explicit/implicit/inherited provider dependencies. + deps := moduleTreeConfigDependencies(root, nil) + + // Next we walk over the resources in the state to catch any additional + // dependencies created by existing resources that are no longer in config. + // Most things we find in state will already be present in 'deps', but + // we're interested in the rare thing that isn't. + moduleTreeMergeStateDependencies(deps, state) + + return deps +} + +func moduleTreeConfigDependencies(root *module.Tree, inheritProviders map[string]*config.ProviderConfig) *moduledeps.Module { + if root == nil { + // If no config is provided, we'll make a synthetic root. + // This isn't necessarily correct if we're called with a nil that + // *isn't* at the root, but in practice that can never happen. + return &moduledeps.Module{ + Name: "root", + } + } + + ret := &moduledeps.Module{ + Name: root.Name(), + } + + cfg := root.Config() + providerConfigs := cfg.ProviderConfigsByFullName() + + // Provider dependencies + { + providers := make(moduledeps.Providers, len(providerConfigs)) + + // Any providerConfigs elements are *explicit* provider dependencies, + // which is the only situation where the user might provide an actual + // version constraint. We'll take care of these first. + for fullName, pCfg := range providerConfigs { + inst := moduledeps.ProviderInstance(fullName) + versionSet := discovery.AllVersions + if pCfg.Version != "" { + versionSet = discovery.ConstraintStr(pCfg.Version).MustParse() + } + providers[inst] = moduledeps.ProviderDependency{ + Constraints: versionSet, + Reason: moduledeps.ProviderDependencyExplicit, + } + } + + // Each resource in the configuration creates an *implicit* provider + // dependency, though we'll only record it if there isn't already + // an explicit dependency on the same provider. + for _, rc := range cfg.Resources { + fullName := rc.ProviderFullName() + inst := moduledeps.ProviderInstance(fullName) + if _, exists := providers[inst]; exists { + // Explicit dependency already present + continue + } + + reason := moduledeps.ProviderDependencyImplicit + if _, inherited := inheritProviders[fullName]; inherited { + reason = moduledeps.ProviderDependencyInherited + } + + providers[inst] = moduledeps.ProviderDependency{ + Constraints: discovery.AllVersions, + Reason: reason, + } + } + + ret.Providers = providers + } + + childInherit := make(map[string]*config.ProviderConfig) + for k, v := range inheritProviders { + childInherit[k] = v + } + for k, v := range providerConfigs { + childInherit[k] = v + } + for _, c := range root.Children() { + ret.Children = append(ret.Children, moduleTreeConfigDependencies(c, childInherit)) + } + + return ret +} + +func moduleTreeMergeStateDependencies(root *moduledeps.Module, state *State) { + // This is really just the same logic as configTreeMergeStateDependencies + // but we retain this old name just to keep the symmetry until we've + // removed all of these "moduleTree..." versions that use the legacy + // configuration structs. + configTreeMergeStateDependencies(root, state) }