terraform: HCL2-flavored module dependency resolver

For the moment this is just a lightly-adapted copy of
ModuleTreeDependencies named ConfigTreeDependencies, with the goal that
the two can live concurrently for the moment while not all callers are yet
updated and then we can drop ModuleTreeDependencies and its helper
functions altogether in a later commit.

This can then be used to make "terraform init" and "terraform providers"
work properly with the HCL2-powered configuration loader.
f-hcl2-planstate
Martin Atkins 8 years ago
parent 8edfe2a84f
commit 902be52976

@ -7,7 +7,6 @@ import (
"sort"
"strings"
multierror "github.com/hashicorp/go-multierror"
"github.com/hashicorp/hcl2/hcl"
"github.com/hashicorp/terraform/backend"
backendinit "github.com/hashicorp/terraform/backend/init"
@ -145,6 +144,8 @@ func (c *InitCommand) Run(args []string) int {
c.showDiagnostics(diags)
return 1
}
c.Ui.Output("")
}
// If our directory is empty, then we're done. We can't get or setup
@ -290,10 +291,10 @@ func (c *InitCommand) Run(args []string) int {
}
// Now that we have loaded all modules, check the module tree for missing providers.
err = c.getProviders(path, state, flagUpgrade)
if err != nil {
// this function provides its own output
log.Printf("[ERROR] %s", err)
providerDiags := c.getProviders(path, state, flagUpgrade)
diags = diags.Append(providerDiags)
if providerDiags.HasErrors() {
c.showDiagnostics(diags)
return 1
}
@ -303,6 +304,11 @@ func (c *InitCommand) Run(args []string) int {
c.Ui.Output("")
}
// If we accumulated any warnings along the way that weren't accompanied
// by errors then we'll output them here so that the success message is
// still the final thing shown.
c.showDiagnostics(diags)
c.Ui.Output(c.Colorize().Color(strings.TrimSpace(outputInitSuccess)))
if !c.RunningInAutomation {
// If we're not running in an automation wrapper, give the user
@ -385,24 +391,25 @@ func (c *InitCommand) backendConfigOverrideBody(flags rawFlags, schema *configsc
// Load the complete module tree, and fetch any missing providers.
// This method outputs its own Ui.
func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade bool) error {
mod, diags := c.Module(path)
func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade bool) tfdiags.Diagnostics {
config, diags := c.loadConfig(path)
if diags.HasErrors() {
c.showDiagnostics(diags)
return diags.Err()
return diags
}
if err := terraform.CheckStateVersion(state); err != nil {
diags = diags.Append(err)
c.showDiagnostics(diags)
return err
return diags
}
if err := terraform.CheckRequiredVersion(mod); err != nil {
diags = diags.Append(err)
c.showDiagnostics(diags)
return err
}
// FIXME: Restore this once terraform.CheckRequiredVersion is updated to
// work with a configs.Config instead of a legacy module.Tree.
/*
if err := terraform.CheckRequiredVersion(mod); err != nil {
diags = diags.Append(err)
return diags
}
*/
var available discovery.PluginMetaSet
if upgrade {
@ -413,7 +420,7 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
available = c.providerPluginSet()
}
requirements := terraform.ModuleTreeDependencies(mod, state).AllPluginRequirements()
requirements := terraform.ConfigTreeDependencies(config, state).AllPluginRequirements()
if len(requirements) == 0 {
// nothing to initialize
return nil
@ -425,7 +432,6 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
missing := c.missingPlugins(available, requirements)
var errs error
if c.getPlugins {
if len(missing) > 0 {
c.Ui.Output(fmt.Sprintf("- Checking for available provider plugins on %s...",
@ -462,12 +468,12 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
c.Ui.Error(fmt.Sprintf(errProviderInstallError, provider, err.Error(), DefaultPluginVendorDir))
}
errs = multierror.Append(errs, err)
diags = diags.Append(err)
}
}
if errs != nil {
return errs
if diags.HasErrors() {
return diags
}
} else if len(missing) > 0 {
// we have missing providers, but aren't going to try and download them
@ -478,11 +484,11 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
} else {
lines = append(lines, fmt.Sprintf("* %s (%s)\n", provider, reqd.Versions))
}
errs = multierror.Append(errs, fmt.Errorf("missing provider %q", provider))
diags = diags.Append(fmt.Errorf("missing provider %q", provider))
}
sort.Strings(lines)
c.Ui.Error(fmt.Sprintf(errMissingProvidersNoInstall, strings.Join(lines, ""), DefaultPluginVendorDir))
return errs
return diags
}
// With all the providers downloaded, we'll generate our lock file
@ -498,8 +504,8 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
for name, meta := range chosen {
digest, err := meta.SHA256()
if err != nil {
c.Ui.Error(fmt.Sprintf("failed to read provider plugin %s: %s", meta.Path, err))
return err
diags = diags.Append(fmt.Errorf("Failed to read provider plugin %s: %s", meta.Path, err))
return diags
}
digests[name] = digest
if c.ignorePluginChecksum {
@ -508,8 +514,8 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
}
err := c.providerPluginsLock().Write(digests)
if err != nil {
c.Ui.Error(fmt.Sprintf("failed to save provider manifest: %s", err))
return err
diags = diags.Append(fmt.Errorf("failed to save provider manifest: %s", err))
return diags
}
{
@ -557,7 +563,7 @@ func (c *InitCommand) getProviders(path string, state *terraform.State, upgrade
}
}
return nil
return diags
}
func (c *InitCommand) AutocompleteArgs() complete.Predictor {

@ -5,6 +5,7 @@ import (
"sort"
"github.com/hashicorp/terraform/moduledeps"
"github.com/hashicorp/terraform/terraform"
"github.com/hashicorp/terraform/tfdiags"
"github.com/xlab/treeprint"
)
@ -69,11 +70,8 @@ func (c *ProvidersCommand) Run(args []string) int {
return 1
}
// FIXME: Restore this once the "terraform" package is updated to deal
// with HCL2 config types.
//s := state.State()
//depTree := terraform.ModuleTreeDependencies(config, s)
var depTree *moduledeps.Module
s := state.State()
depTree := terraform.ConfigTreeDependencies(config, s)
depTree.SortDescendents()
printRoot := treeprint.New()

@ -2,6 +2,7 @@ package configs
import (
"fmt"
"strings"
"github.com/hashicorp/hcl2/gohcl"
"github.com/hashicorp/hcl2/hcl"
@ -39,6 +40,22 @@ func (r *ManagedResource) moduleUniqueKey() string {
return fmt.Sprintf("%s.%s", r.Name, r.Type)
}
// ProviderConfigKey returns a string key for the provider configuration
// that should be used for this resource. This function implements the
// default behavior of extracting the type from the resource type name if
// an explicit "provider" argument was not provided.
func (r *ManagedResource) ProviderConfigKey() string {
if r.ProviderConfigRef == nil {
typeName := r.Type
if under := strings.Index(typeName, "_"); under != -1 {
return typeName[:under]
}
return typeName
}
return r.ProviderConfigRef.String()
}
func decodeResourceBlock(block *hcl.Block) (*ManagedResource, hcl.Diagnostics) {
r := &ManagedResource{
Type: block.Labels[0],
@ -234,6 +251,22 @@ func (r *DataResource) moduleUniqueKey() string {
return fmt.Sprintf("data.%s.%s", r.Name, r.Type)
}
// ProviderConfigKey returns a string key for the provider configuration
// that should be used for this resource. This function implements the
// default behavior of extracting the type from the resource type name if
// an explicit "provider" argument was not provided.
func (r *DataResource) ProviderConfigKey() string {
if r.ProviderConfigRef == nil {
typeName := r.Type
if under := strings.Index(typeName, "_"); under != -1 {
return typeName[:under]
}
return typeName
}
return r.ProviderConfigRef.String()
}
func decodeDataBlock(block *hcl.Block) (*DataResource, hcl.Diagnostics) {
r := &DataResource{
Type: block.Labels[0],
@ -370,6 +403,16 @@ func decodeProviderConfigRef(attr *hcl.Attribute) (*ProviderConfigRef, hcl.Diagn
return ret, diags
}
func (r *ProviderConfigRef) String() string {
if r == nil {
return "<nil>"
}
if r.Alias != "" {
return fmt.Sprintf("%s.%s", r.Name, r.Alias)
}
return r.Name
}
var commonResourceAttributes = []hcl.AttributeSchema{
{
Name: "count",

@ -36,6 +36,11 @@ type Constraints struct {
raw version.Constraints
}
// NewConstraints creates a Constraints based on a version.Constraints.
func NewConstraints(c version.Constraints) Constraints {
return Constraints{c}
}
// AllVersions is a Constraints containing all versions
var AllVersions Constraints

@ -1,36 +1,34 @@
package terraform
import (
version "github.com/hashicorp/go-version"
"github.com/hashicorp/terraform/config"
"github.com/hashicorp/terraform/config/module"
"github.com/hashicorp/terraform/configs"
"github.com/hashicorp/terraform/moduledeps"
"github.com/hashicorp/terraform/plugin/discovery"
)
// ModuleTreeDependencies returns the dependencies of the tree of modules
// described by the given configuration tree and state.
// ConfigTreeDependencies returns the dependencies of the tree of modules
// described by the given configuration and state.
//
// Both configuration and state are required because there can be resources
// implied by instances in the state that no longer exist in config.
//
// This function will panic if any invalid version constraint strings are
// present in the configuration. This is guaranteed not to happen for any
// configuration that has passed a call to Config.Validate().
func ModuleTreeDependencies(root *module.Tree, state *State) *moduledeps.Module {
func ConfigTreeDependencies(root *configs.Config, state *State) *moduledeps.Module {
// First we walk the configuration tree to build the overall structure
// and capture the explicit/implicit/inherited provider dependencies.
deps := moduleTreeConfigDependencies(root, nil)
deps := configTreeConfigDependencies(root, nil)
// Next we walk over the resources in the state to catch any additional
// dependencies created by existing resources that are no longer in config.
// Most things we find in state will already be present in 'deps', but
// we're interested in the rare thing that isn't.
moduleTreeMergeStateDependencies(deps, state)
configTreeMergeStateDependencies(deps, state)
return deps
}
func moduleTreeConfigDependencies(root *module.Tree, inheritProviders map[string]*config.ProviderConfig) *moduledeps.Module {
func configTreeConfigDependencies(root *configs.Config, inheritProviders map[string]*configs.Provider) *moduledeps.Module {
if root == nil {
// If no config is provided, we'll make a synthetic root.
// This isn't necessarily correct if we're called with a nil that
@ -40,37 +38,88 @@ func moduleTreeConfigDependencies(root *module.Tree, inheritProviders map[string
}
}
name := "root"
if len(root.Path) != 0 {
name = root.Path[len(root.Path)-1]
}
ret := &moduledeps.Module{
Name: root.Name(),
Name: name,
}
cfg := root.Config()
providerConfigs := cfg.ProviderConfigsByFullName()
module := root.Module
// Provider dependencies
{
providers := make(moduledeps.Providers, len(providerConfigs))
providers := make(moduledeps.Providers)
// Any providerConfigs elements are *explicit* provider dependencies,
// which is the only situation where the user might provide an actual
// version constraint. We'll take care of these first.
for fullName, pCfg := range providerConfigs {
// The main way to declare a provider dependency is explicitly inside
// the "terraform" block, which allows declaring a requirement without
// also creating a configuration.
for fullName, constraints := range module.ProviderRequirements {
inst := moduledeps.ProviderInstance(fullName)
versionSet := discovery.AllVersions
if pCfg.Version != "" {
versionSet = discovery.ConstraintStr(pCfg.Version).MustParse()
// The handling here is a bit fiddly because the moduledeps package
// was designed around the legacy (pre-0.12) configuration model
// and hasn't yet been revised to handle the new model. As a result,
// we need to do some translation here.
// FIXME: Eventually we should adjust the underlying model so we
// can also retain the source location of each constraint, for
// more informative output from the "terraform providers" command.
var rawConstraints version.Constraints
for _, constraint := range constraints {
rawConstraints = append(rawConstraints, constraint.Required...)
}
discoConstraints := discovery.NewConstraints(rawConstraints)
providers[inst] = moduledeps.ProviderDependency{
Constraints: versionSet,
Constraints: discoConstraints,
Reason: moduledeps.ProviderDependencyExplicit,
}
}
// Provider configurations can also include version constraints,
// allowing for more terse declaration in situations where both a
// configuration and a constraint are defined in the same module.
for fullName, pCfg := range module.ProviderConfigs {
inst := moduledeps.ProviderInstance(fullName)
discoConstraints := discovery.AllVersions
if pCfg.Version.Required != nil {
discoConstraints = discovery.NewConstraints(pCfg.Version.Required)
}
if existing, exists := providers[inst]; exists {
existing.Constraints = existing.Constraints.Append(discoConstraints)
} else {
providers[inst] = moduledeps.ProviderDependency{
Constraints: discoConstraints,
Reason: moduledeps.ProviderDependencyExplicit,
}
}
}
// Each resource in the configuration creates an *implicit* provider
// dependency, though we'll only record it if there isn't already
// an explicit dependency on the same provider.
for _, rc := range cfg.Resources {
fullName := rc.ProviderFullName()
for _, rc := range module.ManagedResources {
fullName := rc.ProviderConfigKey()
inst := moduledeps.ProviderInstance(fullName)
if _, exists := providers[inst]; exists {
// Explicit dependency already present
continue
}
reason := moduledeps.ProviderDependencyImplicit
if _, inherited := inheritProviders[fullName]; inherited {
reason = moduledeps.ProviderDependencyInherited
}
providers[inst] = moduledeps.ProviderDependency{
Constraints: discovery.AllVersions,
Reason: reason,
}
}
for _, rc := range module.DataResources {
fullName := rc.ProviderConfigKey()
inst := moduledeps.ProviderInstance(fullName)
if _, exists := providers[inst]; exists {
// Explicit dependency already present
@ -91,21 +140,21 @@ func moduleTreeConfigDependencies(root *module.Tree, inheritProviders map[string
ret.Providers = providers
}
childInherit := make(map[string]*config.ProviderConfig)
childInherit := make(map[string]*configs.Provider)
for k, v := range inheritProviders {
childInherit[k] = v
}
for k, v := range providerConfigs {
for k, v := range module.ProviderConfigs {
childInherit[k] = v
}
for _, c := range root.Children() {
ret.Children = append(ret.Children, moduleTreeConfigDependencies(c, childInherit))
for _, c := range root.Children {
ret.Children = append(ret.Children, configTreeConfigDependencies(c, childInherit))
}
return ret
}
func moduleTreeMergeStateDependencies(root *moduledeps.Module, state *State) {
func configTreeMergeStateDependencies(root *moduledeps.Module, state *State) {
if state == nil {
return
}
@ -151,5 +200,110 @@ func moduleTreeMergeStateDependencies(root *moduledeps.Module, state *State) {
}
}
}
}
// ModuleTreeDependencies returns the dependencies of the tree of modules
// described by the given configuration tree and state.
//
// Both configuration and state are required because there can be resources
// implied by instances in the state that no longer exist in config.
//
// This function will panic if any invalid version constraint strings are
// present in the configuration. This is guaranteed not to happen for any
// configuration that has passed a call to Config.Validate().
func ModuleTreeDependencies(root *module.Tree, state *State) *moduledeps.Module {
// First we walk the configuration tree to build the overall structure
// and capture the explicit/implicit/inherited provider dependencies.
deps := moduleTreeConfigDependencies(root, nil)
// Next we walk over the resources in the state to catch any additional
// dependencies created by existing resources that are no longer in config.
// Most things we find in state will already be present in 'deps', but
// we're interested in the rare thing that isn't.
moduleTreeMergeStateDependencies(deps, state)
return deps
}
func moduleTreeConfigDependencies(root *module.Tree, inheritProviders map[string]*config.ProviderConfig) *moduledeps.Module {
if root == nil {
// If no config is provided, we'll make a synthetic root.
// This isn't necessarily correct if we're called with a nil that
// *isn't* at the root, but in practice that can never happen.
return &moduledeps.Module{
Name: "root",
}
}
ret := &moduledeps.Module{
Name: root.Name(),
}
cfg := root.Config()
providerConfigs := cfg.ProviderConfigsByFullName()
// Provider dependencies
{
providers := make(moduledeps.Providers, len(providerConfigs))
// Any providerConfigs elements are *explicit* provider dependencies,
// which is the only situation where the user might provide an actual
// version constraint. We'll take care of these first.
for fullName, pCfg := range providerConfigs {
inst := moduledeps.ProviderInstance(fullName)
versionSet := discovery.AllVersions
if pCfg.Version != "" {
versionSet = discovery.ConstraintStr(pCfg.Version).MustParse()
}
providers[inst] = moduledeps.ProviderDependency{
Constraints: versionSet,
Reason: moduledeps.ProviderDependencyExplicit,
}
}
// Each resource in the configuration creates an *implicit* provider
// dependency, though we'll only record it if there isn't already
// an explicit dependency on the same provider.
for _, rc := range cfg.Resources {
fullName := rc.ProviderFullName()
inst := moduledeps.ProviderInstance(fullName)
if _, exists := providers[inst]; exists {
// Explicit dependency already present
continue
}
reason := moduledeps.ProviderDependencyImplicit
if _, inherited := inheritProviders[fullName]; inherited {
reason = moduledeps.ProviderDependencyInherited
}
providers[inst] = moduledeps.ProviderDependency{
Constraints: discovery.AllVersions,
Reason: reason,
}
}
ret.Providers = providers
}
childInherit := make(map[string]*config.ProviderConfig)
for k, v := range inheritProviders {
childInherit[k] = v
}
for k, v := range providerConfigs {
childInherit[k] = v
}
for _, c := range root.Children() {
ret.Children = append(ret.Children, moduleTreeConfigDependencies(c, childInherit))
}
return ret
}
func moduleTreeMergeStateDependencies(root *moduledeps.Module, state *State) {
// This is really just the same logic as configTreeMergeStateDependencies
// but we retain this old name just to keep the symmetry until we've
// removed all of these "moduleTree..." versions that use the legacy
// configuration structs.
configTreeMergeStateDependencies(root, state)
}

Loading…
Cancel
Save