From f77e7a61b039676e7aa8aba49404a93a9230ff6b Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 21 Jun 2018 17:39:27 -0700 Subject: [PATCH] various: helpers for collecting necessary provider types Since schemas are required to interpret provider, resource, and provisioner attributes in configs, states, and plans, these helpers intend to make it easier to gather up the the necessary provider types in order to preload all of the needed schemas before beginning further processing. Config.ProviderTypes returns directly the list of provider types, since at this level further detail is not useful: we've not yet run the provider allocation algorithm, and so the only thing we can reliably extract here is provider types themselves. State.ProviderAddrs and Plan.ProviderAddrs each return a list of absolute provider addresses, which can then be turned into a list of provider types using the new helper providers.AddressedTypesAbs. Since we're already using configs.Config throughout core, this also updates the terraform.LoadSchemas helper to use Config.ProviderTypes to find the necessary providers, rather than implementing its own discovery logic. states.State is not yet plumbed in, so we cannot yet use State.ProviderAddrs to deal with the state but there's a TODO comment to remind us to update that in a later commit when we swap out terraform.State for states.State. A later commit will probably refactor this further so that we can easily obtain schema for the providers needed to interpret a plan too, but that is deferred here because further work is required to make core work with the new plan types first. At that point, terraform.LoadSchemas may become providers.LoadSchemas with a different interface that just accepts lists of provider and provisioner names that have been gathered by the caller using these new helpers. --- configs/config.go | 44 +++++++++++++ configs/config_test.go | 29 ++++++++ .../valid-files/providers-explicit-implied.tf | 15 +++++ plans/plan.go | 38 +++++++++++ plans/plan_test.go | 66 +++++++++++++++++++ providers/addressed_types.go | 47 +++++++++++++ providers/addressed_types_test.go | 49 ++++++++++++++ states/state.go | 36 ++++++++++ terraform/schemas.go | 27 +++----- 9 files changed, 332 insertions(+), 19 deletions(-) create mode 100644 configs/config_test.go create mode 100644 configs/test-fixtures/valid-files/providers-explicit-implied.tf create mode 100644 plans/plan_test.go create mode 100644 providers/addressed_types.go create mode 100644 providers/addressed_types_test.go diff --git a/configs/config.go b/configs/config.go index 5bdeeaadfa..e068cbc34a 100644 --- a/configs/config.go +++ b/configs/config.go @@ -1,6 +1,8 @@ package configs import ( + "sort" + version "github.com/hashicorp/go-version" "github.com/hashicorp/hcl2/hcl" "github.com/hashicorp/terraform/addrs" @@ -159,3 +161,45 @@ func (c *Config) DescendentForInstance(path addrs.ModuleInstance) *Config { } return current } + +// ProviderTypes returns the names of each distinct provider type referenced +// in the receiving configuration. +// +// This is a helper for easily determining which provider types are required +// to fully interpret the configuration, though it does not include version +// information and so callers are expected to have already dealt with +// provider version selection in an earlier step and have identified suitable +// versions for each provider. +func (c *Config) ProviderTypes() []string { + m := make(map[string]struct{}) + c.gatherProviderTypes(m) + + ret := make([]string, 0, len(m)) + for k := range m { + ret = append(ret, k) + } + sort.Strings(ret) + return ret +} +func (c *Config) gatherProviderTypes(m map[string]struct{}) { + if c == nil { + return + } + + for _, pc := range c.Module.ProviderConfigs { + m[pc.Name] = struct{}{} + } + for _, rc := range c.Module.ManagedResources { + providerAddr := rc.ProviderConfigAddr() + m[providerAddr.Type] = struct{}{} + } + for _, rc := range c.Module.DataResources { + providerAddr := rc.ProviderConfigAddr() + m[providerAddr.Type] = struct{}{} + } + + // Must also visit our child modules, recursively. + for _, cc := range c.Children { + cc.gatherProviderTypes(m) + } +} diff --git a/configs/config_test.go b/configs/config_test.go new file mode 100644 index 0000000000..f2c03db8bf --- /dev/null +++ b/configs/config_test.go @@ -0,0 +1,29 @@ +package configs + +import ( + "testing" + + "github.com/go-test/deep" +) + +func TestConfigProviderTypes(t *testing.T) { + mod, diags := testModuleFromFile("test-fixtures/valid-files/providers-explicit-implied.tf") + if diags.HasErrors() { + t.Fatal(diags.Error()) + } + + cfg, diags := BuildConfig(mod, nil) + if diags.HasErrors() { + t.Fatal(diags.Error()) + } + + got := cfg.ProviderTypes() + want := []string{ + "aws", + "null", + "template", + } + for _, problem := range deep.Equal(got, want) { + t.Error(problem) + } +} diff --git a/configs/test-fixtures/valid-files/providers-explicit-implied.tf b/configs/test-fixtures/valid-files/providers-explicit-implied.tf new file mode 100644 index 0000000000..7216e0433e --- /dev/null +++ b/configs/test-fixtures/valid-files/providers-explicit-implied.tf @@ -0,0 +1,15 @@ +provider "aws" { + +} + +provider "template" { + alias = "foo" +} + +resource "aws_instance" "foo" { + +} + +resource "null_resource" "foo" { + +} diff --git a/plans/plan.go b/plans/plan.go index 5cd41f2891..ae119044c4 100644 --- a/plans/plan.go +++ b/plans/plan.go @@ -1,5 +1,11 @@ package plans +import ( + "sort" + + "github.com/hashicorp/terraform/addrs" +) + // Plan is the top-level type representing a planned set of changes. // // A plan is a summary of the set of changes required to move from a current @@ -16,3 +22,35 @@ type Plan struct { Changes *Changes ProviderSHA256s map[string][]byte } + +// ProviderAddrs returns a list of all of the provider configuration addresses +// referenced throughout the receiving plan. +// +// The result is de-duplicated so that each distinct address appears only once. +func (p *Plan) ProviderAddrs() []addrs.AbsProviderConfig { + if p == nil || p.Changes == nil { + return nil + } + + m := map[string]addrs.AbsProviderConfig{} + for _, rc := range p.Changes.Resources { + m[rc.ProviderAddr.String()] = rc.ProviderAddr + } + if len(m) == 0 { + return nil + } + + // This is mainly just so we'll get stable results for testing purposes. + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + + ret := make([]addrs.AbsProviderConfig, len(keys)) + for i, key := range keys { + ret[i] = m[key] + } + + return ret +} diff --git a/plans/plan_test.go b/plans/plan_test.go new file mode 100644 index 0000000000..c65a415241 --- /dev/null +++ b/plans/plan_test.go @@ -0,0 +1,66 @@ +package plans + +import ( + "testing" + + "github.com/go-test/deep" + + "github.com/hashicorp/terraform/addrs" +) + +func TestProviderAddrs(t *testing.T) { + + plan := &Plan{ + VariableValues: map[string]DynamicValue{}, + Changes: &Changes{ + RootOutputs: map[string]*OutputChange{}, + Resources: []*ResourceInstanceChange{ + { + Addr: addrs.Resource{ + Mode: addrs.ManagedResourceMode, + Type: "test_thing", + Name: "woot", + }.Instance(addrs.IntKey(0)).Absolute(addrs.RootModuleInstance), + ProviderAddr: addrs.ProviderConfig{ + Type: "test", + }.Absolute(addrs.RootModuleInstance), + }, + { + Addr: addrs.Resource{ + Mode: addrs.ManagedResourceMode, + Type: "test_thing", + Name: "woot", + }.Instance(addrs.IntKey(0)).Absolute(addrs.RootModuleInstance), + DeposedKey: "foodface", + ProviderAddr: addrs.ProviderConfig{ + Type: "test", + }.Absolute(addrs.RootModuleInstance), + }, + { + Addr: addrs.Resource{ + Mode: addrs.ManagedResourceMode, + Type: "test_thing", + Name: "what", + }.Instance(addrs.IntKey(0)).Absolute(addrs.RootModuleInstance), + ProviderAddr: addrs.ProviderConfig{ + Type: "test", + }.Absolute(addrs.RootModuleInstance.Child("foo", addrs.NoKey)), + }, + }, + }, + } + + got := plan.ProviderAddrs() + want := []addrs.AbsProviderConfig{ + addrs.ProviderConfig{ + Type: "test", + }.Absolute(addrs.RootModuleInstance.Child("foo", addrs.NoKey)), + addrs.ProviderConfig{ + Type: "test", + }.Absolute(addrs.RootModuleInstance), + } + + for _, problem := range deep.Equal(got, want) { + t.Error(problem) + } +} diff --git a/providers/addressed_types.go b/providers/addressed_types.go new file mode 100644 index 0000000000..7ed523f158 --- /dev/null +++ b/providers/addressed_types.go @@ -0,0 +1,47 @@ +package providers + +import ( + "sort" + + "github.com/hashicorp/terraform/addrs" +) + +// AddressedTypes is a helper that extracts all of the distinct provider +// types from the given list of relative provider configuration addresses. +func AddressedTypes(providerAddrs []addrs.ProviderConfig) []string { + if len(providerAddrs) == 0 { + return nil + } + m := map[string]struct{}{} + for _, addr := range providerAddrs { + m[addr.Type] = struct{}{} + } + + names := make([]string, 0, len(m)) + for typeName := range m { + names = append(names, typeName) + } + + sort.Strings(names) // Stable result for tests + return names +} + +// AddressedTypesAbs is a helper that extracts all of the distinct provider +// types from the given list of absolute provider configuration addresses. +func AddressedTypesAbs(providerAddrs []addrs.AbsProviderConfig) []string { + if len(providerAddrs) == 0 { + return nil + } + m := map[string]struct{}{} + for _, addr := range providerAddrs { + m[addr.ProviderConfig.Type] = struct{}{} + } + + names := make([]string, 0, len(m)) + for typeName := range m { + names = append(names, typeName) + } + + sort.Strings(names) // Stable result for tests + return names +} diff --git a/providers/addressed_types_test.go b/providers/addressed_types_test.go new file mode 100644 index 0000000000..80915e3e68 --- /dev/null +++ b/providers/addressed_types_test.go @@ -0,0 +1,49 @@ +package providers + +import ( + "testing" + + "github.com/go-test/deep" + + "github.com/hashicorp/terraform/addrs" +) + +func TestAddressedTypes(t *testing.T) { + providerAddrs := []addrs.ProviderConfig{ + {Type: "aws"}, + {Type: "aws", Alias: "foo"}, + {Type: "azure"}, + {Type: "null"}, + {Type: "null"}, + } + + got := AddressedTypes(providerAddrs) + want := []string{ + "aws", + "azure", + "null", + } + for _, problem := range deep.Equal(got, want) { + t.Error(problem) + } +} + +func TestAddressedTypesAbs(t *testing.T) { + providerAddrs := []addrs.AbsProviderConfig{ + addrs.ProviderConfig{Type: "aws"}.Absolute(addrs.RootModuleInstance), + addrs.ProviderConfig{Type: "aws", Alias: "foo"}.Absolute(addrs.RootModuleInstance), + addrs.ProviderConfig{Type: "azure"}.Absolute(addrs.RootModuleInstance), + addrs.ProviderConfig{Type: "null"}.Absolute(addrs.RootModuleInstance), + addrs.ProviderConfig{Type: "null"}.Absolute(addrs.RootModuleInstance), + } + + got := AddressedTypesAbs(providerAddrs) + want := []string{ + "aws", + "azure", + "null", + } + for _, problem := range deep.Equal(got, want) { + t.Error(problem) + } +} diff --git a/states/state.go b/states/state.go index c1570e24f8..22f4bcdc0f 100644 --- a/states/state.go +++ b/states/state.go @@ -1,6 +1,8 @@ package states import ( + "sort" + "github.com/zclconf/go-cty/cty" "github.com/hashicorp/terraform/addrs" @@ -114,6 +116,40 @@ func (s *State) LocalValue(addr addrs.AbsLocalValue) cty.Value { return ms.LocalValues[addr.LocalValue.Name] } +// ProviderAddrs returns a list of all of the provider configuration addresses +// referenced throughout the receiving state. +// +// The result is de-duplicated so that each distinct address appears only once. +func (s *State) ProviderAddrs() []addrs.AbsProviderConfig { + if s == nil { + return nil + } + + m := map[string]addrs.AbsProviderConfig{} + for _, ms := range s.Modules { + for _, rc := range ms.Resources { + m[rc.ProviderConfig.String()] = rc.ProviderConfig + } + } + if len(m) == 0 { + return nil + } + + // This is mainly just so we'll get stable results for testing purposes. + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + + ret := make([]addrs.AbsProviderConfig, len(keys)) + for i, key := range keys { + ret[i] = m[key] + } + + return ret +} + // SyncWrapper returns a SyncState object wrapping the receiver. func (s *State) SyncWrapper() *SyncState { return &SyncState{ diff --git a/terraform/schemas.go b/terraform/schemas.go index 6f175948ba..cd8d811f27 100644 --- a/terraform/schemas.go +++ b/terraform/schemas.go @@ -81,9 +81,9 @@ func (ss *Schemas) ProvisionerConfig(name string) *configschema.Block { return ss.provisioners[name] } -// LoadSchemas searches the given configuration and state (either of which may -// be nil) for constructs that have an associated schema, requests the -// necessary schemas from the given component factory (which may _not_ be nil), +// LoadSchemas searches the given configuration, state and plan (any of which +// may be nil) for constructs that have an associated schema, requests the +// necessary schemas from the given component factory (which must _not_ be nil), // and returns a single object representing all of the necessary schemas. // // If an error is returned, it may be a wrapped tfdiags.Diagnostics describing @@ -165,26 +165,15 @@ func loadProviderSchemas(schemas map[string]*ProviderSchema, config *configs.Con } if config != nil { - for _, pc := range config.Module.ProviderConfigs { - ensure(pc.Name) - } - for _, rc := range config.Module.ManagedResources { - providerAddr := rc.ProviderConfigAddr() - ensure(providerAddr.Type) - } - for _, rc := range config.Module.DataResources { - providerAddr := rc.ProviderConfigAddr() - ensure(providerAddr.Type) - } - - // Must also visit our child modules, recursively. - for _, cc := range config.Children { - childDiags := loadProviderSchemas(schemas, cc, nil, components) - diags = diags.Append(childDiags) + for _, typeName := range config.ProviderTypes() { + ensure(typeName) } } if state != nil { + // TODO: After adapting this to use *states.State, use + // providers.AddressedTypes(state.ProviderAddrs()) to collect + // our list of required provider types. for _, ms := range state.Modules { for rsKey, rs := range ms.Resources { providerAddrStr := rs.Provider