From a8fcb2d91a0d69b4b8a2f35e86d720d9538ef77d Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Thu, 20 Feb 2020 10:51:34 +0100 Subject: [PATCH] HCL2: add support for dynamic blocks, document for loops and splat expressions (#8720) --- hcl2template/common_test.go | 28 +- hcl2template/internal/mock.go | 8 +- hcl2template/internal/mock.hcl2spec.go | 31 ++- hcl2template/parser.go | 4 +- hcl2template/testdata/complete/build.pkr.hcl | 32 ++- .../testdata/complete/sources.pkr.hcl | 1 + .../testdata/complete/variables.pkr.hcl | 16 +- hcl2template/types.packer_config_test.go | 4 +- .../hashicorp/hcl/v2/ext/dynblock/README.md | 184 ++++++++++++ .../hcl/v2/ext/dynblock/expand_body.go | 262 ++++++++++++++++++ .../hcl/v2/ext/dynblock/expand_spec.go | 215 ++++++++++++++ .../hcl/v2/ext/dynblock/expr_wrap.go | 42 +++ .../hcl/v2/ext/dynblock/iteration.go | 66 +++++ .../hashicorp/hcl/v2/ext/dynblock/public.go | 47 ++++ .../hashicorp/hcl/v2/ext/dynblock/schema.go | 50 ++++ .../hcl/v2/ext/dynblock/unknown_body.go | 84 ++++++ .../hcl/v2/ext/dynblock/variables.go | 209 ++++++++++++++ .../hcl/v2/ext/dynblock/variables_hcldec.go | 43 +++ vendor/modules.txt | 1 + .../from-1.5/expressions.html.md | 220 ++++++++++++++- 20 files changed, 1527 insertions(+), 20 deletions(-) create mode 100644 vendor/github.com/hashicorp/hcl/v2/ext/dynblock/README.md create mode 100644 vendor/github.com/hashicorp/hcl/v2/ext/dynblock/expand_body.go create mode 100644 vendor/github.com/hashicorp/hcl/v2/ext/dynblock/expand_spec.go create mode 100644 vendor/github.com/hashicorp/hcl/v2/ext/dynblock/expr_wrap.go create mode 100644 vendor/github.com/hashicorp/hcl/v2/ext/dynblock/iteration.go create mode 100644 vendor/github.com/hashicorp/hcl/v2/ext/dynblock/public.go create mode 100644 vendor/github.com/hashicorp/hcl/v2/ext/dynblock/schema.go create mode 100644 vendor/github.com/hashicorp/hcl/v2/ext/dynblock/unknown_body.go create mode 100644 vendor/github.com/hashicorp/hcl/v2/ext/dynblock/variables.go create mode 100644 vendor/github.com/hashicorp/hcl/v2/ext/dynblock/variables_hcldec.go diff --git a/hcl2template/common_test.go b/hcl2template/common_test.go index e930c3b74..506a667dc 100644 --- a/hcl2template/common_test.go +++ b/hcl2template/common_test.go @@ -126,6 +126,7 @@ var ( {"a", "b"}, {"c", "d"}, }, + Tags: []MockTag{}, } basicMockBuilder = &MockBuilder{ @@ -145,7 +146,9 @@ var ( NestedMockConfig: basicNestedMockConfig, Nested: basicNestedMockConfig, NestedSlice: []NestedMockConfig{ - {}, + { + Tags: dynamicTagList, + }, }, }, } @@ -154,7 +157,9 @@ var ( NestedMockConfig: basicNestedMockConfig, Nested: basicNestedMockConfig, NestedSlice: []NestedMockConfig{ - {}, + { + Tags: []MockTag{}, + }, }, }, } @@ -163,8 +168,25 @@ var ( NestedMockConfig: basicNestedMockConfig, Nested: basicNestedMockConfig, NestedSlice: []NestedMockConfig{ - {}, + { + Tags: []MockTag{}, + }, }, }, } + + dynamicTagList = []MockTag{ + { + Key: "first_tag_key", + Value: "first_tag_value", + }, + { + Key: "Component", + Value: "user-service", + }, + { + Key: "Environment", + Value: "production", + }, + } ) diff --git a/hcl2template/internal/mock.go b/hcl2template/internal/mock.go index 632465ae0..1c33d8f99 100644 --- a/hcl2template/internal/mock.go +++ b/hcl2template/internal/mock.go @@ -1,4 +1,4 @@ -//go:generate mapstructure-to-hcl2 -type MockConfig,NestedMockConfig +//go:generate mapstructure-to-hcl2 -type MockConfig,NestedMockConfig,MockTag package hcl2template @@ -23,6 +23,12 @@ type NestedMockConfig struct { SliceSliceString [][]string `mapstructure:"slice_slice_string"` NamedMapStringString NamedMapStringString `mapstructure:"named_map_string_string"` NamedString NamedString `mapstructure:"named_string"` + Tags []MockTag `mapstructure:"tag"` +} + +type MockTag struct { + Key string `mapstructure:"key"` + Value string `mapstructure:"value"` } type MockConfig struct { diff --git a/hcl2template/internal/mock.hcl2spec.go b/hcl2template/internal/mock.hcl2spec.go index 0eb205e4b..56b8dd2a9 100644 --- a/hcl2template/internal/mock.hcl2spec.go +++ b/hcl2template/internal/mock.hcl2spec.go @@ -1,4 +1,4 @@ -// Code generated by "mapstructure-to-hcl2 -type MockConfig,NestedMockConfig"; DO NOT EDIT. +// Code generated by "mapstructure-to-hcl2 -type MockConfig,NestedMockConfig,MockTag"; DO NOT EDIT. package hcl2template import ( @@ -21,6 +21,7 @@ type FlatMockConfig struct { SliceSliceString [][]string `mapstructure:"slice_slice_string" cty:"slice_slice_string"` NamedMapStringString NamedMapStringString `mapstructure:"named_map_string_string" cty:"named_map_string_string"` NamedString *NamedString `mapstructure:"named_string" cty:"named_string"` + Tags []FlatMockTag `mapstructure:"tag" cty:"tag"` Nested *FlatNestedMockConfig `mapstructure:"nested" cty:"nested"` NestedSlice []FlatNestedMockConfig `mapstructure:"nested_slice" cty:"nested_slice"` } @@ -49,12 +50,38 @@ func (*FlatMockConfig) HCL2Spec() map[string]hcldec.Spec { "slice_slice_string": &hcldec.AttrSpec{Name: "slice_slice_string", Type: cty.List(cty.List(cty.String)), Required: false}, "named_map_string_string": &hcldec.BlockAttrsSpec{TypeName: "named_map_string_string", ElementType: cty.String, Required: false}, "named_string": &hcldec.AttrSpec{Name: "named_string", Type: cty.String, Required: false}, + "tag": &hcldec.BlockListSpec{TypeName: "tag", Nested: hcldec.ObjectSpec((*FlatMockTag)(nil).HCL2Spec())}, "nested": &hcldec.BlockSpec{TypeName: "nested", Nested: hcldec.ObjectSpec((*FlatNestedMockConfig)(nil).HCL2Spec())}, "nested_slice": &hcldec.BlockListSpec{TypeName: "nested_slice", Nested: hcldec.ObjectSpec((*FlatNestedMockConfig)(nil).HCL2Spec())}, } return s } +// FlatMockTag is an auto-generated flat version of MockTag. +// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up. +type FlatMockTag struct { + Key *string `mapstructure:"key" cty:"key"` + Value *string `mapstructure:"value" cty:"value"` +} + +// FlatMapstructure returns a new FlatMockTag. +// FlatMockTag is an auto-generated flat version of MockTag. +// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up. +func (*MockTag) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } { + return new(FlatMockTag) +} + +// HCL2Spec returns the hcl spec of a MockTag. +// This spec is used by HCL to read the fields of MockTag. +// The decoded values from this spec will then be applied to a FlatMockTag. +func (*FlatMockTag) HCL2Spec() map[string]hcldec.Spec { + s := map[string]hcldec.Spec{ + "key": &hcldec.AttrSpec{Name: "key", Type: cty.String, Required: false}, + "value": &hcldec.AttrSpec{Name: "value", Type: cty.String, Required: false}, + } + return s +} + // FlatNestedMockConfig is an auto-generated flat version of NestedMockConfig. // Where the contents of a field with a `mapstructure:,squash` tag are bubbled up. type FlatNestedMockConfig struct { @@ -69,6 +96,7 @@ type FlatNestedMockConfig struct { SliceSliceString [][]string `mapstructure:"slice_slice_string" cty:"slice_slice_string"` NamedMapStringString NamedMapStringString `mapstructure:"named_map_string_string" cty:"named_map_string_string"` NamedString *NamedString `mapstructure:"named_string" cty:"named_string"` + Tags []FlatMockTag `mapstructure:"tag" cty:"tag"` } // FlatMapstructure returns a new FlatNestedMockConfig. @@ -94,6 +122,7 @@ func (*FlatNestedMockConfig) HCL2Spec() map[string]hcldec.Spec { "slice_slice_string": &hcldec.AttrSpec{Name: "slice_slice_string", Type: cty.List(cty.List(cty.String)), Required: false}, "named_map_string_string": &hcldec.BlockAttrsSpec{TypeName: "named_map_string_string", ElementType: cty.String, Required: false}, "named_string": &hcldec.AttrSpec{Name: "named_string", Type: cty.String, Required: false}, + "tag": &hcldec.BlockListSpec{TypeName: "tag", Nested: hcldec.ObjectSpec((*FlatMockTag)(nil).HCL2Spec())}, } return s } diff --git a/hcl2template/parser.go b/hcl2template/parser.go index 6f7467da5..964a1b0ad 100644 --- a/hcl2template/parser.go +++ b/hcl2template/parser.go @@ -6,6 +6,7 @@ import ( "path/filepath" "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/ext/dynblock" "github.com/hashicorp/hcl/v2/hclparse" "github.com/hashicorp/packer/packer" ) @@ -177,7 +178,8 @@ func (p *Parser) decodeLocalVariables(f *hcl.File, cfg *PackerConfig) hcl.Diagno func (p *Parser) decodeConfig(f *hcl.File, cfg *PackerConfig) hcl.Diagnostics { var diags hcl.Diagnostics - content, moreDiags := f.Body.Content(configSchema) + body := dynblock.Expand(f.Body, cfg.EvalContext()) + content, moreDiags := body.Content(configSchema) diags = append(diags, moreDiags...) for _, block := range content.Blocks { diff --git a/hcl2template/testdata/complete/build.pkr.hcl b/hcl2template/testdata/complete/build.pkr.hcl index 64cf16c2e..d21b9d399 100644 --- a/hcl2template/testdata/complete/build.pkr.hcl +++ b/hcl2template/testdata/complete/build.pkr.hcl @@ -18,7 +18,7 @@ build { a = "b" c = "d" } - slice_string = var.availability_zone_names + slice_string = [for s in var.availability_zone_names : lower(s)] slice_slice_string = [ ["a","b"], ["c","d"] @@ -35,7 +35,7 @@ build { a = "b" c = "d" } - slice_string = var.availability_zone_names + slice_string = [for s in var.availability_zone_names : lower(s)] slice_slice_string = [ ["a","b"], ["c","d"] @@ -43,6 +43,17 @@ build { } nested_slice { + tag { + key = "first_tag_key" + value = "first_tag_value" + } + dynamic "tag" { + for_each = local.standard_tags + content { + key = tag.key + value = tag.value + } + } } } @@ -58,11 +69,7 @@ build { a = "b" c = "d" } - slice_string = [ - "a", - "b", - "c", - ] + slice_string = local.abc_map[*].id slice_slice_string = [ ["a","b"], ["c","d"] @@ -91,6 +98,17 @@ build { } nested_slice { + tag { + key = "first_tag_key" + value = "first_tag_value" + } + dynamic "tag" { + for_each = local.standard_tags + content { + key = tag.key + value = tag.value + } + } } } diff --git a/hcl2template/testdata/complete/sources.pkr.hcl b/hcl2template/testdata/complete/sources.pkr.hcl index dcb14dd04..895221d65 100644 --- a/hcl2template/testdata/complete/sources.pkr.hcl +++ b/hcl2template/testdata/complete/sources.pkr.hcl @@ -5,6 +5,7 @@ source "virtualbox-iso" "ubuntu-1204" { bool = true trilean = true duration = "10s" + map_string_string { a = "b" c = "d" diff --git a/hcl2template/testdata/complete/variables.pkr.hcl b/hcl2template/testdata/complete/variables.pkr.hcl index d144cc1cf..beebda9d3 100644 --- a/hcl2template/testdata/complete/variables.pkr.hcl +++ b/hcl2template/testdata/complete/variables.pkr.hcl @@ -17,9 +17,23 @@ variable "port" { variable "availability_zone_names" { type = list(string) - default = ["a", "b", "c"] + default = ["A", "B", "C"] } locals { feefoo = "${var.foo}_${var.image_id}" } + + +locals { + standard_tags = { + Component = "user-service" + Environment = "production" + } + + abc_map = [ + {id = "a"}, + {id = "b"}, + {id = "c"}, + ] +} diff --git a/hcl2template/types.packer_config_test.go b/hcl2template/types.packer_config_test.go index b76133e0e..4e18155e0 100644 --- a/hcl2template/types.packer_config_test.go +++ b/hcl2template/types.packer_config_test.go @@ -27,7 +27,9 @@ func TestParser_complete(t *testing.T) { "availability_zone_names": &Variable{}, }, LocalVariables: Variables{ - "feefoo": &Variable{}, + "feefoo": &Variable{}, + "standard_tags": &Variable{}, + "abc_map": &Variable{}, }, Sources: map[SourceRef]*SourceBlock{ refVBIsoUbuntu1204: {Type: "virtualbox-iso", Name: "ubuntu-1204"}, diff --git a/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/README.md b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/README.md new file mode 100644 index 000000000..f59ce92e9 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/README.md @@ -0,0 +1,184 @@ +# HCL Dynamic Blocks Extension + +This HCL extension implements a special block type named "dynamic" that can +be used to dynamically generate blocks of other types by iterating over +collection values. + +Normally the block structure in an HCL configuration file is rigid, even +though dynamic expressions can be used within attribute values. This is +convenient for most applications since it allows the overall structure of +the document to be decoded easily, but in some applications it is desirable +to allow dynamic block generation within certain portions of the configuration. + +Dynamic block generation is performed using the `dynamic` block type: + +```hcl +toplevel { + nested { + foo = "static block 1" + } + + dynamic "nested" { + for_each = ["a", "b", "c"] + iterator = nested + content { + foo = "dynamic block ${nested.value}" + } + } + + nested { + foo = "static block 2" + } +} +``` + +The above is interpreted as if it were written as follows: + +```hcl +toplevel { + nested { + foo = "static block 1" + } + + nested { + foo = "dynamic block a" + } + + nested { + foo = "dynamic block b" + } + + nested { + foo = "dynamic block c" + } + + nested { + foo = "static block 2" + } +} +``` + +Since HCL block syntax is not normally exposed to the possibility of unknown +values, this extension must make some compromises when asked to iterate over +an unknown collection. If the length of the collection cannot be statically +recognized (because it is an unknown value of list, map, or set type) then +the `dynamic` construct will generate a _single_ dynamic block whose iterator +key and value are both unknown values of the dynamic pseudo-type, thus causing +any attribute values derived from iteration to appear as unknown values. There +is no explicit representation of the fact that the length of the collection may +eventually be different than one. + +## Usage + +Pass a body to function `Expand` to obtain a new body that will, on access +to its content, evaluate and expand any nested `dynamic` blocks. +Dynamic block processing is also automatically propagated into any nested +blocks that are returned, allowing users to nest dynamic blocks inside +one another and to nest dynamic blocks inside other static blocks. + +HCL structural decoding does not normally have access to an `EvalContext`, so +any variables and functions that should be available to the `for_each` +and `labels` expressions must be passed in when calling `Expand`. Expressions +within the `content` block are evaluated separately and so can be passed a +separate `EvalContext` if desired, during normal attribute expression +evaluation. + +## Detecting Variables + +Some applications dynamically generate an `EvalContext` by analyzing which +variables are referenced by an expression before evaluating it. + +This unfortunately requires some extra effort when this analysis is required +for the context passed to `Expand`: the HCL API requires a schema to be +provided in order to do any analysis of the blocks in a body, but the low-level +schema model provides a description of only one level of nested blocks at +a time, and thus a new schema must be provided for each additional level of +nesting. + +To make this arduous process as convenient as possible, this package provides +a helper function `WalkForEachVariables`, which returns a `WalkVariablesNode` +instance that can be used to find variables directly in a given body and also +determine which nested blocks require recursive calls. Using this mechanism +requires that the caller be able to look up a schema given a nested block type. +For _simple_ formats where a specific block type name always has the same schema +regardless of context, a walk can be implemented as follows: + +```go +func walkVariables(node dynblock.WalkVariablesNode, schema *hcl.BodySchema) []hcl.Traversal { + vars, children := node.Visit(schema) + + for _, child := range children { + var childSchema *hcl.BodySchema + switch child.BlockTypeName { + case "a": + childSchema = &hcl.BodySchema{ + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "b", + LabelNames: []string{"key"}, + }, + }, + } + case "b": + childSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "val", + Required: true, + }, + }, + } + default: + // Should never happen, because the above cases should be exhaustive + // for the application's configuration format. + panic(fmt.Errorf("can't find schema for unknown block type %q", child.BlockTypeName)) + } + + vars = append(vars, testWalkAndAccumVars(child.Node, childSchema)...) + } +} +``` + +### Detecting Variables with `hcldec` Specifications + +For applications that use the higher-level `hcldec` package to decode nested +configuration structures into `cty` values, the same specification can be used +to automatically drive the recursive variable-detection walk described above. + +The helper function `ForEachVariablesHCLDec` allows an entire recursive +configuration structure to be analyzed in a single call given a `hcldec.Spec` +that describes the nested block structure. This means a `hcldec`-based +application can support dynamic blocks with only a little additional effort: + +```go +func decodeBody(body hcl.Body, spec hcldec.Spec) (cty.Value, hcl.Diagnostics) { + // Determine which variables are needed to expand dynamic blocks + neededForDynamic := dynblock.ForEachVariablesHCLDec(body, spec) + + // Build a suitable EvalContext and expand dynamic blocks + dynCtx := buildEvalContext(neededForDynamic) + dynBody := dynblock.Expand(body, dynCtx) + + // Determine which variables are needed to fully decode the expanded body + // This will analyze expressions that came both from static blocks in the + // original body and from blocks that were dynamically added by Expand. + neededForDecode := hcldec.Variables(dynBody, spec) + + // Build a suitable EvalContext and then fully decode the body as per the + // hcldec specification. + decCtx := buildEvalContext(neededForDecode) + return hcldec.Decode(dynBody, spec, decCtx) +} + +func buildEvalContext(needed []hcl.Traversal) *hcl.EvalContext { + // (to be implemented by your application) +} +``` + +# Performance + +This extension is going quite harshly against the grain of the HCL API, and +so it uses lots of wrapping objects and temporary data structures to get its +work done. HCL in general is not suitable for use in high-performance situations +or situations sensitive to memory pressure, but that is _especially_ true for +this extension. diff --git a/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/expand_body.go b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/expand_body.go new file mode 100644 index 000000000..65a9eab2d --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/expand_body.go @@ -0,0 +1,262 @@ +package dynblock + +import ( + "fmt" + + "github.com/hashicorp/hcl/v2" + "github.com/zclconf/go-cty/cty" +) + +// expandBody wraps another hcl.Body and expands any "dynamic" blocks found +// inside whenever Content or PartialContent is called. +type expandBody struct { + original hcl.Body + forEachCtx *hcl.EvalContext + iteration *iteration // non-nil if we're nested inside another "dynamic" block + + // These are used with PartialContent to produce a "remaining items" + // body to return. They are nil on all bodies fresh out of the transformer. + // + // Note that this is re-implemented here rather than delegating to the + // existing support required by the underlying body because we need to + // retain access to the entire original body on subsequent decode operations + // so we can retain any "dynamic" blocks for types we didn't take consume + // on the first pass. + hiddenAttrs map[string]struct{} + hiddenBlocks map[string]hcl.BlockHeaderSchema +} + +func (b *expandBody) Content(schema *hcl.BodySchema) (*hcl.BodyContent, hcl.Diagnostics) { + extSchema := b.extendSchema(schema) + rawContent, diags := b.original.Content(extSchema) + + blocks, blockDiags := b.expandBlocks(schema, rawContent.Blocks, false) + diags = append(diags, blockDiags...) + attrs := b.prepareAttributes(rawContent.Attributes) + + content := &hcl.BodyContent{ + Attributes: attrs, + Blocks: blocks, + MissingItemRange: b.original.MissingItemRange(), + } + + return content, diags +} + +func (b *expandBody) PartialContent(schema *hcl.BodySchema) (*hcl.BodyContent, hcl.Body, hcl.Diagnostics) { + extSchema := b.extendSchema(schema) + rawContent, _, diags := b.original.PartialContent(extSchema) + // We discard the "remain" argument above because we're going to construct + // our own remain that also takes into account remaining "dynamic" blocks. + + blocks, blockDiags := b.expandBlocks(schema, rawContent.Blocks, true) + diags = append(diags, blockDiags...) + attrs := b.prepareAttributes(rawContent.Attributes) + + content := &hcl.BodyContent{ + Attributes: attrs, + Blocks: blocks, + MissingItemRange: b.original.MissingItemRange(), + } + + remain := &expandBody{ + original: b.original, + forEachCtx: b.forEachCtx, + iteration: b.iteration, + hiddenAttrs: make(map[string]struct{}), + hiddenBlocks: make(map[string]hcl.BlockHeaderSchema), + } + for name := range b.hiddenAttrs { + remain.hiddenAttrs[name] = struct{}{} + } + for typeName, blockS := range b.hiddenBlocks { + remain.hiddenBlocks[typeName] = blockS + } + for _, attrS := range schema.Attributes { + remain.hiddenAttrs[attrS.Name] = struct{}{} + } + for _, blockS := range schema.Blocks { + remain.hiddenBlocks[blockS.Type] = blockS + } + + return content, remain, diags +} + +func (b *expandBody) extendSchema(schema *hcl.BodySchema) *hcl.BodySchema { + // We augment the requested schema to also include our special "dynamic" + // block type, since then we'll get instances of it interleaved with + // all of the literal child blocks we must also include. + extSchema := &hcl.BodySchema{ + Attributes: schema.Attributes, + Blocks: make([]hcl.BlockHeaderSchema, len(schema.Blocks), len(schema.Blocks)+len(b.hiddenBlocks)+1), + } + copy(extSchema.Blocks, schema.Blocks) + extSchema.Blocks = append(extSchema.Blocks, dynamicBlockHeaderSchema) + + // If we have any hiddenBlocks then we also need to register those here + // so that a call to "Content" on the underlying body won't fail. + // (We'll filter these out again once we process the result of either + // Content or PartialContent.) + for _, blockS := range b.hiddenBlocks { + extSchema.Blocks = append(extSchema.Blocks, blockS) + } + + // If we have any hiddenAttrs then we also need to register these, for + // the same reason as we deal with hiddenBlocks above. + if len(b.hiddenAttrs) != 0 { + newAttrs := make([]hcl.AttributeSchema, len(schema.Attributes), len(schema.Attributes)+len(b.hiddenAttrs)) + copy(newAttrs, extSchema.Attributes) + for name := range b.hiddenAttrs { + newAttrs = append(newAttrs, hcl.AttributeSchema{ + Name: name, + Required: false, + }) + } + extSchema.Attributes = newAttrs + } + + return extSchema +} + +func (b *expandBody) prepareAttributes(rawAttrs hcl.Attributes) hcl.Attributes { + if len(b.hiddenAttrs) == 0 && b.iteration == nil { + // Easy path: just pass through the attrs from the original body verbatim + return rawAttrs + } + + // Otherwise we have some work to do: we must filter out any attributes + // that are hidden (since a previous PartialContent call already saw these) + // and wrap the expressions of the inner attributes so that they will + // have access to our iteration variables. + attrs := make(hcl.Attributes, len(rawAttrs)) + for name, rawAttr := range rawAttrs { + if _, hidden := b.hiddenAttrs[name]; hidden { + continue + } + if b.iteration != nil { + attr := *rawAttr // shallow copy so we can mutate it + attr.Expr = exprWrap{ + Expression: attr.Expr, + i: b.iteration, + } + attrs[name] = &attr + } else { + // If we have no active iteration then no wrapping is required. + attrs[name] = rawAttr + } + } + return attrs +} + +func (b *expandBody) expandBlocks(schema *hcl.BodySchema, rawBlocks hcl.Blocks, partial bool) (hcl.Blocks, hcl.Diagnostics) { + var blocks hcl.Blocks + var diags hcl.Diagnostics + + for _, rawBlock := range rawBlocks { + switch rawBlock.Type { + case "dynamic": + realBlockType := rawBlock.Labels[0] + if _, hidden := b.hiddenBlocks[realBlockType]; hidden { + continue + } + + var blockS *hcl.BlockHeaderSchema + for _, candidate := range schema.Blocks { + if candidate.Type == realBlockType { + blockS = &candidate + break + } + } + if blockS == nil { + // Not a block type that the caller requested. + if !partial { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Unsupported block type", + Detail: fmt.Sprintf("Blocks of type %q are not expected here.", realBlockType), + Subject: &rawBlock.LabelRanges[0], + }) + } + continue + } + + spec, specDiags := b.decodeSpec(blockS, rawBlock) + diags = append(diags, specDiags...) + if specDiags.HasErrors() { + continue + } + + if spec.forEachVal.IsKnown() { + for it := spec.forEachVal.ElementIterator(); it.Next(); { + key, value := it.Element() + i := b.iteration.MakeChild(spec.iteratorName, key, value) + + block, blockDiags := spec.newBlock(i, b.forEachCtx) + diags = append(diags, blockDiags...) + if block != nil { + // Attach our new iteration context so that attributes + // and other nested blocks can refer to our iterator. + block.Body = b.expandChild(block.Body, i) + blocks = append(blocks, block) + } + } + } else { + // If our top-level iteration value isn't known then we're forced + // to compromise since HCL doesn't have any concept of an + // "unknown block". In this case then, we'll produce a single + // dynamic block with the iterator values set to DynamicVal, + // which at least makes the potential for a block visible + // in our result, even though it's not represented in a fully-accurate + // way. + i := b.iteration.MakeChild(spec.iteratorName, cty.DynamicVal, cty.DynamicVal) + block, blockDiags := spec.newBlock(i, b.forEachCtx) + diags = append(diags, blockDiags...) + if block != nil { + block.Body = b.expandChild(block.Body, i) + + // We additionally force all of the leaf attribute values + // in the result to be unknown so the calling application + // can, if necessary, use that as a heuristic to detect + // when a single nested block might be standing in for + // multiple blocks yet to be expanded. This retains the + // structure of the generated body but forces all of its + // leaf attribute values to be unknown. + block.Body = unknownBody{block.Body} + + blocks = append(blocks, block) + } + } + + default: + if _, hidden := b.hiddenBlocks[rawBlock.Type]; !hidden { + // A static block doesn't create a new iteration context, but + // it does need to inherit _our own_ iteration context in + // case it contains expressions that refer to our inherited + // iterators, or nested "dynamic" blocks. + expandedBlock := *rawBlock // shallow copy + expandedBlock.Body = b.expandChild(rawBlock.Body, b.iteration) + blocks = append(blocks, &expandedBlock) + } + } + } + + return blocks, diags +} + +func (b *expandBody) expandChild(child hcl.Body, i *iteration) hcl.Body { + chiCtx := i.EvalContext(b.forEachCtx) + ret := Expand(child, chiCtx) + ret.(*expandBody).iteration = i + return ret +} + +func (b *expandBody) JustAttributes() (hcl.Attributes, hcl.Diagnostics) { + // blocks aren't allowed in JustAttributes mode and this body can + // only produce blocks, so we'll just pass straight through to our + // underlying body here. + return b.original.JustAttributes() +} + +func (b *expandBody) MissingItemRange() hcl.Range { + return b.original.MissingItemRange() +} diff --git a/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/expand_spec.go b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/expand_spec.go new file mode 100644 index 000000000..98a51eadd --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/expand_spec.go @@ -0,0 +1,215 @@ +package dynblock + +import ( + "fmt" + + "github.com/hashicorp/hcl/v2" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/convert" +) + +type expandSpec struct { + blockType string + blockTypeRange hcl.Range + defRange hcl.Range + forEachVal cty.Value + iteratorName string + labelExprs []hcl.Expression + contentBody hcl.Body + inherited map[string]*iteration +} + +func (b *expandBody) decodeSpec(blockS *hcl.BlockHeaderSchema, rawSpec *hcl.Block) (*expandSpec, hcl.Diagnostics) { + var diags hcl.Diagnostics + + var schema *hcl.BodySchema + if len(blockS.LabelNames) != 0 { + schema = dynamicBlockBodySchemaLabels + } else { + schema = dynamicBlockBodySchemaNoLabels + } + + specContent, specDiags := rawSpec.Body.Content(schema) + diags = append(diags, specDiags...) + if specDiags.HasErrors() { + return nil, diags + } + + //// for_each attribute + + eachAttr := specContent.Attributes["for_each"] + eachVal, eachDiags := eachAttr.Expr.Value(b.forEachCtx) + diags = append(diags, eachDiags...) + + if !eachVal.CanIterateElements() && eachVal.Type() != cty.DynamicPseudoType { + // We skip this error for DynamicPseudoType because that means we either + // have a null (which is checked immediately below) or an unknown + // (which is handled in the expandBody Content methods). + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid dynamic for_each value", + Detail: fmt.Sprintf("Cannot use a %s value in for_each. An iterable collection is required.", eachVal.Type().FriendlyName()), + Subject: eachAttr.Expr.Range().Ptr(), + Expression: eachAttr.Expr, + EvalContext: b.forEachCtx, + }) + return nil, diags + } + if eachVal.IsNull() { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid dynamic for_each value", + Detail: "Cannot use a null value in for_each.", + Subject: eachAttr.Expr.Range().Ptr(), + Expression: eachAttr.Expr, + EvalContext: b.forEachCtx, + }) + return nil, diags + } + + //// iterator attribute + + iteratorName := blockS.Type + if iteratorAttr := specContent.Attributes["iterator"]; iteratorAttr != nil { + itTraversal, itDiags := hcl.AbsTraversalForExpr(iteratorAttr.Expr) + diags = append(diags, itDiags...) + if itDiags.HasErrors() { + return nil, diags + } + + if len(itTraversal) != 1 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid dynamic iterator name", + Detail: "Dynamic iterator must be a single variable name.", + Subject: itTraversal.SourceRange().Ptr(), + }) + return nil, diags + } + + iteratorName = itTraversal.RootName() + } + + var labelExprs []hcl.Expression + if labelsAttr := specContent.Attributes["labels"]; labelsAttr != nil { + var labelDiags hcl.Diagnostics + labelExprs, labelDiags = hcl.ExprList(labelsAttr.Expr) + diags = append(diags, labelDiags...) + if labelDiags.HasErrors() { + return nil, diags + } + + if len(labelExprs) > len(blockS.LabelNames) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Extraneous dynamic block label", + Detail: fmt.Sprintf("Blocks of type %q require %d label(s).", blockS.Type, len(blockS.LabelNames)), + Subject: labelExprs[len(blockS.LabelNames)].Range().Ptr(), + }) + return nil, diags + } else if len(labelExprs) < len(blockS.LabelNames) { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Insufficient dynamic block labels", + Detail: fmt.Sprintf("Blocks of type %q require %d label(s).", blockS.Type, len(blockS.LabelNames)), + Subject: labelsAttr.Expr.Range().Ptr(), + }) + return nil, diags + } + } + + // Since our schema requests only blocks of type "content", we can assume + // that all entries in specContent.Blocks are content blocks. + if len(specContent.Blocks) == 0 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Missing dynamic content block", + Detail: "A dynamic block must have a nested block of type \"content\" to describe the body of each generated block.", + Subject: &specContent.MissingItemRange, + }) + return nil, diags + } + if len(specContent.Blocks) > 1 { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Extraneous dynamic content block", + Detail: "Only one nested content block is allowed for each dynamic block.", + Subject: &specContent.Blocks[1].DefRange, + }) + return nil, diags + } + + return &expandSpec{ + blockType: blockS.Type, + blockTypeRange: rawSpec.LabelRanges[0], + defRange: rawSpec.DefRange, + forEachVal: eachVal, + iteratorName: iteratorName, + labelExprs: labelExprs, + contentBody: specContent.Blocks[0].Body, + }, diags +} + +func (s *expandSpec) newBlock(i *iteration, ctx *hcl.EvalContext) (*hcl.Block, hcl.Diagnostics) { + var diags hcl.Diagnostics + var labels []string + var labelRanges []hcl.Range + lCtx := i.EvalContext(ctx) + for _, labelExpr := range s.labelExprs { + labelVal, labelDiags := labelExpr.Value(lCtx) + diags = append(diags, labelDiags...) + if labelDiags.HasErrors() { + return nil, diags + } + + var convErr error + labelVal, convErr = convert.Convert(labelVal, cty.String) + if convErr != nil { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid dynamic block label", + Detail: fmt.Sprintf("Cannot use this value as a dynamic block label: %s.", convErr), + Subject: labelExpr.Range().Ptr(), + Expression: labelExpr, + EvalContext: lCtx, + }) + return nil, diags + } + if labelVal.IsNull() { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid dynamic block label", + Detail: "Cannot use a null value as a dynamic block label.", + Subject: labelExpr.Range().Ptr(), + Expression: labelExpr, + EvalContext: lCtx, + }) + return nil, diags + } + if !labelVal.IsKnown() { + diags = append(diags, &hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "Invalid dynamic block label", + Detail: "This value is not yet known. Dynamic block labels must be immediately-known values.", + Subject: labelExpr.Range().Ptr(), + Expression: labelExpr, + EvalContext: lCtx, + }) + return nil, diags + } + + labels = append(labels, labelVal.AsString()) + labelRanges = append(labelRanges, labelExpr.Range()) + } + + block := &hcl.Block{ + Type: s.blockType, + TypeRange: s.blockTypeRange, + Labels: labels, + LabelRanges: labelRanges, + DefRange: s.defRange, + Body: s.contentBody, + } + + return block, diags +} diff --git a/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/expr_wrap.go b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/expr_wrap.go new file mode 100644 index 000000000..460a1d2a3 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/expr_wrap.go @@ -0,0 +1,42 @@ +package dynblock + +import ( + "github.com/hashicorp/hcl/v2" + "github.com/zclconf/go-cty/cty" +) + +type exprWrap struct { + hcl.Expression + i *iteration +} + +func (e exprWrap) Variables() []hcl.Traversal { + raw := e.Expression.Variables() + ret := make([]hcl.Traversal, 0, len(raw)) + + // Filter out traversals that refer to our iterator name or any + // iterator we've inherited; we're going to provide those in + // our Value wrapper, so the caller doesn't need to know about them. + for _, traversal := range raw { + rootName := traversal.RootName() + if rootName == e.i.IteratorName { + continue + } + if _, inherited := e.i.Inherited[rootName]; inherited { + continue + } + ret = append(ret, traversal) + } + return ret +} + +func (e exprWrap) Value(ctx *hcl.EvalContext) (cty.Value, hcl.Diagnostics) { + extCtx := e.i.EvalContext(ctx) + return e.Expression.Value(extCtx) +} + +// UnwrapExpression returns the expression being wrapped by this instance. +// This allows the original expression to be recovered by hcl.UnwrapExpression. +func (e exprWrap) UnwrapExpression() hcl.Expression { + return e.Expression +} diff --git a/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/iteration.go b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/iteration.go new file mode 100644 index 000000000..c56638868 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/iteration.go @@ -0,0 +1,66 @@ +package dynblock + +import ( + "github.com/hashicorp/hcl/v2" + "github.com/zclconf/go-cty/cty" +) + +type iteration struct { + IteratorName string + Key cty.Value + Value cty.Value + Inherited map[string]*iteration +} + +func (s *expandSpec) MakeIteration(key, value cty.Value) *iteration { + return &iteration{ + IteratorName: s.iteratorName, + Key: key, + Value: value, + Inherited: s.inherited, + } +} + +func (i *iteration) Object() cty.Value { + return cty.ObjectVal(map[string]cty.Value{ + "key": i.Key, + "value": i.Value, + }) +} + +func (i *iteration) EvalContext(base *hcl.EvalContext) *hcl.EvalContext { + new := base.NewChild() + + if i != nil { + new.Variables = map[string]cty.Value{} + for name, otherIt := range i.Inherited { + new.Variables[name] = otherIt.Object() + } + new.Variables[i.IteratorName] = i.Object() + } + + return new +} + +func (i *iteration) MakeChild(iteratorName string, key, value cty.Value) *iteration { + if i == nil { + // Create entirely new root iteration, then + return &iteration{ + IteratorName: iteratorName, + Key: key, + Value: value, + } + } + + inherited := map[string]*iteration{} + for name, otherIt := range i.Inherited { + inherited[name] = otherIt + } + inherited[i.IteratorName] = i + return &iteration{ + IteratorName: iteratorName, + Key: key, + Value: value, + Inherited: inherited, + } +} diff --git a/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/public.go b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/public.go new file mode 100644 index 000000000..a5bfd94ec --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/public.go @@ -0,0 +1,47 @@ +// Package dynblock provides an extension to HCL that allows dynamic +// declaration of nested blocks in certain contexts via a special block type +// named "dynamic". +package dynblock + +import ( + "github.com/hashicorp/hcl/v2" +) + +// Expand "dynamic" blocks in the given body, returning a new body that +// has those blocks expanded. +// +// The given EvalContext is used when evaluating "for_each" and "labels" +// attributes within dynamic blocks, allowing those expressions access to +// variables and functions beyond the iterator variable created by the +// iteration. +// +// Expand returns no diagnostics because no blocks are actually expanded +// until a call to Content or PartialContent on the returned body, which +// will then expand only the blocks selected by the schema. +// +// "dynamic" blocks are also expanded automatically within nested blocks +// in the given body, including within other dynamic blocks, thus allowing +// multi-dimensional iteration. However, it is not possible to +// dynamically-generate the "dynamic" blocks themselves except through nesting. +// +// parent { +// dynamic "child" { +// for_each = child_objs +// content { +// dynamic "grandchild" { +// for_each = child.value.children +// labels = [grandchild.key] +// content { +// parent_key = child.key +// value = grandchild.value +// } +// } +// } +// } +// } +func Expand(body hcl.Body, ctx *hcl.EvalContext) hcl.Body { + return &expandBody{ + original: body, + forEachCtx: ctx, + } +} diff --git a/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/schema.go b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/schema.go new file mode 100644 index 000000000..b3907d6ea --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/schema.go @@ -0,0 +1,50 @@ +package dynblock + +import "github.com/hashicorp/hcl/v2" + +var dynamicBlockHeaderSchema = hcl.BlockHeaderSchema{ + Type: "dynamic", + LabelNames: []string{"type"}, +} + +var dynamicBlockBodySchemaLabels = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "for_each", + Required: true, + }, + { + Name: "iterator", + Required: false, + }, + { + Name: "labels", + Required: true, + }, + }, + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "content", + LabelNames: nil, + }, + }, +} + +var dynamicBlockBodySchemaNoLabels = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "for_each", + Required: true, + }, + { + Name: "iterator", + Required: false, + }, + }, + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "content", + LabelNames: nil, + }, + }, +} diff --git a/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/unknown_body.go b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/unknown_body.go new file mode 100644 index 000000000..ce98259a5 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/unknown_body.go @@ -0,0 +1,84 @@ +package dynblock + +import ( + "github.com/hashicorp/hcl/v2" + "github.com/zclconf/go-cty/cty" +) + +// unknownBody is a funny body that just reports everything inside it as +// unknown. It uses a given other body as a sort of template for what attributes +// and blocks are inside -- including source location information -- but +// subsitutes unknown values of unknown type for all attributes. +// +// This rather odd process is used to handle expansion of dynamic blocks whose +// for_each expression is unknown. Since a block cannot itself be unknown, +// we instead arrange for everything _inside_ the block to be unknown instead, +// to give the best possible approximation. +type unknownBody struct { + template hcl.Body +} + +var _ hcl.Body = unknownBody{} + +func (b unknownBody) Content(schema *hcl.BodySchema) (*hcl.BodyContent, hcl.Diagnostics) { + content, diags := b.template.Content(schema) + content = b.fixupContent(content) + + // We're intentionally preserving the diagnostics reported from the + // inner body so that we can still report where the template body doesn't + // match the requested schema. + return content, diags +} + +func (b unknownBody) PartialContent(schema *hcl.BodySchema) (*hcl.BodyContent, hcl.Body, hcl.Diagnostics) { + content, remain, diags := b.template.PartialContent(schema) + content = b.fixupContent(content) + remain = unknownBody{remain} // remaining content must also be wrapped + + // We're intentionally preserving the diagnostics reported from the + // inner body so that we can still report where the template body doesn't + // match the requested schema. + return content, remain, diags +} + +func (b unknownBody) JustAttributes() (hcl.Attributes, hcl.Diagnostics) { + attrs, diags := b.template.JustAttributes() + attrs = b.fixupAttrs(attrs) + + // We're intentionally preserving the diagnostics reported from the + // inner body so that we can still report where the template body doesn't + // match the requested schema. + return attrs, diags +} + +func (b unknownBody) MissingItemRange() hcl.Range { + return b.template.MissingItemRange() +} + +func (b unknownBody) fixupContent(got *hcl.BodyContent) *hcl.BodyContent { + ret := &hcl.BodyContent{} + ret.Attributes = b.fixupAttrs(got.Attributes) + if len(got.Blocks) > 0 { + ret.Blocks = make(hcl.Blocks, 0, len(got.Blocks)) + for _, gotBlock := range got.Blocks { + new := *gotBlock // shallow copy + new.Body = unknownBody{gotBlock.Body} // nested content must also be marked unknown + ret.Blocks = append(ret.Blocks, &new) + } + } + + return ret +} + +func (b unknownBody) fixupAttrs(got hcl.Attributes) hcl.Attributes { + if len(got) == 0 { + return nil + } + ret := make(hcl.Attributes, len(got)) + for name, gotAttr := range got { + new := *gotAttr // shallow copy + new.Expr = hcl.StaticExpr(cty.DynamicVal, gotAttr.Expr.Range()) + ret[name] = &new + } + return ret +} diff --git a/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/variables.go b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/variables.go new file mode 100644 index 000000000..192339295 --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/variables.go @@ -0,0 +1,209 @@ +package dynblock + +import ( + "github.com/hashicorp/hcl/v2" + "github.com/zclconf/go-cty/cty" +) + +// WalkVariables begins the recursive process of walking all expressions and +// nested blocks in the given body and its child bodies while taking into +// account any "dynamic" blocks. +// +// This function requires that the caller walk through the nested block +// structure in the given body level-by-level so that an appropriate schema +// can be provided at each level to inform further processing. This workflow +// is thus easiest to use for calling applications that have some higher-level +// schema representation available with which to drive this multi-step +// process. If your application uses the hcldec package, you may be able to +// use VariablesHCLDec instead for a more automatic approach. +func WalkVariables(body hcl.Body) WalkVariablesNode { + return WalkVariablesNode{ + body: body, + includeContent: true, + } +} + +// WalkExpandVariables is like Variables but it includes only the variables +// required for successful block expansion, ignoring any variables referenced +// inside block contents. The result is the minimal set of all variables +// required for a call to Expand, excluding variables that would only be +// needed to subsequently call Content or PartialContent on the expanded +// body. +func WalkExpandVariables(body hcl.Body) WalkVariablesNode { + return WalkVariablesNode{ + body: body, + } +} + +type WalkVariablesNode struct { + body hcl.Body + it *iteration + + includeContent bool +} + +type WalkVariablesChild struct { + BlockTypeName string + Node WalkVariablesNode +} + +// Body returns the HCL Body associated with the child node, in case the caller +// wants to do some sort of inspection of it in order to decide what schema +// to pass to Visit. +// +// Most implementations should just fetch a fixed schema based on the +// BlockTypeName field and not access this. Deciding on a schema dynamically +// based on the body is a strange thing to do and generally necessary only if +// your caller is already doing other bizarre things with HCL bodies. +func (c WalkVariablesChild) Body() hcl.Body { + return c.Node.body +} + +// Visit returns the variable traversals required for any "dynamic" blocks +// directly in the body associated with this node, and also returns any child +// nodes that must be visited in order to continue the walk. +// +// Each child node has its associated block type name given in its BlockTypeName +// field, which the calling application should use to determine the appropriate +// schema for the content of each child node and pass it to the child node's +// own Visit method to continue the walk recursively. +func (n WalkVariablesNode) Visit(schema *hcl.BodySchema) (vars []hcl.Traversal, children []WalkVariablesChild) { + extSchema := n.extendSchema(schema) + container, _, _ := n.body.PartialContent(extSchema) + if container == nil { + return vars, children + } + + children = make([]WalkVariablesChild, 0, len(container.Blocks)) + + if n.includeContent { + for _, attr := range container.Attributes { + for _, traversal := range attr.Expr.Variables() { + var ours, inherited bool + if n.it != nil { + ours = traversal.RootName() == n.it.IteratorName + _, inherited = n.it.Inherited[traversal.RootName()] + } + + if !(ours || inherited) { + vars = append(vars, traversal) + } + } + } + } + + for _, block := range container.Blocks { + switch block.Type { + + case "dynamic": + blockTypeName := block.Labels[0] + inner, _, _ := block.Body.PartialContent(variableDetectionInnerSchema) + if inner == nil { + continue + } + + iteratorName := blockTypeName + if attr, exists := inner.Attributes["iterator"]; exists { + iterTraversal, _ := hcl.AbsTraversalForExpr(attr.Expr) + if len(iterTraversal) == 0 { + // Ignore this invalid dynamic block, since it'll produce + // an error if someone tries to extract content from it + // later anyway. + continue + } + iteratorName = iterTraversal.RootName() + } + blockIt := n.it.MakeChild(iteratorName, cty.DynamicVal, cty.DynamicVal) + + if attr, exists := inner.Attributes["for_each"]; exists { + // Filter out iterator names inherited from parent blocks + for _, traversal := range attr.Expr.Variables() { + if _, inherited := blockIt.Inherited[traversal.RootName()]; !inherited { + vars = append(vars, traversal) + } + } + } + if attr, exists := inner.Attributes["labels"]; exists { + // Filter out both our own iterator name _and_ those inherited + // from parent blocks, since we provide _both_ of these to the + // label expressions. + for _, traversal := range attr.Expr.Variables() { + ours := traversal.RootName() == iteratorName + _, inherited := blockIt.Inherited[traversal.RootName()] + + if !(ours || inherited) { + vars = append(vars, traversal) + } + } + } + + for _, contentBlock := range inner.Blocks { + // We only request "content" blocks in our schema, so we know + // any blocks we find here will be content blocks. We require + // exactly one content block for actual expansion, but we'll + // be more liberal here so that callers can still collect + // variables from erroneous "dynamic" blocks. + children = append(children, WalkVariablesChild{ + BlockTypeName: blockTypeName, + Node: WalkVariablesNode{ + body: contentBlock.Body, + it: blockIt, + includeContent: n.includeContent, + }, + }) + } + + default: + children = append(children, WalkVariablesChild{ + BlockTypeName: block.Type, + Node: WalkVariablesNode{ + body: block.Body, + it: n.it, + includeContent: n.includeContent, + }, + }) + + } + } + + return vars, children +} + +func (n WalkVariablesNode) extendSchema(schema *hcl.BodySchema) *hcl.BodySchema { + // We augment the requested schema to also include our special "dynamic" + // block type, since then we'll get instances of it interleaved with + // all of the literal child blocks we must also include. + extSchema := &hcl.BodySchema{ + Attributes: schema.Attributes, + Blocks: make([]hcl.BlockHeaderSchema, len(schema.Blocks), len(schema.Blocks)+1), + } + copy(extSchema.Blocks, schema.Blocks) + extSchema.Blocks = append(extSchema.Blocks, dynamicBlockHeaderSchema) + + return extSchema +} + +// This is a more relaxed schema than what's in schema.go, since we +// want to maximize the amount of variables we can find even if there +// are erroneous blocks. +var variableDetectionInnerSchema = &hcl.BodySchema{ + Attributes: []hcl.AttributeSchema{ + { + Name: "for_each", + Required: false, + }, + { + Name: "labels", + Required: false, + }, + { + Name: "iterator", + Required: false, + }, + }, + Blocks: []hcl.BlockHeaderSchema{ + { + Type: "content", + }, + }, +} diff --git a/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/variables_hcldec.go b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/variables_hcldec.go new file mode 100644 index 000000000..907ef3eba --- /dev/null +++ b/vendor/github.com/hashicorp/hcl/v2/ext/dynblock/variables_hcldec.go @@ -0,0 +1,43 @@ +package dynblock + +import ( + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hcldec" +) + +// VariablesHCLDec is a wrapper around WalkVariables that uses the given hcldec +// specification to automatically drive the recursive walk through nested +// blocks in the given body. +// +// This is a drop-in replacement for hcldec.Variables which is able to treat +// blocks of type "dynamic" in the same special way that dynblock.Expand would, +// exposing both the variables referenced in the "for_each" and "labels" +// arguments and variables used in the nested "content" block. +func VariablesHCLDec(body hcl.Body, spec hcldec.Spec) []hcl.Traversal { + rootNode := WalkVariables(body) + return walkVariablesWithHCLDec(rootNode, spec) +} + +// ExpandVariablesHCLDec is like VariablesHCLDec but it includes only the +// minimal set of variables required to call Expand, ignoring variables that +// are referenced only inside normal block contents. See WalkExpandVariables +// for more information. +func ExpandVariablesHCLDec(body hcl.Body, spec hcldec.Spec) []hcl.Traversal { + rootNode := WalkExpandVariables(body) + return walkVariablesWithHCLDec(rootNode, spec) +} + +func walkVariablesWithHCLDec(node WalkVariablesNode, spec hcldec.Spec) []hcl.Traversal { + vars, children := node.Visit(hcldec.ImpliedSchema(spec)) + + if len(children) > 0 { + childSpecs := hcldec.ChildBlockTypes(spec) + for _, child := range children { + if childSpec, exists := childSpecs[child.BlockTypeName]; exists { + vars = append(vars, walkVariablesWithHCLDec(child.Node, childSpec)...) + } + } + } + + return vars +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 8c6a868dd..59b8dce5e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -357,6 +357,7 @@ github.com/hashicorp/hcl/json/token # github.com/hashicorp/hcl/v2 v2.3.0 github.com/hashicorp/hcl/v2 github.com/hashicorp/hcl/v2/ext/customdecode +github.com/hashicorp/hcl/v2/ext/dynblock github.com/hashicorp/hcl/v2/ext/tryfunc github.com/hashicorp/hcl/v2/ext/typeexpr github.com/hashicorp/hcl/v2/gohcl diff --git a/website/source/docs/configuration/from-1.5/expressions.html.md b/website/source/docs/configuration/from-1.5/expressions.html.md index 8c2069a26..348ed7f7e 100644 --- a/website/source/docs/configuration/from-1.5/expressions.html.md +++ b/website/source/docs/configuration/from-1.5/expressions.html.md @@ -4,7 +4,7 @@ page_title: "Expressions - Configuration Language" sidebar_current: configuration-expressions description: |- HCL allows the use of expressions to access data exported - by resources and to transform and combine that data to produce other values. + by sources and to transform and combine that data to produce other values. --- # Expressions @@ -12,7 +12,7 @@ description: |- _Expressions_ are used to refer to or compute values within a configuration. The simplest expressions are just literal values, like `"hello"` or `5`, but HCL also allows more complex expressions such as references to data exported by -resources, arithmetic, conditional evaluation, and a number of built-in +sources, arithmetic, conditional evaluation, and a number of built-in functions. Expressions can be used in a number of places in HCL, but some contexts limit @@ -70,9 +70,9 @@ source arguments. ### Type Conversion -Expressions are most often used to set values for the arguments of resources and -child modules. In these cases, the argument has an expected type and the given -expression must produce a value of that type. +Expressions are most often used to set values for arguments. In these cases, +the argument has an expected type and the given expression must produce a value +of that type. Where possible, Packer automatically converts values from one type to another in order to produce the expected type. If this isn't possible, Packer @@ -137,6 +137,216 @@ The following named values are available: [source](./sources.html) of the given type and name. +### Available Functions + +For a full list of available functions, see [the function +reference](/docs/configuration/from-1.5/functions.html). + +## `for` Expressions + +A _`for` expression_ creates a complex type value by transforming +another complex type value. Each element in the input value +can correspond to either one or zero values in the result, and an arbitrary +expression can be used to transform each input element into an output element. + +For example, if `var.list` is a list of strings, then the following expression +produces a list of strings with all-uppercase letters: + +```hcl +[for s in var.list : upper(s)] +``` + +This `for` expression iterates over each element of `var.list`, and then +evaluates the expression `upper(s)` with `s` set to each respective element. +It then builds a new tuple value with all of the results of executing that +expression in the same order. + +The type of brackets around the `for` expression decide what type of result +it produces. The above example uses `[` and `]`, which produces a tuple. If +`{` and `}` are used instead, the result is an object, and two result +expressions must be provided separated by the `=>` symbol: + +```hcl +{for s in var.list : s => upper(s)} +``` + +This expression produces an object whose attributes are the original elements +from `var.list` and their corresponding values are the uppercase versions. + +A `for` expression can also include an optional `if` clause to filter elements +from the source collection, which can produce a value with fewer elements than +the source: + +``` +[for s in var.list : upper(s) if s != ""] +``` + +The source value can also be an object or map value, in which case two +temporary variable names can be provided to access the keys and values +respectively: + +``` +[for k, v in var.map : length(k) + length(v)] +``` + +Finally, if the result type is an object (using `{` and `}` delimiters) then +the value result expression can be followed by the `...` symbol to group +together results that have a common key: + +``` +{for s in var.list : substr(s, 0, 1) => s... if s != ""} +``` + +## Splat Expressions + +A _splat expression_ provides a more concise way to express a common operation +that could otherwise be performed with a `for` expression. + +If `var.list` is a list of objects that all have an attribute `id`, then a list +of the ids could be produced with the following `for` expression: + +```hcl +[for o in var.list : o.id] +``` + +This is equivalent to the following _splat expression:_ + +```hcl +var.list[*].id +``` + +The special `[*]` symbol iterates over all of the elements of the list given to +its left and accesses from each one the attribute name given on its right. A +splat expression can also be used to access attributes and indexes from lists +of complex types by extending the sequence of operations to the right of the +symbol: + +```hcl +var.list[*].interfaces[0].name +``` + +The above expression is equivalent to the following `for` expression: + +```hcl +[for o in var.list : o.interfaces[0].name] +``` + +Splat expressions are for lists only (and thus cannot be used [to reference +resources created with +`for_each`](/docs/configuration/resources.html#referring-to-instances-1), which +are represented as maps). However, if a splat expression is applied to a value +that is _not_ a list or tuple then the value is automatically wrapped in a +single-element list before processing. + +For example, `var.single_object[*].id` is equivalent to +`[var.single_object][*].id`, or effectively `[var.single_object.id]`. This +behavior is not interesting in most cases, but it is particularly useful when +referring to resources that may or may not have `count` set, and thus may or +may not produce a tuple value: + +```hcl +aws_instance.example[*].id +``` + +The above will produce a list of ids whether `aws_instance.example` has `count` +set or not, avoiding the need to revise various other expressions in the +configuration when a particular resource switches to and from having `count` +set. + +## `dynamic` blocks + +Within top-level block constructs like sources, expressions can usually be used +only when assigning a value to an argument using the `name = expression` form. +This covers many uses, but some source types include repeatable _nested +blocks_ in their arguments, which do not accept expressions: + +```hcl +source "amazon-ebs" "example" { + name = "pkr-test-name" # can use expressions here + + tag { + # but the "tag" block is always a literal block + } +} +``` + +You can dynamically construct repeatable nested blocks like `tag` using a +special `dynamic` block type, which is supported anywhere, example: + +```hcl +locals { + standard_tags = { + Component = "user-service" + Environment = "production" + } +} + +source "amazon-ebs" "example" { + # ... + + tag { + key = "Name" + value = "example-asg-name" + } + + dynamic "tag" { + for_each = local.standard_tags + + content { + key = tag.key + value = tag.value + } + } +} +``` + +A `dynamic` block acts much like a `for` expression, but produces nested blocks +instead of a complex typed value. It iterates over a given complex value, and +generates a nested block for each element of that complex value. + +- The label of the dynamic block (`"tag"` in the example above) specifies + what kind of nested block to generate. +- The `for_each` argument provides the complex value to iterate over. +- The `iterator` argument (optional) sets the name of a temporary variable + that represents the current element of the complex value. If omitted, the name + of the variable defaults to the label of the `dynamic` block (`"tag"` in + the example above). +- The `labels` argument (optional) is a list of strings that specifies the block + labels, in order, to use for each generated block. You can use the temporary + iterator variable in this value. +- The nested `content` block defines the body of each generated block. You can + use the temporary iterator variable inside this block. + +Since the `for_each` argument accepts any collection or structural value, +you can use a `for` expression or splat expression to transform an existing +collection. + +The iterator object (`tag` in the example above) has two attributes: + +* `key` is the map key or list element index for the current element. If the + `for_each` expression produces a _set_ value then `key` is identical to + `value` and should not be used. +* `value` is the value of the current element. + +A `dynamic` block can only generate arguments that belong to the source type, +data source or provisioner being configured. + +The `for_each` value must be a map or set with one element per desired nested +block. If you need to declare resource instances based on a nested data +structure or combinations of elements from multiple data structures you can use +expressions and functions to derive a suitable value. For some common examples +of such situations, see the +[`flatten`](/docs/configuration/from-1.5/functions/collection/flatten.html) and +[`setproduct`](/docs/configuration/from-1.5/functions/collection/setproduct.html) +functions. + +### Best Practices for `dynamic` Blocks + +Overuse of `dynamic` blocks can make configuration hard to read and maintain, +so we recommend using them only when you need to hide details in order to build +a clean user interface for a re-usable code. Always write nested blocks out +literally where possible. + ## String Literals HCL has two different syntaxes for string literals. The