command/cliconfig/syntax: Just enough HCL 2 API for CLI Config decoding

For compatibility reasons we can't use the HCL 2 parser to parse the CLI
config: our historical format uses an incompatible syntax for environment
variable substitutions in certain values.

However, the HCL 1 decoding API makes it impossible to deal with certain
configuration structures involving sequences of blocks of different types
where ordering must be preserved, which we need for a forthcoming feature
to specify provider installation locations.

To deal with this empasse, here we have an implementation of the relevant
parts of the HCL 2 API in terms of the HCL 1 AST, mimicking the important
HCL 1 decoder behaviors, so that we can use HCL 2 patterns for decoding
while still using HCL 1 syntax and the custom environment variable
substitution syntax for backward-compatibility.

The main goal here is that it be possible to write a CLI configuration
that both the new and old implementations can consume, without creating
any significant ambiguities for either implementation. This might have
some slight incompatibilities with existing CLI configs as written due to
how liberal the HCL 1 decoder tends to be, but it should be broadly
compatible with CLI configuration syntax as documented and with reasonable
variants.
f-cliconfig-hcl2api
Martin Atkins 6 years ago
parent 2aac8cf812
commit 7e87b640bc

@ -0,0 +1,300 @@
package syntax
import (
"fmt"
"reflect"
hcl1 "github.com/hashicorp/hcl"
hcl1ast "github.com/hashicorp/hcl/hcl/ast"
hcl2 "github.com/hashicorp/hcl/v2"
)
// LegacyBody is an implementation of hcl.Body in terms of the HCL 1 AST,
// mimicking HCL 1's decoding rules while working with an HCL 2 body schema.
//
// Unlike a normal HCL body, it does not support arbitrary expressions in
// attribute values. Instead, it treats most attribute values as literal
// constants but selectively processes some specific arguments using
// os.Expand, assuming that the evaluation context contains environment
// variables for substitution.
type LegacyBody struct {
node *hcl1ast.ObjectType
filename string
// expandAttrs is a set of attribute names that are subject to environment
// variable expansion using os.Expand.
expandAttrs map[string]struct{}
// hidden is a set of attribute names and block type names that have
// already been processed by an earlier call to PartialContent and are thus
// ineligible for further processing. This is set only in the body returned
// in the "remain" return value from Body.PartialContent.
hidden map[string]struct{}
}
var _ hcl2.Body = (*LegacyBody)(nil)
// JustAttributes implements hcl.Body.JustAttributes by interpreting all of
// the items in the enclosed HCL 1 object as attributes.
func (b *LegacyBody) JustAttributes() (hcl2.Attributes, hcl2.Diagnostics) {
// The HCL 1 equivalent of JustAttributes was to decode the object into
// a map[string]interface{}, so we'll do that here to ensure that we
// get an equivalent result.
var vals map[string]interface{}
err := hcl1.DecodeObject(&vals, b.node)
if err != nil {
return nil, hcl1ErrorAsDiagnostics(err, hcl1PosDefaultFilename(b.node.Lbrace, b.filename))
}
// NOTE: decoding lost details, so this position is inaccurate: it's the
// containing object rather than the individual item.
// It's not possible in general to find a single source range for an
// attribute in HCL 1, because it allows piecemeal definition of an
// attribute across multiple items that might not even be consecutive in
// the body.
pos := hcl1PosDefaultFilename(b.node.Lbrace, b.filename)
rng := hcl1PosAsHCL2Range(pos)
ret := make(hcl2.Attributes, len(vals))
var diags hcl2.Diagnostics
for k, raw := range vals {
ret[k] = &hcl2.Attribute{
Name: k,
Range: rng,
NameRange: rng,
}
if _, expand := b.expandAttrs[k]; expand {
ret[k].Expr = &ExpandExpression{
raw: raw,
pos: hcl1PosDefaultFilename(b.node.Lbrace, b.filename),
}
} else {
expr, moreDiags := literalExpr(raw, pos)
diags = append(diags, moreDiags...)
ret[k].Expr = expr
}
}
return ret, diags
}
// PartialContent implements hcl.Body.PartialContent against the enclosed
// HCL 1 object.
//
// This implementation preserves the HCL 1 decoding behaviors as closely as
// possible within the HCL 2 API, including ignoring attribute and block type
// names that don't appear in the schema at all.
func (b *LegacyBody) PartialContent(schema *hcl2.BodySchema) (*hcl2.BodyContent, hcl2.Body, hcl2.Diagnostics) {
newHidden := map[string]struct{}{}
for k, v := range b.hidden { // propagate anything already hidden
newHidden[k] = v
}
remain := *b // shallow copy
remain.hidden = newHidden
var diags hcl2.Diagnostics
content := &hcl2.BodyContent{
Attributes: hcl2.Attributes{},
Blocks: hcl2.Blocks{},
}
pos := hcl1PosDefaultFilename(b.node.Lbrace, b.filename)
rng := hcl1PosAsHCL2Range(pos)
for _, attrS := range schema.Attributes {
newHidden[attrS.Name] = struct{}{}
// HCL 1 "Filter" looks for all of the nested items whose key sequences
// start with the given name.
filter := b.node.List.Filter(attrS.Name)
var raw interface{} // where we'll put our result; like decoding into a struct field of type interface{}
// Here we're mimicking what HCL 1's decoder would do when decoding
// into a struct field: it goes hunting both for direct assignments
// of the given name and nested blocks whose first key is the given
// name and then just processes both, letting the cards fall where
// they may if both are set. The sequence of operations below is the
// same as in HCL 1 so we can achieve as close a result as possible.
// See: https://github.com/hashicorp/hcl/blob/914dc3f8dd7c463188c73fc47e9ced82a6e421ca/decoder.go#L700-L726
prefixMatches := filter.Children()
matches := filter.Elem()
if len(matches.Items) == 0 && len(prefixMatches.Items) == 0 {
continue
}
if len(prefixMatches.Items) > 0 {
if err := hcl1.DecodeObject(raw, prefixMatches); err != nil {
return content, &remain, hcl1ErrorAsDiagnostics(err, pos)
}
}
for _, match := range matches.Items {
var decodeNode hcl1ast.Node = match.Val
if ot, ok := decodeNode.(*hcl1ast.ObjectType); ok {
decodeNode = &hcl1ast.ObjectList{Items: ot.List.Items}
}
if err := hcl1.DecodeObject(raw, decodeNode); err != nil {
return content, &remain, hcl1ErrorAsDiagnostics(err, pos)
}
}
expr, moreDiags := literalExpr(raw, pos)
diags = append(diags, moreDiags...)
attr := &hcl2.Attribute{
Name: attrS.Name,
Expr: expr,
Range: rng,
NameRange: rng,
}
content.Attributes[attrS.Name] = attr
}
blockSByType := make(map[string]hcl2.BlockHeaderSchema)
for _, blockS := range schema.Blocks {
if len(blockS.LabelNames) > 1 {
// This is currently not supported because assuming only one
// level makes this decidedly simpler and the CLI config doesn't
// currently have any multi-label block types.
diags = diags.Append(&hcl2.Diagnostic{
Severity: hcl2.DiagError,
Summary: "Unsupported block type in CLI config schema",
Detail: fmt.Sprintf("The block type %q is defined with more than one label. The CLI config decoder doesn't currently support that. This is a bug in Terraform.", blockS.Type),
Subject: rng.Ptr(),
})
continue
}
blockSByType[blockS.Type] = blockS
}
// We must iterate all the items here, rather than iterating the schema,
// because we want to preserve the lexical ordering of the nested blocks.
for _, item := range b.node.List.Items {
blockS, ok := blockSByType[item.Keys[0].Token.Value().(string)]
if !ok {
continue // not a block type
}
newHidden[blockS.Type] = struct{}{}
if len(blockS.LabelNames) > 0 {
// Deal with JSON key ambiguity: the JSON parser tries to guess
// what the user might mean but it doesn't have any schema to
// base that on, so we now need to give it a hint that we're
// trying to expand nested objects.
item = expandObject(item)
}
labelKeys := item.Keys[1:] // the first key signifies the block type
if len(labelKeys) != len(blockS.LabelNames) {
diags = diags.Append(&hcl2.Diagnostic{
Severity: hcl2.DiagError,
Summary: "Wrong number of block labels",
Detail: fmt.Sprintf("The block type %q expects %d label(s), but we found %d here.", blockS.Type, len(blockS.LabelNames), len(labelKeys)),
Subject: rng.Ptr(),
})
continue
}
nested, ok := item.Val.(*hcl1ast.ObjectType)
if !ok {
diags = diags.Append(&hcl2.Diagnostic{
Severity: hcl2.DiagError,
Summary: "Attribute where block was expected",
Detail: fmt.Sprintf("The name %q must be a nested block, not an attribute.", blockS.Type),
Subject: rng.Ptr(),
})
continue
}
var labels []string
if len(labelKeys) > 0 {
labels = make([]string, len(labelKeys))
}
for i, k := range labelKeys {
labels[i] = k.Token.Value().(string)
}
block := &hcl2.Block{
Type: blockS.Type,
Labels: labels,
Body: &LegacyBody{
node: nested,
filename: b.filename,
},
DefRange: rng,
TypeRange: rng,
}
content.Blocks = append(content.Blocks, block)
}
// By the time we get here we should've assigned values to all of our
// required attributes.
for _, attrS := range schema.Attributes {
if !attrS.Required {
continue
}
if _, exists := content.Attributes[attrS.Name]; !exists {
diags = diags.Append(&hcl2.Diagnostic{
Severity: hcl2.DiagError,
Summary: "Missing required argument",
Detail: fmt.Sprintf("The argument %q is required.", attrS.Name),
Subject: rng.Ptr(),
})
}
}
return content, &remain, diags
}
// MissingItemRange returns the location of the opening brace of the body,
// if any. Otherwise, it returns an invalid range.
func (b *LegacyBody) MissingItemRange() hcl2.Range {
return hcl1PosAsHCL2Range(hcl1PosDefaultFilename(b.node.Lbrace, b.filename))
}
// Content implements hcl.Body.Content against the enclosed HCL 1 object.
//
// Because this body implementation ignores names that are not specified in
// the schema, Content is really just PartialContent but discarding any
// remaining items in the body.
func (b *LegacyBody) Content(schema *hcl2.BodySchema) (*hcl2.BodyContent, hcl2.Diagnostics) {
ret, _, diags := b.PartialContent(schema)
return ret, diags
}
// expandObject detects if an ambiguous JSON object was flattened to a List which
// should be decoded into a struct, and expands the ast to properly decode.
// This is based on the function of the same name in HCL 1.
func expandObject(item *hcl1ast.ObjectItem) *hcl1ast.ObjectItem {
// A list value will have a key and field name. If it had more fields,
// it wouldn't have been flattened.
if len(item.Keys) != 2 {
return item
}
keyToken := item.Keys[0].Token
item.Keys = item.Keys[1:]
// we need to un-flatten the ast enough to decode
newNode := &hcl1ast.ObjectItem{
Keys: []*hcl1ast.ObjectKey{
{
Token: keyToken,
},
},
Val: &hcl1ast.ObjectType{
List: &hcl1ast.ObjectList{
Items: []*hcl1ast.ObjectItem{item},
},
},
}
return newNode
}
// Getting hold of the empty interface type _itself_ requires some indirection,
// because otherwise reflect.TypeOf will try to take the dynamic type of this
// nil value and will panic.
var emptyInterfaceType = reflect.TypeOf((*interface{})(nil)).Elem()

@ -0,0 +1,78 @@
package syntax
import (
"fmt"
hcl1parser "github.com/hashicorp/hcl/hcl/parser"
hcl1token "github.com/hashicorp/hcl/hcl/token"
hcl2 "github.com/hashicorp/hcl/v2"
)
// hcl1ErrorAsDiagnostic converts the given error, assumed to be returned from
// the HCL 1 parser or decoder, into an HCL 2 diagnostic that is as high-quality
// as possible given the limitations of HCL 1's error reporting.
//
// Not all HCL 1 errors carry accurate position information, so the caller must
// provide a default position to use for errors that lack one of their own.
// This default position should have a filename, which might need to be added
// using hcl1PosDefaultFilename before calling.
func hcl1ErrorAsDiagnostic(err error, defaultPos hcl1token.Pos) *hcl2.Diagnostic {
switch err := err.(type) {
case *hcl1parser.PosError:
pos := hcl1PosDefaultFilename(err.Pos, defaultPos.Filename)
return &hcl2.Diagnostic{
Severity: hcl2.DiagError,
Summary: "Invalid CLI configuration",
Detail: fmt.Sprintf("Error while processing the CLI configuration: %s.", err),
Subject: hcl1PosAsHCL2Range(pos).Ptr(),
}
default:
return &hcl2.Diagnostic{
Severity: hcl2.DiagError,
Summary: "Invalid CLI configuration",
Detail: fmt.Sprintf("Error while processing the CLI configuration: %s.", err),
Subject: hcl1PosAsHCL2Range(defaultPos).Ptr(),
}
}
}
// hcl1ErrorAsDiagnostics is a helper wrapper around hcl1ErrorAsDiagnostic that
// also wraps the result in a single-element HCL 2 Diagnostics, to ease the
// common case where a single HCL 1 error terminates all further processing.
func hcl1ErrorAsDiagnostics(err error, defaultPos hcl1token.Pos) hcl2.Diagnostics {
return hcl2.Diagnostics{
hcl1ErrorAsDiagnostic(err, defaultPos),
}
}
// hcl1PosDefaultFilename inserts a default filename into the given position
// and returns it, unless the position already has a filename. HCL 1 rarely
// generates filenames in practice, so this is often necessary.
func hcl1PosDefaultFilename(given hcl1token.Pos, defaultFilename string) hcl1token.Pos {
if given.Filename == "" {
given.Filename = defaultFilename
}
return given
}
// hcl1PosAsHCL2Range converts the given HCL 1 position into an HCL 2 range.
// The given position should have a filename, which might require preprocessing
// it with hcl1PosDefaultFilename.
func hcl1PosAsHCL2Range(given hcl1token.Pos) hcl2.Range {
// A single-byte/character range starting at the given position, just so
// there's something for the recipient of the diagnostic to highlight as
// the error.
return hcl2.Range{
Filename: given.Filename,
Start: hcl2.Pos{
Line: given.Line,
Column: given.Column, // Note: will be incorrect in the presence of multi-rune characters, cause HCL 1 has a rune-based idea of columns.
Byte: given.Offset,
},
End: hcl2.Pos{
Line: given.Line,
Column: given.Column + 1,
Byte: given.Offset + 1,
},
}
}

@ -0,0 +1,17 @@
// Package syntax deals with the low-level syntax details of the CLI
// configuration, exposing an HCL-compatible API to callers.
//
// The CLI configuration continutes to use HCL 1 syntax primitives because it
// must remain compatible with the processing done by earlier Terraform
// versions, but the HCL 1 API makes it difficult to write a robust
// configuration decoder that gives good user feedback.
//
// Therefore this package takes the rather unusual strategy of implementing
// HCL 2's syntax-agnostic decoding API on top of a subset of the HCL 1 API
// that was exercised by previous versions of the CLI config decoder. Using
// the HCL 2 API conventions here might allow mixed-mode parsing in future
// versions where some files can use real HCL 2 syntax, but for now the CLI
// config format doesn't need any HCL-2-unique features and so we're supporting
// HCL 1 syntax (with environment variable substitution in a few specific spots)
// only.
package syntax

@ -0,0 +1,120 @@
package syntax
import (
"fmt"
"os"
hcl1token "github.com/hashicorp/hcl/hcl/token"
hcl2 "github.com/hashicorp/hcl/v2"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/gocty"
"github.com/hashicorp/terraform/tfdiags"
)
// ExpandExpression is an hcl.Expression implementation that accepts the
// variable syntax defined by os.Expand on any string that appears inside its
// value.
//
// Note that it currently supports only direct strings and slices of strings,
// because those are the only situations currently used in the CLI config
// language. To use environment variable expansion in other constructs will
// require first expanding the implementation to include other types.
type ExpandExpression struct {
raw interface{}
pos hcl1token.Pos
}
var _ hcl2.Expression = (*ExpandExpression)(nil)
// Value implements hcl.Expression.Value by calling os.Expand on any string
// appearing inside the expression value, assuming that the variables in the
// given context are environment variables.
func (e *ExpandExpression) Value(ctx *hcl2.EvalContext) (cty.Value, hcl2.Diagnostics) {
switch raw := e.raw.(type) {
case string:
return cty.StringVal(expandFromEvalContext(raw, ctx)), nil
case []string:
vals := make([]cty.Value, len(raw))
for i, s := range raw {
vals[i] = cty.StringVal(expandFromEvalContext(s, ctx))
}
return cty.ListVal(vals), nil
default:
var diags hcl2.Diagnostics
// FIXME: We don't have enough context here to produce a good error
// message, because we don't know if the caller wants a string or a
// list of string. We know it's one or the other, but telling that to
// the user would be confusing because the user must still know which
// one in order to fix it.
diags = diags.Append(&hcl2.Diagnostic{
Severity: hcl2.DiagError,
Summary: "Unsupported value type",
Detail: "This value is of the wrong type for this CLI configuration argument.",
Subject: hcl1PosAsHCL2Range(e.pos).Ptr(),
})
return cty.DynamicVal, diags
}
}
// Variables always returns an empty set of traversals, contrary to the
// definition of hcl.Expression.Variables, because we know that in practice
// the CLI configuration decoder never needs to inspect references prior to
// evaluation.
func (e *ExpandExpression) Variables() []hcl2.Traversal {
return nil
}
func expandFromEvalContext(str string, ctx *hcl2.EvalContext) string {
return os.Expand(str, func(name string) string {
v, ok := ctx.Variables[name]
if !ok {
return ""
}
if v.Type() != cty.String || v.IsNull() || !v.IsKnown() {
// Should never happen, because CLI Config only supports environment
// variables as referenceable values and they are always known strings.
return ""
}
return v.AsString()
})
}
// Range implements hcl.Expression.Range, returning an approximate range
// derived from the underlying HCL 1 value.
func (e *ExpandExpression) Range() hcl2.Range {
return hcl1PosAsHCL2Range(e.pos)
}
// StartRange is an alias for Range.
func (e *ExpandExpression) StartRange() hcl2.Range {
return e.Range()
}
func literalExpr(raw interface{}, pos hcl1token.Pos) (hcl2.Expression, hcl2.Diagnostics) {
ty, err := gocty.ImpliedType(raw)
if err != nil {
var diags hcl2.Diagnostics
diags = diags.Append(&hcl2.Diagnostic{
Severity: hcl2.DiagError,
Summary: "Invalid argument value",
Detail: fmt.Sprintf("Cannot decode this argument value: %s.", tfdiags.FormatError(err)),
Subject: hcl1PosAsHCL2Range(pos).Ptr(),
})
return hcl2.StaticExpr(cty.DynamicVal, hcl1PosAsHCL2Range(pos)), diags
}
val, err := gocty.ToCtyValue(raw, ty)
if err != nil {
var diags hcl2.Diagnostics
diags = diags.Append(&hcl2.Diagnostic{
Severity: hcl2.DiagError,
Summary: "Invalid argument value",
Detail: fmt.Sprintf("Cannot decode this argument value: %s.", tfdiags.FormatError(err)),
Subject: hcl1PosAsHCL2Range(pos).Ptr(),
})
return hcl2.StaticExpr(cty.DynamicVal, hcl1PosAsHCL2Range(pos)), diags
}
return hcl2.StaticExpr(val, hcl1PosAsHCL2Range(pos)), nil
}
Loading…
Cancel
Save