Sensitive locals (#10509)

* Allow locals to be delcared as individual blocks, and give them the Sensitive flag

* add docs for new local block

* linting

* add tests

* modified parsing to use schema, check for dupes properly

* update comment

fix wording a liiitle

* add tests for duplicate variables definition in two different files

* remove unnecessary slice initialisation

* fix crash by returning when decode error is hit

* parseLocalVariables: only treat a local vars if its not nil

also return in case of error
return locals in case of error too

* fix duplicate_locals test for windows

Co-authored-by: Adrien Delorme <azr@users.noreply.github.com>

hcl2template/parser.go

@@ -20,6 +20,7 @@ const (
 	variablesLabel    = "variables"
 	variableLabel     = "variable"
 	localsLabel       = "locals"
+	localLabel        = "local"
 	dataSourceLabel   = "data"
 	buildLabel        = "build"
 	communicatorLabel = "communicator"
@@ -32,6 +33,7 @@ var configSchema = &hcl.BodySchema{
 		{Type: variablesLabel},
 		{Type: variableLabel, LabelNames: []string{"name"}},
 		{Type: localsLabel},
+		{Type: localLabel, LabelNames: []string{"name"}},
 		{Type: dataSourceLabel, LabelNames: []string{"type", "name"}},
 		{Type: buildLabel},
 		{Type: communicatorLabel, LabelNames: []string{"type", "name"}},
@@ -257,17 +259,8 @@ func sniffCoreVersionRequirements(body hcl.Body) ([]VersionConstraint, hcl.Diagn
 	return constraints, diags
 }
 
-func (cfg *PackerConfig) Initialize(opts packer.InitializeOptions) hcl.Diagnostics {
-	var diags hcl.Diagnostics
-
-	_, moreDiags := cfg.InputVariables.Values()
-	diags = append(diags, moreDiags...)
-	_, moreDiags = cfg.LocalVariables.Values()
-	diags = append(diags, moreDiags...)
-	diags = append(diags, cfg.evaluateDatasources(opts.SkipDatasourcesExecution)...)
-	diags = append(diags, cfg.evaluateLocalVariables(cfg.LocalBlocks)...)
-
-	for _, variable := range cfg.InputVariables {
+func filterVarsFromLogs(inputOrLocal Variables) {
+	for _, variable := range inputOrLocal {
 		if !variable.Sensitive {
 			continue
 		}
@@ -279,6 +272,20 @@ func (cfg *PackerConfig) Initialize(opts packer.InitializeOptions) hcl.Diagnosti
 			return true, nil
 		})
 	}
+}
+
+func (cfg *PackerConfig) Initialize(opts packer.InitializeOptions) hcl.Diagnostics {
+	var diags hcl.Diagnostics
+
+	_, moreDiags := cfg.InputVariables.Values()
+	diags = append(diags, moreDiags...)
+	_, moreDiags = cfg.LocalVariables.Values()
+	diags = append(diags, moreDiags...)
+	diags = append(diags, cfg.evaluateDatasources(opts.SkipDatasourcesExecution)...)
+	diags = append(diags, cfg.evaluateLocalVariables(cfg.LocalBlocks)...)
+
+	filterVarsFromLogs(cfg.InputVariables)
+	filterVarsFromLogs(cfg.LocalVariables)
 
 	// decode the actual content
 	for _, file := range cfg.files {

hcl2template/testdata/complete/variables.pkr.hcl

@@ -38,3 +38,8 @@ locals {
     {id = "c"},
   ]
 }
+
+local "supersecret" {
+  expression = "${var.image_id}-password"
+  sensitive = true
+}

hcl2template/testdata/variables/basic.pkr.hcl

@@ -36,3 +36,8 @@ locals {
   service_name = "forum"
   owner        = "Community Team"
 }
+
+local "supersecret" {
+  sensitive = true
+  expression = "secretvar"
+}

hcl2template/testdata/variables/duplicate_locals/one.pkr.hcl

@@ -0,0 +1,4 @@
+
+local "sensible" {
+    expression = "something"
+}

hcl2template/testdata/variables/duplicate_locals/one_copy.pkr.hcl

@@ -0,0 +1,4 @@
+
+local "sensible" {
+    expression = "something"
+}

hcl2template/types.packer_config.go

@@ -150,10 +150,20 @@ func (c *PackerConfig) parseLocalVariables(f *hcl.File) ([]*LocalBlock, hcl.Diag
 
 	content, moreDiags := f.Body.Content(configSchema)
 	diags = append(diags, moreDiags...)
-	var locals []*LocalBlock
+
+	locals := c.LocalBlocks
 
 	for _, block := range content.Blocks {
 		switch block.Type {
+		case localLabel:
+			l, moreDiags := decodeLocalBlock(block, locals)
+			diags = append(diags, moreDiags...)
+			if l != nil {
+				locals = append(locals, l)
+			}
+			if moreDiags.HasErrors() {
+				return locals, diags
+			}
 		case localsLabel:
 			attrs, moreDiags := block.Body.JustAttributes()
 			diags = append(diags, moreDiags...)
@@ -166,7 +176,7 @@ func (c *PackerConfig) parseLocalVariables(f *hcl.File) ([]*LocalBlock, hcl.Diag
 						Subject:  attr.NameRange.Ptr(),
 						Context:  block.DefRange.Ptr(),
 					})
-					return nil, diags
+					return locals, diags
 				}
 				locals = append(locals, &LocalBlock{
 					Name: name,
@@ -176,6 +186,7 @@ func (c *PackerConfig) parseLocalVariables(f *hcl.File) ([]*LocalBlock, hcl.Diag
 		}
 	}
 
+	c.LocalBlocks = locals
 	return locals, diags
 }
 
@@ -221,14 +232,14 @@ func (c *PackerConfig) evaluateLocalVariables(locals []*LocalBlock) hcl.Diagnost
 
 func (c *PackerConfig) evaluateLocalVariable(local *LocalBlock) hcl.Diagnostics {
 	var diags hcl.Diagnostics
-
 	value, moreDiags := local.Expr.Value(c.EvalContext(nil))
 	diags = append(diags, moreDiags...)
 	if moreDiags.HasErrors() {
 		return diags
 	}
 	c.LocalVariables[local.Name] = &Variable{
-		Name: local.Name,
+		Name:      local.Name,
+		Sensitive: local.Sensitive,
 		Values: []VariableAssignment{{
 			Value: value,
 			Expr:  local.Expr,

hcl2template/types.packer_config_test.go

@@ -104,6 +104,13 @@ func TestParser_complete(t *testing.T) {
 							}),
 						}),
 					},
+					"supersecret": &Variable{
+						Name: "supersecret",
+						Values: []VariableAssignment{{From: "default",
+							Value: cty.StringVal("image-id-default-password")}},
+						Type:      cty.String,
+						Sensitive: true,
+					},
 				},
 				Datasources: Datasources{
 					DatasourceRef{Type: "amazon-ami", Name: "test"}: Datasource{

hcl2template/types.variables.go

@@ -24,6 +24,9 @@ const badIdentifierDetail = "A name must start with a letter or underscore and m
 type LocalBlock struct {
 	Name string
 	Expr hcl.Expression
+	// When Sensitive is set to true Packer will try its best to hide/obfuscate
+	// the variable from the output stream. By replacing the text.
+	Sensitive bool
 }
 
 // VariableAssignment represents a way a variable was set: the expression
@@ -246,6 +249,56 @@ var variableBlockSchema = &hcl.BodySchema{
 	},
 }
 
+var localBlockSchema = &hcl.BodySchema{
+	Attributes: []hcl.AttributeSchema{
+		{
+			Name: "expression",
+		},
+		{
+			Name: "sensitive",
+		},
+	},
+}
+
+func decodeLocalBlock(block *hcl.Block, locals []*LocalBlock) (*LocalBlock, hcl.Diagnostics) {
+	name := block.Labels[0]
+	for _, loc := range locals {
+		if loc.Name == name {
+			return nil, []*hcl.Diagnostic{{
+				Severity: hcl.DiagError,
+				Summary:  "Duplicate variable",
+				Detail:   "Duplicate " + block.Labels[0] + " variable definition found.",
+				Context:  block.DefRange.Ptr(),
+			}}
+		}
+	}
+
+	content, diags := block.Body.Content(localBlockSchema)
+	if !hclsyntax.ValidIdentifier(name) {
+		diags = append(diags, &hcl.Diagnostic{
+			Severity: hcl.DiagError,
+			Summary:  "Invalid local name",
+			Detail:   badIdentifierDetail,
+			Subject:  &block.LabelRanges[0],
+		})
+	}
+
+	l := &LocalBlock{
+		Name: name,
+	}
+
+	if attr, exists := content.Attributes["sensitive"]; exists {
+		valDiags := gohcl.DecodeExpression(attr.Expr, nil, &l.Sensitive)
+		diags = append(diags, valDiags...)
+	}
+
+	if def, ok := content.Attributes["expression"]; ok {
+		l.Expr = def.Expr
+	}
+
+	return l, diags
+}
+
 // decodeVariableBlock decodes a "variable" block
 // ectx is passed only in the evaluation of the default value.
 func (variables *Variables) decodeVariableBlock(block *hcl.Block, ectx *hcl.EvalContext) hcl.Diagnostics {
@@ -254,7 +307,6 @@ func (variables *Variables) decodeVariableBlock(block *hcl.Block, ectx *hcl.Eval
 	}
 
 	if _, found := (*variables)[block.Labels[0]]; found {
-
 		return []*hcl.Diagnostic{{
 			Severity: hcl.DiagError,
 			Summary:  "Duplicate variable",

hcl2template/types.variables_test.go

@@ -89,6 +89,15 @@ func TestParse_variables(t *testing.T) {
 						}},
 						Type: cty.String,
 					},
+					"supersecret": &Variable{
+						Name: "supersecret",
+						Values: []VariableAssignment{{
+							From:  "default",
+							Value: cty.StringVal("secretvar"),
+						}},
+						Type:      cty.String,
+						Sensitive: true,
+					},
 				},
 			},
 			false, false,
@@ -135,6 +144,28 @@ func TestParse_variables(t *testing.T) {
 			[]packersdk.Build{},
 			false,
 		},
+		{"duplicate local block",
+			defaultParser,
+			parseTestArgs{"testdata/variables/duplicate_locals", nil, nil},
+			&PackerConfig{
+				Basedir: "testdata/variables/duplicate_locals",
+				LocalVariables: Variables{
+					"sensible": &Variable{
+						Values: []VariableAssignment{
+							{
+								From:  "default",
+								Value: cty.StringVal("something"),
+							},
+						},
+						Type: cty.String,
+						Name: "sensible",
+					},
+				},
+			},
+			true, true,
+			[]packersdk.Build{},
+			false,
+		},
 		{"invalid default type",
 			defaultParser,
 			parseTestArgs{"testdata/variables/invalid_default.pkr.hcl", nil, nil},

website/content/docs/templates/hcl_templates/locals.mdx

@@ -21,9 +21,17 @@ Guide_](/guides/hcl/variables).
 
 ## Examples
 
-Local values are defined in `locals` blocks:
+Local values are defined in `local` or `locals` blocks:
 
 ```hcl
+# Using the local block allows you to mark locals as sensitive, which will
+# filter their values from logs.
+local "mylocal" {
+  expression = "${var.secret_api_key}"
+  sensitive  = true
+}
+
+# Using the locals block is more compact and efficient for declaring many locals
 # Ids for multiple sets of EC2 instances, merged together
 locals {
   instance_ids = "${concat(aws_instance.blue.*.id, aws_instance.green.*.id)}"
@@ -72,6 +80,12 @@ source "amazon-ebs" "server" {
 
 ## Description
 
+The `local` block defines exactly one local variable within a folder. The block
+label is the name of the local, and the "expression" is the expression that
+should be evaluated to create the local. Using this block, you can optionally
+supply a "sensitive" boolean to mark the variable as sensitive and filter it
+from logs.
+
 The `locals` block defines one or more local variables within a folder.
 
 The names given for the items in the `locals` block must be unique throughout a

website/content/partials/from-1.5/locals/example-block.mdx

@@ -6,4 +6,10 @@ locals {
     # locals can also be set with other variables :
     baz = "Foo is '${var.foo}' but not '${local.wee}'"
 }
+
+# Use the singular local block if you need to mark a local as sensitive
+local "mylocal" {
+  expression = "${var.secret_api_key}"
+  sensitive  = true
+}
 ```