Merge pull request #10780 from hashicorp/fix_10728

add legacy_isotime hcl function

command/hcl2_upgrade.go

@@ -120,6 +120,7 @@ var (
 	amazonSecretsManagerMap = map[string]map[string]interface{}{}
 	localsVariableMap       = map[string]string{}
 	timestamp               = false
+	isotime                 = false
 )
 
 type BlockParser interface {
@@ -269,18 +270,18 @@ func (uc UnhandleableArgumentError) Error() string {
 # Visit %s for more infos.`, uc.Call, uc.Correspondance, uc.Docs)
 }
 
+func fallbackReturn(err error, s []byte) []byte {
+	if strings.Contains(err.Error(), "unhandled") {
+		return append([]byte(fmt.Sprintf("\n# %s\n", err)), s...)
+	}
+
+	return append([]byte(fmt.Sprintf("\n# could not parse template for following block: %q\n", err)), s...)
+}
+
 // transposeTemplatingCalls executes parts of blocks as go template files and replaces
 // their result with their hcl2 variant. If something goes wrong the template
 // containing the go template string is returned.
 func transposeTemplatingCalls(s []byte) []byte {
-	fallbackReturn := func(err error) []byte {
-		if strings.Contains(err.Error(), "unhandled") {
-			return append([]byte(fmt.Sprintf("\n# %s\n", err)), s...)
-		}
-
-		return append([]byte(fmt.Sprintf("\n# could not parse template for following block: %q\n", err)), s...)
-	}
-
 	funcErrors := &multierror.Error{
 		ErrorFormat: func(es []error) string {
 			if len(es) == 1 {
@@ -336,8 +337,14 @@ func transposeTemplatingCalls(s []byte) []byte {
 			return "${local.timestamp}"
 		},
 		"isotime": func(a ...string) string {
-			timestamp = true
-			return "${local.timestamp}"
+			if len(a) == 0 {
+				// returns rfc3339 formatted string.
+				return "${timestamp()}"
+			}
+			// otherwise a valid isotime func has one input.
+			isotime = true
+			return fmt.Sprintf("${legacy_isotime(\"%s\")}", a[0])
+
 		},
 		"user": func(in string) string {
 			if _, ok := localsVariableMap[in]; ok {
@@ -430,14 +437,14 @@ func transposeTemplatingCalls(s []byte) []byte {
 		Parse(string(s))
 
 	if err != nil {
-		return fallbackReturn(err)
+		return fallbackReturn(err, s)
 	}
 
 	str := &bytes.Buffer{}
 	// PASSTHROUGHS is a map of variable-specific golang text template fields
 	// that should remain in the text template format.
 	if err := tpl.Execute(str, PASSTHROUGHS); err != nil {
-		return fallbackReturn(err)
+		return fallbackReturn(err, s)
 	}
 
 	out := str.Bytes()
@@ -454,14 +461,6 @@ func transposeTemplatingCalls(s []byte) []byte {
 // In variableTransposeTemplatingCalls the definition of aws_secretsmanager function will create a data source
 // with the same name as the variable.
 func variableTransposeTemplatingCalls(s []byte) (isLocal bool, body []byte) {
-	fallbackReturn := func(err error) []byte {
-		if strings.Contains(err.Error(), "unhandled") {
-			return append([]byte(fmt.Sprintf("\n# %s\n", err)), s...)
-		}
-
-		return append([]byte(fmt.Sprintf("\n# could not parse template for following block: %q\n", err)), s...)
-	}
-
 	setIsLocal := func(a ...string) string {
 		isLocal = true
 		return ""
@@ -503,14 +502,14 @@ func variableTransposeTemplatingCalls(s []byte) (isLocal bool, body []byte) {
 		Parse(string(s))
 
 	if err != nil {
-		return isLocal, fallbackReturn(err)
+		return isLocal, fallbackReturn(err, s)
 	}
 
 	str := &bytes.Buffer{}
 	// PASSTHROUGHS is a map of variable-specific golang text template fields
 	// that should remain in the text template format.
 	if err := tpl.Execute(str, PASSTHROUGHS); err != nil {
-		return isLocal, fallbackReturn(err)
+		return isLocal, fallbackReturn(err, s)
 	}
 
 	return isLocal, str.Bytes()
@@ -675,12 +674,49 @@ type VariableParser struct {
 	localsOut       []byte
 }
 
+func makeLocal(variable *template.Variable, sensitive bool, localBody *hclwrite.Body, localsContent *hclwrite.File, hasLocals *bool) []byte {
+	if sensitive {
+		// Create Local block because this is sensitive
+		sensitiveLocalContent := hclwrite.NewEmptyFile()
+		body := sensitiveLocalContent.Body()
+		body.AppendNewline()
+		sensitiveLocalBody := body.AppendNewBlock("local", []string{variable.Key}).Body()
+		sensitiveLocalBody.SetAttributeValue("sensitive", cty.BoolVal(true))
+		sensitiveLocalBody.SetAttributeValue("expression", hcl2shim.HCL2ValueFromConfigValue(variable.Default))
+		localsVariableMap[variable.Key] = "local"
+		return sensitiveLocalContent.Bytes()
+	}
+	localBody.SetAttributeValue(variable.Key, hcl2shim.HCL2ValueFromConfigValue(variable.Default))
+	localsVariableMap[variable.Key] = "locals"
+	*hasLocals = true
+	return []byte{}
+}
+
+func makeVariable(variable *template.Variable, sensitive bool) []byte {
+	variablesContent := hclwrite.NewEmptyFile()
+	variablesBody := variablesContent.Body()
+	variablesBody.AppendNewline()
+	variableBody := variablesBody.AppendNewBlock("variable", []string{variable.Key}).Body()
+	variableBody.SetAttributeRaw("type", hclwrite.Tokens{&hclwrite.Token{Bytes: []byte("string")}})
+
+	if variable.Default != "" || !variable.Required {
+		shimmed := hcl2shim.HCL2ValueFromConfigValue(variable.Default)
+		variableBody.SetAttributeValue("default", shimmed)
+	}
+	if sensitive {
+		variableBody.SetAttributeValue("sensitive", cty.BoolVal(true))
+	}
+
+	return variablesContent.Bytes()
+}
+
 func (p *VariableParser) Parse(tpl *template.Template) error {
-	// OutPut Locals and Local blocks
+	// Output Locals and Local blocks
 	localsContent := hclwrite.NewEmptyFile()
 	localsBody := localsContent.Body()
 	localsBody.AppendNewline()
 	localBody := localsBody.AppendNewBlock("locals", nil).Body()
+	hasLocals := false
 
 	if len(p.variablesOut) == 0 {
 		p.variablesOut = []byte{}
@@ -700,47 +736,34 @@ func (p *VariableParser) Parse(tpl *template.Template) error {
 		})
 	}
 
-	hasLocals := false
 	for _, variable := range variables {
-		variablesContent := hclwrite.NewEmptyFile()
-		variablesBody := variablesContent.Body()
-		variablesBody.AppendNewline()
-		variableBody := variablesBody.AppendNewBlock("variable", []string{variable.Key}).Body()
-		variableBody.SetAttributeRaw("type", hclwrite.Tokens{&hclwrite.Token{Bytes: []byte("string")}})
-
-		if variable.Default != "" || !variable.Required {
-			variableBody.SetAttributeValue("default", hcl2shim.HCL2ValueFromConfigValue(variable.Default))
-		}
+		// Create new HCL2 "variables" block, and populate the "value"
+		// field with the "Default" value from the JSON variable.
+
+		// Interpolate Jsonval first as an hcl variable to determine if it is
+		// a local.
+		isLocal, _ := variableTransposeTemplatingCalls([]byte(variable.Default))
 		sensitive := false
 		if isSensitiveVariable(variable.Key, tpl.SensitiveVariables) {
 			sensitive = true
-			variableBody.SetAttributeValue("sensitive", cty.BoolVal(true))
 		}
-		isLocal, out := variableTransposeTemplatingCalls(variablesContent.Bytes())
+		// Create final HCL block and append.
 		if isLocal {
-			if sensitive {
-				// Create Local block because this is sensitive
-				localContent := hclwrite.NewEmptyFile()
-				body := localContent.Body()
-				body.AppendNewline()
-				localBody := body.AppendNewBlock("local", []string{variable.Key}).Body()
-				localBody.SetAttributeValue("sensitive", cty.BoolVal(true))
-				localBody.SetAttributeValue("expression", hcl2shim.HCL2ValueFromConfigValue(variable.Default))
-				p.localsOut = append(p.localsOut, transposeTemplatingCalls(localContent.Bytes())...)
-				localsVariableMap[variable.Key] = "local"
-				continue
+			sensitiveBlocks := makeLocal(variable, sensitive, localBody, localsContent, &hasLocals)
+			if len(sensitiveBlocks) > 0 {
+				p.localsOut = append(p.localsOut, transposeTemplatingCalls(sensitiveBlocks)...)
 			}
-			localBody.SetAttributeValue(variable.Key, hcl2shim.HCL2ValueFromConfigValue(variable.Default))
-			localsVariableMap[variable.Key] = "locals"
-			hasLocals = true
 			continue
 		}
+		varbytes := makeVariable(variable, sensitive)
+		_, out := variableTransposeTemplatingCalls(varbytes)
 		p.variablesOut = append(p.variablesOut, out...)
 	}
 
-	if hasLocals {
+	if hasLocals == true {
 		p.localsOut = append(p.localsOut, transposeTemplatingCalls(localsContent.Bytes())...)
 	}
+
 	return nil
 }
 
@@ -771,6 +794,9 @@ func (p *LocalsParser) Write(out *bytes.Buffer) {
 		}
 		fmt.Fprintln(out, `locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") }`)
 	}
+	if isotime {
+		fmt.Fprintln(out, `# The "legacy_isotime" function has been provided for backwards compatability, but we recommend switching to the timestamp and formatdate functions.`)
+	}
 	if len(p.LocalsOut) > 0 {
 		if p.WithAnnotations {
 			out.Write([]byte(localsVarHeader))

command/hcl2_upgrade_test.go

@@ -46,8 +46,8 @@ func Test_hcl2_upgrade(t *testing.T) {
 			if err != nil {
 				t.Fatalf("%v %s", err, bs)
 			}
-			expected := mustBytes(ioutil.ReadFile(expectedPath))
-			actual := mustBytes(ioutil.ReadFile(outputPath))
+			expected := string(mustBytes(ioutil.ReadFile(expectedPath)))
+			actual := string(mustBytes(ioutil.ReadFile(outputPath)))
 
 			if diff := cmp.Diff(expected, actual); diff != "" {
 				t.Fatalf("unexpected output: %s", diff)

command/test-fixtures/hcl2_upgrade/complete-variables-with-template-engine/expected.pkr.hcl

@@ -5,6 +5,7 @@ variable "env_test" {
 }
 
 locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") }
+# The "legacy_isotime" function has been provided for backwards compatability, but we recommend switching to the timestamp and formatdate functions.
 
 # 5 errors occurred upgrading the following block:
 # unhandled "lower" call:
@@ -33,7 +34,7 @@ locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") }
 # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/upper for more infos.
 locals {
   build_timestamp = "${local.timestamp}"
-  iso_datetime    = "${local.timestamp}"
+  iso_datetime    = "${legacy_isotime("2006-01-02T15:04:05Z07:00")}"
   lower           = "{{ lower `HELLO` }}"
   pwd             = "${path.cwd}"
   replace         = "{{ replace `b` `c` `ababa` 2 }}"

command/test-fixtures/hcl2_upgrade/complete/expected.pkr.hcl

@@ -198,7 +198,7 @@ build {
   # Please manually upgrade to use custom validation rules, `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)`
   # Visit https://packer.io/docs/templates/hcl_templates/variables#custom-validation-rules , https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos.
   provisioner "shell" {
-    inline = ["echo mybuild-{{ clean_resource_name `${local.timestamp}` }}"]
+    inline = ["echo mybuild-{{ clean_resource_name `${timestamp()}` }}"]
   }
 
 

command/test-fixtures/hcl2_upgrade/without-annotations/expected.pkr.hcl

@@ -151,7 +151,7 @@ build {
   # Please manually upgrade to use custom validation rules, `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)`
   # Visit https://packer.io/docs/templates/hcl_templates/variables#custom-validation-rules , https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos.
   provisioner "shell" {
-    inline = ["echo mybuild-{{ clean_resource_name `${local.timestamp}` }}"]
+    inline = ["echo mybuild-{{ clean_resource_name `${timestamp()}` }}"]
   }
 
 

hcl2template/function/datetime.go

@@ -1,12 +1,22 @@
 package function
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/zclconf/go-cty/cty"
 	"github.com/zclconf/go-cty/cty/function"
 )
 
+// InitTime is the UTC time when this package was initialized. It is
+// used as the timestamp for all configuration templates so that they
+// match for a single build.
+var InitTime time.Time
+
+func init() {
+	InitTime = time.Now().UTC()
+}
+
 // TimestampFunc constructs a function that returns a string representation of the current date and time.
 var TimestampFunc = function.New(&function.Spec{
 	Params: []function.Parameter{},
@@ -24,3 +34,44 @@ var TimestampFunc = function.New(&function.Spec{
 func Timestamp() (cty.Value, error) {
 	return TimestampFunc.Call([]cty.Value{})
 }
+
+// LegacyIsotimeFunc constructs a function that returns a string representation
+// of the current date and time using golang's datetime formatting.
+var LegacyIsotimeFunc = function.New(&function.Spec{
+	Params: []function.Parameter{},
+	VarParam: &function.Parameter{
+		Name: "format",
+		Type: cty.String,
+	},
+	Type: function.StaticReturnType(cty.String),
+	Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
+		if len(args) > 1 {
+			return cty.StringVal(""), fmt.Errorf("too many values, 1 needed: %v", args)
+		} else if len(args) == 0 {
+			return cty.StringVal(InitTime.Format(time.RFC3339)), nil
+		}
+		format := args[0].AsString()
+		return cty.StringVal(InitTime.Format(format)), nil
+	},
+})
+
+// LegacyIsotimeFunc returns a string representation of the current date and
+// time using the given format string. The format string follows golang's
+// datetime formatting. See
+// https://www.packer.io/docs/templates/legacy_json_templates/engine#isotime-function-format-reference
+// for more details.
+//
+// This function has been provided to create backwards compatability with
+// Packer's legacy JSON templates. However, we recommend that you upgrade your
+// HCL Packer template to use Timestamp and FormatDate together as soon as is
+// convenient.
+//
+// Please note that if you are using a large number of builders, provisioners
+// or post-processors, the isotime may be slightly different for each one
+// because it is from when the plugin is launched not the initial Packer
+// process. In order to avoid this and make the timestamp consistent across all
+// plugins, set it as a user variable and then access the user variable within
+// your plugins.
+func LegacyIsotime(format cty.Value) (cty.Value, error) {
+	return LegacyIsotimeFunc.Call([]cty.Value{format})
+}

hcl2template/function/datetime_test.go

@@ -0,0 +1,64 @@
+package function
+
+import (
+	"fmt"
+	"regexp"
+	"testing"
+	"time"
+
+	"github.com/zclconf/go-cty/cty"
+)
+
+// HCL template usage example:
+//
+// locals {
+//   emptyformat  = legacy_isotime()
+//   golangformat = legacy_isotime("01-02-2006")
+// }
+
+func TestLegacyIsotime_empty(t *testing.T) {
+	got, err := LegacyIsotimeFunc.Call([]cty.Value{})
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+
+	_, err = time.Parse(time.RFC3339, got.AsString())
+	if err != nil {
+		t.Fatalf("Didn't get an RFC3339 string from empty case: %s", err)
+	}
+
+}
+
+func TestLegacyIsotime_inputs(t *testing.T) {
+	tests := []struct {
+		Value cty.Value
+		Want  string
+	}{
+		{
+			cty.StringVal("01-02-2006"),
+			`^\d{2}-\d{2}-\d{4}$`,
+		},
+		{
+			cty.StringVal("Mon Jan 02, 2006"),
+			`^(Mon|Tues|Wed|Thu|Fri|Sat|Sun){1} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec){1} \d{2}, \d{4}$`,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("legacy_isotime(%#v)", test.Value), func(t *testing.T) {
+			got, err := LegacyIsotime(test.Value)
+
+			if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+			re, err := regexp.Compile(test.Want)
+			if err != nil {
+				t.Fatalf("Bad regular expression test string: %#v", err)
+			}
+
+			if !re.MatchString(got.AsString()) {
+				t.Errorf("wrong result\ngot:  %#v\nwant: %#v", got, test.Want)
+			}
+		})
+	}
+}

hcl2template/functions.go

@@ -68,6 +68,7 @@ func Functions(basedir string) map[string]function.Function {
 		"jsondecode":         stdlib.JSONDecodeFunc,
 		"jsonencode":         stdlib.JSONEncodeFunc,
 		"keys":               stdlib.KeysFunc,
+		"legacy_isotime":     pkrfunction.LegacyIsotimeFunc,
 		"length":             pkrfunction.LengthFunc,
 		"log":                stdlib.LogFunc,
 		"lookup":             stdlib.LookupFunc,