diff --git a/command/hcl2_upgrade.go b/command/hcl2_upgrade.go index bcc90a6cd..7192c193b 100644 --- a/command/hcl2_upgrade.go +++ b/command/hcl2_upgrade.go @@ -120,6 +120,7 @@ var ( amazonSecretsManagerMap = map[string]map[string]interface{}{} localsVariableMap = map[string]string{} timestamp = false + isotime = false ) type BlockParser interface { @@ -269,18 +270,18 @@ func (uc UnhandleableArgumentError) Error() string { # Visit %s for more infos.`, uc.Call, uc.Correspondance, uc.Docs) } +func fallbackReturn(err error, s []byte) []byte { + if strings.Contains(err.Error(), "unhandled") { + return append([]byte(fmt.Sprintf("\n# %s\n", err)), s...) + } + + return append([]byte(fmt.Sprintf("\n# could not parse template for following block: %q\n", err)), s...) +} + // transposeTemplatingCalls executes parts of blocks as go template files and replaces // their result with their hcl2 variant. If something goes wrong the template // containing the go template string is returned. func transposeTemplatingCalls(s []byte) []byte { - fallbackReturn := func(err error) []byte { - if strings.Contains(err.Error(), "unhandled") { - return append([]byte(fmt.Sprintf("\n# %s\n", err)), s...) - } - - return append([]byte(fmt.Sprintf("\n# could not parse template for following block: %q\n", err)), s...) - } - funcErrors := &multierror.Error{ ErrorFormat: func(es []error) string { if len(es) == 1 { @@ -336,8 +337,14 @@ func transposeTemplatingCalls(s []byte) []byte { return "${local.timestamp}" }, "isotime": func(a ...string) string { - timestamp = true - return "${local.timestamp}" + if len(a) == 0 { + // returns rfc3339 formatted string. + return "${timestamp()}" + } + // otherwise a valid isotime func has one input. + isotime = true + return fmt.Sprintf("${legacy_isotime(\"%s\")}", a[0]) + }, "user": func(in string) string { if _, ok := localsVariableMap[in]; ok { @@ -430,14 +437,14 @@ func transposeTemplatingCalls(s []byte) []byte { Parse(string(s)) if err != nil { - return fallbackReturn(err) + return fallbackReturn(err, s) } str := &bytes.Buffer{} // PASSTHROUGHS is a map of variable-specific golang text template fields // that should remain in the text template format. if err := tpl.Execute(str, PASSTHROUGHS); err != nil { - return fallbackReturn(err) + return fallbackReturn(err, s) } out := str.Bytes() @@ -454,14 +461,6 @@ func transposeTemplatingCalls(s []byte) []byte { // In variableTransposeTemplatingCalls the definition of aws_secretsmanager function will create a data source // with the same name as the variable. func variableTransposeTemplatingCalls(s []byte) (isLocal bool, body []byte) { - fallbackReturn := func(err error) []byte { - if strings.Contains(err.Error(), "unhandled") { - return append([]byte(fmt.Sprintf("\n# %s\n", err)), s...) - } - - return append([]byte(fmt.Sprintf("\n# could not parse template for following block: %q\n", err)), s...) - } - setIsLocal := func(a ...string) string { isLocal = true return "" @@ -503,14 +502,14 @@ func variableTransposeTemplatingCalls(s []byte) (isLocal bool, body []byte) { Parse(string(s)) if err != nil { - return isLocal, fallbackReturn(err) + return isLocal, fallbackReturn(err, s) } str := &bytes.Buffer{} // PASSTHROUGHS is a map of variable-specific golang text template fields // that should remain in the text template format. if err := tpl.Execute(str, PASSTHROUGHS); err != nil { - return isLocal, fallbackReturn(err) + return isLocal, fallbackReturn(err, s) } return isLocal, str.Bytes() @@ -675,12 +674,49 @@ type VariableParser struct { localsOut []byte } +func makeLocal(variable *template.Variable, sensitive bool, localBody *hclwrite.Body, localsContent *hclwrite.File, hasLocals *bool) []byte { + if sensitive { + // Create Local block because this is sensitive + sensitiveLocalContent := hclwrite.NewEmptyFile() + body := sensitiveLocalContent.Body() + body.AppendNewline() + sensitiveLocalBody := body.AppendNewBlock("local", []string{variable.Key}).Body() + sensitiveLocalBody.SetAttributeValue("sensitive", cty.BoolVal(true)) + sensitiveLocalBody.SetAttributeValue("expression", hcl2shim.HCL2ValueFromConfigValue(variable.Default)) + localsVariableMap[variable.Key] = "local" + return sensitiveLocalContent.Bytes() + } + localBody.SetAttributeValue(variable.Key, hcl2shim.HCL2ValueFromConfigValue(variable.Default)) + localsVariableMap[variable.Key] = "locals" + *hasLocals = true + return []byte{} +} + +func makeVariable(variable *template.Variable, sensitive bool) []byte { + variablesContent := hclwrite.NewEmptyFile() + variablesBody := variablesContent.Body() + variablesBody.AppendNewline() + variableBody := variablesBody.AppendNewBlock("variable", []string{variable.Key}).Body() + variableBody.SetAttributeRaw("type", hclwrite.Tokens{&hclwrite.Token{Bytes: []byte("string")}}) + + if variable.Default != "" || !variable.Required { + shimmed := hcl2shim.HCL2ValueFromConfigValue(variable.Default) + variableBody.SetAttributeValue("default", shimmed) + } + if sensitive { + variableBody.SetAttributeValue("sensitive", cty.BoolVal(true)) + } + + return variablesContent.Bytes() +} + func (p *VariableParser) Parse(tpl *template.Template) error { - // OutPut Locals and Local blocks + // Output Locals and Local blocks localsContent := hclwrite.NewEmptyFile() localsBody := localsContent.Body() localsBody.AppendNewline() localBody := localsBody.AppendNewBlock("locals", nil).Body() + hasLocals := false if len(p.variablesOut) == 0 { p.variablesOut = []byte{} @@ -700,47 +736,34 @@ func (p *VariableParser) Parse(tpl *template.Template) error { }) } - hasLocals := false for _, variable := range variables { - variablesContent := hclwrite.NewEmptyFile() - variablesBody := variablesContent.Body() - variablesBody.AppendNewline() - variableBody := variablesBody.AppendNewBlock("variable", []string{variable.Key}).Body() - variableBody.SetAttributeRaw("type", hclwrite.Tokens{&hclwrite.Token{Bytes: []byte("string")}}) - - if variable.Default != "" || !variable.Required { - variableBody.SetAttributeValue("default", hcl2shim.HCL2ValueFromConfigValue(variable.Default)) - } + // Create new HCL2 "variables" block, and populate the "value" + // field with the "Default" value from the JSON variable. + + // Interpolate Jsonval first as an hcl variable to determine if it is + // a local. + isLocal, _ := variableTransposeTemplatingCalls([]byte(variable.Default)) sensitive := false if isSensitiveVariable(variable.Key, tpl.SensitiveVariables) { sensitive = true - variableBody.SetAttributeValue("sensitive", cty.BoolVal(true)) } - isLocal, out := variableTransposeTemplatingCalls(variablesContent.Bytes()) + // Create final HCL block and append. if isLocal { - if sensitive { - // Create Local block because this is sensitive - localContent := hclwrite.NewEmptyFile() - body := localContent.Body() - body.AppendNewline() - localBody := body.AppendNewBlock("local", []string{variable.Key}).Body() - localBody.SetAttributeValue("sensitive", cty.BoolVal(true)) - localBody.SetAttributeValue("expression", hcl2shim.HCL2ValueFromConfigValue(variable.Default)) - p.localsOut = append(p.localsOut, transposeTemplatingCalls(localContent.Bytes())...) - localsVariableMap[variable.Key] = "local" - continue + sensitiveBlocks := makeLocal(variable, sensitive, localBody, localsContent, &hasLocals) + if len(sensitiveBlocks) > 0 { + p.localsOut = append(p.localsOut, transposeTemplatingCalls(sensitiveBlocks)...) } - localBody.SetAttributeValue(variable.Key, hcl2shim.HCL2ValueFromConfigValue(variable.Default)) - localsVariableMap[variable.Key] = "locals" - hasLocals = true continue } + varbytes := makeVariable(variable, sensitive) + _, out := variableTransposeTemplatingCalls(varbytes) p.variablesOut = append(p.variablesOut, out...) } - if hasLocals { + if hasLocals == true { p.localsOut = append(p.localsOut, transposeTemplatingCalls(localsContent.Bytes())...) } + return nil } @@ -771,6 +794,9 @@ func (p *LocalsParser) Write(out *bytes.Buffer) { } fmt.Fprintln(out, `locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") }`) } + if isotime { + fmt.Fprintln(out, `# The "legacy_isotime" function has been provided for backwards compatability, but we recommend switching to the timestamp and formatdate functions.`) + } if len(p.LocalsOut) > 0 { if p.WithAnnotations { out.Write([]byte(localsVarHeader)) diff --git a/command/hcl2_upgrade_test.go b/command/hcl2_upgrade_test.go index 1e3ce491f..fea33f365 100644 --- a/command/hcl2_upgrade_test.go +++ b/command/hcl2_upgrade_test.go @@ -46,8 +46,8 @@ func Test_hcl2_upgrade(t *testing.T) { if err != nil { t.Fatalf("%v %s", err, bs) } - expected := mustBytes(ioutil.ReadFile(expectedPath)) - actual := mustBytes(ioutil.ReadFile(outputPath)) + expected := string(mustBytes(ioutil.ReadFile(expectedPath))) + actual := string(mustBytes(ioutil.ReadFile(outputPath))) if diff := cmp.Diff(expected, actual); diff != "" { t.Fatalf("unexpected output: %s", diff) diff --git a/command/test-fixtures/hcl2_upgrade/complete-variables-with-template-engine/expected.pkr.hcl b/command/test-fixtures/hcl2_upgrade/complete-variables-with-template-engine/expected.pkr.hcl index 1116310d6..1bf776b6f 100644 --- a/command/test-fixtures/hcl2_upgrade/complete-variables-with-template-engine/expected.pkr.hcl +++ b/command/test-fixtures/hcl2_upgrade/complete-variables-with-template-engine/expected.pkr.hcl @@ -5,6 +5,7 @@ variable "env_test" { } locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") } +# The "legacy_isotime" function has been provided for backwards compatability, but we recommend switching to the timestamp and formatdate functions. # 5 errors occurred upgrading the following block: # unhandled "lower" call: @@ -33,7 +34,7 @@ locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") } # Visit https://www.packer.io/docs/templates/hcl_templates/functions/string/upper for more infos. locals { build_timestamp = "${local.timestamp}" - iso_datetime = "${local.timestamp}" + iso_datetime = "${legacy_isotime("2006-01-02T15:04:05Z07:00")}" lower = "{{ lower `HELLO` }}" pwd = "${path.cwd}" replace = "{{ replace `b` `c` `ababa` 2 }}" diff --git a/command/test-fixtures/hcl2_upgrade/complete/expected.pkr.hcl b/command/test-fixtures/hcl2_upgrade/complete/expected.pkr.hcl index 67e127138..3f1a46150 100644 --- a/command/test-fixtures/hcl2_upgrade/complete/expected.pkr.hcl +++ b/command/test-fixtures/hcl2_upgrade/complete/expected.pkr.hcl @@ -198,7 +198,7 @@ build { # Please manually upgrade to use custom validation rules, `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)` # Visit https://packer.io/docs/templates/hcl_templates/variables#custom-validation-rules , https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos. provisioner "shell" { - inline = ["echo mybuild-{{ clean_resource_name `${local.timestamp}` }}"] + inline = ["echo mybuild-{{ clean_resource_name `${timestamp()}` }}"] } diff --git a/command/test-fixtures/hcl2_upgrade/without-annotations/expected.pkr.hcl b/command/test-fixtures/hcl2_upgrade/without-annotations/expected.pkr.hcl index 93c51bdc4..e332c21e0 100644 --- a/command/test-fixtures/hcl2_upgrade/without-annotations/expected.pkr.hcl +++ b/command/test-fixtures/hcl2_upgrade/without-annotations/expected.pkr.hcl @@ -151,7 +151,7 @@ build { # Please manually upgrade to use custom validation rules, `replace(string, substring, replacement)` or `regex_replace(string, substring, replacement)` # Visit https://packer.io/docs/templates/hcl_templates/variables#custom-validation-rules , https://www.packer.io/docs/templates/hcl_templates/functions/string/replace or https://www.packer.io/docs/templates/hcl_templates/functions/string/regex_replace for more infos. provisioner "shell" { - inline = ["echo mybuild-{{ clean_resource_name `${local.timestamp}` }}"] + inline = ["echo mybuild-{{ clean_resource_name `${timestamp()}` }}"] } diff --git a/hcl2template/function/datetime.go b/hcl2template/function/datetime.go index c3b9acc2d..e53454f98 100644 --- a/hcl2template/function/datetime.go +++ b/hcl2template/function/datetime.go @@ -1,12 +1,22 @@ package function import ( + "fmt" "time" "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/function" ) +// InitTime is the UTC time when this package was initialized. It is +// used as the timestamp for all configuration templates so that they +// match for a single build. +var InitTime time.Time + +func init() { + InitTime = time.Now().UTC() +} + // TimestampFunc constructs a function that returns a string representation of the current date and time. var TimestampFunc = function.New(&function.Spec{ Params: []function.Parameter{}, @@ -24,3 +34,44 @@ var TimestampFunc = function.New(&function.Spec{ func Timestamp() (cty.Value, error) { return TimestampFunc.Call([]cty.Value{}) } + +// LegacyIsotimeFunc constructs a function that returns a string representation +// of the current date and time using golang's datetime formatting. +var LegacyIsotimeFunc = function.New(&function.Spec{ + Params: []function.Parameter{}, + VarParam: &function.Parameter{ + Name: "format", + Type: cty.String, + }, + Type: function.StaticReturnType(cty.String), + Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { + if len(args) > 1 { + return cty.StringVal(""), fmt.Errorf("too many values, 1 needed: %v", args) + } else if len(args) == 0 { + return cty.StringVal(InitTime.Format(time.RFC3339)), nil + } + format := args[0].AsString() + return cty.StringVal(InitTime.Format(format)), nil + }, +}) + +// LegacyIsotimeFunc returns a string representation of the current date and +// time using the given format string. The format string follows golang's +// datetime formatting. See +// https://www.packer.io/docs/templates/legacy_json_templates/engine#isotime-function-format-reference +// for more details. +// +// This function has been provided to create backwards compatability with +// Packer's legacy JSON templates. However, we recommend that you upgrade your +// HCL Packer template to use Timestamp and FormatDate together as soon as is +// convenient. +// +// Please note that if you are using a large number of builders, provisioners +// or post-processors, the isotime may be slightly different for each one +// because it is from when the plugin is launched not the initial Packer +// process. In order to avoid this and make the timestamp consistent across all +// plugins, set it as a user variable and then access the user variable within +// your plugins. +func LegacyIsotime(format cty.Value) (cty.Value, error) { + return LegacyIsotimeFunc.Call([]cty.Value{format}) +} diff --git a/hcl2template/function/datetime_test.go b/hcl2template/function/datetime_test.go new file mode 100644 index 000000000..1d90ce53e --- /dev/null +++ b/hcl2template/function/datetime_test.go @@ -0,0 +1,64 @@ +package function + +import ( + "fmt" + "regexp" + "testing" + "time" + + "github.com/zclconf/go-cty/cty" +) + +// HCL template usage example: +// +// locals { +// emptyformat = legacy_isotime() +// golangformat = legacy_isotime("01-02-2006") +// } + +func TestLegacyIsotime_empty(t *testing.T) { + got, err := LegacyIsotimeFunc.Call([]cty.Value{}) + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + _, err = time.Parse(time.RFC3339, got.AsString()) + if err != nil { + t.Fatalf("Didn't get an RFC3339 string from empty case: %s", err) + } + +} + +func TestLegacyIsotime_inputs(t *testing.T) { + tests := []struct { + Value cty.Value + Want string + }{ + { + cty.StringVal("01-02-2006"), + `^\d{2}-\d{2}-\d{4}$`, + }, + { + cty.StringVal("Mon Jan 02, 2006"), + `^(Mon|Tues|Wed|Thu|Fri|Sat|Sun){1} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec){1} \d{2}, \d{4}$`, + }, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("legacy_isotime(%#v)", test.Value), func(t *testing.T) { + got, err := LegacyIsotime(test.Value) + + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + re, err := regexp.Compile(test.Want) + if err != nil { + t.Fatalf("Bad regular expression test string: %#v", err) + } + + if !re.MatchString(got.AsString()) { + t.Errorf("wrong result\ngot: %#v\nwant: %#v", got, test.Want) + } + }) + } +} diff --git a/hcl2template/functions.go b/hcl2template/functions.go index 2e103b95a..fdd900ee4 100644 --- a/hcl2template/functions.go +++ b/hcl2template/functions.go @@ -68,6 +68,7 @@ func Functions(basedir string) map[string]function.Function { "jsondecode": stdlib.JSONDecodeFunc, "jsonencode": stdlib.JSONEncodeFunc, "keys": stdlib.KeysFunc, + "legacy_isotime": pkrfunction.LegacyIsotimeFunc, "length": pkrfunction.LengthFunc, "log": stdlib.LogFunc, "lookup": stdlib.LookupFunc,