From 3c249c0d4294dc594c1d84c2e708585660439f86 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Fri, 21 Oct 2022 16:54:21 -0400 Subject: [PATCH] hcl2_upgrade: replace passthroughs by pre-visit In previous versions of hcl2_upgrade, we referenced a list of passthrough statements from the original JSON template to be replaced by their equivalent Go-template in HCL2. This works but requires us to explicitely accept every possible templated variable in order to not ever encounter in our generated HCL2 templates. This is suboptimal, hence this commit changes approach by pre-visting the AST from the original Go Template, escaping every sequence that is not covered by the list of functions we can migrate. Pipes are also excluded from this function. --- command/hcl2_upgrade.go | 182 +++++++----------- command/hcl2_upgrade_test.go | 2 + .../hcl2_upgrade/ami_test/expected.pkr.hcl | 24 +++ .../hcl2_upgrade/ami_test/input.json | 21 ++ .../placeholders/expected.pkr.hcl | 20 ++ .../hcl2_upgrade/placeholders/input.json | 13 ++ 6 files changed, 151 insertions(+), 111 deletions(-) create mode 100644 command/test-fixtures/hcl2_upgrade/ami_test/expected.pkr.hcl create mode 100644 command/test-fixtures/hcl2_upgrade/ami_test/input.json create mode 100644 command/test-fixtures/hcl2_upgrade/placeholders/expected.pkr.hcl create mode 100644 command/test-fixtures/hcl2_upgrade/placeholders/input.json diff --git a/command/hcl2_upgrade.go b/command/hcl2_upgrade.go index 0f119567a..21dfef428 100644 --- a/command/hcl2_upgrade.go +++ b/command/hcl2_upgrade.go @@ -13,6 +13,7 @@ import ( "strconv" "strings" texttemplate "text/template" + "text/template/parse" "github.com/hashicorp/go-multierror" "github.com/hashicorp/hcl/v2/hclwrite" @@ -282,6 +283,55 @@ func fallbackReturn(err error, s []byte) []byte { return append([]byte(fmt.Sprintf("\n# could not parse template for following block: %q\n", err)), s...) } +// reTemplate writes a new template to `out` and escapes all unknown variables +// so that we don't interpret them later on when interpreting the template +func reTemplate(nd parse.Node, out io.Writer, funcs texttemplate.FuncMap) error { + switch node := nd.(type) { + case *parse.ActionNode: + // Leave pipes as-is + if len(node.Pipe.Cmds) > 1 { + fmt.Fprintf(out, "%s", node.String()) + return nil + } + + cmd := node.Pipe.Cmds[0] + args := cmd.Args + if len(args) > 1 { + // Function calls with parameters are left aside + fmt.Fprintf(out, "%s", node.String()) + return nil + } + + _, ok := funcs[args[0].String()] + if ok { + // Known functions left as-is + fmt.Fprintf(out, "%s", node.String()) + return nil + } + + // Escape anything that isn't in the func map + fmt.Fprintf(out, "{{ \"{{\" }} %s {{ \"}}\" }}", cmd.String()) + + // TODO maybe node.Pipe.Decls? Though in Packer templates they're not + // supported officially so they can be left aside for now + case *parse.ListNode: + for _, child := range node.Nodes { + err := reTemplate(child, out, funcs) + if err != nil { + return err + } + } + case *parse.TextNode: + _, err := fmt.Fprintf(out, "%s", node.Text) + if err != nil { + return err + } + default: + return fmt.Errorf("unhandled node type %s", reflect.TypeOf(nd)) + } + return nil +} + // transposeTemplatingCalls executes parts of blocks as go template files and replaces // their result with their hcl2 variant. If something goes wrong the template // containing the go template string is returned. @@ -467,10 +517,18 @@ func transposeTemplatingCalls(s []byte) []byte { } } + retempl := &bytes.Buffer{} + if err := reTemplate(tpl.Root, retempl, funcMap); err != nil { + return fallbackReturn(err, s) + } + + tpl, err = texttemplate.New("hcl2_upgrade"). + Funcs(funcMap). + Parse(retempl.String()) + str := &bytes.Buffer{} - // PASSTHROUGHS is a map of variable-specific golang text template fields - // that should remain in the text template format. - if err := tpl.Execute(str, PASSTHROUGHS); err != nil { + + if err := tpl.Execute(str, nil); err != nil { return fallbackReturn(err, s) } @@ -548,10 +606,17 @@ func variableTransposeTemplatingCalls(s []byte) (isLocal bool, body []byte) { } } + retempl := &bytes.Buffer{} + if err := reTemplate(tpl.Root, retempl, funcMap); err != nil { + return isLocal, fallbackReturn(err, s) + } + + tpl, err = texttemplate.New("hcl2_upgrade"). + Funcs(funcMap). + Parse(retempl.String()) + str := &bytes.Buffer{} - // PASSTHROUGHS is a map of variable-specific golang text template fields - // that should remain in the text template format. - if err := tpl.Execute(str, PASSTHROUGHS); err != nil { + if err := tpl.Execute(str, nil); err != nil { return isLocal, fallbackReturn(err, s) } @@ -1198,111 +1263,6 @@ func (p *PostProcessorParser) Write(out *bytes.Buffer) { } } -var PASSTHROUGHS = map[string]string{"NVME_Present": "{{ .NVME_Present }}", - "Usb_Present": "{{ .Usb_Present }}", - "Serial_Type": "{{ .Serial_Type }}", - "MapKey": "{{ .MapKey }}", - "HostAlias": "{{ .HostAlias }}", - "BoxName": "{{ .BoxName }}", - "Port": "{{ .Port }}", - "Header": "{{ .Header }}", - "HTTPIP": "{{ .HTTPIP }}", - "Host": "{{ .Host }}", - "PACKER_TEST_TEMP": "{{ .PACKER_TEST_TEMP }}", - "SCSI_diskAdapterType": "{{ .SCSI_diskAdapterType }}", - "VHDBlockSizeBytes": "{{ .VHDBlockSizeBytes }}", - "Parallel_Auto": "{{ .Parallel_Auto }}", - "KTyp": "{{ .KTyp }}", - "MemorySize": "{{ .MemorySize }}", - "APIURL": "{{ .APIURL }}", - "SourcePath": "{{ .SourcePath }}", - "CDROMType": "{{ .CDROMType }}", - "Parallel_Present": "{{ .Parallel_Present }}", - "HTTPPort": "{{ .HTTPPort }}", - "BuildName": "{{ .BuildName }}", - "Network_Device": "{{ .Network_Device }}", - "Flavor": "{{ .Flavor }}", - "Image": "{{ .Image }}", - "Os": "{{ .Os }}", - "Network_Type": "{{ .Network_Type }}", - "SourceOMIName": "{{ .SourceOMIName }}", - "Serial_Yield": "{{ .Serial_Yield }}", - "SourceAMI": "{{ .SourceAMI }}", - "SSHHostPort": "{{ .SSHHostPort }}", - "Vars": "{{ .Vars }}", - "Slice": "{{ .Slice }}", - "Version": "{{ .Version }}", - "Parallel_Bidirectional": "{{ .Parallel_Bidirectional }}", - "Serial_Auto": "{{ .Serial_Auto }}", - "VHDX": "{{ .VHDX }}", - "WinRMPassword": "{{ .WinRMPassword }}", - "DefaultOrganizationID": "{{ .DefaultOrganizationID }}", - "HTTPDir": "{{ .HTTPDir }}", - "HTTPContent": "{{ .HTTPContent }}", - "SegmentPath": "{{ .SegmentPath }}", - "NewVHDSizeBytes": "{{ .NewVHDSizeBytes }}", - "CTyp": "{{ .CTyp }}", - "VMName": "{{ .VMName }}", - "Serial_Present": "{{ .Serial_Present }}", - "Varname": "{{ .Varname }}", - "DiskNumber": "{{ .DiskNumber }}", - "SecondID": "{{ .SecondID }}", - "Typ": "{{ .Typ }}", - "SourceAMIName": "{{ .SourceAMIName }}", - "ActiveProfile": "{{ .ActiveProfile }}", - "Primitive": "{{ .Primitive }}", - "Elem": "{{ .Elem }}", - "Network_Adapter": "{{ .Network_Adapter }}", - "Minor": "{{ .Minor }}", - "ProjectName": "{{ .ProjectName }}", - "Generation": "{{ .Generation }}", - "User": "{{ .User }}", - "Size": "{{ .Size }}", - "Parallel_Filename": "{{ .Parallel_Filename }}", - "ID": "{{ .ID }}", - "FastpathLen": "{{ .FastpathLen }}", - "Tag": "{{ .Tag }}", - "Serial_Endpoint": "{{ .Serial_Endpoint }}", - "GuestOS": "{{ .GuestOS }}", - "Major": "{{ .Major }}", - "Serial_Filename": "{{ .Serial_Filename }}", - "Name": "{{ .Name }}", - "SourceOMI": "{{ .SourceOMI }}", - "SCSI_Present": "{{ .SCSI_Present }}", - "CpuCount": "{{ .CpuCount }}", - "DefaultProjectID": "{{ .DefaultProjectID }}", - "CDROMType_PrimarySecondary": "{{ .CDROMType_PrimarySecondary }}", - "Arch": "{{ .Arch }}", - "ImageFile": "{{ .ImageFile }}", - "SATA_Present": "{{ .SATA_Present }}", - "Serial_Host": "{{ .Serial_Host }}", - "BuildRegion": "{{ .BuildRegion }}", - "Id": "{{ .Id }}", - "SyncedFolder": "{{ .SyncedFolder }}", - "Network_Name": "{{ .Network_Name }}", - "AccountID": "{{ .AccountID }}", - "OPTION": "{{ .OPTION }}", - "Type": "{{ .Type }}", - "CustomVagrantfile": "{{ .CustomVagrantfile }}", - "SendTelemetry": "{{ .SendTelemetry }}", - "DiskType": "{{ .DiskType }}", - "Password": "{{ .Password }}", - "HardDrivePath": "{{ .HardDrivePath }}", - "ISOPath": "{{ .ISOPath }}", - "Insecure": "{{ .Insecure }}", - "Region": "{{ .Region }}", - "SecretKey": "{{ .SecretKey }}", - "DefaultRegion": "{{ .DefaultRegion }}", - "MemoryStartupBytes": "{{ .MemoryStartupBytes }}", - "SwitchName": "{{ .SwitchName }}", - "Path": "{{ .Path }}", - "Username": "{{ .Username }}", - "OutputDir": "{{ .OutputDir }}", - "DiskName": "{{ .DiskName }}", - "ProviderVagrantfile": "{{ .ProviderVagrantfile }}", - "Sound_Present": "{{ .Sound_Present }}", -} - func fixQuoting(old string) string { // This regex captures golang template functions that use escaped quotes: // {{ env \"myvar\" }} diff --git a/command/hcl2_upgrade_test.go b/command/hcl2_upgrade_test.go index 0ad176d92..101b5b528 100644 --- a/command/hcl2_upgrade_test.go +++ b/command/hcl2_upgrade_test.go @@ -30,6 +30,8 @@ func Test_hcl2_upgrade(t *testing.T) { {folder: "escaping", flags: []string{}}, {folder: "vsphere_linux_options_and_network_interface", exitCode: 1, flags: []string{}}, {folder: "nonexistent", flags: []string{}, exitCode: 1, exitEarly: true}, + {folder: "placeholders", flags: []string{}, exitCode: 0}, + {folder: "ami_test", flags: []string{}, exitCode: 0}, } for _, tc := range tc { diff --git a/command/test-fixtures/hcl2_upgrade/ami_test/expected.pkr.hcl b/command/test-fixtures/hcl2_upgrade/ami_test/expected.pkr.hcl new file mode 100644 index 000000000..72ccb2038 --- /dev/null +++ b/command/test-fixtures/hcl2_upgrade/ami_test/expected.pkr.hcl @@ -0,0 +1,24 @@ + +source "amazon-ebs" "autogenerated_1" { + run_tags = { + SourceAMI = "{{ .SourceAMI }}" + SourceAMICreationDate = "{{ .SourceAMICreationDate }}" + SourceAMIName = "{{ .SourceAMIName }}" + SourceAMIOwner = "{{ .SourceAMIOwner }}" + SourceAMIOwnerName = "{{ .SourceAMIOwnerName }}" + SourceAMITags = "{{ .SourceAMITags.TagName }}" + } + tags = { + SourceAMI = "{{ .SourceAMI }}" + SourceAMICreationDate = "{{ .SourceAMICreationDate }}" + SourceAMIName = "{{ .SourceAMIName }}" + SourceAMIOwner = "{{ .SourceAMIOwner }}" + SourceAMIOwnerName = "{{ .SourceAMIOwnerName }}" + SourceAMITags = "{{ .SourceAMITags.TagName }}" + } +} + +build { + sources = ["source.amazon-ebs.autogenerated_1"] + +} diff --git a/command/test-fixtures/hcl2_upgrade/ami_test/input.json b/command/test-fixtures/hcl2_upgrade/ami_test/input.json new file mode 100644 index 000000000..b66524c5d --- /dev/null +++ b/command/test-fixtures/hcl2_upgrade/ami_test/input.json @@ -0,0 +1,21 @@ +{ + "builders": [{ + "type": "amazon-ebs", + "tags": { + "SourceAMIName" : "{{ .SourceAMIName }}", + "SourceAMI" : "{{ .SourceAMI }}", + "SourceAMICreationDate" : "{{ .SourceAMICreationDate }}", + "SourceAMIOwner" : "{{ .SourceAMIOwner }}", + "SourceAMIOwnerName" : "{{ .SourceAMIOwnerName }}", + "SourceAMITags": "{{ .SourceAMITags.TagName }}" + }, + "run_tags": { + "SourceAMIName" : "{{.SourceAMIName}}", + "SourceAMI" : "{{.SourceAMI}}", + "SourceAMICreationDate" : "{{.SourceAMICreationDate}}", + "SourceAMIOwner" : "{{.SourceAMIOwner}}", + "SourceAMIOwnerName" : "{{.SourceAMIOwnerName}}", + "SourceAMITags": "{{ .SourceAMITags.TagName }}" + } + }] +} diff --git a/command/test-fixtures/hcl2_upgrade/placeholders/expected.pkr.hcl b/command/test-fixtures/hcl2_upgrade/placeholders/expected.pkr.hcl new file mode 100644 index 000000000..04cd477f4 --- /dev/null +++ b/command/test-fixtures/hcl2_upgrade/placeholders/expected.pkr.hcl @@ -0,0 +1,20 @@ + +variable "envtest" { + type = string + default = "${env("Something")}" +} + +variable "test" { + type = string + default = "{{ .Something }}" +} + +source "null" "autogenerated_1" { + communicator = "none" + other_prop = "{{ .Else }}" +} + +build { + sources = ["source.null.autogenerated_1"] + +} diff --git a/command/test-fixtures/hcl2_upgrade/placeholders/input.json b/command/test-fixtures/hcl2_upgrade/placeholders/input.json new file mode 100644 index 000000000..cf7d44271 --- /dev/null +++ b/command/test-fixtures/hcl2_upgrade/placeholders/input.json @@ -0,0 +1,13 @@ +{ + "variables": { + "test": "{{ .Something }}", + "envtest": "{{ env \"Something\"}}" + }, + "builders": [ + { + "type": "null", + "communicator": "none", + "other_prop": "{{ .Else }}" + } + ] +}