From 7969b690de7dd09ec2ed7f09c41aa3870d064e18 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Thu, 2 Jul 2020 15:07:29 +0200 Subject: [PATCH 01/24] Revert "Share build info with Provisioner and Post-Processor via HCL2 variables (#9444)" This reverts commit 99046c917822ede71b32b46a841fd6b4f2fa5d79. --- command/build_test.go | 115 +++--------------- .../basic_template.pkr.hcl | 15 --- .../basic_template_with_hcl_func.pkr.hcl | 21 ---- .../multiple_build_blocks.pkr.hcl | 46 ------- .../multiple_source_build.pkr.hcl | 32 ----- .../named_build.pkr.hcl | 16 --- hcl2template/common_test.go | 32 ----- hcl2template/types.build.post-processor.go | 48 +------- hcl2template/types.build.provisioners.go | 38 +----- hcl2template/types.packer_config.go | 69 ++++------- packer/build.go | 23 ---- packer/provisioner.go | 31 +---- post-processor/shell-local/post-processor.go | 6 +- website/data/docs-navigation.js | 1 - website/pages/docs/builders/amazon/chroot.mdx | 45 +------ website/pages/docs/builders/amazon/ebs.mdx | 41 +------ .../docs/builders/amazon/ebssurrogate.mdx | 41 +------ .../pages/docs/builders/amazon/ebsvolume.mdx | 41 +------ .../pages/docs/builders/amazon/instance.mdx | 41 +------ .../from-1.5/blocks/build/post-processor.mdx | 5 - .../from-1.5/blocks/build/provisioner.mdx | 7 +- .../docs/from-1.5/contextual-variables.mdx | 71 ----------- website/pages/docs/templates/engine.mdx | 16 +-- 23 files changed, 69 insertions(+), 732 deletions(-) delete mode 100644 command/test-fixtures/build-variable-sharing/basic_template.pkr.hcl delete mode 100644 command/test-fixtures/build-variable-sharing/basic_template_with_hcl_func.pkr.hcl delete mode 100644 command/test-fixtures/build-variable-sharing/multiple_build_blocks.pkr.hcl delete mode 100644 command/test-fixtures/build-variable-sharing/multiple_source_build.pkr.hcl delete mode 100644 command/test-fixtures/build-variable-sharing/named_build.pkr.hcl delete mode 100644 website/pages/docs/from-1.5/contextual-variables.mdx diff --git a/command/build_test.go b/command/build_test.go index 5cc9ba10a..e464b3401 100644 --- a/command/build_test.go +++ b/command/build_test.go @@ -228,8 +228,8 @@ func TestBuild(t *testing.T) { testFixture("hcl-only-except"), }, fileCheck: fileCheck{ - expected: []string{"chocolate.txt", "vanilla.txt"}, notExpected: []string{"cherry.txt"}, + expected: []string{"chocolate.txt", "vanilla.txt"}, }, }, } @@ -341,109 +341,26 @@ func TestBuildProvisionAndPosProcessWithBuildVariablesSharing(t *testing.T) { c := &BuildCommand{ Meta: testMetaFile(t), } - tc := []struct { - name string - args []string - expectedFiles []string - notExpectedFiles []string - }{ - { - name: "JSON: basic template", - args: []string{ - filepath.Join(testFixture("build-variable-sharing"), "template.json"), - }, - expectedFiles: []string{ - "provisioner.Null.txt", - "post-processor.Null.txt", - }, - notExpectedFiles: []string{}, - }, - { - name: "HCL2: basic template", - args: []string{ - filepath.Join(testFixture("build-variable-sharing"), "basic_template.pkr.hcl"), - }, - expectedFiles: []string{ - "provisioner.Null.txt", - "post-processor.Null.txt", - }, - notExpectedFiles: []string{}, - }, - { - name: "HCL2: basic template with build variables within HCL function", - args: []string{ - filepath.Join(testFixture("build-variable-sharing"), "basic_template_with_hcl_func.pkr.hcl"), - }, - expectedFiles: []string{ - "provisioner.Null.txt", - "provisioner.NULL.txt", - "post-processor.Null.txt", - "post-processor.NULL.txt", - }, - notExpectedFiles: []string{}, - }, - { - name: "HCL2: basic template with named build", - args: []string{ - filepath.Join(testFixture("build-variable-sharing"), "named_build.pkr.hcl"), - }, - expectedFiles: []string{ - "provisioner.Null.txt", - "post-processor.Null.txt", - }, - notExpectedFiles: []string{}, - }, - { - name: "HCL2: multiple build block sharing same sources", - args: []string{ - filepath.Join(testFixture("build-variable-sharing"), "multiple_build_blocks.pkr.hcl"), - }, - expectedFiles: []string{ - "vanilla.chocolate.provisioner.Null.txt", - "vanilla.chocolate.post-processor.Null.txt", - "apple.chocolate.provisioner.Null.txt", - "apple.chocolate.post-processor.Null.txt", - "sugar.banana.provisioner.Null.txt", - "sugar.banana.post-processor.Null.txt", - }, - notExpectedFiles: []string{}, - }, - { - name: "HCL2: multiple sources build with only/except set for provisioner and post-processors", - args: []string{ - filepath.Join(testFixture("build-variable-sharing"), "multiple_source_build.pkr.hcl"), - }, - expectedFiles: []string{ - "all.Null.txt", - }, - notExpectedFiles: []string{ - "chocolate.Null.txt", - "banana.Null.txt", - }, - }, + + args := []string{ + filepath.Join(testFixture("build-variable-sharing"), "template.json"), } - for _, tt := range tc { - t.Run(tt.name, func(t *testing.T) { - defer cleanup(tt.expectedFiles...) - defer cleanup(tt.notExpectedFiles...) + files := []string{ + "provisioner.Null.txt", + "post-processor.Null.txt", + } - if code := c.Run(tt.args); code != 0 { - fatalCommand(t, c.Meta) - } + defer cleanup(files...) - for _, f := range tt.expectedFiles { - if !fileExists(f) { - t.Errorf("Expected to find %s", f) - } - } + if code := c.Run(args); code != 0 { + fatalCommand(t, c.Meta) + } - for _, f := range tt.notExpectedFiles { - if fileExists(f) { - t.Errorf("Not expected to find %s", f) - } - } - }) + for _, f := range files { + if !fileExists(f) { + t.Errorf("Expected to find %s", f) + } } } diff --git a/command/test-fixtures/build-variable-sharing/basic_template.pkr.hcl b/command/test-fixtures/build-variable-sharing/basic_template.pkr.hcl deleted file mode 100644 index 052a9b072..000000000 --- a/command/test-fixtures/build-variable-sharing/basic_template.pkr.hcl +++ /dev/null @@ -1,15 +0,0 @@ -source "null" "chocolate" { - communicator = "none" -} - -build { - sources = ["null.chocolate"] - - provisioner "shell-local" { - inline = ["echo hi > provisioner.${build.ID}.txt"] - } - - post-processor "shell-local" { - inline = ["echo hi > post-processor.${build.ID}.txt"] - } -} \ No newline at end of file diff --git a/command/test-fixtures/build-variable-sharing/basic_template_with_hcl_func.pkr.hcl b/command/test-fixtures/build-variable-sharing/basic_template_with_hcl_func.pkr.hcl deleted file mode 100644 index 218c28f2a..000000000 --- a/command/test-fixtures/build-variable-sharing/basic_template_with_hcl_func.pkr.hcl +++ /dev/null @@ -1,21 +0,0 @@ -source "null" "chocolate" { - communicator = "none" -} - -build { - sources = ["null.chocolate"] - - provisioner "shell-local" { - inline = [ - "echo hi > provisioner.${build.ID}.txt", - "echo hi > provisioner.${upper(build.ID)}.txt" - ] - } - - post-processor "shell-local" { - inline = [ - "echo hi > post-processor.${build.ID}.txt", - "echo hi > post-processor.${upper(build.ID)}.txt" - ] - } -} \ No newline at end of file diff --git a/command/test-fixtures/build-variable-sharing/multiple_build_blocks.pkr.hcl b/command/test-fixtures/build-variable-sharing/multiple_build_blocks.pkr.hcl deleted file mode 100644 index 4af6bcc5b..000000000 --- a/command/test-fixtures/build-variable-sharing/multiple_build_blocks.pkr.hcl +++ /dev/null @@ -1,46 +0,0 @@ -source "null" "chocolate" { - communicator = "none" -} - -source "null" "banana" { - communicator = "none" -} - -build { - name = "vanilla" - sources = ["null.chocolate"] - - provisioner "shell-local" { - inline = ["echo hi > vanilla.chocolate.provisioner.${build.ID}.txt"] - } - - post-processor "shell-local" { - inline = ["echo hi > vanilla.chocolate.post-processor.${build.ID}.txt"] - } -} - -build { - name = "apple" - sources = ["null.chocolate"] - - provisioner "shell-local" { - inline = ["echo hi > apple.chocolate.provisioner.${build.ID}.txt"] - } - - post-processor "shell-local" { - inline = ["echo hi > apple.chocolate.post-processor.${build.ID}.txt"] - } -} - -build { - name = "sugar" - sources = ["null.banana"] - - provisioner "shell-local" { - inline = ["echo hi > sugar.banana.provisioner.${build.ID}.txt"] - } - - post-processor "shell-local" { - inline = ["echo hi > sugar.banana.post-processor.${build.ID}.txt"] - } -} \ No newline at end of file diff --git a/command/test-fixtures/build-variable-sharing/multiple_source_build.pkr.hcl b/command/test-fixtures/build-variable-sharing/multiple_source_build.pkr.hcl deleted file mode 100644 index 8e1f0e97a..000000000 --- a/command/test-fixtures/build-variable-sharing/multiple_source_build.pkr.hcl +++ /dev/null @@ -1,32 +0,0 @@ -source "null" "chocolate" { - communicator = "none" -} - -source "null" "banana" { - communicator = "none" -} - -build { - name = "vanilla" - sources = [ - "null.chocolate", - "null.banana", - ] - - provisioner "shell-local" { - inline = [ - "echo hi > all.${build.ID}.txt", - "echo hi > chocolate.${build.ID}.txt", - "echo hi > banana.${build.ID}.txt" - ] - } - - post-processor "shell-local" { - only = ["null.chocolate"] - inline = ["rm chocolate.${build.ID}.txt"] - } - post-processor "shell-local" { - except = ["null.chocolate"] - inline = ["rm banana.${build.ID}.txt"] - } -} \ No newline at end of file diff --git a/command/test-fixtures/build-variable-sharing/named_build.pkr.hcl b/command/test-fixtures/build-variable-sharing/named_build.pkr.hcl deleted file mode 100644 index 6e110365f..000000000 --- a/command/test-fixtures/build-variable-sharing/named_build.pkr.hcl +++ /dev/null @@ -1,16 +0,0 @@ -source "null" "chocolate" { - communicator = "none" -} - -build { - name = "vanilla" - sources = ["null.chocolate"] - - provisioner "shell-local" { - inline = ["echo hi > provisioner.${build.ID}.txt"] - } - - post-processor "shell-local" { - inline = ["echo hi > post-processor.${build.ID}.txt"] - } -} \ No newline at end of file diff --git a/hcl2template/common_test.go b/hcl2template/common_test.go index 541d077b2..5db6dc6c2 100644 --- a/hcl2template/common_test.go +++ b/hcl2template/common_test.go @@ -121,37 +121,6 @@ func testParse(t *testing.T, tests []parseTest) { if tt.getBuildsWantDiags == (gotDiags == nil) { t.Fatalf("Parser.getBuilds() unexpected diagnostics. %s", gotDiags) } - - // Validates implementation of HCL2ProvisionerPrepare and HCL2PostProcessorsPrepare - for _, build := range gotBuilds { - coreBuild, ok := build.(*packer.CoreBuild) - if !ok { - t.Fatalf("build %s should implement CoreBuild", build.Name()) - } - if coreBuild.HCL2ProvisionerPrepare == nil { - t.Fatalf("build %s should have HCL2ProvisionerPrepare implementation", build.Name()) - } - if coreBuild.HCL2PostProcessorsPrepare == nil { - t.Fatalf("build %s should have HCL2PostProcessorsPrepare implementation", build.Name()) - } - - provisioners, diags := coreBuild.HCL2ProvisionerPrepare(nil) - if diags.HasErrors() { - t.Fatalf("build %s: HCL2ProvisionerPrepare should prepare provisioners", build.Name()) - } - coreBuild.Provisioners = provisioners - - postProcessors, diags := coreBuild.HCL2PostProcessorsPrepare(nil) - if diags.HasErrors() { - t.Fatalf("build %s: HCL2PostProcessorsPrepare should prepare post-processors", build.Name()) - } - if len(postProcessors) > 0 { - coreBuild.PostProcessors = [][]packer.CoreBuildPostProcessor{postProcessors} - } else { - coreBuild.PostProcessors = [][]packer.CoreBuildPostProcessor{} - } - } - if diff := cmp.Diff(tt.getBuildsWantBuilds, gotBuilds, cmpopts.IgnoreUnexported( cty.Value{}, @@ -161,7 +130,6 @@ func testParse(t *testing.T, tests []parseTest) { packer.CoreBuildPostProcessor{}, null.Builder{}, ), - cmpopts.IgnoreFields(packer.CoreBuild{}, "HCL2ProvisionerPrepare", "HCL2PostProcessorsPrepare"), ); diff != "" { t.Fatalf("Parser.getBuilds() wrong packer builds. %s", diff) } diff --git a/hcl2template/types.build.post-processor.go b/hcl2template/types.build.post-processor.go index 6fa249f00..b45b682f5 100644 --- a/hcl2template/types.build.post-processor.go +++ b/hcl2template/types.build.post-processor.go @@ -6,7 +6,6 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/packer/packer" - "github.com/zclconf/go-cty/cty" ) // ProvisionerBlock references a detected but unparsed post processor @@ -62,7 +61,7 @@ func (p *Parser) decodePostProcessor(block *hcl.Block) (*PostProcessorBlock, hcl return postProcessor, diags } -func (cfg *PackerConfig) startPostProcessor(source SourceBlock, pp *PostProcessorBlock, ectx *hcl.EvalContext) (packer.PostProcessor, hcl.Diagnostics) { +func (cfg *PackerConfig) startPostProcessor(source SourceBlock, pp *PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]string) (packer.PostProcessor, hcl.Diagnostics) { // ProvisionerBlock represents a detected but unparsed provisioner var diags hcl.Diagnostics @@ -77,7 +76,7 @@ func (cfg *PackerConfig) startPostProcessor(source SourceBlock, pp *PostProcesso } flatProvisinerCfg, moreDiags := decodeHCL2Spec(pp.Rest, ectx, postProcessor) diags = append(diags, moreDiags...) - err = postProcessor.Configure(source.builderVariables(), flatProvisinerCfg) + err = postProcessor.Configure(source.builderVariables(), flatProvisinerCfg, generatedVars) if err != nil { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, @@ -89,46 +88,3 @@ func (cfg *PackerConfig) startPostProcessor(source SourceBlock, pp *PostProcesso } return postProcessor, diags } - -type postProcessorsPrepare struct { - cfg *PackerConfig - postProcessorBlock []*PostProcessorBlock - src SourceRef -} - -// HCL2PostProcessorsPrepare is used by the CoreBuild at the runtime, after running the build and before running the post-processors, -// to interpolate any build variable by decoding and preparing it. -func (pp *postProcessorsPrepare) HCL2PostProcessorsPrepare(builderArtifact packer.Artifact) ([]packer.CoreBuildPostProcessor, hcl.Diagnostics) { - src := pp.cfg.Sources[pp.src.Ref()] - - generatedData := make(map[string]interface{}) - if builderArtifact != nil { - artifactStateData := builderArtifact.State("generated_data") - if artifactStateData != nil { - for k, v := range artifactStateData.(map[interface{}]interface{}) { - generatedData[k.(string)] = v - } - } - } - - variables := make(Variables) - for k, v := range generatedData { - if value, ok := v.(string); ok { - variables[k] = &Variable{ - DefaultValue: cty.StringVal(value), - Type: cty.String, - } - } - } - variablesVal, _ := variables.Values() - - generatedVariables := map[string]cty.Value{ - sourcesAccessor: cty.ObjectVal(map[string]cty.Value{ - "type": cty.StringVal(src.Type), - "name": cty.StringVal(src.Name), - }), - buildAccessor: cty.ObjectVal(variablesVal), - } - - return pp.cfg.getCoreBuildPostProcessors(src, pp.postProcessorBlock, pp.cfg.EvalContext(generatedVariables)) -} diff --git a/hcl2template/types.build.provisioners.go b/hcl2template/types.build.provisioners.go index a2eeb2ee9..70756f39e 100644 --- a/hcl2template/types.build.provisioners.go +++ b/hcl2template/types.build.provisioners.go @@ -7,7 +7,6 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/packer/packer" - "github.com/zclconf/go-cty/cty" ) // OnlyExcept is a struct that is meant to be embedded that contains the @@ -133,7 +132,7 @@ func (p *Parser) decodeProvisioner(block *hcl.Block) (*ProvisionerBlock, hcl.Dia return provisioner, diags } -func (cfg *PackerConfig) startProvisioner(source SourceBlock, pb *ProvisionerBlock, ectx *hcl.EvalContext) (packer.Provisioner, hcl.Diagnostics) { +func (cfg *PackerConfig) startProvisioner(source SourceBlock, pb *ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]string) (packer.Provisioner, hcl.Diagnostics) { var diags hcl.Diagnostics provisioner, err := cfg.provisionersSchemas.Start(pb.PType) @@ -156,7 +155,7 @@ func (cfg *PackerConfig) startProvisioner(source SourceBlock, pb *ProvisionerBlo // configs := make([]interface{}, 2) // configs = append(, flatProvisionerCfg) // configs = append(configs, generatedVars) - err = provisioner.Prepare(source.builderVariables(), flatProvisionerCfg) + err = provisioner.Prepare(source.builderVariables(), flatProvisionerCfg, generatedVars) if err != nil { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, @@ -168,36 +167,3 @@ func (cfg *PackerConfig) startProvisioner(source SourceBlock, pb *ProvisionerBlo } return provisioner, diags } - -type provisionerPrepare struct { - cfg *PackerConfig - provisionerBlock []*ProvisionerBlock - src SourceRef -} - -// HCL2ProvisionerPrepare is used by the ProvisionHook at the runtime in the provision step -// to interpolate any build variable by decoding and preparing it again. -func (pp *provisionerPrepare) HCL2ProvisionerPrepare(data map[string]interface{}) ([]packer.CoreBuildProvisioner, hcl.Diagnostics) { - src := pp.cfg.Sources[pp.src.Ref()] - - variables := make(Variables) - for k, v := range data { - if value, ok := v.(string); ok { - variables[k] = &Variable{ - DefaultValue: cty.StringVal(value), - Type: cty.String, - } - } - } - variablesVal, _ := variables.Values() - - generatedVariables := map[string]cty.Value{ - sourcesAccessor: cty.ObjectVal(map[string]cty.Value{ - "type": cty.StringVal(src.Type), - "name": cty.StringVal(src.Name), - }), - buildAccessor: cty.ObjectVal(variablesVal), - } - - return pp.cfg.getCoreBuildProvisioners(src, pp.provisionerBlock, pp.cfg.EvalContext(generatedVariables)) -} diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index b791dabd1..2758c5ed1 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -7,6 +7,7 @@ import ( "github.com/gobwas/glob" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/hashicorp/packer/helper/common" "github.com/hashicorp/packer/packer" "github.com/zclconf/go-cty/cty" ) @@ -40,7 +41,6 @@ type PackerConfig struct { except []glob.Glob only []glob.Glob - debug bool } type ValidationOptions struct { @@ -51,7 +51,6 @@ const ( inputVariablesAccessor = "var" localsAccessor = "local" sourcesAccessor = "source" - buildAccessor = "build" ) // EvalContext returns the *hcl.EvalContext that will be passed to an hcl @@ -199,18 +198,19 @@ func (c *PackerConfig) evaluateLocalVariable(local *Local) hcl.Diagnostics { // getCoreBuildProvisioners takes a list of provisioner block, starts according // provisioners and sends parsed HCL2 over to it. -func (cfg *PackerConfig) getCoreBuildProvisioners(source SourceBlock, blocks []*ProvisionerBlock, ectx *hcl.EvalContext) ([]packer.CoreBuildProvisioner, hcl.Diagnostics) { +func (cfg *PackerConfig) getCoreBuildProvisioners(source SourceBlock, blocks []*ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]string) ([]packer.CoreBuildProvisioner, hcl.Diagnostics) { var diags hcl.Diagnostics res := []packer.CoreBuildProvisioner{} for _, pb := range blocks { if pb.OnlyExcept.Skip(source.String()) { continue } - provisioner, moreDiags := cfg.startProvisioner(source, pb, ectx) + provisioner, moreDiags := cfg.startProvisioner(source, pb, ectx, generatedVars) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { continue } + // If we're pausing, we wrap the provisioner in a special pauser. if pb.PauseBefore != 0 { provisioner = &packer.PausedProvisioner{ @@ -229,11 +229,6 @@ func (cfg *PackerConfig) getCoreBuildProvisioners(source SourceBlock, blocks []* Provisioner: provisioner, } } - if cfg.debug { - provisioner = &packer.DebuggedProvisioner{ - Provisioner: provisioner, - } - } res = append(res, packer.CoreBuildProvisioner{ PType: pb.PType, @@ -246,7 +241,7 @@ func (cfg *PackerConfig) getCoreBuildProvisioners(source SourceBlock, blocks []* // getCoreBuildProvisioners takes a list of post processor block, starts // according provisioners and sends parsed HCL2 over to it. -func (cfg *PackerConfig) getCoreBuildPostProcessors(source SourceBlock, blocks []*PostProcessorBlock, ectx *hcl.EvalContext) ([]packer.CoreBuildPostProcessor, hcl.Diagnostics) { +func (cfg *PackerConfig) getCoreBuildPostProcessors(source SourceBlock, blocks []*PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]string) ([]packer.CoreBuildPostProcessor, hcl.Diagnostics) { var diags hcl.Diagnostics res := []packer.CoreBuildPostProcessor{} for _, ppb := range blocks { @@ -270,7 +265,7 @@ func (cfg *PackerConfig) getCoreBuildPostProcessors(source SourceBlock, blocks [ break } - postProcessor, moreDiags := cfg.startPostProcessor(source, ppb, ectx) + postProcessor, moreDiags := cfg.startPostProcessor(source, ppb, ectx, generatedVars) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { continue @@ -293,8 +288,6 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packer.Build res := []packer.Build{} var diags hcl.Diagnostics - cfg.debug = opts.Debug - for _, build := range cfg.Builds { for _, from := range build.Sources { src, found := cfg.Sources[from.Ref()] @@ -361,6 +354,13 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packer.Build continue } + variables := map[string]cty.Value{ + sourcesAccessor: cty.ObjectVal(map[string]cty.Value{ + "type": cty.StringVal(src.Type), + "name": cty.StringVal(src.Name), + }), + } + // If the builder has provided a list of to-be-generated variables that // should be made accessible to provisioners, pass that list into // the provisioner prepare() so that the provisioner can appropriately @@ -369,42 +369,30 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packer.Build generatedPlaceholderMap := packer.BasicPlaceholderData() if generatedVars != nil { for _, k := range generatedVars { - generatedPlaceholderMap[k] = "" + generatedPlaceholderMap[k] = fmt.Sprintf("Build_%s. "+ + common.PlaceholderMsg, k) } } - buildVariables, _ := setBuildVariables(generatedPlaceholderMap).Values() - variables := map[string]cty.Value{ - sourcesAccessor: cty.ObjectVal(map[string]cty.Value{ - "type": cty.StringVal(src.Type), - "name": cty.StringVal(src.Name), - }), - buildAccessor: cty.ObjectVal(buildVariables), - } - - _, moreDiags = cfg.getCoreBuildProvisioners(src, build.ProvisionerBlocks, cfg.EvalContext(variables)) + provisioners, moreDiags := cfg.getCoreBuildProvisioners(src, build.ProvisionerBlocks, cfg.EvalContext(variables), generatedPlaceholderMap) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { continue } - _, moreDiags = cfg.getCoreBuildPostProcessors(src, build.PostProcessors, cfg.EvalContext(variables)) + postProcessors, moreDiags := cfg.getCoreBuildPostProcessors(src, build.PostProcessors, cfg.EvalContext(variables), generatedPlaceholderMap) + pps := [][]packer.CoreBuildPostProcessor{} + if len(postProcessors) > 0 { + pps = [][]packer.CoreBuildPostProcessor{postProcessors} + } diags = append(diags, moreDiags...) if moreDiags.HasErrors() { continue } pcb.Builder = builder + pcb.Provisioners = provisioners + pcb.PostProcessors = pps pcb.Prepared = true - pcb.HCL2ProvisionerPrepare = (&provisionerPrepare{ - cfg: cfg, - provisionerBlock: build.ProvisionerBlocks, - src: from, - }).HCL2ProvisionerPrepare - pcb.HCL2PostProcessorsPrepare = (&postProcessorsPrepare{ - cfg: cfg, - postProcessorBlock: build.PostProcessors, - src: from, - }).HCL2PostProcessorsPrepare // Prepare just sets the "prepareCalled" flag on CoreBuild, since // we did all the prep here. @@ -425,17 +413,6 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packer.Build return res, diags } -func setBuildVariables(generatedVars map[string]string) Variables { - variables := make(Variables) - for k := range generatedVars { - variables[k] = &Variable{ - DefaultValue: cty.StringVal("unknown"), - Type: cty.String, - } - } - return variables -} - var PackerConsoleHelp = strings.TrimSpace(` Packer console HCL2 Mode. The Packer console allows you to experiment with Packer interpolations. diff --git a/packer/build.go b/packer/build.go index 78b16ec3f..ca900797f 100644 --- a/packer/build.go +++ b/packer/build.go @@ -6,7 +6,6 @@ import ( "log" "sync" - "github.com/hashicorp/hcl/v2" "github.com/hashicorp/packer/helper/common" ) @@ -102,11 +101,6 @@ type CoreBuild struct { // Indicates whether the build is already initialized before calling Prepare(..) Prepared bool - // HCL2ProvisionerPrepare and HCL2PostProcessorsPrepare are used to interpolate any build variable by decoding and preparing - // the Provisioners and Post-Processors at runtime for HCL2 templates. - HCL2ProvisionerPrepare func(data map[string]interface{}) ([]CoreBuildProvisioner, hcl.Diagnostics) - HCL2PostProcessorsPrepare func(builderArtifact Artifact) ([]CoreBuildPostProcessor, hcl.Diagnostics) - debug bool force bool onError string @@ -274,12 +268,6 @@ func (b *CoreBuild) Run(ctx context.Context, originalUi Ui) ([]Artifact, error) }) } - if b.HCL2ProvisionerPrepare != nil { - hooks[HookProvision] = append(hooks[HookProvision], &ProvisionHook{ - HCL2Prepare: b.HCL2ProvisionerPrepare, - }) - } - if b.CleanupProvisioner.PType != "" { hookedCleanupProvisioner := &HookedProvisioner{ b.CleanupProvisioner.Provisioner, @@ -315,17 +303,6 @@ func (b *CoreBuild) Run(ctx context.Context, originalUi Ui) ([]Artifact, error) } errors := make([]error, 0) - - if b.HCL2PostProcessorsPrepare != nil { - // For HCL2, decode and prepare Post-Processors to interpolate build variables. - postProcessors, diags := b.HCL2PostProcessorsPrepare(builderArtifact) - if diags.HasErrors() { - errors = append(errors, diags) - } else if len(postProcessors) > 0 { - b.PostProcessors = [][]CoreBuildPostProcessor{postProcessors} - } - } - keepOriginalArtifact := len(b.PostProcessors) == 0 // Run the post-processors diff --git a/packer/provisioner.go b/packer/provisioner.go index d549c431b..905b28971 100644 --- a/packer/provisioner.go +++ b/packer/provisioner.go @@ -7,7 +7,6 @@ import ( "sync" "time" - "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hcldec" "github.com/hashicorp/packer/helper/common" ) @@ -41,8 +40,6 @@ type ProvisionHook struct { // The provisioners to run as part of the hook. These should already // be prepared (by calling Prepare) at some earlier stage. Provisioners []*HookedProvisioner - - HCL2Prepare func(data map[string]interface{}) ([]CoreBuildProvisioner, hcl.Diagnostics) } // Provisioners interpolate most of their fields in the prepare stage; this @@ -116,7 +113,7 @@ func CastDataToMap(data interface{}) map[string]interface{} { // Runs the provisioners in order. func (h *ProvisionHook) Run(ctx context.Context, name string, ui Ui, comm Communicator, data interface{}) error { // Shortcut - if len(h.Provisioners) == 0 && h.HCL2Prepare == nil { + if len(h.Provisioners) == 0 { return nil } @@ -126,34 +123,10 @@ func (h *ProvisionHook) Run(ctx context.Context, name string, ui Ui, comm Commun "`communicator` config was set to \"none\". If you have any provisioners\n" + "then a communicator is required. Please fix this to continue.") } - - cast := CastDataToMap(data) - - if h.HCL2Prepare != nil { - // For HCL2, decode and prepare Provisioners now to interpolate build variables - coreP, diags := h.HCL2Prepare(cast) - if diags.HasErrors() { - return diags - } - hookedProvisioners := make([]*HookedProvisioner, len(coreP)) - for i, p := range coreP { - var pConfig interface{} - if len(p.config) > 0 { - pConfig = p.config[0] - } - - hookedProvisioners[i] = &HookedProvisioner{ - Provisioner: p.Provisioner, - Config: pConfig, - TypeName: p.PType, - } - } - h.Provisioners = hookedProvisioners - } - for _, p := range h.Provisioners { ts := CheckpointReporter.AddSpan(p.TypeName, "provisioner", p.Config) + cast := CastDataToMap(data) err := p.Provisioner.Provision(ctx, ui, comm, cast) ts.End(err) diff --git a/post-processor/shell-local/post-processor.go b/post-processor/shell-local/post-processor.go index a17ce67e3..0cbf9fe71 100644 --- a/post-processor/shell-local/post-processor.go +++ b/post-processor/shell-local/post-processor.go @@ -42,9 +42,9 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) { generatedData := make(map[string]interface{}) - artifactStateData := artifact.State("generated_data") - if artifactStateData != nil { - for k, v := range artifactStateData.(map[interface{}]interface{}) { + artifactSateData := artifact.State("generated_data") + if artifactSateData != nil { + for k, v := range artifactSateData.(map[interface{}]interface{}) { generatedData[k.(string)] = v } } diff --git a/website/data/docs-navigation.js b/website/data/docs-navigation.js index 0669763f6..a05fe8525 100644 --- a/website/data/docs-navigation.js +++ b/website/data/docs-navigation.js @@ -140,7 +140,6 @@ export default [ }, 'variables', 'locals', - 'contextual-variables', 'syntax', 'onlyexcept', 'expressions', diff --git a/website/pages/docs/builders/amazon/chroot.mdx b/website/pages/docs/builders/amazon/chroot.mdx index 7bf474b22..d5165d6f3 100644 --- a/website/pages/docs/builders/amazon/chroot.mdx +++ b/website/pages/docs/builders/amazon/chroot.mdx @@ -328,52 +328,13 @@ variables are available: - `SourceAMIOwnerName` - The source AMI owner alias/name (for example `amazon`). - `SourceAMITags` - The source AMI Tags, as a `map[string]string` object. -## Build Shared Information Variables +## Build function template engine variables -This builder generates data that are shared with provisioner and post-processor via build function of [template engine](/docs/templates/engine) for JSON and [contextual variables](/docs/from-1.5/contextual-variables) for HCL2. - -The generated variables available for this builder are: +For the build function of [template engine](/docs/templates/engine), the following +variables are available: - `SourceAMIName` - The source AMI Name (for example `ubuntu/images/ebs-ssd/ubuntu-xenial-16.04-amd64-server-20180306`) used to build the AMI. - `Device` - Root device path. - `MountPath` - Device mounting path. - -Usage example: - - - - -```json -"post-processors": [ - { - "type": "manifest", - "output": "manifest.json", - "strip_path": true, - "custom_data": { - "source_ami_name": "{{ build `SourceAMIName` }}", - "device": "{{ build `Device` }}", - "mount_path": "{{ build `MountPath` }}" - } - } -] -``` - - - - -```hcl -post-processor "manifest" { - output = "manifest.json" - strip_path = true - custom_data = { - source_ami_name = "${build.SourceAMIName}" - device = "${build.Device}" - mount_path = "${build.MountPath}" - } -} -``` - - - diff --git a/website/pages/docs/builders/amazon/ebs.mdx b/website/pages/docs/builders/amazon/ebs.mdx index e0db95037..351f97915 100644 --- a/website/pages/docs/builders/amazon/ebs.mdx +++ b/website/pages/docs/builders/amazon/ebs.mdx @@ -235,50 +235,15 @@ variables are available: - `SourceAMIOwnerName` - The source AMI owner alias/name (for example `amazon`). - `SourceAMITags` - The source AMI Tags, as a `map[string]string` object. -## Build Shared Information Variables +## Build function template engine variables -This builder generates data that are shared with provisioner and post-processor via build function of [template engine](/docs/templates/engine) for JSON and [contextual variables](/docs/from-1.5/contextual-variables) for HCL2. - -The generated variables available for this builder are: +For the build function of [template engine](/docs/templates/engine), the following +variables are available: - `SourceAMIName` - The source AMI Name (for example `ubuntu/images/ebs-ssd/ubuntu-xenial-16.04-amd64-server-20180306`) used to build the AMI. -Usage example: - - - - -```json -"post-processors": [ - { - "type": "manifest", - "output": "manifest.json", - "strip_path": true, - "custom_data": { - "source_ami_name": "{{ build `SourceAMIName` }}" - } - } -] -``` - - - - -```hcl -post-processor "manifest" { - output = "manifest.json" - strip_path = true - custom_data = { - source_ami_name = "${build.SourceAMIName}" - } -} -``` - - - - ## Tag Example Here is an example using the optional AMI tags. This will add the tags diff --git a/website/pages/docs/builders/amazon/ebssurrogate.mdx b/website/pages/docs/builders/amazon/ebssurrogate.mdx index 529db51e0..deb506e13 100644 --- a/website/pages/docs/builders/amazon/ebssurrogate.mdx +++ b/website/pages/docs/builders/amazon/ebssurrogate.mdx @@ -207,50 +207,15 @@ variables are available: - `SourceAMIOwnerName` - The source AMI owner alias/name (for example `amazon`). - `SourceAMITags` - The source AMI Tags, as a `map[string]string` object. -## Build Shared Information Variables +## Build function template engine variables -This builder generates data that are shared with provisioner and post-processor via build function of [template engine](/docs/templates/engine) for JSON and [contextual variables](/docs/from-1.5/contextual-variables) for HCL2. - -The generated variables available for this builder are: +For the build function of [template engine](/docs/templates/engine), the following +variables are available: - `SourceAMIName` - The source AMI Name (for example `ubuntu/images/ebs-ssd/ubuntu-xenial-16.04-amd64-server-20180306`) used to build the AMI. -Usage example: - - - - -```json -"post-processors": [ - { - "type": "manifest", - "output": "manifest.json", - "strip_path": true, - "custom_data": { - "source_ami_name": "{{ build `SourceAMIName` }}" - } - } -] -``` - - - - -```hcl -post-processor "manifest" { - output = "manifest.json" - strip_path = true - custom_data = { - source_ami_name = "${build.SourceAMIName}" - } -} -``` - - - - -> **Note:** Packer uses pre-built AMIs as the source for building images. These source AMIs may include volumes that are not flagged to be destroyed on termination of the instance building the new image. In addition to those diff --git a/website/pages/docs/builders/amazon/ebsvolume.mdx b/website/pages/docs/builders/amazon/ebsvolume.mdx index 4e25e3dcf..1b5b6aabc 100644 --- a/website/pages/docs/builders/amazon/ebsvolume.mdx +++ b/website/pages/docs/builders/amazon/ebsvolume.mdx @@ -243,48 +243,13 @@ termination of the instance building the new image. In addition to those volumes created by this builder, any volumes in the source AMI which are not marked for deletion on termination will remain in your account. -## Build Shared Information Variables +## Build function template engine variables -This builder generates data that are shared with provisioner and post-processor via build function of [template engine](/docs/templates/engine) for JSON and [contextual variables](/docs/from-1.5/contextual-variables) for HCL2. - -The generated variables available for this builder are: +For the build function of [template engine](/docs/templates/engine), the following +variables are available: - `SourceAMIName` - The source AMI Name (for example `ubuntu/images/ebs-ssd/ubuntu-xenial-16.04-amd64-server-20180306`) used to build the AMI. -Usage example: - - - - -```json -"post-processors": [ - { - "type": "manifest", - "output": "manifest.json", - "strip_path": true, - "custom_data": { - "source_ami_name": "{{ build `SourceAMIName` }}" - } - } -] -``` - - - - -```hcl -post-processor "manifest" { - output = "manifest.json" - strip_path = true - custom_data = { - source_ami_name = "${build.SourceAMIName}" - } -} -``` - - - - @include 'builders/aws-ssh-differentiation-table.mdx' diff --git a/website/pages/docs/builders/amazon/instance.mdx b/website/pages/docs/builders/amazon/instance.mdx index 9abf80da1..69410bbcf 100644 --- a/website/pages/docs/builders/amazon/instance.mdx +++ b/website/pages/docs/builders/amazon/instance.mdx @@ -199,50 +199,15 @@ variables are available: - `SourceAMIOwnerName` - The source AMI owner alias/name (for example `amazon`). - `SourceAMITags` - The source AMI Tags, as a `map[string]string` object. -## Build Shared Information Variables +## Build function template engine variables -This builder generates data that are shared with provisioner and post-processor via build function of [template engine](/docs/templates/engine) for JSON and [contextual variables](/docs/from-1.5/contextual-variables) for HCL2. - -The generated variables available for this builder are: +For the build function of [template engine](/docs/templates/engine), the following +variables are available: - `SourceAMIName` - The source AMI Name (for example `ubuntu/images/ebs-ssd/ubuntu-xenial-16.04-amd64-server-20180306`) used to build the AMI. -Usage example: - - - - -```json -"post-processors": [ - { - "type": "manifest", - "output": "manifest.json", - "strip_path": true, - "custom_data": { - "source_ami_name": "{{ build `SourceAMIName` }}" - } - } -] -``` - - - - -```hcl -post-processor "manifest" { - output = "manifest.json" - strip_path = true - custom_data = { - source_ami_name = "${build.SourceAMIName}" - } -} -``` - - - - ## Custom Bundle Commands A lot of the process required for creating an instance-store backed AMI diff --git a/website/pages/docs/from-1.5/blocks/build/post-processor.mdx b/website/pages/docs/from-1.5/blocks/build/post-processor.mdx index cb94f1ef9..47fccaabb 100644 --- a/website/pages/docs/from-1.5/blocks/build/post-processor.mdx +++ b/website/pages/docs/from-1.5/blocks/build/post-processor.mdx @@ -71,8 +71,3 @@ build { ``` The values within `only` or `except` are _source names_, not builder types. - -## Build Contextual Variables - -Packer allows to access connection information and basic instance state information from a post-processor. These information are stored in the `build` variable. -Check out the [Contextual Variables](/docs/from-1.5/contextual-variables) documentation to learn more about and see some examples of how to use them. diff --git a/website/pages/docs/from-1.5/blocks/build/provisioner.mdx b/website/pages/docs/from-1.5/blocks/build/provisioner.mdx index 5ab488405..0c06e36f8 100644 --- a/website/pages/docs/from-1.5/blocks/build/provisioner.mdx +++ b/website/pages/docs/from-1.5/blocks/build/provisioner.mdx @@ -59,7 +59,7 @@ build { ] provisioner "shell" { # This provisioner only runs for the 'first-example' source. - only = ["source.amazon-ebs.first-example"] + only = ["source.amazon-ebs.first-example"] inline = [ "echo provisioning all the things", @@ -160,8 +160,3 @@ For the above provisioner, Packer will cancel the script if it takes more than 5 minutes. Timeout has no effect in debug mode. - -## Build Contextual Variables - -Packer allows to access connection information and basic instance state information from a provisioner. These information are stored in the `build` variable. -Check out the [Contextual Variables](/docs/from-1.5/contextual-variables) documentation to learn more about and see some examples of how to use them. diff --git a/website/pages/docs/from-1.5/contextual-variables.mdx b/website/pages/docs/from-1.5/contextual-variables.mdx deleted file mode 100644 index 48c91472d..000000000 --- a/website/pages/docs/from-1.5/contextual-variables.mdx +++ /dev/null @@ -1,71 +0,0 @@ ---- -layout: docs -page_title: Contextual Variables - HCL Configuration Language -sidebar_title: Contextual Variables -description: |- - Special variables provide connection information and basic instance state information. - This page covers all existing special variables. ---- - -# Build Variables - -`@include 'from-1.5/beta-hcl2-note.mdx'` - -Build variables will allow you to access connection information and basic instance state information for a builder. -All special build variables are stored in the `build` variable: - -```hcl - provisioner "shell" { - environment_vars = ["TESTVAR=${build.PackerRunUUID}"] - inline = ["echo $TESTVAR"] - } -``` - -Here is the list of available build variables: - -- **ID**: Represents the vm being provisioned. For example, in Amazon it is the instance id; in digitalocean, - it is the droplet id; in Vmware, it is the vm name. - -- **Host**, **Port**, **User** and **Password**: The host, port, user, and password that Packer uses to access the machine. - Useful for using the shell local provisioner to run Ansible or Inspec against the provisioned instance. - -- **ConnType**: Type of communicator being used. For example, for SSH communicator this will be "ssh". - -- **PackerRunUUID**: Current build's unique id. Can be used to specify build artifacts. - An example of that, is when multiple builds runs at the same time producing the same artifact. - It's possible to differentiate these artifacts by naming them with the builds' unique ids. - -- **PackerHTTPIP**, **PackerHTTPPort**, and **PackerHTTPAddr**: HTTP IP, port, and address of the file server Packer creates to serve items in the "http" dir to the vm. The HTTP address is displayed in the format `IP:PORT`. - -- **SSHPublicKey** and **SSHPrivateKey**: The public and private key that Packer uses to connect to the instance. - These are unique to the SSH communicator and are unset when using other communicators. - **SSHPublicKey** and **SSHPrivateKey** can have escape sequences and special characters so their output should be single quoted to avoid surprises. For example: - - ```hcl - provisioner "shell" { - inline = ["echo '${build.SSHPrivateKey}' > /tmp/packer-session.pem"] - } - ``` - -For backwards compatibility, `WinRMPassword` is also available through this -engine, though it is no different than using the more general `Password`. - -All build variables are valid to use with any of the [HCL2 functions](/docs/from-1.5/functions). -Example of using [upper](/docs/from-1.5/functions/string/upper) to upper case the build ID: - -```hcl - post-processor "shell-local" { - inline = ["echo ${upper(build.ID)}"] - } -``` - -For builder-specific builder variables, please also refer to the builder docs: - - - Amazon EC2: [chroot](/docs/builders/amazon/chroot#build-shared-information-variables), - [EBS Volume](/docs/builders/amazon/ebsvolume#build-shared-information-variables), - [EBS](/docs/builders/amazon/ebs#build-shared-information-variables), - [EBS Surrogate](/docs/builders/amazon/ebssurrogate#build-shared-information-variables), - [Instance](/docs/builders/amazon/instance#build-shared-information-variables). - -The HCL2 Special Build Variables is in beta; please report any issues or requests on the Packer -issue tracker on GitHub. diff --git a/website/pages/docs/templates/engine.mdx b/website/pages/docs/templates/engine.mdx index 11ec5f9f2..ebf50fd2e 100644 --- a/website/pages/docs/templates/engine.mdx +++ b/website/pages/docs/templates/engine.mdx @@ -83,8 +83,6 @@ Here is a full list of the available functions for reference. - **ConnType**: Type of communicator being used. For example, for SSH communicator this will be "ssh". - **PackerRunUUID**: Current build's unique id. Can be used to specify build artifacts. - An example of that, is when multiple builds runs at the same time producing the same artifact. - It's possible to differentiate these artifacts by naming them with the builds' unique ids. - **PackerHTTPIP**, **PackerHTTPPort**, and **PackerHTTPAddr**: HTTP IP, port, and address of the file server Packer creates to serve items in the "http" dir to the vm. The HTTP address is displayed in the format `IP:PORT`. @@ -100,20 +98,16 @@ Here is a full list of the available functions for reference. } ``` - For backwards compatibility, `WinRMPassword` is also available through this + For backwards compatability, `WinRMPassword` is also available through this engine, though it is no different than using the more general `Password`. This function is only for use within specific options inside of _provisioners_ -- these options will be listed as being template engines - in the provisioner documentation. + in the provisioner documentation. This feature does not yet work + if the provisioners are being used in conjunction with our chroot builders + or with lxc/lxd builders. - For builder-specific builder variables, please also refer to the builder docs: - - - Amazon EC2: [chroot](/docs/builders/amazon/chroot#build-shared-information-variables), - [EBS Volume](/docs/builders/amazon/ebsvolume#build-shared-information-variables), - [EBS](/docs/builders/amazon/ebs#build-shared-information-variables), - [EBS Surrogate](/docs/builders/amazon/ebssurrogate#build-shared-information-variables), - [Instance](/docs/builders/amazon/instance#build-shared-information-variables). + For builder-specific engine variables, please also refer to the builder docs. This engine is in beta; please report any issues or requests on the Packer issue tracker on GitHub. From 284d46a1e0e3c54bbdf8bca399e7e716849c22be Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Thu, 2 Jul 2020 15:56:49 +0200 Subject: [PATCH 02/24] WIP --- hcl2template/internal/mock.go | 2 +- hcl2template/testdata/complete/build.pkr.hcl | 2 +- hcl2template/types.packer_config.go | 26 +++++++++++--------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/hcl2template/internal/mock.go b/hcl2template/internal/mock.go index 3c36528f3..bf746ed5c 100644 --- a/hcl2template/internal/mock.go +++ b/hcl2template/internal/mock.go @@ -74,7 +74,7 @@ var _ packer.Builder = new(MockBuilder) func (b *MockBuilder) ConfigSpec() hcldec.ObjectSpec { return b.Config.FlatMapstructure().HCL2Spec() } func (b *MockBuilder) Prepare(raws ...interface{}) ([]string, []string, error) { - return nil, nil, b.Config.Prepare(raws...) + return []string{"ID"}, nil, b.Config.Prepare(raws...) } func (b *MockBuilder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (packer.Artifact, error) { diff --git a/hcl2template/testdata/complete/build.pkr.hcl b/hcl2template/testdata/complete/build.pkr.hcl index d26ecb4ed..ef73e65f9 100644 --- a/hcl2template/testdata/complete/build.pkr.hcl +++ b/hcl2template/testdata/complete/build.pkr.hcl @@ -11,7 +11,7 @@ build { provisioner "shell" { name = "provisioner that does something" - not_squashed = var.foo + not_squashed = "${var.foo} ${lower(build.ID)}" string = "string" int = "${41 + 1}" int64 = "${42 + 1}" diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index 2758c5ed1..3908c91fe 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -51,6 +51,7 @@ const ( inputVariablesAccessor = "var" localsAccessor = "local" sourcesAccessor = "source" + buildAccessor = "build" ) // EvalContext returns the *hcl.EvalContext that will be passed to an hcl @@ -354,24 +355,25 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packer.Build continue } - variables := map[string]cty.Value{ - sourcesAccessor: cty.ObjectVal(map[string]cty.Value{ - "type": cty.StringVal(src.Type), - "name": cty.StringVal(src.Name), - }), - } - // If the builder has provided a list of to-be-generated variables that // should be made accessible to provisioners, pass that list into // the provisioner prepare() so that the provisioner can appropriately // validate user input against what will become available. Otherwise, // only pass the default variables, using the basic placeholder data. generatedPlaceholderMap := packer.BasicPlaceholderData() - if generatedVars != nil { - for _, k := range generatedVars { - generatedPlaceholderMap[k] = fmt.Sprintf("Build_%s. "+ - common.PlaceholderMsg, k) - } + unknownBuildValues := map[string]cty.Value{} + for _, k := range generatedVars { + generatedPlaceholderMap[k] = fmt.Sprintf("Build_%s. "+ + common.PlaceholderMsg, k) + unknownBuildValues[k] = cty.StringVal("") + } + + variables := map[string]cty.Value{ + sourcesAccessor: cty.ObjectVal(map[string]cty.Value{ + "type": cty.StringVal(src.Type), + "name": cty.StringVal(src.Name), + }), + buildAccessor: cty.ObjectVal(unknownBuildValues), } provisioners, moreDiags := cfg.getCoreBuildProvisioners(src, build.ProvisionerBlocks, cfg.EvalContext(variables), generatedPlaceholderMap) From 6a108fa3f6c30d943835a9b5bb8772954d9210cd Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Thu, 2 Jul 2020 16:03:23 +0200 Subject: [PATCH 03/24] bring docs from master --- website/data/docs-navigation.js | 1 + website/pages/docs/builders/amazon/chroot.mdx | 45 +++++++++++- website/pages/docs/builders/amazon/ebs.mdx | 41 ++++++++++- .../docs/builders/amazon/ebssurrogate.mdx | 41 ++++++++++- .../pages/docs/builders/amazon/ebsvolume.mdx | 41 ++++++++++- .../pages/docs/builders/amazon/instance.mdx | 41 ++++++++++- .../from-1.5/blocks/build/post-processor.mdx | 5 ++ .../from-1.5/blocks/build/provisioner.mdx | 7 +- .../docs/from-1.5/contextual-variables.mdx | 71 +++++++++++++++++++ website/pages/docs/templates/engine.mdx | 16 +++-- 10 files changed, 288 insertions(+), 21 deletions(-) create mode 100644 website/pages/docs/from-1.5/contextual-variables.mdx diff --git a/website/data/docs-navigation.js b/website/data/docs-navigation.js index a05fe8525..0669763f6 100644 --- a/website/data/docs-navigation.js +++ b/website/data/docs-navigation.js @@ -140,6 +140,7 @@ export default [ }, 'variables', 'locals', + 'contextual-variables', 'syntax', 'onlyexcept', 'expressions', diff --git a/website/pages/docs/builders/amazon/chroot.mdx b/website/pages/docs/builders/amazon/chroot.mdx index d5165d6f3..7bf474b22 100644 --- a/website/pages/docs/builders/amazon/chroot.mdx +++ b/website/pages/docs/builders/amazon/chroot.mdx @@ -328,13 +328,52 @@ variables are available: - `SourceAMIOwnerName` - The source AMI owner alias/name (for example `amazon`). - `SourceAMITags` - The source AMI Tags, as a `map[string]string` object. -## Build function template engine variables +## Build Shared Information Variables -For the build function of [template engine](/docs/templates/engine), the following -variables are available: +This builder generates data that are shared with provisioner and post-processor via build function of [template engine](/docs/templates/engine) for JSON and [contextual variables](/docs/from-1.5/contextual-variables) for HCL2. + +The generated variables available for this builder are: - `SourceAMIName` - The source AMI Name (for example `ubuntu/images/ebs-ssd/ubuntu-xenial-16.04-amd64-server-20180306`) used to build the AMI. - `Device` - Root device path. - `MountPath` - Device mounting path. + +Usage example: + + + + +```json +"post-processors": [ + { + "type": "manifest", + "output": "manifest.json", + "strip_path": true, + "custom_data": { + "source_ami_name": "{{ build `SourceAMIName` }}", + "device": "{{ build `Device` }}", + "mount_path": "{{ build `MountPath` }}" + } + } +] +``` + + + + +```hcl +post-processor "manifest" { + output = "manifest.json" + strip_path = true + custom_data = { + source_ami_name = "${build.SourceAMIName}" + device = "${build.Device}" + mount_path = "${build.MountPath}" + } +} +``` + + + diff --git a/website/pages/docs/builders/amazon/ebs.mdx b/website/pages/docs/builders/amazon/ebs.mdx index 351f97915..e0db95037 100644 --- a/website/pages/docs/builders/amazon/ebs.mdx +++ b/website/pages/docs/builders/amazon/ebs.mdx @@ -235,15 +235,50 @@ variables are available: - `SourceAMIOwnerName` - The source AMI owner alias/name (for example `amazon`). - `SourceAMITags` - The source AMI Tags, as a `map[string]string` object. -## Build function template engine variables +## Build Shared Information Variables -For the build function of [template engine](/docs/templates/engine), the following -variables are available: +This builder generates data that are shared with provisioner and post-processor via build function of [template engine](/docs/templates/engine) for JSON and [contextual variables](/docs/from-1.5/contextual-variables) for HCL2. + +The generated variables available for this builder are: - `SourceAMIName` - The source AMI Name (for example `ubuntu/images/ebs-ssd/ubuntu-xenial-16.04-amd64-server-20180306`) used to build the AMI. +Usage example: + + + + +```json +"post-processors": [ + { + "type": "manifest", + "output": "manifest.json", + "strip_path": true, + "custom_data": { + "source_ami_name": "{{ build `SourceAMIName` }}" + } + } +] +``` + + + + +```hcl +post-processor "manifest" { + output = "manifest.json" + strip_path = true + custom_data = { + source_ami_name = "${build.SourceAMIName}" + } +} +``` + + + + ## Tag Example Here is an example using the optional AMI tags. This will add the tags diff --git a/website/pages/docs/builders/amazon/ebssurrogate.mdx b/website/pages/docs/builders/amazon/ebssurrogate.mdx index deb506e13..529db51e0 100644 --- a/website/pages/docs/builders/amazon/ebssurrogate.mdx +++ b/website/pages/docs/builders/amazon/ebssurrogate.mdx @@ -207,15 +207,50 @@ variables are available: - `SourceAMIOwnerName` - The source AMI owner alias/name (for example `amazon`). - `SourceAMITags` - The source AMI Tags, as a `map[string]string` object. -## Build function template engine variables +## Build Shared Information Variables -For the build function of [template engine](/docs/templates/engine), the following -variables are available: +This builder generates data that are shared with provisioner and post-processor via build function of [template engine](/docs/templates/engine) for JSON and [contextual variables](/docs/from-1.5/contextual-variables) for HCL2. + +The generated variables available for this builder are: - `SourceAMIName` - The source AMI Name (for example `ubuntu/images/ebs-ssd/ubuntu-xenial-16.04-amd64-server-20180306`) used to build the AMI. +Usage example: + + + + +```json +"post-processors": [ + { + "type": "manifest", + "output": "manifest.json", + "strip_path": true, + "custom_data": { + "source_ami_name": "{{ build `SourceAMIName` }}" + } + } +] +``` + + + + +```hcl +post-processor "manifest" { + output = "manifest.json" + strip_path = true + custom_data = { + source_ami_name = "${build.SourceAMIName}" + } +} +``` + + + + -> **Note:** Packer uses pre-built AMIs as the source for building images. These source AMIs may include volumes that are not flagged to be destroyed on termination of the instance building the new image. In addition to those diff --git a/website/pages/docs/builders/amazon/ebsvolume.mdx b/website/pages/docs/builders/amazon/ebsvolume.mdx index 1b5b6aabc..4e25e3dcf 100644 --- a/website/pages/docs/builders/amazon/ebsvolume.mdx +++ b/website/pages/docs/builders/amazon/ebsvolume.mdx @@ -243,13 +243,48 @@ termination of the instance building the new image. In addition to those volumes created by this builder, any volumes in the source AMI which are not marked for deletion on termination will remain in your account. -## Build function template engine variables +## Build Shared Information Variables -For the build function of [template engine](/docs/templates/engine), the following -variables are available: +This builder generates data that are shared with provisioner and post-processor via build function of [template engine](/docs/templates/engine) for JSON and [contextual variables](/docs/from-1.5/contextual-variables) for HCL2. + +The generated variables available for this builder are: - `SourceAMIName` - The source AMI Name (for example `ubuntu/images/ebs-ssd/ubuntu-xenial-16.04-amd64-server-20180306`) used to build the AMI. +Usage example: + + + + +```json +"post-processors": [ + { + "type": "manifest", + "output": "manifest.json", + "strip_path": true, + "custom_data": { + "source_ami_name": "{{ build `SourceAMIName` }}" + } + } +] +``` + + + + +```hcl +post-processor "manifest" { + output = "manifest.json" + strip_path = true + custom_data = { + source_ami_name = "${build.SourceAMIName}" + } +} +``` + + + + @include 'builders/aws-ssh-differentiation-table.mdx' diff --git a/website/pages/docs/builders/amazon/instance.mdx b/website/pages/docs/builders/amazon/instance.mdx index 69410bbcf..9abf80da1 100644 --- a/website/pages/docs/builders/amazon/instance.mdx +++ b/website/pages/docs/builders/amazon/instance.mdx @@ -199,15 +199,50 @@ variables are available: - `SourceAMIOwnerName` - The source AMI owner alias/name (for example `amazon`). - `SourceAMITags` - The source AMI Tags, as a `map[string]string` object. -## Build function template engine variables +## Build Shared Information Variables -For the build function of [template engine](/docs/templates/engine), the following -variables are available: +This builder generates data that are shared with provisioner and post-processor via build function of [template engine](/docs/templates/engine) for JSON and [contextual variables](/docs/from-1.5/contextual-variables) for HCL2. + +The generated variables available for this builder are: - `SourceAMIName` - The source AMI Name (for example `ubuntu/images/ebs-ssd/ubuntu-xenial-16.04-amd64-server-20180306`) used to build the AMI. +Usage example: + + + + +```json +"post-processors": [ + { + "type": "manifest", + "output": "manifest.json", + "strip_path": true, + "custom_data": { + "source_ami_name": "{{ build `SourceAMIName` }}" + } + } +] +``` + + + + +```hcl +post-processor "manifest" { + output = "manifest.json" + strip_path = true + custom_data = { + source_ami_name = "${build.SourceAMIName}" + } +} +``` + + + + ## Custom Bundle Commands A lot of the process required for creating an instance-store backed AMI diff --git a/website/pages/docs/from-1.5/blocks/build/post-processor.mdx b/website/pages/docs/from-1.5/blocks/build/post-processor.mdx index 47fccaabb..cb94f1ef9 100644 --- a/website/pages/docs/from-1.5/blocks/build/post-processor.mdx +++ b/website/pages/docs/from-1.5/blocks/build/post-processor.mdx @@ -71,3 +71,8 @@ build { ``` The values within `only` or `except` are _source names_, not builder types. + +## Build Contextual Variables + +Packer allows to access connection information and basic instance state information from a post-processor. These information are stored in the `build` variable. +Check out the [Contextual Variables](/docs/from-1.5/contextual-variables) documentation to learn more about and see some examples of how to use them. diff --git a/website/pages/docs/from-1.5/blocks/build/provisioner.mdx b/website/pages/docs/from-1.5/blocks/build/provisioner.mdx index 0c06e36f8..5ab488405 100644 --- a/website/pages/docs/from-1.5/blocks/build/provisioner.mdx +++ b/website/pages/docs/from-1.5/blocks/build/provisioner.mdx @@ -59,7 +59,7 @@ build { ] provisioner "shell" { # This provisioner only runs for the 'first-example' source. - only = ["source.amazon-ebs.first-example"] + only = ["source.amazon-ebs.first-example"] inline = [ "echo provisioning all the things", @@ -160,3 +160,8 @@ For the above provisioner, Packer will cancel the script if it takes more than 5 minutes. Timeout has no effect in debug mode. + +## Build Contextual Variables + +Packer allows to access connection information and basic instance state information from a provisioner. These information are stored in the `build` variable. +Check out the [Contextual Variables](/docs/from-1.5/contextual-variables) documentation to learn more about and see some examples of how to use them. diff --git a/website/pages/docs/from-1.5/contextual-variables.mdx b/website/pages/docs/from-1.5/contextual-variables.mdx new file mode 100644 index 000000000..48c91472d --- /dev/null +++ b/website/pages/docs/from-1.5/contextual-variables.mdx @@ -0,0 +1,71 @@ +--- +layout: docs +page_title: Contextual Variables - HCL Configuration Language +sidebar_title: Contextual Variables +description: |- + Special variables provide connection information and basic instance state information. + This page covers all existing special variables. +--- + +# Build Variables + +`@include 'from-1.5/beta-hcl2-note.mdx'` + +Build variables will allow you to access connection information and basic instance state information for a builder. +All special build variables are stored in the `build` variable: + +```hcl + provisioner "shell" { + environment_vars = ["TESTVAR=${build.PackerRunUUID}"] + inline = ["echo $TESTVAR"] + } +``` + +Here is the list of available build variables: + +- **ID**: Represents the vm being provisioned. For example, in Amazon it is the instance id; in digitalocean, + it is the droplet id; in Vmware, it is the vm name. + +- **Host**, **Port**, **User** and **Password**: The host, port, user, and password that Packer uses to access the machine. + Useful for using the shell local provisioner to run Ansible or Inspec against the provisioned instance. + +- **ConnType**: Type of communicator being used. For example, for SSH communicator this will be "ssh". + +- **PackerRunUUID**: Current build's unique id. Can be used to specify build artifacts. + An example of that, is when multiple builds runs at the same time producing the same artifact. + It's possible to differentiate these artifacts by naming them with the builds' unique ids. + +- **PackerHTTPIP**, **PackerHTTPPort**, and **PackerHTTPAddr**: HTTP IP, port, and address of the file server Packer creates to serve items in the "http" dir to the vm. The HTTP address is displayed in the format `IP:PORT`. + +- **SSHPublicKey** and **SSHPrivateKey**: The public and private key that Packer uses to connect to the instance. + These are unique to the SSH communicator and are unset when using other communicators. + **SSHPublicKey** and **SSHPrivateKey** can have escape sequences and special characters so their output should be single quoted to avoid surprises. For example: + + ```hcl + provisioner "shell" { + inline = ["echo '${build.SSHPrivateKey}' > /tmp/packer-session.pem"] + } + ``` + +For backwards compatibility, `WinRMPassword` is also available through this +engine, though it is no different than using the more general `Password`. + +All build variables are valid to use with any of the [HCL2 functions](/docs/from-1.5/functions). +Example of using [upper](/docs/from-1.5/functions/string/upper) to upper case the build ID: + +```hcl + post-processor "shell-local" { + inline = ["echo ${upper(build.ID)}"] + } +``` + +For builder-specific builder variables, please also refer to the builder docs: + + - Amazon EC2: [chroot](/docs/builders/amazon/chroot#build-shared-information-variables), + [EBS Volume](/docs/builders/amazon/ebsvolume#build-shared-information-variables), + [EBS](/docs/builders/amazon/ebs#build-shared-information-variables), + [EBS Surrogate](/docs/builders/amazon/ebssurrogate#build-shared-information-variables), + [Instance](/docs/builders/amazon/instance#build-shared-information-variables). + +The HCL2 Special Build Variables is in beta; please report any issues or requests on the Packer +issue tracker on GitHub. diff --git a/website/pages/docs/templates/engine.mdx b/website/pages/docs/templates/engine.mdx index ebf50fd2e..11ec5f9f2 100644 --- a/website/pages/docs/templates/engine.mdx +++ b/website/pages/docs/templates/engine.mdx @@ -83,6 +83,8 @@ Here is a full list of the available functions for reference. - **ConnType**: Type of communicator being used. For example, for SSH communicator this will be "ssh". - **PackerRunUUID**: Current build's unique id. Can be used to specify build artifacts. + An example of that, is when multiple builds runs at the same time producing the same artifact. + It's possible to differentiate these artifacts by naming them with the builds' unique ids. - **PackerHTTPIP**, **PackerHTTPPort**, and **PackerHTTPAddr**: HTTP IP, port, and address of the file server Packer creates to serve items in the "http" dir to the vm. The HTTP address is displayed in the format `IP:PORT`. @@ -98,16 +100,20 @@ Here is a full list of the available functions for reference. } ``` - For backwards compatability, `WinRMPassword` is also available through this + For backwards compatibility, `WinRMPassword` is also available through this engine, though it is no different than using the more general `Password`. This function is only for use within specific options inside of _provisioners_ -- these options will be listed as being template engines - in the provisioner documentation. This feature does not yet work - if the provisioners are being used in conjunction with our chroot builders - or with lxc/lxd builders. + in the provisioner documentation. - For builder-specific engine variables, please also refer to the builder docs. + For builder-specific builder variables, please also refer to the builder docs: + + - Amazon EC2: [chroot](/docs/builders/amazon/chroot#build-shared-information-variables), + [EBS Volume](/docs/builders/amazon/ebsvolume#build-shared-information-variables), + [EBS](/docs/builders/amazon/ebs#build-shared-information-variables), + [EBS Surrogate](/docs/builders/amazon/ebssurrogate#build-shared-information-variables), + [Instance](/docs/builders/amazon/instance#build-shared-information-variables). This engine is in beta; please report any issues or requests on the Packer issue tracker on GitHub. From c88cb098a167f3005919eb87421c4c6c00481a1f Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Thu, 2 Jul 2020 16:04:38 +0200 Subject: [PATCH 04/24] Update post-processor.go --- post-processor/shell-local/post-processor.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/post-processor/shell-local/post-processor.go b/post-processor/shell-local/post-processor.go index 0cbf9fe71..a17ce67e3 100644 --- a/post-processor/shell-local/post-processor.go +++ b/post-processor/shell-local/post-processor.go @@ -42,9 +42,9 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { func (p *PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) { generatedData := make(map[string]interface{}) - artifactSateData := artifact.State("generated_data") - if artifactSateData != nil { - for k, v := range artifactSateData.(map[interface{}]interface{}) { + artifactStateData := artifact.State("generated_data") + if artifactStateData != nil { + for k, v := range artifactStateData.(map[interface{}]interface{}) { generatedData[k.(string)] = v } } From db6c3adbbaaf90090ed1c7a86b629c6c7ee42ee0 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Thu, 2 Jul 2020 18:02:19 +0200 Subject: [PATCH 05/24] add hcl2 provisioner type that reprepares itself --- hcl2template/common_test.go | 3 +- hcl2template/testdata/complete/build.pkr.hcl | 4 +- hcl2template/types.build.post-processor.go | 4 +- hcl2template/types.build.provisioners.go | 21 +++---- hcl2template/types.build_test.go | 40 ++++++++----- hcl2template/types.hcl_provisioner.go | 61 ++++++++++++++++++++ hcl2template/types.packer_config.go | 12 ++-- hcl2template/types.packer_config_test.go | 30 +++++++--- packer/provisioner.go | 4 +- 9 files changed, 129 insertions(+), 50 deletions(-) create mode 100644 hcl2template/types.hcl_provisioner.go diff --git a/hcl2template/common_test.go b/hcl2template/common_test.go index 5db6dc6c2..610e178ce 100644 --- a/hcl2template/common_test.go +++ b/hcl2template/common_test.go @@ -129,6 +129,7 @@ func testParse(t *testing.T, tests []parseTest) { packer.CoreBuildProvisioner{}, packer.CoreBuildPostProcessor{}, null.Builder{}, + HCL2Provisioner{}, ), ); diff != "" { t.Fatalf("Parser.getBuilds() wrong packer builds. %s", diff) @@ -176,7 +177,7 @@ var ( basicMockProvisioner = &MockProvisioner{ Config: MockConfig{ - NotSquashed: "value", + NotSquashed: "value ", NestedMockConfig: basicNestedMockConfig, Nested: basicNestedMockConfig, NestedSlice: []NestedMockConfig{ diff --git a/hcl2template/testdata/complete/build.pkr.hcl b/hcl2template/testdata/complete/build.pkr.hcl index ef73e65f9..5d6745a01 100644 --- a/hcl2template/testdata/complete/build.pkr.hcl +++ b/hcl2template/testdata/complete/build.pkr.hcl @@ -11,7 +11,7 @@ build { provisioner "shell" { name = "provisioner that does something" - not_squashed = "${var.foo} ${lower(build.ID)}" + not_squashed = "${var.foo} ${upper(build.ID)}" string = "string" int = "${41 + 1}" int64 = "${42 + 1}" @@ -62,7 +62,7 @@ build { } provisioner "file" { - not_squashed = "${var.foo}" + not_squashed = "${var.foo} ${upper(build.ID)}" string = "string" int = 42 int64 = 43 diff --git a/hcl2template/types.build.post-processor.go b/hcl2template/types.build.post-processor.go index b45b682f5..f80b40ecf 100644 --- a/hcl2template/types.build.post-processor.go +++ b/hcl2template/types.build.post-processor.go @@ -61,7 +61,7 @@ func (p *Parser) decodePostProcessor(block *hcl.Block) (*PostProcessorBlock, hcl return postProcessor, diags } -func (cfg *PackerConfig) startPostProcessor(source SourceBlock, pp *PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]string) (packer.PostProcessor, hcl.Diagnostics) { +func (cfg *PackerConfig) startPostProcessor(source SourceBlock, pp *PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]interface{}) (packer.PostProcessor, hcl.Diagnostics) { // ProvisionerBlock represents a detected but unparsed provisioner var diags hcl.Diagnostics @@ -76,7 +76,7 @@ func (cfg *PackerConfig) startPostProcessor(source SourceBlock, pp *PostProcesso } flatProvisinerCfg, moreDiags := decodeHCL2Spec(pp.Rest, ectx, postProcessor) diags = append(diags, moreDiags...) - err = postProcessor.Configure(source.builderVariables(), flatProvisinerCfg, generatedVars) + err = postProcessor.Configure(source.builderVariables(), flatProvisinerCfg) if err != nil { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, diff --git a/hcl2template/types.build.provisioners.go b/hcl2template/types.build.provisioners.go index 70756f39e..19b699155 100644 --- a/hcl2template/types.build.provisioners.go +++ b/hcl2template/types.build.provisioners.go @@ -132,7 +132,7 @@ func (p *Parser) decodeProvisioner(block *hcl.Block) (*ProvisionerBlock, hcl.Dia return provisioner, diags } -func (cfg *PackerConfig) startProvisioner(source SourceBlock, pb *ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]string) (packer.Provisioner, hcl.Diagnostics) { +func (cfg *PackerConfig) startProvisioner(source SourceBlock, pb *ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]interface{}) (packer.Provisioner, hcl.Diagnostics) { var diags hcl.Diagnostics provisioner, err := cfg.provisionersSchemas.Start(pb.PType) @@ -144,18 +144,13 @@ func (cfg *PackerConfig) startProvisioner(source SourceBlock, pb *ProvisionerBlo }) return nil, diags } - flatProvisionerCfg, moreDiags := decodeHCL2Spec(pb.HCL2Ref.Rest, ectx, provisioner) - diags = append(diags, moreDiags...) - if diags.HasErrors() { - return nil, diags + p := &HCL2Provisioner{ + Provisioner: provisioner, + provisionerBlock: pb, + evalContext: ectx, + builderVariables: source.builderVariables(), } - // manipulate generatedVars from builder to add to the interfaces being - // passed to the provisioner Prepare() - - // configs := make([]interface{}, 2) - // configs = append(, flatProvisionerCfg) - // configs = append(configs, generatedVars) - err = provisioner.Prepare(source.builderVariables(), flatProvisionerCfg, generatedVars) + err = p.HCL2Prepare(generatedVars) if err != nil { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, @@ -165,5 +160,5 @@ func (cfg *PackerConfig) startProvisioner(source SourceBlock, pb *ProvisionerBlo }) return nil, diags } - return provisioner, diags + return p, diags } diff --git a/hcl2template/types.build_test.go b/hcl2template/types.build_test.go index c3d5dc9e5..f7a69e844 100644 --- a/hcl2template/types.build_test.go +++ b/hcl2template/types.build_test.go @@ -266,19 +266,23 @@ func TestParse_build(t *testing.T) { Provisioners: []packer.CoreBuildProvisioner{ { PType: "shell", - Provisioner: &MockProvisioner{ - Config: MockConfig{ - NestedMockConfig: NestedMockConfig{Tags: []MockTag{}}, - NestedSlice: []NestedMockConfig{}, + Provisioner: &HCL2Provisioner{ + Provisioner: &MockProvisioner{ + Config: MockConfig{ + NestedMockConfig: NestedMockConfig{Tags: []MockTag{}}, + NestedSlice: []NestedMockConfig{}, + }, }, }, }, { PType: "file", - Provisioner: &MockProvisioner{ - Config: MockConfig{ - NestedMockConfig: NestedMockConfig{Tags: []MockTag{}}, - NestedSlice: []NestedMockConfig{}, + Provisioner: &HCL2Provisioner{ + Provisioner: &MockProvisioner{ + Config: MockConfig{ + NestedMockConfig: NestedMockConfig{Tags: []MockTag{}}, + NestedSlice: []NestedMockConfig{}, + }, }, }, }, @@ -292,19 +296,23 @@ func TestParse_build(t *testing.T) { Provisioners: []packer.CoreBuildProvisioner{ { PType: "file", - Provisioner: &MockProvisioner{ - Config: MockConfig{ - NestedMockConfig: NestedMockConfig{Tags: []MockTag{}}, - NestedSlice: []NestedMockConfig{}, + Provisioner: &HCL2Provisioner{ + Provisioner: &MockProvisioner{ + Config: MockConfig{ + NestedMockConfig: NestedMockConfig{Tags: []MockTag{}}, + NestedSlice: []NestedMockConfig{}, + }, }, }, }, { PType: "shell", - Provisioner: &MockProvisioner{ - Config: MockConfig{ - NestedMockConfig: NestedMockConfig{Tags: []MockTag{}}, - NestedSlice: []NestedMockConfig{}, + Provisioner: &HCL2Provisioner{ + Provisioner: &MockProvisioner{ + Config: MockConfig{ + NestedMockConfig: NestedMockConfig{Tags: []MockTag{}}, + NestedSlice: []NestedMockConfig{}, + }, }, }, }, diff --git a/hcl2template/types.hcl_provisioner.go b/hcl2template/types.hcl_provisioner.go new file mode 100644 index 000000000..795497874 --- /dev/null +++ b/hcl2template/types.hcl_provisioner.go @@ -0,0 +1,61 @@ +package hcl2template + +import ( + "context" + "fmt" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/hashicorp/packer/packer" + "github.com/zclconf/go-cty/cty" +) + +type HCL2Provisioner struct { + Provisioner packer.Provisioner + provisionerBlock *ProvisionerBlock + evalContext *hcl.EvalContext + builderVariables map[string]string +} + +func (p *HCL2Provisioner) ConfigSpec() hcldec.ObjectSpec { + return p.Provisioner.ConfigSpec() +} + +func (p *HCL2Provisioner) HCL2Prepare(buildVars map[string]interface{}) error { + var diags hcl.Diagnostics + ectx := p.evalContext + if len(buildVars) > 0 { + ectx = p.evalContext.NewChild() + buildValues := map[string]cty.Value{} + for k, v := range buildVars { + switch v := v.(type) { + case string: + buildValues[k] = cty.StringVal(v) + default: + return fmt.Errorf("unhandled builvar type: %T", v) + } + } + ectx.Variables = map[string]cty.Value{ + buildAccessor: cty.ObjectVal(buildValues), + } + } + + flatProvisionerCfg, moreDiags := decodeHCL2Spec(p.provisionerBlock.HCL2Ref.Rest, ectx, p.Provisioner) + diags = append(diags, moreDiags...) + if diags.HasErrors() { + return diags + } + return p.Provisioner.Prepare(p.builderVariables, flatProvisionerCfg) +} + +func (p *HCL2Provisioner) Prepare(args ...interface{}) error { + return p.Provisioner.Prepare(args...) +} + +func (p *HCL2Provisioner) Provision(ctx context.Context, ui packer.Ui, c packer.Communicator, vars map[string]interface{}) error { + err := p.HCL2Prepare(vars) + if err != nil { + return err + } + return p.Provisioner.Provision(ctx, ui, c, vars) +} diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index 3908c91fe..069175ca2 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -199,7 +199,7 @@ func (c *PackerConfig) evaluateLocalVariable(local *Local) hcl.Diagnostics { // getCoreBuildProvisioners takes a list of provisioner block, starts according // provisioners and sends parsed HCL2 over to it. -func (cfg *PackerConfig) getCoreBuildProvisioners(source SourceBlock, blocks []*ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]string) ([]packer.CoreBuildProvisioner, hcl.Diagnostics) { +func (cfg *PackerConfig) getCoreBuildProvisioners(source SourceBlock, blocks []*ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]interface{}) ([]packer.CoreBuildProvisioner, hcl.Diagnostics) { var diags hcl.Diagnostics res := []packer.CoreBuildProvisioner{} for _, pb := range blocks { @@ -242,7 +242,7 @@ func (cfg *PackerConfig) getCoreBuildProvisioners(source SourceBlock, blocks []* // getCoreBuildProvisioners takes a list of post processor block, starts // according provisioners and sends parsed HCL2 over to it. -func (cfg *PackerConfig) getCoreBuildPostProcessors(source SourceBlock, blocks []*PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]string) ([]packer.CoreBuildPostProcessor, hcl.Diagnostics) { +func (cfg *PackerConfig) getCoreBuildPostProcessors(source SourceBlock, blocks []*PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]interface{}) ([]packer.CoreBuildPostProcessor, hcl.Diagnostics) { var diags hcl.Diagnostics res := []packer.CoreBuildPostProcessor{} for _, ppb := range blocks { @@ -365,7 +365,7 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packer.Build for _, k := range generatedVars { generatedPlaceholderMap[k] = fmt.Sprintf("Build_%s. "+ common.PlaceholderMsg, k) - unknownBuildValues[k] = cty.StringVal("") + unknownBuildValues[k] = cty.StringVal("") } variables := map[string]cty.Value{ @@ -376,16 +376,16 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packer.Build buildAccessor: cty.ObjectVal(unknownBuildValues), } - provisioners, moreDiags := cfg.getCoreBuildProvisioners(src, build.ProvisionerBlocks, cfg.EvalContext(variables), generatedPlaceholderMap) + provisioners, moreDiags := cfg.getCoreBuildProvisioners(src, build.ProvisionerBlocks, cfg.EvalContext(variables), nil) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { continue } - postProcessors, moreDiags := cfg.getCoreBuildPostProcessors(src, build.PostProcessors, cfg.EvalContext(variables), generatedPlaceholderMap) + postProcessors, moreDiags := cfg.getCoreBuildPostProcessors(src, build.PostProcessors, cfg.EvalContext(variables), nil) pps := [][]packer.CoreBuildPostProcessor{} if len(postProcessors) > 0 { pps = [][]packer.CoreBuildPostProcessor{postProcessors} - } + } // TODO(azr): remove this diags = append(diags, moreDiags...) if moreDiags.HasErrors() { continue diff --git a/hcl2template/types.packer_config_test.go b/hcl2template/types.packer_config_test.go index 74df297e4..9849f06df 100644 --- a/hcl2template/types.packer_config_test.go +++ b/hcl2template/types.packer_config_test.go @@ -110,11 +110,18 @@ func TestParser_complete(t *testing.T) { Builder: basicMockBuilder, Provisioners: []packer.CoreBuildProvisioner{ { - PType: "shell", - PName: "provisioner that does something", - Provisioner: basicMockProvisioner, + PType: "shell", + PName: "provisioner that does something", + Provisioner: &HCL2Provisioner{ + Provisioner: basicMockProvisioner, + }, + }, + { + PType: "file", + Provisioner: &HCL2Provisioner{ + Provisioner: basicMockProvisioner, + }, }, - {PType: "file", Provisioner: basicMockProvisioner}, }, PostProcessors: [][]packer.CoreBuildPostProcessor{ { @@ -146,11 +153,18 @@ func TestParser_complete(t *testing.T) { }, Provisioners: []packer.CoreBuildProvisioner{ { - PType: "shell", - PName: "provisioner that does something", - Provisioner: basicMockProvisioner, + PType: "shell", + PName: "provisioner that does something", + Provisioner: &HCL2Provisioner{ + Provisioner: basicMockProvisioner, + }, + }, + { + PType: "file", + Provisioner: &HCL2Provisioner{ + Provisioner: basicMockProvisioner, + }, }, - {PType: "file", Provisioner: basicMockProvisioner}, }, PostProcessors: [][]packer.CoreBuildPostProcessor{ { diff --git a/packer/provisioner.go b/packer/provisioner.go index 905b28971..8caffc1a9 100644 --- a/packer/provisioner.go +++ b/packer/provisioner.go @@ -51,8 +51,8 @@ type ProvisionHook struct { // custom generated data could be passed into provisioners from builders to // enable specialized builder-specific (but still validated!!) access to builder // data. -func BasicPlaceholderData() map[string]string { - placeholderData := map[string]string{} +func BasicPlaceholderData() map[string]interface{} { + placeholderData := map[string]interface{}{} msg := "Build_%s. " + common.PlaceholderMsg placeholderData["ID"] = fmt.Sprintf(msg, "ID") // The following correspond to communicator-agnostic functions that are From 39261f367149c8e76f8d0a73b9e2b28af322cb4a Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Mon, 6 Jul 2020 10:32:39 +0200 Subject: [PATCH 06/24] add hcl2 post processor type that reprepares itself --- hcl2template/common_test.go | 1 + hcl2template/types.build.post-processor.go | 12 ++-- hcl2template/types.build.provisioners.go | 6 +- hcl2template/types.build_test.go | 40 ++++++++----- hcl2template/types.hcl_post-processor.go | 68 ++++++++++++++++++++++ hcl2template/types.packer_config_test.go | 28 +++++---- 6 files changed, 122 insertions(+), 33 deletions(-) create mode 100644 hcl2template/types.hcl_post-processor.go diff --git a/hcl2template/common_test.go b/hcl2template/common_test.go index 610e178ce..4a4889e8b 100644 --- a/hcl2template/common_test.go +++ b/hcl2template/common_test.go @@ -130,6 +130,7 @@ func testParse(t *testing.T, tests []parseTest) { packer.CoreBuildPostProcessor{}, null.Builder{}, HCL2Provisioner{}, + HCL2PostProcessor{}, ), ); diff != "" { t.Fatalf("Parser.getBuilds() wrong packer builds. %s", diff) diff --git a/hcl2template/types.build.post-processor.go b/hcl2template/types.build.post-processor.go index f80b40ecf..c54626639 100644 --- a/hcl2template/types.build.post-processor.go +++ b/hcl2template/types.build.post-processor.go @@ -74,9 +74,13 @@ func (cfg *PackerConfig) startPostProcessor(source SourceBlock, pp *PostProcesso }) return nil, diags } - flatProvisinerCfg, moreDiags := decodeHCL2Spec(pp.Rest, ectx, postProcessor) - diags = append(diags, moreDiags...) - err = postProcessor.Configure(source.builderVariables(), flatProvisinerCfg) + hclPostProcessor := &HCL2PostProcessor{ + PostProcessor: postProcessor, + postProcessorBlock: pp, + evalContext: ectx, + builderVariables: source.builderVariables(), + } + err = hclPostProcessor.HCL2Prepare(generatedVars) if err != nil { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, @@ -86,5 +90,5 @@ func (cfg *PackerConfig) startPostProcessor(source SourceBlock, pp *PostProcesso }) return nil, diags } - return postProcessor, diags + return hclPostProcessor, diags } diff --git a/hcl2template/types.build.provisioners.go b/hcl2template/types.build.provisioners.go index 19b699155..ebc89600b 100644 --- a/hcl2template/types.build.provisioners.go +++ b/hcl2template/types.build.provisioners.go @@ -144,13 +144,13 @@ func (cfg *PackerConfig) startProvisioner(source SourceBlock, pb *ProvisionerBlo }) return nil, diags } - p := &HCL2Provisioner{ + hclProvisioner := &HCL2Provisioner{ Provisioner: provisioner, provisionerBlock: pb, evalContext: ectx, builderVariables: source.builderVariables(), } - err = p.HCL2Prepare(generatedVars) + err = hclProvisioner.HCL2Prepare(generatedVars) if err != nil { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, @@ -160,5 +160,5 @@ func (cfg *PackerConfig) startProvisioner(source SourceBlock, pb *ProvisionerBlo }) return nil, diags } - return p, diags + return hclProvisioner, diags } diff --git a/hcl2template/types.build_test.go b/hcl2template/types.build_test.go index f7a69e844..658c79c68 100644 --- a/hcl2template/types.build_test.go +++ b/hcl2template/types.build_test.go @@ -171,19 +171,23 @@ func TestParse_build(t *testing.T) { { { PType: "amazon-import", - PostProcessor: &MockPostProcessor{ - Config: MockConfig{ - NestedMockConfig: NestedMockConfig{Tags: []MockTag{}}, - NestedSlice: []NestedMockConfig{}, + PostProcessor: &HCL2PostProcessor{ + PostProcessor: &MockPostProcessor{ + Config: MockConfig{ + NestedMockConfig: NestedMockConfig{Tags: []MockTag{}}, + NestedSlice: []NestedMockConfig{}, + }, }, }, }, { PType: "manifest", - PostProcessor: &MockPostProcessor{ - Config: MockConfig{ - NestedMockConfig: NestedMockConfig{Tags: []MockTag{}}, - NestedSlice: []NestedMockConfig{}, + PostProcessor: &HCL2PostProcessor{ + PostProcessor: &MockPostProcessor{ + Config: MockConfig{ + NestedMockConfig: NestedMockConfig{Tags: []MockTag{}}, + NestedSlice: []NestedMockConfig{}, + }, }, }, }, @@ -199,19 +203,23 @@ func TestParse_build(t *testing.T) { { { PType: "manifest", - PostProcessor: &MockPostProcessor{ - Config: MockConfig{ - NestedMockConfig: NestedMockConfig{Tags: []MockTag{}}, - NestedSlice: []NestedMockConfig{}, + PostProcessor: &HCL2PostProcessor{ + PostProcessor: &MockPostProcessor{ + Config: MockConfig{ + NestedMockConfig: NestedMockConfig{Tags: []MockTag{}}, + NestedSlice: []NestedMockConfig{}, + }, }, }, }, { PType: "amazon-import", - PostProcessor: &MockPostProcessor{ - Config: MockConfig{ - NestedMockConfig: NestedMockConfig{Tags: []MockTag{}}, - NestedSlice: []NestedMockConfig{}, + PostProcessor: &HCL2PostProcessor{ + PostProcessor: &MockPostProcessor{ + Config: MockConfig{ + NestedMockConfig: NestedMockConfig{Tags: []MockTag{}}, + NestedSlice: []NestedMockConfig{}, + }, }, }, }, diff --git a/hcl2template/types.hcl_post-processor.go b/hcl2template/types.hcl_post-processor.go new file mode 100644 index 000000000..6698b3f5a --- /dev/null +++ b/hcl2template/types.hcl_post-processor.go @@ -0,0 +1,68 @@ +package hcl2template + +import ( + "context" + "fmt" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hcldec" + "github.com/hashicorp/packer/packer" + "github.com/zclconf/go-cty/cty" +) + +type HCL2PostProcessor struct { + PostProcessor packer.PostProcessor + postProcessorBlock *PostProcessorBlock + evalContext *hcl.EvalContext + builderVariables map[string]string +} + +func (p *HCL2PostProcessor) ConfigSpec() hcldec.ObjectSpec { + return p.PostProcessor.ConfigSpec() +} + +func (p *HCL2PostProcessor) HCL2Prepare(buildVars map[string]interface{}) error { + var diags hcl.Diagnostics + ectx := p.evalContext + if len(buildVars) > 0 { + ectx = p.evalContext.NewChild() + buildValues := map[string]cty.Value{} + for k, v := range buildVars { + switch v := v.(type) { + case string: + buildValues[k] = cty.StringVal(v) + default: + return fmt.Errorf("unhandled builvar type: %T", v) + } + } + ectx.Variables = map[string]cty.Value{ + buildAccessor: cty.ObjectVal(buildValues), + } + } + + flatPostProcessorCfg, moreDiags := decodeHCL2Spec(p.postProcessorBlock.HCL2Ref.Rest, ectx, p.PostProcessor) + diags = append(diags, moreDiags...) + if diags.HasErrors() { + return diags + } + return p.PostProcessor.Configure(p.builderVariables, flatPostProcessorCfg) +} + +func (p *HCL2PostProcessor) Configure(args ...interface{}) error { + return p.PostProcessor.Configure(args...) +} + +func (p *HCL2PostProcessor) PostProcess(ctx context.Context, ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, bool, error) { + generatedData := make(map[string]interface{}) + if artifactStateData, ok := artifact.State("generated_data").(map[interface{}]interface{}); ok { + for k, v := range artifactStateData { + generatedData[k.(string)] = v + } + } + + err := p.HCL2Prepare(generatedData) + if err != nil { + return nil, false, false, err + } + return p.PostProcessor.PostProcess(ctx, ui, artifact) +} diff --git a/hcl2template/types.packer_config_test.go b/hcl2template/types.packer_config_test.go index 9849f06df..09efb0b3b 100644 --- a/hcl2template/types.packer_config_test.go +++ b/hcl2template/types.packer_config_test.go @@ -126,14 +126,18 @@ func TestParser_complete(t *testing.T) { PostProcessors: [][]packer.CoreBuildPostProcessor{ { { - PType: "amazon-import", - PName: "something", - PostProcessor: basicMockPostProcessor, + PType: "amazon-import", + PName: "something", + PostProcessor: &HCL2PostProcessor{ + PostProcessor: basicMockPostProcessor, + }, KeepInputArtifact: pTrue, }, { - PType: "amazon-import", - PostProcessor: basicMockPostProcessor, + PType: "amazon-import", + PostProcessor: &HCL2PostProcessor{ + PostProcessor: basicMockPostProcessor, + }, }, }, }, @@ -169,14 +173,18 @@ func TestParser_complete(t *testing.T) { PostProcessors: [][]packer.CoreBuildPostProcessor{ { { - PType: "amazon-import", - PName: "something", - PostProcessor: basicMockPostProcessor, + PType: "amazon-import", + PName: "something", + PostProcessor: &HCL2PostProcessor{ + PostProcessor: basicMockPostProcessor, + }, KeepInputArtifact: pTrue, }, { - PType: "amazon-import", - PostProcessor: basicMockPostProcessor, + PType: "amazon-import", + PostProcessor: &HCL2PostProcessor{ + PostProcessor: basicMockPostProcessor, + }, }, }, }, From 1b669ff2ab27f5b8ebc3f6ecbfce43e6b8824480 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Mon, 6 Jul 2020 11:18:02 +0200 Subject: [PATCH 07/24] woops --- packer/provisioner.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packer/provisioner.go b/packer/provisioner.go index 8caffc1a9..905b28971 100644 --- a/packer/provisioner.go +++ b/packer/provisioner.go @@ -51,8 +51,8 @@ type ProvisionHook struct { // custom generated data could be passed into provisioners from builders to // enable specialized builder-specific (but still validated!!) access to builder // data. -func BasicPlaceholderData() map[string]interface{} { - placeholderData := map[string]interface{}{} +func BasicPlaceholderData() map[string]string { + placeholderData := map[string]string{} msg := "Build_%s. " + common.PlaceholderMsg placeholderData["ID"] = fmt.Sprintf(msg, "ID") // The following correspond to communicator-agnostic functions that are From 90610dcf04104b12fb175a9596da4300e696dd09 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Mon, 6 Jul 2020 11:38:12 +0200 Subject: [PATCH 08/24] docs --- hcl2template/types.hcl_post-processor.go | 4 ++++ hcl2template/types.hcl_provisioner.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/hcl2template/types.hcl_post-processor.go b/hcl2template/types.hcl_post-processor.go index 6698b3f5a..0afec4d4b 100644 --- a/hcl2template/types.hcl_post-processor.go +++ b/hcl2template/types.hcl_post-processor.go @@ -10,6 +10,10 @@ import ( "github.com/zclconf/go-cty/cty" ) +// HCL2PostProcessor has a reference to the part of the HCL2 body where it is +// defined, allowing to completely reconfigure the PostProcessor right before +// calling PostProcess: with contextual variables. +// This permits using "${build.ID}" values for example. type HCL2PostProcessor struct { PostProcessor packer.PostProcessor postProcessorBlock *PostProcessorBlock diff --git a/hcl2template/types.hcl_provisioner.go b/hcl2template/types.hcl_provisioner.go index 795497874..3ea1e35dc 100644 --- a/hcl2template/types.hcl_provisioner.go +++ b/hcl2template/types.hcl_provisioner.go @@ -10,6 +10,10 @@ import ( "github.com/zclconf/go-cty/cty" ) +// HCL2Provisioner has a reference to the part of the HCL2 body where it is +// defined, allowing to completely reconfigure the Provisioner right before +// calling Provision: with contextual variables. +// This permits using "${build.ID}" values for example. type HCL2Provisioner struct { Provisioner packer.Provisioner provisionerBlock *ProvisionerBlock From 0ed10f921fab558355d0d40162d5f90d8a021134 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Mon, 6 Jul 2020 11:48:39 +0200 Subject: [PATCH 09/24] add test --- hcl2template/common_test.go | 1 + hcl2template/testdata/complete/build.pkr.hcl | 2 ++ 2 files changed, 3 insertions(+) diff --git a/hcl2template/common_test.go b/hcl2template/common_test.go index 4a4889e8b..781febd90 100644 --- a/hcl2template/common_test.go +++ b/hcl2template/common_test.go @@ -190,6 +190,7 @@ var ( } basicMockPostProcessor = &MockPostProcessor{ Config: MockConfig{ + NotSquashed: "value ", NestedMockConfig: basicNestedMockConfig, Nested: basicNestedMockConfig, NestedSlice: []NestedMockConfig{ diff --git a/hcl2template/testdata/complete/build.pkr.hcl b/hcl2template/testdata/complete/build.pkr.hcl index 5d6745a01..0077d98e5 100644 --- a/hcl2template/testdata/complete/build.pkr.hcl +++ b/hcl2template/testdata/complete/build.pkr.hcl @@ -138,6 +138,7 @@ build { ["a","b"], ["c","d"] ] + not_squashed = "${var.foo} ${upper(build.ID)}" nested { string = "string" @@ -185,6 +186,7 @@ build { ["a","b"], ["c","d"] ] + not_squashed = "${var.foo} ${upper(build.ID)}" nested { string = "string" From cf514d31f2a0f80d8841307f0f2e6252f466fd22 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Mon, 6 Jul 2020 15:14:17 +0200 Subject: [PATCH 10/24] avoid passing generatedVars down during first prepare --- hcl2template/types.build.post-processor.go | 4 ++-- hcl2template/types.build.provisioners.go | 4 ++-- hcl2template/types.packer_config.go | 12 ++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/hcl2template/types.build.post-processor.go b/hcl2template/types.build.post-processor.go index c54626639..fcfa2dc0e 100644 --- a/hcl2template/types.build.post-processor.go +++ b/hcl2template/types.build.post-processor.go @@ -61,7 +61,7 @@ func (p *Parser) decodePostProcessor(block *hcl.Block) (*PostProcessorBlock, hcl return postProcessor, diags } -func (cfg *PackerConfig) startPostProcessor(source SourceBlock, pp *PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]interface{}) (packer.PostProcessor, hcl.Diagnostics) { +func (cfg *PackerConfig) startPostProcessor(source SourceBlock, pp *PostProcessorBlock, ectx *hcl.EvalContext) (packer.PostProcessor, hcl.Diagnostics) { // ProvisionerBlock represents a detected but unparsed provisioner var diags hcl.Diagnostics @@ -80,7 +80,7 @@ func (cfg *PackerConfig) startPostProcessor(source SourceBlock, pp *PostProcesso evalContext: ectx, builderVariables: source.builderVariables(), } - err = hclPostProcessor.HCL2Prepare(generatedVars) + err = hclPostProcessor.HCL2Prepare(nil) if err != nil { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, diff --git a/hcl2template/types.build.provisioners.go b/hcl2template/types.build.provisioners.go index ebc89600b..519a72e72 100644 --- a/hcl2template/types.build.provisioners.go +++ b/hcl2template/types.build.provisioners.go @@ -132,7 +132,7 @@ func (p *Parser) decodeProvisioner(block *hcl.Block) (*ProvisionerBlock, hcl.Dia return provisioner, diags } -func (cfg *PackerConfig) startProvisioner(source SourceBlock, pb *ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]interface{}) (packer.Provisioner, hcl.Diagnostics) { +func (cfg *PackerConfig) startProvisioner(source SourceBlock, pb *ProvisionerBlock, ectx *hcl.EvalContext) (packer.Provisioner, hcl.Diagnostics) { var diags hcl.Diagnostics provisioner, err := cfg.provisionersSchemas.Start(pb.PType) @@ -150,7 +150,7 @@ func (cfg *PackerConfig) startProvisioner(source SourceBlock, pb *ProvisionerBlo evalContext: ectx, builderVariables: source.builderVariables(), } - err = hclProvisioner.HCL2Prepare(generatedVars) + err = hclProvisioner.HCL2Prepare(nil) if err != nil { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index 069175ca2..b40df364b 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -199,14 +199,14 @@ func (c *PackerConfig) evaluateLocalVariable(local *Local) hcl.Diagnostics { // getCoreBuildProvisioners takes a list of provisioner block, starts according // provisioners and sends parsed HCL2 over to it. -func (cfg *PackerConfig) getCoreBuildProvisioners(source SourceBlock, blocks []*ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]interface{}) ([]packer.CoreBuildProvisioner, hcl.Diagnostics) { +func (cfg *PackerConfig) getCoreBuildProvisioners(source SourceBlock, blocks []*ProvisionerBlock, ectx *hcl.EvalContext) ([]packer.CoreBuildProvisioner, hcl.Diagnostics) { var diags hcl.Diagnostics res := []packer.CoreBuildProvisioner{} for _, pb := range blocks { if pb.OnlyExcept.Skip(source.String()) { continue } - provisioner, moreDiags := cfg.startProvisioner(source, pb, ectx, generatedVars) + provisioner, moreDiags := cfg.startProvisioner(source, pb, ectx) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { continue @@ -242,7 +242,7 @@ func (cfg *PackerConfig) getCoreBuildProvisioners(source SourceBlock, blocks []* // getCoreBuildProvisioners takes a list of post processor block, starts // according provisioners and sends parsed HCL2 over to it. -func (cfg *PackerConfig) getCoreBuildPostProcessors(source SourceBlock, blocks []*PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]interface{}) ([]packer.CoreBuildPostProcessor, hcl.Diagnostics) { +func (cfg *PackerConfig) getCoreBuildPostProcessors(source SourceBlock, blocks []*PostProcessorBlock, ectx *hcl.EvalContext) ([]packer.CoreBuildPostProcessor, hcl.Diagnostics) { var diags hcl.Diagnostics res := []packer.CoreBuildPostProcessor{} for _, ppb := range blocks { @@ -266,7 +266,7 @@ func (cfg *PackerConfig) getCoreBuildPostProcessors(source SourceBlock, blocks [ break } - postProcessor, moreDiags := cfg.startPostProcessor(source, ppb, ectx, generatedVars) + postProcessor, moreDiags := cfg.startPostProcessor(source, ppb, ectx) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { continue @@ -376,12 +376,12 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packer.Build buildAccessor: cty.ObjectVal(unknownBuildValues), } - provisioners, moreDiags := cfg.getCoreBuildProvisioners(src, build.ProvisionerBlocks, cfg.EvalContext(variables), nil) + provisioners, moreDiags := cfg.getCoreBuildProvisioners(src, build.ProvisionerBlocks, cfg.EvalContext(variables)) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { continue } - postProcessors, moreDiags := cfg.getCoreBuildPostProcessors(src, build.PostProcessors, cfg.EvalContext(variables), nil) + postProcessors, moreDiags := cfg.getCoreBuildPostProcessors(src, build.PostProcessors, cfg.EvalContext(variables)) pps := [][]packer.CoreBuildPostProcessor{} if len(postProcessors) > 0 { pps = [][]packer.CoreBuildPostProcessor{postProcessors} From eaf63e3bb5f493f392887918f61ec0e151a22413 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Mon, 6 Jul 2020 16:07:29 +0200 Subject: [PATCH 11/24] pass source name from local source name too --- hcl2template/types.packer_config.go | 7 ++----- hcl2template/types.source.go | 19 ++++++++++++++++--- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index b40df364b..82e4374f3 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -369,11 +369,8 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packer.Build } variables := map[string]cty.Value{ - sourcesAccessor: cty.ObjectVal(map[string]cty.Value{ - "type": cty.StringVal(src.Type), - "name": cty.StringVal(src.Name), - }), - buildAccessor: cty.ObjectVal(unknownBuildValues), + sourcesAccessor: cty.ObjectVal(src.ctyValues()), + buildAccessor: cty.ObjectVal(unknownBuildValues), } provisioners, moreDiags := cfg.getCoreBuildProvisioners(src, build.ProvisionerBlocks, cfg.EvalContext(variables)) diff --git a/hcl2template/types.source.go b/hcl2template/types.source.go index 51f275aa0..076d83d1b 100644 --- a/hcl2template/types.source.go +++ b/hcl2template/types.source.go @@ -7,6 +7,7 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/gohcl" "github.com/hashicorp/packer/packer" + "github.com/zclconf/go-cty/cty" ) // SourceBlock references an HCL 'source' block. @@ -26,11 +27,23 @@ type SourceBlock struct { LocalName string } -func (b *SourceBlock) String() string { +func (b *SourceBlock) name() string { if b.LocalName != "" { - return fmt.Sprintf("%s.%s", b.Type, b.LocalName) + return b.LocalName + } + return b.Name +} + +func (b *SourceBlock) String() string { + return fmt.Sprintf("%s.%s", b.Type, b.name()) +} + +// EvalContext adds the values of the source to the passed eval context. +func (b *SourceBlock) ctyValues() map[string]cty.Value { + return map[string]cty.Value{ + "type": cty.StringVal(b.Type), + "name": cty.StringVal(b.name()), } - return fmt.Sprintf("%s.%s", b.Type, b.Name) } // decodeBuildSource reads a used source block from a build: From 0293f69fa12223c6bc5f18d82d0e4cecee24d778 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Mon, 6 Jul 2020 16:07:46 +0200 Subject: [PATCH 12/24] default build values to be unknown --- hcl2template/types.packer_config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index 82e4374f3..a19dba2dd 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -69,6 +69,7 @@ func (cfg *PackerConfig) EvalContext(variables map[string]cty.Value) *hcl.EvalCo "type": cty.UnknownVal(cty.String), "name": cty.UnknownVal(cty.String), }), + buildAccessor: cty.UnknownVal(cty.EmptyObject), }, } for k, v := range variables { From 8e923977f055e9662cc9fcc78a843e0388f47895 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Mon, 6 Jul 2020 16:53:06 +0200 Subject: [PATCH 13/24] null builder: say that we are returnin the ID value --- builder/null/builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/null/builder.go b/builder/null/builder.go index 1d2fc40e4..f68263069 100644 --- a/builder/null/builder.go +++ b/builder/null/builder.go @@ -25,7 +25,7 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, []string, error) { return nil, warnings, errs } - return nil, warnings, nil + return []string{"ID"}, warnings, nil } func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (packer.Artifact, error) { From 0deaa5d2a5682b8cd4901158f91ec184106b3eec Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Mon, 6 Jul 2020 17:01:55 +0200 Subject: [PATCH 14/24] add a complete tests for builder variables + only/except --- command/build_test.go | 67 +++++++++++++++++++ command/command_test.go | 6 +- .../test-fixtures/hcl/complete/build.pkr.hcl | 54 +++++++++++++++ .../hcl/complete/sources.pkr.hcl | 3 + 4 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 command/test-fixtures/hcl/complete/build.pkr.hcl create mode 100644 command/test-fixtures/hcl/complete/sources.pkr.hcl diff --git a/command/build_test.go b/command/build_test.go index e464b3401..651ae988e 100644 --- a/command/build_test.go +++ b/command/build_test.go @@ -20,6 +20,17 @@ import ( shell_local "github.com/hashicorp/packer/provisioner/shell-local" ) +var ( + spaghettiCarbonara = `spaghetti +carbonara +` + lasagna = `lasagna +tomato +mozza +cooking... +` +) + func TestBuild(t *testing.T) { tc := []struct { name string @@ -232,6 +243,62 @@ func TestBuild(t *testing.T) { expected: []string{"chocolate.txt", "vanilla.txt"}, }, }, + + // complete + { + name: "hcl - complete", + args: []string{ + testFixture("hcl", "complete"), + }, + fileCheck: fileCheck{ + expectedContent: map[string]string{ + "NULL.spaghetti_carbonara.txt": spaghettiCarbonara, + "NULL.lasagna.txt": lasagna, + }, + }, + }, + + { + name: "hcl - complete - except carbonara", + args: []string{ + "-except", "recipes.null.spaghetti_carbonara", + testFixture("hcl", "complete"), + }, + fileCheck: fileCheck{ + notExpected: []string{"NULL.spaghetti_carbonara.txt"}, + expectedContent: map[string]string{ + "NULL.lasagna.txt": lasagna, + }, + }, + }, + + { + name: "hcl - complete - only lasagna", + args: []string{ + "-only", "*lasagna", + testFixture("hcl", "complete"), + }, + fileCheck: fileCheck{ + notExpected: []string{"NULL.spaghetti_carbonara.txt"}, + expectedContent: map[string]string{ + "NULL.lasagna.txt": lasagna, + }, + }, + }, + + { + name: "hcl - complete - only recipes", + args: []string{ + "-only", "recipes.*", + testFixture("hcl", "complete"), + }, + fileCheck: fileCheck{ + expectedContent: map[string]string{ + "NULL.spaghetti_carbonara.txt": spaghettiCarbonara, + "NULL.lasagna.txt": lasagna, + }, + }, + }, } for _, tt := range tc { diff --git a/command/command_test.go b/command/command_test.go index 31f1e3d54..34d044292 100644 --- a/command/command_test.go +++ b/command/command_test.go @@ -27,8 +27,10 @@ func outputCommand(t *testing.T, m Meta) (string, string) { return out.String(), err.String() } -func testFixture(n string) string { - return filepath.Join(fixturesDir, n) +func testFixture(n ...string) string { + paths := []string{fixturesDir} + paths = append(paths, n...) + return filepath.Join(paths...) } func testMeta(t *testing.T) Meta { diff --git a/command/test-fixtures/hcl/complete/build.pkr.hcl b/command/test-fixtures/hcl/complete/build.pkr.hcl new file mode 100644 index 000000000..ceb0a58d3 --- /dev/null +++ b/command/test-fixtures/hcl/complete/build.pkr.hcl @@ -0,0 +1,54 @@ + +build { + source "source.null.base" { + name = "tiramisu" + } +} + +build { + name = "recipes" + source "source.null.base" { + name = "spaghetti_carbonara" + } + source "source.null.base" { + name = "lasagna" + } + + provisioner "shell-local" { + name = "add_spaghetti" + inline = [ "echo spaghetti > ${upper(build.ID)}.${source.name}.txt" ] + only = ["null.spaghetti_carbonara"] + } + + post-processor "shell-local" { + name = "carbonara_it" + inline = [ "echo carbonara >> ${upper(build.ID)}.${source.name}.txt" ] + except = ["null.lasagna"] + } + + + provisioner "shell-local" { + name = "add_lasagna" + inline = [ "echo lasagna > ${upper(build.ID)}.${source.name}.txt" ] + only = ["null.lasagna"] + } + + provisioner "shell-local" { + name = "add_tomato" + inline = [ "echo tomato >> ${upper(build.ID)}.${source.name}.txt" ] + except = ["null.spaghetti_carbonara"] + } + + provisioner "shell-local" { + name = "add_mozza" + inline = [ "echo mozza >> ${upper(build.ID)}.${source.name}.txt" ] + except = ["null.spaghetti_carbonara"] + } + + post-processor "shell-local" { + name = "cook" + inline = [ "echo 'cooking...' >> ${upper(build.ID)}.${source.name}.txt" ] + except = ["null.spaghetti_carbonara"] + } + +} diff --git a/command/test-fixtures/hcl/complete/sources.pkr.hcl b/command/test-fixtures/hcl/complete/sources.pkr.hcl new file mode 100644 index 000000000..baeae65fb --- /dev/null +++ b/command/test-fixtures/hcl/complete/sources.pkr.hcl @@ -0,0 +1,3 @@ +source "null" "base" { + communicator = "none" +} From 0475b0529d8bb8fba4ec4cb1bdcf44f43e874dc1 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Mon, 6 Jul 2020 17:53:08 +0200 Subject: [PATCH 15/24] tests: fix line endings in windows --- command/build_windows_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 command/build_windows_test.go diff --git a/command/build_windows_test.go b/command/build_windows_test.go new file mode 100644 index 000000000..fb8a52222 --- /dev/null +++ b/command/build_windows_test.go @@ -0,0 +1,12 @@ +package command + +import "strings" + +func init() { + spaghettiCarbonara = fixWindowsLineEndings(spaghettiCarbonara) + lasagna = fixWindowsLineEndings(lasagna) +} + +func fixWindowsLineEndings(s string) string { + return strings.ReplaceAll(s, "\n", " \r\n") +} From 10e5d952ac010fa90445e14b5215a965710aeffe Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Mon, 6 Jul 2020 18:08:51 +0200 Subject: [PATCH 16/24] unquote 'cooking...' --- command/test-fixtures/hcl/complete/build.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command/test-fixtures/hcl/complete/build.pkr.hcl b/command/test-fixtures/hcl/complete/build.pkr.hcl index ceb0a58d3..f19bc805d 100644 --- a/command/test-fixtures/hcl/complete/build.pkr.hcl +++ b/command/test-fixtures/hcl/complete/build.pkr.hcl @@ -47,7 +47,7 @@ build { post-processor "shell-local" { name = "cook" - inline = [ "echo 'cooking...' >> ${upper(build.ID)}.${source.name}.txt" ] + inline = [ "echo cooking... >> ${upper(build.ID)}.${source.name}.txt" ] except = ["null.spaghetti_carbonara"] } From ab54f48247da0d36d2e1c0bca65a1ae7630809b5 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Tue, 7 Jul 2020 11:28:07 +0200 Subject: [PATCH 17/24] add BuilderDataCommonKeys var to see what common keys a builder can return --- packer/provisioner.go | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/packer/provisioner.go b/packer/provisioner.go index 905b28971..af1772e83 100644 --- a/packer/provisioner.go +++ b/packer/provisioner.go @@ -42,6 +42,24 @@ type ProvisionHook struct { Provisioners []*HookedProvisioner } +// BuilderDataCommonKeys is the list of common keys that all builder will +// return +var BuilderDataCommonKeys = []string{ + "ID", + "Host", + "Port", + "User", + "Password", + "ConnType", + "PackerRunUUID", + "PackerHTTPPort", + "PackerHTTPIP", + "PackerHTTPAddr", + "SSHPublicKey", + "SSHPrivateKey", + "WinRMPassword", +} + // Provisioners interpolate most of their fields in the prepare stage; this // placeholder map helps keep fields that are only generated at build time from // accidentally being interpolated into empty strings at prepare time. @@ -53,26 +71,9 @@ type ProvisionHook struct { // data. func BasicPlaceholderData() map[string]string { placeholderData := map[string]string{} - msg := "Build_%s. " + common.PlaceholderMsg - placeholderData["ID"] = fmt.Sprintf(msg, "ID") - // The following correspond to communicator-agnostic functions that are - // part of the SSH and WinRM communicator implementations. These functions - // are not part of the communicator interface, but are stored on the - // Communicator Config and return the appropriate values rather than - // depending on the actual communicator config values. E.g "Password" - // reprosents either WinRMPassword or SSHPassword, which makes this more - // useful if a template contains multiple builds. - placeholderData["Host"] = fmt.Sprintf(msg, "Host") - placeholderData["Port"] = fmt.Sprintf(msg, "Port") - placeholderData["User"] = fmt.Sprintf(msg, "User") - placeholderData["Password"] = fmt.Sprintf(msg, "Password") - placeholderData["ConnType"] = fmt.Sprintf(msg, "Type") - placeholderData["PackerRunUUID"] = fmt.Sprintf(msg, "PackerRunUUID") - placeholderData["PackerHTTPPort"] = fmt.Sprintf(msg, "PackerHTTPPort") - placeholderData["PackerHTTPIP"] = fmt.Sprintf(msg, "PackerHTTPIP") - placeholderData["PackerHTTPAddr"] = fmt.Sprintf(msg, "PackerHTTPAddr") - placeholderData["SSHPublicKey"] = fmt.Sprintf(msg, "SSHPublicKey") - placeholderData["SSHPrivateKey"] = fmt.Sprintf(msg, "SSHPrivateKey") + for _, key := range BuilderDataCommonKeys { + placeholderData[key] = fmt.Sprintf("Build_%s. "+common.PlaceholderMsg, key) + } // Backwards-compatability: WinRM Password can get through without forcing // the generated func validation. From 34ff3d0c8f2ed67e3a23bd06ef5a407a6754fd79 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Tue, 7 Jul 2020 11:29:27 +0200 Subject: [PATCH 18/24] hcl2 GetBuilds: use packer.BuilderDataCommonKeys to avoid forcing all builders to return exactly what they will return It would have been a bit nicer if all builder were more specific in order for hcl to tell wether a value will be set during the first prepare time but that also totally works ! --- hcl2template/types.packer_config.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index a19dba2dd..dd72a6955 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -7,7 +7,6 @@ import ( "github.com/gobwas/glob" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" - "github.com/hashicorp/packer/helper/common" "github.com/hashicorp/packer/packer" "github.com/zclconf/go-cty/cty" ) @@ -361,11 +360,8 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packer.Build // the provisioner prepare() so that the provisioner can appropriately // validate user input against what will become available. Otherwise, // only pass the default variables, using the basic placeholder data. - generatedPlaceholderMap := packer.BasicPlaceholderData() unknownBuildValues := map[string]cty.Value{} - for _, k := range generatedVars { - generatedPlaceholderMap[k] = fmt.Sprintf("Build_%s. "+ - common.PlaceholderMsg, k) + for _, k := range append(packer.BuilderDataCommonKeys, generatedVars...) { unknownBuildValues[k] = cty.StringVal("") } From c1c8b8d22ad961b55065e2e62708cae0b67f5701 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Tue, 7 Jul 2020 11:31:19 +0200 Subject: [PATCH 19/24] complete -> recipes --- command/build_test.go | 18 +++++++++--------- .../hcl/{complete => recipes}/build.pkr.hcl | 0 .../hcl/{complete => recipes}/sources.pkr.hcl | 0 3 files changed, 9 insertions(+), 9 deletions(-) rename command/test-fixtures/hcl/{complete => recipes}/build.pkr.hcl (100%) rename command/test-fixtures/hcl/{complete => recipes}/sources.pkr.hcl (100%) diff --git a/command/build_test.go b/command/build_test.go index 651ae988e..0724ff036 100644 --- a/command/build_test.go +++ b/command/build_test.go @@ -244,11 +244,11 @@ func TestBuild(t *testing.T) { }, }, - // complete + // recipes { - name: "hcl - complete", + name: "hcl - recipes", args: []string{ - testFixture("hcl", "complete"), + testFixture("hcl", "recipes"), }, fileCheck: fileCheck{ expectedContent: map[string]string{ @@ -259,10 +259,10 @@ func TestBuild(t *testing.T) { }, { - name: "hcl - complete - except carbonara", + name: "hcl - recipes - except carbonara", args: []string{ "-except", "recipes.null.spaghetti_carbonara", - testFixture("hcl", "complete"), + testFixture("hcl", "recipes"), }, fileCheck: fileCheck{ notExpected: []string{"NULL.spaghetti_carbonara.txt"}, @@ -273,10 +273,10 @@ func TestBuild(t *testing.T) { }, { - name: "hcl - complete - only lasagna", + name: "hcl - recipes - only lasagna", args: []string{ "-only", "*lasagna", - testFixture("hcl", "complete"), + testFixture("hcl", "recipes"), }, fileCheck: fileCheck{ notExpected: []string{"NULL.spaghetti_carbonara.txt"}, @@ -287,10 +287,10 @@ func TestBuild(t *testing.T) { }, { - name: "hcl - complete - only recipes", + name: "hcl - recipes - only recipes", args: []string{ "-only", "recipes.*", - testFixture("hcl", "complete"), + testFixture("hcl", "recipes"), }, fileCheck: fileCheck{ expectedContent: map[string]string{ diff --git a/command/test-fixtures/hcl/complete/build.pkr.hcl b/command/test-fixtures/hcl/recipes/build.pkr.hcl similarity index 100% rename from command/test-fixtures/hcl/complete/build.pkr.hcl rename to command/test-fixtures/hcl/recipes/build.pkr.hcl diff --git a/command/test-fixtures/hcl/complete/sources.pkr.hcl b/command/test-fixtures/hcl/recipes/sources.pkr.hcl similarity index 100% rename from command/test-fixtures/hcl/complete/sources.pkr.hcl rename to command/test-fixtures/hcl/recipes/sources.pkr.hcl From a15ad19411b07f2fd3c59c1782dcbe19fb7301ed Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Tue, 7 Jul 2020 11:41:24 +0200 Subject: [PATCH 20/24] test builds can be 'excepted' --- command/build_test.go | 17 +++++++++++++++-- command/test-fixtures/hcl/recipes/build.pkr.hcl | 17 +++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/command/build_test.go b/command/build_test.go index 0724ff036..1a4351d08 100644 --- a/command/build_test.go +++ b/command/build_test.go @@ -28,6 +28,11 @@ carbonara tomato mozza cooking... +` + tiramisu = `whip_york +mascarpone +whipped_egg_white +dress ` ) @@ -254,6 +259,7 @@ func TestBuild(t *testing.T) { expectedContent: map[string]string{ "NULL.spaghetti_carbonara.txt": spaghettiCarbonara, "NULL.lasagna.txt": lasagna, + "NULL.tiramisu.txt": tiramisu, }, }, }, @@ -267,7 +273,8 @@ func TestBuild(t *testing.T) { fileCheck: fileCheck{ notExpected: []string{"NULL.spaghetti_carbonara.txt"}, expectedContent: map[string]string{ - "NULL.lasagna.txt": lasagna, + "NULL.lasagna.txt": lasagna, + "NULL.tiramisu.txt": tiramisu, }, }, }, @@ -279,7 +286,10 @@ func TestBuild(t *testing.T) { testFixture("hcl", "recipes"), }, fileCheck: fileCheck{ - notExpected: []string{"NULL.spaghetti_carbonara.txt"}, + notExpected: []string{ + "NULL.spaghetti_carbonara.txt", + "NULL.tiramisu.txt", + }, expectedContent: map[string]string{ "NULL.lasagna.txt": lasagna, }, @@ -293,6 +303,9 @@ func TestBuild(t *testing.T) { testFixture("hcl", "recipes"), }, fileCheck: fileCheck{ + notExpected: []string{ + "NULL.tiramisu.txt", + }, expectedContent: map[string]string{ "NULL.spaghetti_carbonara.txt": spaghettiCarbonara, "NULL.lasagna.txt": lasagna, diff --git a/command/test-fixtures/hcl/recipes/build.pkr.hcl b/command/test-fixtures/hcl/recipes/build.pkr.hcl index f19bc805d..25a0a5acc 100644 --- a/command/test-fixtures/hcl/recipes/build.pkr.hcl +++ b/command/test-fixtures/hcl/recipes/build.pkr.hcl @@ -3,6 +3,23 @@ build { source "source.null.base" { name = "tiramisu" } + + provisioner "shell-local" { + name = "whipped_york" + inline = [ "echo whip_york > ${upper(build.ID)}.${source.name}.txt" ] + } + provisioner "shell-local" { + name = "mascarpone" + inline = [ "echo mascarpone >> ${upper(build.ID)}.${source.name}.txt" ] + } + post-processor "shell-local" { + name = "whipped_egg_white" + inline = [ "echo whipped_egg_white >> ${upper(build.ID)}.${source.name}.txt" ] + } + post-processor "shell-local" { + name = "dress_with_coffeed_boudoirs" + inline = [ "echo dress >> ${upper(build.ID)}.${source.name}.txt" ] + } } build { From d84d956e3612fe29d7bffd16f4445f6999534546 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Tue, 7 Jul 2020 11:43:13 +0200 Subject: [PATCH 21/24] add language coment ! --- command/test-fixtures/hcl/recipes/build.pkr.hcl | 1 + 1 file changed, 1 insertion(+) diff --git a/command/test-fixtures/hcl/recipes/build.pkr.hcl b/command/test-fixtures/hcl/recipes/build.pkr.hcl index 25a0a5acc..782bf10a2 100644 --- a/command/test-fixtures/hcl/recipes/build.pkr.hcl +++ b/command/test-fixtures/hcl/recipes/build.pkr.hcl @@ -2,6 +2,7 @@ build { source "source.null.base" { name = "tiramisu" + // pull me up ! } provisioner "shell-local" { From 12c9959db2ec7ddc0498f83a92ceeefdee5ec3d5 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Tue, 7 Jul 2020 11:46:43 +0200 Subject: [PATCH 22/24] Revert "null builder: say that we are returnin the ID value" This reverts commit 8e923977f055e9662cc9fcc78a843e0388f47895. --- builder/null/builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builder/null/builder.go b/builder/null/builder.go index f68263069..1d2fc40e4 100644 --- a/builder/null/builder.go +++ b/builder/null/builder.go @@ -25,7 +25,7 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, []string, error) { return nil, warnings, errs } - return []string{"ID"}, warnings, nil + return nil, warnings, nil } func (b *Builder) Run(ctx context.Context, ui packer.Ui, hook packer.Hook) (packer.Artifact, error) { From 3fb7ddf5095aff02cdf47220e17622de3ef1cb51 Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Tue, 7 Jul 2020 11:52:08 +0200 Subject: [PATCH 23/24] Update provisioner.go --- packer/provisioner.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packer/provisioner.go b/packer/provisioner.go index af1772e83..834f7433f 100644 --- a/packer/provisioner.go +++ b/packer/provisioner.go @@ -46,6 +46,13 @@ type ProvisionHook struct { // return var BuilderDataCommonKeys = []string{ "ID", + // The following correspond to communicator-agnostic functions that are } + // part of the SSH and WinRM communicator implementations. These functions + // are not part of the communicator interface, but are stored on the + // Communicator Config and return the appropriate values rather than + // depending on the actual communicator config values. E.g "Password" + // reprosents either WinRMPassword or SSHPassword, which makes this more + // useful if a template contains multiple builds. "Host", "Port", "User", From b0238e6f4a3f4b688a2d0010d6633e86a31c03cb Mon Sep 17 00:00:00 2001 From: Adrien Delorme Date: Tue, 7 Jul 2020 12:00:36 +0200 Subject: [PATCH 24/24] Update build_windows_test.go --- command/build_windows_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/command/build_windows_test.go b/command/build_windows_test.go index fb8a52222..829cf987f 100644 --- a/command/build_windows_test.go +++ b/command/build_windows_test.go @@ -5,6 +5,7 @@ import "strings" func init() { spaghettiCarbonara = fixWindowsLineEndings(spaghettiCarbonara) lasagna = fixWindowsLineEndings(lasagna) + tiramisu = fixWindowsLineEndings(tiramisu) } func fixWindowsLineEndings(s string) string {