diff --git a/hcl2template/parser.go b/hcl2template/parser.go index 59efd69c0..3f006f537 100644 --- a/hcl2template/parser.go +++ b/hcl2template/parser.go @@ -2,6 +2,7 @@ package hcl2template import ( "fmt" + "log" "os" "path/filepath" @@ -191,7 +192,7 @@ func (p *Parser) decodeConfig(f *hcl.File, cfg *PackerConfig) hcl.Diagnostics { } ref := source.Ref() - if existing := cfg.Sources[ref]; existing != nil { + if existing, found := cfg.Sources[ref]; found { diags = append(diags, &hcl.Diagnostic{ Severity: hcl.DiagError, Summary: "Duplicate " + sourceLabel + " block", @@ -205,9 +206,10 @@ func (p *Parser) decodeConfig(f *hcl.File, cfg *PackerConfig) hcl.Diagnostics { } if cfg.Sources == nil { - cfg.Sources = map[SourceRef]*SourceBlock{} + cfg.Sources = map[SourceRef]SourceBlock{} } cfg.Sources[ref] = source + log.Printf("adding %s source to Available sources", ref) case buildLabel: build, moreDiags := p.decodeBuildConfig(block) diff --git a/hcl2template/testdata/complete/build.pkr.hcl b/hcl2template/testdata/complete/build.pkr.hcl index 5641e540b..327d2399f 100644 --- a/hcl2template/testdata/complete/build.pkr.hcl +++ b/hcl2template/testdata/complete/build.pkr.hcl @@ -5,6 +5,10 @@ build { "source.virtualbox-iso.ubuntu-1204", ] + source "source.amazon-ebs.ubuntu-1604" { + string = "setting from build section" + } + provisioner "shell" { name = "provisioner that does something" not_squashed = var.foo diff --git a/hcl2template/testdata/complete/sources.pkr.hcl b/hcl2template/testdata/complete/sources.pkr.hcl index 2575929c7..8b05624d5 100644 --- a/hcl2template/testdata/complete/sources.pkr.hcl +++ b/hcl2template/testdata/complete/sources.pkr.hcl @@ -1,3 +1,9 @@ + +source "amazon-ebs" "ubuntu-1604" { + int = 42 +} + + source "virtualbox-iso" "ubuntu-1204" { string = "string" int = 42 @@ -85,4 +91,4 @@ source "virtualbox-iso" "ubuntu-1204" { ["c","d"] ] } -} \ No newline at end of file +} diff --git a/hcl2template/types.build.go b/hcl2template/types.build.go index 09b663525..a90ad67b6 100644 --- a/hcl2template/types.build.go +++ b/hcl2template/types.build.go @@ -19,6 +19,7 @@ const ( var buildSchema = &hcl.BodySchema{ Blocks: []hcl.BlockHeaderSchema{ {Type: buildFromLabel, LabelNames: []string{"type"}}, + {Type: sourceLabel, LabelNames: []string{"reference"}}, {Type: buildProvisionerLabel, LabelNames: []string{"type"}}, {Type: buildPostProcessorLabel, LabelNames: []string{"type"}}, }, @@ -92,6 +93,13 @@ func (p *Parser) decodeBuildConfig(block *hcl.Block) (*BuildBlock, hcl.Diagnosti } for _, block := range content.Blocks { switch block.Type { + case sourceLabel: + ref, moreDiags := p.decodeBuildSource(block) + diags = append(diags, moreDiags...) + if moreDiags.HasErrors() { + continue + } + build.Sources = append(build.Sources, ref) case buildProvisionerLabel: p, moreDiags := p.decodeProvisioner(block) diags = append(diags, moreDiags...) diff --git a/hcl2template/types.build.post-processor.go b/hcl2template/types.build.post-processor.go index 8add11514..bedbb1ad0 100644 --- a/hcl2template/types.build.post-processor.go +++ b/hcl2template/types.build.post-processor.go @@ -49,7 +49,7 @@ func (p *Parser) decodePostProcessor(block *hcl.Block) (*PostProcessorBlock, hcl return postProcessor, diags } -func (cfg *PackerConfig) startPostProcessor(source *SourceBlock, pp *PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]string) (packer.PostProcessor, hcl.Diagnostics) { +func (cfg *PackerConfig) startPostProcessor(source SourceBlock, pp *PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]string) (packer.PostProcessor, hcl.Diagnostics) { // ProvisionerBlock represents a detected but unparsed provisioner var diags hcl.Diagnostics diff --git a/hcl2template/types.build.provisioners.go b/hcl2template/types.build.provisioners.go index df10d77dc..c41ba2eb4 100644 --- a/hcl2template/types.build.provisioners.go +++ b/hcl2template/types.build.provisioners.go @@ -77,7 +77,7 @@ func (p *Parser) decodeProvisioner(block *hcl.Block) (*ProvisionerBlock, hcl.Dia return provisioner, diags } -func (cfg *PackerConfig) startProvisioner(source *SourceBlock, pb *ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]string) (packer.Provisioner, hcl.Diagnostics) { +func (cfg *PackerConfig) startProvisioner(source SourceBlock, pb *ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]string) (packer.Provisioner, hcl.Diagnostics) { var diags hcl.Diagnostics provisioner, err := cfg.provisionersSchemas.Start(pb.PType) diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index 41580d3af..6940b76d1 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -18,7 +18,7 @@ type PackerConfig struct { Basedir string // Available Source blocks - Sources map[SourceRef]*SourceBlock + Sources map[SourceRef]SourceBlock // InputVariables and LocalVariables are the list of defined input and // local variables. They are of the same type but are not used in the same @@ -194,7 +194,7 @@ func (c *PackerConfig) evaluateLocalVariable(local *Local) hcl.Diagnostics { // getCoreBuildProvisioners takes a list of provisioner block, starts according // provisioners and sends parsed HCL2 over to it. -func (cfg *PackerConfig) getCoreBuildProvisioners(source *SourceBlock, blocks []*ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]string) ([]packer.CoreBuildProvisioner, hcl.Diagnostics) { +func (cfg *PackerConfig) getCoreBuildProvisioners(source SourceBlock, blocks []*ProvisionerBlock, ectx *hcl.EvalContext, generatedVars map[string]string) ([]packer.CoreBuildProvisioner, hcl.Diagnostics) { var diags hcl.Diagnostics res := []packer.CoreBuildProvisioner{} for _, pb := range blocks { @@ -234,7 +234,7 @@ func (cfg *PackerConfig) getCoreBuildProvisioners(source *SourceBlock, blocks [] // getCoreBuildProvisioners takes a list of post processor block, starts // according provisioners and sends parsed HCL2 over to it. -func (cfg *PackerConfig) getCoreBuildPostProcessors(source *SourceBlock, blocks []*PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]string) ([]packer.CoreBuildPostProcessor, hcl.Diagnostics) { +func (cfg *PackerConfig) getCoreBuildPostProcessors(source SourceBlock, blocks []*PostProcessorBlock, ectx *hcl.EvalContext, generatedVars map[string]string) ([]packer.CoreBuildPostProcessor, hcl.Diagnostics) { var diags hcl.Diagnostics res := []packer.CoreBuildPostProcessor{} for _, ppb := range blocks { @@ -262,15 +262,17 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packer.Build for _, build := range cfg.Builds { for _, from := range build.Sources { - src, found := cfg.Sources[from] + src, found := cfg.Sources[from.Ref()] if !found { diags = append(diags, &hcl.Diagnostic{ Summary: "Unknown " + sourceLabel + " " + from.String(), Subject: build.HCL2Ref.DefRange.Ptr(), Severity: hcl.DiagError, + Detail: fmt.Sprintf("Known: %v", cfg.Sources), }) continue } + src.addition = from.addition // Apply the -only and -except command-line options to exclude matching builds. buildName := fmt.Sprintf("%s.%s", src.Type, src.Name) diff --git a/hcl2template/types.packer_config_test.go b/hcl2template/types.packer_config_test.go index 844038f19..e3675d478 100644 --- a/hcl2template/types.packer_config_test.go +++ b/hcl2template/types.packer_config_test.go @@ -1,10 +1,9 @@ package hcl2template import ( - "path/filepath" "testing" - "time" + . "github.com/hashicorp/packer/hcl2template/internal" "github.com/hashicorp/packer/packer" "github.com/zclconf/go-cty/cty" ) @@ -72,12 +71,16 @@ func TestParser_complete(t *testing.T) { }), }, }, - Sources: map[SourceRef]*SourceBlock{ - refVBIsoUbuntu1204: {Type: "virtualbox-iso", Name: "ubuntu-1204"}, + Sources: map[SourceRef]SourceBlock{ + refVBIsoUbuntu1204: {Type: "virtualbox-iso", Name: "ubuntu-1204"}, + refAWSEBSUbuntu1204: {Type: "amazon-ebs", Name: "ubuntu-1604"}, }, Builds: Builds{ &BuildBlock{ - Sources: []SourceRef{refVBIsoUbuntu1204}, + Sources: []SourceRef{ + refVBIsoUbuntu1204, + refAWSEBSUbuntu1204, + }, ProvisionerBlocks: []*ProvisionerBlock{ { PType: "shell", @@ -125,136 +128,40 @@ func TestParser_complete(t *testing.T) { }, }, }, - }, - false, - }, - {"dir with no config files", - defaultParser, - parseTestArgs{"testdata/empty", nil, nil}, - nil, - true, true, - nil, - false, - }, - {name: "inexistent dir", - parser: defaultParser, - args: parseTestArgs{"testdata/inexistent", nil, nil}, - parseWantCfg: nil, - parseWantDiags: true, - parseWantDiagHasErrors: true, - }, - {name: "folder named build.pkr.hcl with an unknown src", - parser: defaultParser, - args: parseTestArgs{"testdata/build.pkr.hcl", nil, nil}, - parseWantCfg: &PackerConfig{ - Basedir: "testdata/build.pkr.hcl", - Builds: Builds{ - &BuildBlock{ - Sources: []SourceRef{refAWSEBSUbuntu1204, refVBIsoUbuntu1204}, - ProvisionerBlocks: []*ProvisionerBlock{ - {PType: "shell"}, - {PType: "file"}, - }, - PostProcessors: []*PostProcessorBlock{ - {PType: "amazon-import"}, - }, - }, - }, - }, - parseWantDiags: false, - parseWantDiagHasErrors: false, - getBuildsWantBuilds: []packer.Build{}, - getBuildsWantDiags: true, - }, - {name: "unknown block type", - parser: defaultParser, - args: parseTestArgs{"testdata/unknown", nil, nil}, - parseWantCfg: &PackerConfig{ - Basedir: "testdata/unknown", - }, - parseWantDiags: true, - parseWantDiagHasErrors: true, - }, - {"provisioner with wrappers pause_before and max_retriers", - defaultParser, - parseTestArgs{"testdata/build/provisioner_paused_before_retry.pkr.hcl", nil, nil}, - &PackerConfig{ - Basedir: filepath.Join("testdata", "build"), - Sources: map[SourceRef]*SourceBlock{ - refVBIsoUbuntu1204: {Type: "virtualbox-iso", Name: "ubuntu-1204"}, - }, - Builds: Builds{ - &BuildBlock{ - Sources: []SourceRef{refVBIsoUbuntu1204}, - ProvisionerBlocks: []*ProvisionerBlock{ - { - PType: "shell", - PauseBefore: time.Second * 10, - MaxRetries: 5, + &packer.CoreBuild{ + Type: "amazon-ebs.ubuntu-1604", + Prepared: true, + Builder: &MockBuilder{ + Config: MockConfig{ + NestedMockConfig: NestedMockConfig{ + String: "setting from build section", + Int: 42, + Tags: []MockTag{}, }, + NestedSlice: []NestedMockConfig{}, }, }, - }, - }, - false, false, - []packer.Build{ - &packer.CoreBuild{ - Type: "virtualbox-iso.ubuntu-1204", - Prepared: true, - Builder: emptyMockBuilder, Provisioners: []packer.CoreBuildProvisioner{ { - PType: "shell", - Provisioner: &packer.RetriedProvisioner{ - MaxRetries: 5, - Provisioner: &packer.PausedProvisioner{ - PauseBefore: time.Second * 10, - Provisioner: emptyMockProvisioner, - }, - }, + PType: "shell", + PName: "provisioner that does something", + Provisioner: basicMockProvisioner, }, + {PType: "file", Provisioner: basicMockProvisioner}, }, - PostProcessors: [][]packer.CoreBuildPostProcessor{}, - }, - }, - false, - }, - {"provisioner with wrappers timeout", - defaultParser, - parseTestArgs{"testdata/build/provisioner_timeout.pkr.hcl", nil, nil}, - &PackerConfig{ - Basedir: filepath.Join("testdata", "build"), - Sources: map[SourceRef]*SourceBlock{ - refVBIsoUbuntu1204: {Type: "virtualbox-iso", Name: "ubuntu-1204"}, - }, - Builds: Builds{ - &BuildBlock{ - Sources: []SourceRef{refVBIsoUbuntu1204}, - ProvisionerBlocks: []*ProvisionerBlock{ + PostProcessors: [][]packer.CoreBuildPostProcessor{ + { { - PType: "shell", - Timeout: time.Second * 10, + PType: "amazon-import", + PName: "something", + PostProcessor: basicMockPostProcessor, }, - }, - }, - }, - }, - false, false, - []packer.Build{ - &packer.CoreBuild{ - Type: "virtualbox-iso.ubuntu-1204", - Prepared: true, - Builder: emptyMockBuilder, - Provisioners: []packer.CoreBuildProvisioner{ - { - PType: "shell", - Provisioner: &packer.TimeoutProvisioner{ - Timeout: time.Second * 10, - Provisioner: emptyMockProvisioner, + { + PType: "amazon-import", + PostProcessor: basicMockPostProcessor, }, }, }, - PostProcessors: [][]packer.CoreBuildPostProcessor{}, }, }, false, diff --git a/hcl2template/types.source.go b/hcl2template/types.source.go index dbc5a9724..3d91ace4c 100644 --- a/hcl2template/types.source.go +++ b/hcl2template/types.source.go @@ -16,10 +16,24 @@ type SourceBlock struct { Name string block *hcl.Block + + // addition will be merged into block to allow to override builder settings + // per build.source block. + addition hcl.Body +} + +// decodeBuildSource reads a used source block from a build: +// build { +// source "type.example" {} +// } +func (p *Parser) decodeBuildSource(block *hcl.Block) (SourceRef, hcl.Diagnostics) { + ref := sourceRefFromString(block.Labels[0]) + ref.addition = block.Body + return ref, nil } -func (p *Parser) decodeSource(block *hcl.Block) (*SourceBlock, hcl.Diagnostics) { - source := &SourceBlock{ +func (p *Parser) decodeSource(block *hcl.Block) (SourceBlock, hcl.Diagnostics) { + source := SourceBlock{ Type: block.Labels[0], Name: block.Labels[1], block: block, @@ -33,13 +47,13 @@ func (p *Parser) decodeSource(block *hcl.Block) (*SourceBlock, hcl.Diagnostics) Detail: fmt.Sprintf("known builders: %v", p.BuilderSchemas.List()), Severity: hcl.DiagError, }) - return nil, diags + return source, diags } return source, diags } -func (cfg *PackerConfig) startBuilder(source *SourceBlock, ectx *hcl.EvalContext, opts packer.GetBuildsOptions) (packer.Builder, hcl.Diagnostics, []string) { +func (cfg *PackerConfig) startBuilder(source SourceBlock, ectx *hcl.EvalContext, opts packer.GetBuildsOptions) (packer.Builder, hcl.Diagnostics, []string) { var diags hcl.Diagnostics builder, err := cfg.builderSchemas.Start(source.Type) @@ -52,7 +66,12 @@ func (cfg *PackerConfig) startBuilder(source *SourceBlock, ectx *hcl.EvalContext return builder, diags, nil } - decoded, moreDiags := decodeHCL2Spec(source.block.Body, ectx, builder) + body := source.block.Body + if source.addition != nil { + body = hcl.MergeBodies([]hcl.Body{source.block.Body, source.addition}) + } + + decoded, moreDiags := decodeHCL2Spec(body, ectx, builder) diags = append(diags, moreDiags...) if moreDiags.HasErrors() { return nil, diags, nil @@ -92,6 +111,19 @@ func (source *SourceBlock) Ref() SourceRef { type SourceRef struct { Type string Name string + + // The content of this body will be merged into a new block to allow to + // override builder settings per build section. + addition hcl.Body +} + +// the 'addition' field makes of ref a different entry in the soruces map, so +// Ref is here to make sure only one is returned. +func (r *SourceRef) Ref() SourceRef { + return SourceRef{ + Type: r.Type, + Name: r.Name, + } } // NoSource is the zero value of sourceRef, representing the absense of an diff --git a/hcl2template/types.source_test.go b/hcl2template/types.source_test.go index 423d20422..9a344765c 100644 --- a/hcl2template/types.source_test.go +++ b/hcl2template/types.source_test.go @@ -16,7 +16,7 @@ func TestParse_source(t *testing.T) { parseTestArgs{"testdata/sources/basic.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "sources"), - Sources: map[SourceRef]*SourceBlock{ + Sources: map[SourceRef]SourceBlock{ { Type: "virtualbox-iso", Name: "ubuntu-1204", @@ -65,7 +65,7 @@ func TestParse_source(t *testing.T) { parseTestArgs{"testdata/sources/duplicate.pkr.hcl", nil, nil}, &PackerConfig{ Basedir: filepath.Join("testdata", "sources"), - Sources: map[SourceRef]*SourceBlock{ + Sources: map[SourceRef]SourceBlock{ { Type: "virtualbox-iso", Name: "ubuntu-1204", diff --git a/hcl2template/types.variables_test.go b/hcl2template/types.variables_test.go index 15f5fc9f6..df87594eb 100644 --- a/hcl2template/types.variables_test.go +++ b/hcl2template/types.variables_test.go @@ -162,15 +162,17 @@ func TestParse_variables(t *testing.T) { Name: "foo", }, }, - Sources: map[SourceRef]*SourceBlock{ - SourceRef{"null", "null-builder"}: &SourceBlock{ + Sources: map[SourceRef]SourceBlock{ + SourceRef{"null", "null-builder", nil}: SourceBlock{ Name: "null-builder", Type: "null", }, }, Builds: Builds{ &BuildBlock{ - Sources: []SourceRef{SourceRef{"null", "null-builder"}}, + Sources: []SourceRef{ + {"null", "null-builder", nil}, + }, }, }, },