From 16f2205ec78dc4da9548a6e38554f3e62ba79965 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Fri, 17 Mar 2023 14:26:41 +0000 Subject: [PATCH] backport of commit 7ff112a3130dbe8f9799e00c1e5fb4206ff915df --- hcl2template/common_test.go | 9 ++++ hcl2template/types.packer_config.go | 15 +++++++ packer/build.go | 56 ++++++++++++++++++----- packer/build_test.go | 23 +++++----- packer/telemetry.go | 70 ++++++++++++++++++++++++----- 5 files changed, 140 insertions(+), 33 deletions(-) diff --git a/hcl2template/common_test.go b/hcl2template/common_test.go index 5b1c68fbd..3d398708b 100644 --- a/hcl2template/common_test.go +++ b/hcl2template/common_test.go @@ -364,6 +364,15 @@ var cmpOpts = []cmp.Option{ cmpopts.IgnoreFields(VariableAssignment{}, "Expr", // its an interface ), + cmpopts.IgnoreFields(packer.CoreBuild{}, + "HCLConfig", + ), + cmpopts.IgnoreFields(packer.CoreBuildProvisioner{}, + "HCLConfig", + ), + cmpopts.IgnoreFields(packer.CoreBuildPostProcessor{}, + "HCLConfig", + ), cmpopts.IgnoreTypes(hcl2template.MockBuilder{}), cmpopts.IgnoreTypes(HCL2Ref{}), cmpopts.IgnoreTypes([]*LocalBlock{}), diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index 3ac4b0c51..b84109d1f 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -359,7 +359,10 @@ func (cfg *PackerConfig) evaluateDatasources(skipExecution bool) hcl.Diagnostics continue } + dsOpts, _ := decodeHCL2Spec(body, cfg.EvalContext(DatasourceContext, nil), datasource) + sp := packer.CheckpointReporter.AddSpan(ref.Type, "datasource", dsOpts) realValue, err := datasource.Execute() + sp.End(err) if err != nil { diags = append(diags, &hcl.Diagnostic{ Summary: err.Error(), @@ -441,7 +444,10 @@ func (cfg *PackerConfig) recursivelyEvaluateDatasources(ref DatasourceRef, depen return dependencies, diags, shouldContinue } + opts, _ := decodeHCL2Spec(ds.block.Body, cfg.EvalContext(DatasourceContext, nil), datasource) + sp := packer.CheckpointReporter.AddSpan(ref.Type, "datasource", opts) realValue, err := datasource.Execute() + sp.End(err) if err != nil { diags = append(diags, &hcl.Diagnostic{ Summary: err.Error(), @@ -486,6 +492,8 @@ func (cfg *PackerConfig) getCoreBuildProvisioner(source SourceUseBlock, pb *Prov return packer.CoreBuildProvisioner{}, diags } + flatProvisionerCfg, _ := decodeHCL2Spec(pb.HCL2Ref.Rest, ectx, provisioner) + // If we're pausing, we wrap the provisioner in a special pauser. if pb.PauseBefore != 0 { provisioner = &packer.PausedProvisioner{ @@ -509,6 +517,7 @@ func (cfg *PackerConfig) getCoreBuildProvisioner(source SourceUseBlock, pb *Prov PType: pb.PType, PName: pb.PName, Provisioner: provisioner, + HCLConfig: flatProvisionerCfg, }, diags } @@ -547,10 +556,13 @@ func (cfg *PackerConfig) getCoreBuildPostProcessors(source SourceUseBlock, block continue } + flatPostProcessorCfg, moreDiags := decodeHCL2Spec(ppb.HCL2Ref.Rest, ectx, postProcessor) + pps = append(pps, packer.CoreBuildPostProcessor{ PostProcessor: postProcessor, PName: ppb.PName, PType: ppb.PType, + HCLConfig: flatPostProcessorCfg, KeepInputArtifact: ppb.KeepInputArtifact, }) } @@ -653,6 +665,9 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packersdk.Bu continue } + decoded, _ := decodeHCL2Spec(srcUsage.Body, cfg.EvalContext(BuildContext, nil), builder) + pcb.HCLConfig = decoded + // If the builder has provided a list of to-be-generated variables that // should be made accessible to provisioners, pass that list into // the provisioner prepare() so that the provisioner can appropriately diff --git a/packer/build.go b/packer/build.go index e5ff54b72..d48174477 100644 --- a/packer/build.go +++ b/packer/build.go @@ -10,6 +10,7 @@ import ( packersdk "github.com/hashicorp/packer-plugin-sdk/packer" "github.com/hashicorp/packer-plugin-sdk/packerbuilderdata" "github.com/hashicorp/packer/version" + "github.com/zclconf/go-cty/cty" ) // A CoreBuild struct represents a single build job, the result of which should @@ -17,10 +18,19 @@ import ( // multiple files, of course, but it should be for only a single provider (such // as VirtualBox, EC2, etc.). type CoreBuild struct { - BuildName string - Type string - Builder packersdk.Builder - BuilderConfig interface{} + BuildName string + Type string + Builder packersdk.Builder + // BuilderConfig is the config for the builder. + // + // Is is deserialised directly from the JSON template, + // and is only populated for legacy JSON templates. + BuilderConfig interface{} + // HCLConfig is the HCL config for the builder + // + // Its only use is for telemetry, since we use it to extract the + // field names from it. + HCLConfig cty.Value BuilderType string hooks map[string][]packersdk.Hook Provisioners []CoreBuildProvisioner @@ -42,9 +52,16 @@ type CoreBuild struct { // CoreBuildPostProcessor Keeps track of the post-processor and the // configuration of the post-processor used within a build. type CoreBuildPostProcessor struct { - PostProcessor packersdk.PostProcessor - PType string - PName string + PostProcessor packersdk.PostProcessor + PType string + PName string + // HCLConfig is the HCL config for the post-processor + // + // Its only use is for telemetry, since we use it to extract the + // field names from it. + HCLConfig cty.Value + // config is JSON-specific, the configuration for the post-processor + // deserialised directly from the JSON template config map[string]interface{} KeepInputArtifact *bool } @@ -55,7 +72,14 @@ type CoreBuildProvisioner struct { PType string PName string Provisioner packersdk.Provisioner - config []interface{} + // HCLConfig is the HCL config for the provisioner + // + // Its only use is for telemetry, since we use it to extract the + // field names from it. + HCLConfig cty.Value + // config is JSON-specific, and is the configuration of the + // provisioner, with overrides + config []interface{} } // Returns the name of the build. @@ -173,6 +197,8 @@ func (b *CoreBuild) Run(ctx context.Context, originalUi packersdk.Ui) ([]packers var pConfig interface{} if len(p.config) > 0 { pConfig = p.config[0] + } else { + pConfig = p.HCLConfig } if b.debug { hookedProvisioners[i] = &HookedProvisioner{ @@ -218,8 +244,13 @@ func (b *CoreBuild) Run(ctx context.Context, originalUi packersdk.Ui) ([]packers Ui: originalUi, } + var ts *TelemetrySpan log.Printf("Running builder: %s", b.BuilderType) - ts := CheckpointReporter.AddSpan(b.Type, "builder", b.BuilderConfig) + if b.BuilderConfig != nil { + ts = CheckpointReporter.AddSpan(b.Type, "builder", b.BuilderConfig) + } else { + ts = CheckpointReporter.AddSpan(b.Type, "builder", b.HCLConfig) + } builderArtifact, err := b.Builder.Run(ctx, builderUi, hook) ts.End(err) if err != nil { @@ -257,7 +288,12 @@ PostProcessorRunSeqLoop: } else { builderUi.Say(fmt.Sprintf("Running post-processor: %s (type %s)", corePP.PName, corePP.PType)) } - ts := CheckpointReporter.AddSpan(corePP.PType, "post-processor", corePP.config) + var ts *TelemetrySpan + if corePP.config != nil { + ts = CheckpointReporter.AddSpan(corePP.PType, "post-processor", corePP.config) + } else { + ts = CheckpointReporter.AddSpan(corePP.PType, "post-processor", corePP.HCLConfig) + } artifact, defaultKeep, forceOverride, err := corePP.PostProcessor.PostProcess(ctx, ppUi, priorArtifact) ts.End(err) if err != nil { diff --git a/packer/build_test.go b/packer/build_test.go index 9b4812b35..46fad2bf1 100644 --- a/packer/build_test.go +++ b/packer/build_test.go @@ -9,6 +9,7 @@ import ( packersdk "github.com/hashicorp/packer-plugin-sdk/packer" "github.com/hashicorp/packer-plugin-sdk/packerbuilderdata" "github.com/hashicorp/packer/version" + "github.com/zclconf/go-cty/cty" ) func boolPointer(tf bool) *bool { @@ -32,7 +33,7 @@ func testBuild() *CoreBuild { }, PostProcessors: [][]CoreBuildPostProcessor{ { - {&MockPostProcessor{ArtifactId: "pp"}, "testPP", "testPPName", make(map[string]interface{}), boolPointer(true)}, + {&MockPostProcessor{ArtifactId: "pp"}, "testPP", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(true)}, }, }, Variables: make(map[string]string), @@ -302,7 +303,7 @@ func TestBuild_Run_Artifacts(t *testing.T) { build = testBuild() build.PostProcessors = [][]CoreBuildPostProcessor{ { - {&MockPostProcessor{ArtifactId: "pp"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false)}, + {&MockPostProcessor{ArtifactId: "pp"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false)}, }, } @@ -327,10 +328,10 @@ func TestBuild_Run_Artifacts(t *testing.T) { build = testBuild() build.PostProcessors = [][]CoreBuildPostProcessor{ { - {&MockPostProcessor{ArtifactId: "pp1"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false)}, + {&MockPostProcessor{ArtifactId: "pp1"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false)}, }, { - {&MockPostProcessor{ArtifactId: "pp2"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(true)}, + {&MockPostProcessor{ArtifactId: "pp2"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(true)}, }, } @@ -355,12 +356,12 @@ func TestBuild_Run_Artifacts(t *testing.T) { build = testBuild() build.PostProcessors = [][]CoreBuildPostProcessor{ { - {&MockPostProcessor{ArtifactId: "pp1a"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false)}, - {&MockPostProcessor{ArtifactId: "pp1b"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(true)}, + {&MockPostProcessor{ArtifactId: "pp1a"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false)}, + {&MockPostProcessor{ArtifactId: "pp1b"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(true)}, }, { - {&MockPostProcessor{ArtifactId: "pp2a"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false)}, - {&MockPostProcessor{ArtifactId: "pp2b"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false)}, + {&MockPostProcessor{ArtifactId: "pp2a"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false)}, + {&MockPostProcessor{ArtifactId: "pp2b"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false)}, }, } @@ -386,7 +387,7 @@ func TestBuild_Run_Artifacts(t *testing.T) { build.PostProcessors = [][]CoreBuildPostProcessor{ { { - &MockPostProcessor{ArtifactId: "pp", Keep: true, ForceOverride: true}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false), + &MockPostProcessor{ArtifactId: "pp", Keep: true, ForceOverride: true}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false), }, }, } @@ -414,7 +415,7 @@ func TestBuild_Run_Artifacts(t *testing.T) { build.PostProcessors = [][]CoreBuildPostProcessor{ { { - &MockPostProcessor{ArtifactId: "pp", Keep: true, ForceOverride: false}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false), + &MockPostProcessor{ArtifactId: "pp", Keep: true, ForceOverride: false}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false), }, }, } @@ -441,7 +442,7 @@ func TestBuild_Run_Artifacts(t *testing.T) { build.PostProcessors = [][]CoreBuildPostProcessor{ { { - &MockPostProcessor{ArtifactId: "pp", Keep: true, ForceOverride: false}, "pp", "testPPName", make(map[string]interface{}), nil, + &MockPostProcessor{ArtifactId: "pp", Keep: true, ForceOverride: false}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), nil, }, }, } diff --git a/packer/telemetry.go b/packer/telemetry.go index 92711d6a3..4ec60a400 100644 --- a/packer/telemetry.go +++ b/packer/telemetry.go @@ -2,6 +2,7 @@ package packer import ( "context" + "fmt" "log" "os" "path/filepath" @@ -11,6 +12,7 @@ import ( checkpoint "github.com/hashicorp/go-checkpoint" "github.com/hashicorp/packer-plugin-sdk/pathing" packerVersion "github.com/hashicorp/packer/version" + "github.com/zclconf/go-cty/cty" ) const TelemetryVersion string = "beta/packer/5" @@ -154,22 +156,66 @@ func flattenConfigKeys(options interface{}) []string { var flatten func(string, interface{}) []string flatten = func(prefix string, options interface{}) (strOpts []string) { - if m, ok := options.(map[string]interface{}); ok { - for k, v := range m { - if prefix != "" { - k = prefix + "/" + k - } - if n, ok := v.(map[string]interface{}); ok { - strOpts = append(strOpts, flatten(k, n)...) - } else { - strOpts = append(strOpts, k) - } - } + switch opt := options.(type) { + case map[string]interface{}: + return flattenJSON(prefix, options) + case cty.Value: + return flattenHCL(prefix, opt) + default: + return nil } - return } flattened := flatten("", options) sort.Strings(flattened) return flattened } + +func flattenJSON(prefix string, options interface{}) (strOpts []string) { + if m, ok := options.(map[string]interface{}); ok { + for k, v := range m { + if prefix != "" { + k = prefix + "/" + k + } + if n, ok := v.(map[string]interface{}); ok { + strOpts = append(strOpts, flattenJSON(k, n)...) + } else { + strOpts = append(strOpts, k) + } + } + } + return +} + +func flattenHCL(prefix string, v cty.Value) (args []string) { + if v.IsNull() { + return []string{} + } + t := v.Type() + switch { + case t.IsObjectType(), t.IsMapType(): + if !v.IsKnown() { + return []string{} + } + it := v.ElementIterator() + for it.Next() { + key, val := it.Element() + keyStr := key.AsString() + + if val.IsNull() { + continue + } + + if prefix != "" { + keyStr = fmt.Sprintf("%s/%s", prefix, keyStr) + } + + if val.Type().IsObjectType() || val.Type().IsMapType() { + args = append(args, flattenHCL(keyStr, val)...) + } else { + args = append(args, keyStr) + } + } + } + return args +}