From f1a41b488fce1a892dd946ce191a49fdfe08e5f5 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Fri, 17 Mar 2023 10:26:41 -0400 Subject: [PATCH] packer: add HCL2 support for telemetry Packer's telemetry package captures the field names from a configuration on legacy JSON templates, but did not contain any code for handling HCL2 template configurations. This commit adds a routine to extract the field names from a HCL2 configuration once flattened, and adds some glue code to propagate the configurations to the telemetry structures. --- hcl2template/common_test.go | 9 ++++ hcl2template/types.packer_config.go | 15 +++++++ packer/build.go | 56 ++++++++++++++++++----- packer/build_test.go | 23 +++++----- packer/telemetry.go | 70 ++++++++++++++++++++++++----- 5 files changed, 140 insertions(+), 33 deletions(-) diff --git a/hcl2template/common_test.go b/hcl2template/common_test.go index 62f937da9..cb37a7bf9 100644 --- a/hcl2template/common_test.go +++ b/hcl2template/common_test.go @@ -367,6 +367,15 @@ var cmpOpts = []cmp.Option{ cmpopts.IgnoreFields(VariableAssignment{}, "Expr", // its an interface ), + cmpopts.IgnoreFields(packer.CoreBuild{}, + "HCLConfig", + ), + cmpopts.IgnoreFields(packer.CoreBuildProvisioner{}, + "HCLConfig", + ), + cmpopts.IgnoreFields(packer.CoreBuildPostProcessor{}, + "HCLConfig", + ), cmpopts.IgnoreTypes(hcl2template.MockBuilder{}), cmpopts.IgnoreTypes(HCL2Ref{}), cmpopts.IgnoreTypes([]*LocalBlock{}), diff --git a/hcl2template/types.packer_config.go b/hcl2template/types.packer_config.go index b3802ea97..84d1fe8de 100644 --- a/hcl2template/types.packer_config.go +++ b/hcl2template/types.packer_config.go @@ -362,7 +362,10 @@ func (cfg *PackerConfig) evaluateDatasources(skipExecution bool) hcl.Diagnostics continue } + dsOpts, _ := decodeHCL2Spec(body, cfg.EvalContext(DatasourceContext, nil), datasource) + sp := packer.CheckpointReporter.AddSpan(ref.Type, "datasource", dsOpts) realValue, err := datasource.Execute() + sp.End(err) if err != nil { diags = append(diags, &hcl.Diagnostic{ Summary: err.Error(), @@ -444,7 +447,10 @@ func (cfg *PackerConfig) recursivelyEvaluateDatasources(ref DatasourceRef, depen return dependencies, diags, shouldContinue } + opts, _ := decodeHCL2Spec(ds.block.Body, cfg.EvalContext(DatasourceContext, nil), datasource) + sp := packer.CheckpointReporter.AddSpan(ref.Type, "datasource", opts) realValue, err := datasource.Execute() + sp.End(err) if err != nil { diags = append(diags, &hcl.Diagnostic{ Summary: err.Error(), @@ -489,6 +495,8 @@ func (cfg *PackerConfig) getCoreBuildProvisioner(source SourceUseBlock, pb *Prov return packer.CoreBuildProvisioner{}, diags } + flatProvisionerCfg, _ := decodeHCL2Spec(pb.HCL2Ref.Rest, ectx, provisioner) + // If we're pausing, we wrap the provisioner in a special pauser. if pb.PauseBefore != 0 { provisioner = &packer.PausedProvisioner{ @@ -512,6 +520,7 @@ func (cfg *PackerConfig) getCoreBuildProvisioner(source SourceUseBlock, pb *Prov PType: pb.PType, PName: pb.PName, Provisioner: provisioner, + HCLConfig: flatProvisionerCfg, }, diags } @@ -550,10 +559,13 @@ func (cfg *PackerConfig) getCoreBuildPostProcessors(source SourceUseBlock, block continue } + flatPostProcessorCfg, moreDiags := decodeHCL2Spec(ppb.HCL2Ref.Rest, ectx, postProcessor) + pps = append(pps, packer.CoreBuildPostProcessor{ PostProcessor: postProcessor, PName: ppb.PName, PType: ppb.PType, + HCLConfig: flatPostProcessorCfg, KeepInputArtifact: ppb.KeepInputArtifact, }) } @@ -656,6 +668,9 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packersdk.Bu continue } + decoded, _ := decodeHCL2Spec(srcUsage.Body, cfg.EvalContext(BuildContext, nil), builder) + pcb.HCLConfig = decoded + // If the builder has provided a list of to-be-generated variables that // should be made accessible to provisioners, pass that list into // the provisioner prepare() so that the provisioner can appropriately diff --git a/packer/build.go b/packer/build.go index 688692c4c..3309bb194 100644 --- a/packer/build.go +++ b/packer/build.go @@ -13,6 +13,7 @@ import ( packersdk "github.com/hashicorp/packer-plugin-sdk/packer" "github.com/hashicorp/packer-plugin-sdk/packerbuilderdata" "github.com/hashicorp/packer/version" + "github.com/zclconf/go-cty/cty" ) // A CoreBuild struct represents a single build job, the result of which should @@ -20,10 +21,19 @@ import ( // multiple files, of course, but it should be for only a single provider (such // as VirtualBox, EC2, etc.). type CoreBuild struct { - BuildName string - Type string - Builder packersdk.Builder - BuilderConfig interface{} + BuildName string + Type string + Builder packersdk.Builder + // BuilderConfig is the config for the builder. + // + // Is is deserialised directly from the JSON template, + // and is only populated for legacy JSON templates. + BuilderConfig interface{} + // HCLConfig is the HCL config for the builder + // + // Its only use is for telemetry, since we use it to extract the + // field names from it. + HCLConfig cty.Value BuilderType string hooks map[string][]packersdk.Hook Provisioners []CoreBuildProvisioner @@ -45,9 +55,16 @@ type CoreBuild struct { // CoreBuildPostProcessor Keeps track of the post-processor and the // configuration of the post-processor used within a build. type CoreBuildPostProcessor struct { - PostProcessor packersdk.PostProcessor - PType string - PName string + PostProcessor packersdk.PostProcessor + PType string + PName string + // HCLConfig is the HCL config for the post-processor + // + // Its only use is for telemetry, since we use it to extract the + // field names from it. + HCLConfig cty.Value + // config is JSON-specific, the configuration for the post-processor + // deserialised directly from the JSON template config map[string]interface{} KeepInputArtifact *bool } @@ -58,7 +75,14 @@ type CoreBuildProvisioner struct { PType string PName string Provisioner packersdk.Provisioner - config []interface{} + // HCLConfig is the HCL config for the provisioner + // + // Its only use is for telemetry, since we use it to extract the + // field names from it. + HCLConfig cty.Value + // config is JSON-specific, and is the configuration of the + // provisioner, with overrides + config []interface{} } // Returns the name of the build. @@ -176,6 +200,8 @@ func (b *CoreBuild) Run(ctx context.Context, originalUi packersdk.Ui) ([]packers var pConfig interface{} if len(p.config) > 0 { pConfig = p.config[0] + } else { + pConfig = p.HCLConfig } if b.debug { hookedProvisioners[i] = &HookedProvisioner{ @@ -221,8 +247,13 @@ func (b *CoreBuild) Run(ctx context.Context, originalUi packersdk.Ui) ([]packers Ui: originalUi, } + var ts *TelemetrySpan log.Printf("Running builder: %s", b.BuilderType) - ts := CheckpointReporter.AddSpan(b.Type, "builder", b.BuilderConfig) + if b.BuilderConfig != nil { + ts = CheckpointReporter.AddSpan(b.Type, "builder", b.BuilderConfig) + } else { + ts = CheckpointReporter.AddSpan(b.Type, "builder", b.HCLConfig) + } builderArtifact, err := b.Builder.Run(ctx, builderUi, hook) ts.End(err) if err != nil { @@ -260,7 +291,12 @@ PostProcessorRunSeqLoop: } else { builderUi.Say(fmt.Sprintf("Running post-processor: %s (type %s)", corePP.PName, corePP.PType)) } - ts := CheckpointReporter.AddSpan(corePP.PType, "post-processor", corePP.config) + var ts *TelemetrySpan + if corePP.config != nil { + ts = CheckpointReporter.AddSpan(corePP.PType, "post-processor", corePP.config) + } else { + ts = CheckpointReporter.AddSpan(corePP.PType, "post-processor", corePP.HCLConfig) + } artifact, defaultKeep, forceOverride, err := corePP.PostProcessor.PostProcess(ctx, ppUi, priorArtifact) ts.End(err) if err != nil { diff --git a/packer/build_test.go b/packer/build_test.go index 4ae5f2537..86531d8ca 100644 --- a/packer/build_test.go +++ b/packer/build_test.go @@ -12,6 +12,7 @@ import ( packersdk "github.com/hashicorp/packer-plugin-sdk/packer" "github.com/hashicorp/packer-plugin-sdk/packerbuilderdata" "github.com/hashicorp/packer/version" + "github.com/zclconf/go-cty/cty" ) func boolPointer(tf bool) *bool { @@ -35,7 +36,7 @@ func testBuild() *CoreBuild { }, PostProcessors: [][]CoreBuildPostProcessor{ { - {&MockPostProcessor{ArtifactId: "pp"}, "testPP", "testPPName", make(map[string]interface{}), boolPointer(true)}, + {&MockPostProcessor{ArtifactId: "pp"}, "testPP", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(true)}, }, }, Variables: make(map[string]string), @@ -305,7 +306,7 @@ func TestBuild_Run_Artifacts(t *testing.T) { build = testBuild() build.PostProcessors = [][]CoreBuildPostProcessor{ { - {&MockPostProcessor{ArtifactId: "pp"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false)}, + {&MockPostProcessor{ArtifactId: "pp"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false)}, }, } @@ -330,10 +331,10 @@ func TestBuild_Run_Artifacts(t *testing.T) { build = testBuild() build.PostProcessors = [][]CoreBuildPostProcessor{ { - {&MockPostProcessor{ArtifactId: "pp1"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false)}, + {&MockPostProcessor{ArtifactId: "pp1"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false)}, }, { - {&MockPostProcessor{ArtifactId: "pp2"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(true)}, + {&MockPostProcessor{ArtifactId: "pp2"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(true)}, }, } @@ -358,12 +359,12 @@ func TestBuild_Run_Artifacts(t *testing.T) { build = testBuild() build.PostProcessors = [][]CoreBuildPostProcessor{ { - {&MockPostProcessor{ArtifactId: "pp1a"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false)}, - {&MockPostProcessor{ArtifactId: "pp1b"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(true)}, + {&MockPostProcessor{ArtifactId: "pp1a"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false)}, + {&MockPostProcessor{ArtifactId: "pp1b"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(true)}, }, { - {&MockPostProcessor{ArtifactId: "pp2a"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false)}, - {&MockPostProcessor{ArtifactId: "pp2b"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false)}, + {&MockPostProcessor{ArtifactId: "pp2a"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false)}, + {&MockPostProcessor{ArtifactId: "pp2b"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false)}, }, } @@ -389,7 +390,7 @@ func TestBuild_Run_Artifacts(t *testing.T) { build.PostProcessors = [][]CoreBuildPostProcessor{ { { - &MockPostProcessor{ArtifactId: "pp", Keep: true, ForceOverride: true}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false), + &MockPostProcessor{ArtifactId: "pp", Keep: true, ForceOverride: true}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false), }, }, } @@ -417,7 +418,7 @@ func TestBuild_Run_Artifacts(t *testing.T) { build.PostProcessors = [][]CoreBuildPostProcessor{ { { - &MockPostProcessor{ArtifactId: "pp", Keep: true, ForceOverride: false}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false), + &MockPostProcessor{ArtifactId: "pp", Keep: true, ForceOverride: false}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false), }, }, } @@ -444,7 +445,7 @@ func TestBuild_Run_Artifacts(t *testing.T) { build.PostProcessors = [][]CoreBuildPostProcessor{ { { - &MockPostProcessor{ArtifactId: "pp", Keep: true, ForceOverride: false}, "pp", "testPPName", make(map[string]interface{}), nil, + &MockPostProcessor{ArtifactId: "pp", Keep: true, ForceOverride: false}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), nil, }, }, } diff --git a/packer/telemetry.go b/packer/telemetry.go index 2dd7a6f2b..9148ea9f1 100644 --- a/packer/telemetry.go +++ b/packer/telemetry.go @@ -5,6 +5,7 @@ package packer import ( "context" + "fmt" "log" "os" "path/filepath" @@ -14,6 +15,7 @@ import ( checkpoint "github.com/hashicorp/go-checkpoint" "github.com/hashicorp/packer-plugin-sdk/pathing" packerVersion "github.com/hashicorp/packer/version" + "github.com/zclconf/go-cty/cty" ) const TelemetryVersion string = "beta/packer/5" @@ -157,22 +159,66 @@ func flattenConfigKeys(options interface{}) []string { var flatten func(string, interface{}) []string flatten = func(prefix string, options interface{}) (strOpts []string) { - if m, ok := options.(map[string]interface{}); ok { - for k, v := range m { - if prefix != "" { - k = prefix + "/" + k - } - if n, ok := v.(map[string]interface{}); ok { - strOpts = append(strOpts, flatten(k, n)...) - } else { - strOpts = append(strOpts, k) - } - } + switch opt := options.(type) { + case map[string]interface{}: + return flattenJSON(prefix, options) + case cty.Value: + return flattenHCL(prefix, opt) + default: + return nil } - return } flattened := flatten("", options) sort.Strings(flattened) return flattened } + +func flattenJSON(prefix string, options interface{}) (strOpts []string) { + if m, ok := options.(map[string]interface{}); ok { + for k, v := range m { + if prefix != "" { + k = prefix + "/" + k + } + if n, ok := v.(map[string]interface{}); ok { + strOpts = append(strOpts, flattenJSON(k, n)...) + } else { + strOpts = append(strOpts, k) + } + } + } + return +} + +func flattenHCL(prefix string, v cty.Value) (args []string) { + if v.IsNull() { + return []string{} + } + t := v.Type() + switch { + case t.IsObjectType(), t.IsMapType(): + if !v.IsKnown() { + return []string{} + } + it := v.ElementIterator() + for it.Next() { + key, val := it.Element() + keyStr := key.AsString() + + if val.IsNull() { + continue + } + + if prefix != "" { + keyStr = fmt.Sprintf("%s/%s", prefix, keyStr) + } + + if val.Type().IsObjectType() || val.Type().IsMapType() { + args = append(args, flattenHCL(keyStr, val)...) + } else { + args = append(args, keyStr) + } + } + } + return args +}