aherman
/
packer
mirror of https://github.com/hashicorp/packer
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

packer: add HCL2 support for telemetry

Browse Source 
Packer's telemetry package captures the field names from a configuration
on legacy JSON templates, but did not contain any code for handling HCL2
template configurations.

This commit adds a routine to extract the field names from a HCL2
configuration once flattened, and adds some glue code to propagate the
configurations to the telemetry structures.
pull/12374/head
Lucas Bajolet 3 years ago committed by Lucas Bajolet
parent 26af7901fe
commit f1a41b488f
5 changed files with 140 additions and 33 deletions
Show Stats Download Patch File Download Diff File

9
hcl2template/common_test.go
Unescape View File

@ -367,6 +367,15 @@ var cmpOpts = []cmp.Option{
cmpopts.IgnoreFields(VariableAssignment{},
"Expr", // its an interface
),
cmpopts.IgnoreFields(packer.CoreBuild{},
"HCLConfig",
),
cmpopts.IgnoreFields(packer.CoreBuildProvisioner{},
"HCLConfig",
),
cmpopts.IgnoreFields(packer.CoreBuildPostProcessor{},
"HCLConfig",
),
cmpopts.IgnoreTypes(hcl2template.MockBuilder{}),
cmpopts.IgnoreTypes(HCL2Ref{}),
cmpopts.IgnoreTypes([]*LocalBlock{}),

15
hcl2template/types.packer_config.go
Unescape View File

@ -362,7 +362,10 @@ func (cfg *PackerConfig) evaluateDatasources(skipExecution bool) hcl.Diagnostics
continue
}
dsOpts, _ := decodeHCL2Spec(body, cfg.EvalContext(DatasourceContext, nil), datasource)
sp := packer.CheckpointReporter.AddSpan(ref.Type, "datasource", dsOpts)
realValue, err := datasource.Execute()
sp.End(err)
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Summary: err.Error(),
@ -444,7 +447,10 @@ func (cfg *PackerConfig) recursivelyEvaluateDatasources(ref DatasourceRef, depen
return dependencies, diags, shouldContinue
}
opts, _ := decodeHCL2Spec(ds.block.Body, cfg.EvalContext(DatasourceContext, nil), datasource)
sp := packer.CheckpointReporter.AddSpan(ref.Type, "datasource", opts)
realValue, err := datasource.Execute()
sp.End(err)
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Summary: err.Error(),
@ -489,6 +495,8 @@ func (cfg *PackerConfig) getCoreBuildProvisioner(source SourceUseBlock, pb *Prov
return packer.CoreBuildProvisioner{}, diags
}
flatProvisionerCfg, _ := decodeHCL2Spec(pb.HCL2Ref.Rest, ectx, provisioner)
// If we're pausing, we wrap the provisioner in a special pauser.
if pb.PauseBefore != 0 {
provisioner = &packer.PausedProvisioner{
@ -512,6 +520,7 @@ func (cfg *PackerConfig) getCoreBuildProvisioner(source SourceUseBlock, pb *Prov
PType: pb.PType,
PName: pb.PName,
Provisioner: provisioner,
HCLConfig: flatProvisionerCfg,
}, diags
}
@ -550,10 +559,13 @@ func (cfg *PackerConfig) getCoreBuildPostProcessors(source SourceUseBlock, block
continue
}
flatPostProcessorCfg, moreDiags := decodeHCL2Spec(ppb.HCL2Ref.Rest, ectx, postProcessor)
pps = append(pps, packer.CoreBuildPostProcessor{
PostProcessor: postProcessor,
PName: ppb.PName,
PType: ppb.PType,
HCLConfig: flatPostProcessorCfg,
KeepInputArtifact: ppb.KeepInputArtifact,
})
}
@ -656,6 +668,9 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packersdk.Bu
continue
}
decoded, _ := decodeHCL2Spec(srcUsage.Body, cfg.EvalContext(BuildContext, nil), builder)
pcb.HCLConfig = decoded
// If the builder has provided a list of to-be-generated variables that
// should be made accessible to provisioners, pass that list into
// the provisioner prepare() so that the provisioner can appropriately

56
packer/build.go
Unescape View File

@ -13,6 +13,7 @@ import (
packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
"github.com/hashicorp/packer-plugin-sdk/packerbuilderdata"
"github.com/hashicorp/packer/version"
"github.com/zclconf/go-cty/cty"
)
// A CoreBuild struct represents a single build job, the result of which should
@ -20,10 +21,19 @@ import (
// multiple files, of course, but it should be for only a single provider (such
// as VirtualBox, EC2, etc.).
type CoreBuild struct {
BuildName string
Type string
Builder packersdk.Builder
BuilderConfig interface{}
BuildName string
Type string
Builder packersdk.Builder
// BuilderConfig is the config for the builder.
//
// Is is deserialised directly from the JSON template,
// and is only populated for legacy JSON templates.
BuilderConfig interface{}
// HCLConfig is the HCL config for the builder
//
// Its only use is for telemetry, since we use it to extract the
// field names from it.
HCLConfig cty.Value
BuilderType string
hooks map[string][]packersdk.Hook
Provisioners []CoreBuildProvisioner
@ -45,9 +55,16 @@ type CoreBuild struct {
// CoreBuildPostProcessor Keeps track of the post-processor and the
// configuration of the post-processor used within a build.
type CoreBuildPostProcessor struct {
PostProcessor packersdk.PostProcessor
PType string
PName string
PostProcessor packersdk.PostProcessor
PType string
PName string
// HCLConfig is the HCL config for the post-processor
//
// Its only use is for telemetry, since we use it to extract the
// field names from it.
HCLConfig cty.Value
// config is JSON-specific, the configuration for the post-processor
// deserialised directly from the JSON template
config map[string]interface{}
KeepInputArtifact *bool
}
@ -58,7 +75,14 @@ type CoreBuildProvisioner struct {
PType string
PName string
Provisioner packersdk.Provisioner
config []interface{}
// HCLConfig is the HCL config for the provisioner
//
// Its only use is for telemetry, since we use it to extract the
// field names from it.
HCLConfig cty.Value
// config is JSON-specific, and is the configuration of the
// provisioner, with overrides
config []interface{}
}
// Returns the name of the build.
@ -176,6 +200,8 @@ func (b *CoreBuild) Run(ctx context.Context, originalUi packersdk.Ui) ([]packers
var pConfig interface{}
if len(p.config) > 0 {
pConfig = p.config[0]
} else {
pConfig = p.HCLConfig
}
if b.debug {
hookedProvisioners[i] = &HookedProvisioner{
@ -221,8 +247,13 @@ func (b *CoreBuild) Run(ctx context.Context, originalUi packersdk.Ui) ([]packers
Ui: originalUi,
}
var ts *TelemetrySpan
log.Printf("Running builder: %s", b.BuilderType)
ts := CheckpointReporter.AddSpan(b.Type, "builder", b.BuilderConfig)
if b.BuilderConfig != nil {
ts = CheckpointReporter.AddSpan(b.Type, "builder", b.BuilderConfig)
} else {
ts = CheckpointReporter.AddSpan(b.Type, "builder", b.HCLConfig)
}
builderArtifact, err := b.Builder.Run(ctx, builderUi, hook)
ts.End(err)
if err != nil {
@ -260,7 +291,12 @@ PostProcessorRunSeqLoop:
} else {
builderUi.Say(fmt.Sprintf("Running post-processor: %s (type %s)", corePP.PName, corePP.PType))
}
ts := CheckpointReporter.AddSpan(corePP.PType, "post-processor", corePP.config)
var ts *TelemetrySpan
if corePP.config != nil {
ts = CheckpointReporter.AddSpan(corePP.PType, "post-processor", corePP.config)
} else {
ts = CheckpointReporter.AddSpan(corePP.PType, "post-processor", corePP.HCLConfig)
}
artifact, defaultKeep, forceOverride, err := corePP.PostProcessor.PostProcess(ctx, ppUi, priorArtifact)
ts.End(err)
if err != nil {

23
packer/build_test.go
Unescape View File

@ -12,6 +12,7 @@ import (
packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
"github.com/hashicorp/packer-plugin-sdk/packerbuilderdata"
"github.com/hashicorp/packer/version"
"github.com/zclconf/go-cty/cty"
)
func boolPointer(tf bool) *bool {
@ -35,7 +36,7 @@ func testBuild() *CoreBuild {
},
PostProcessors: [][]CoreBuildPostProcessor{
{
{&MockPostProcessor{ArtifactId: "pp"}, "testPP", "testPPName", make(map[string]interface{}), boolPointer(true)},
{&MockPostProcessor{ArtifactId: "pp"}, "testPP", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(true)},
},
},
Variables: make(map[string]string),
@ -305,7 +306,7 @@ func TestBuild_Run_Artifacts(t *testing.T) {
build = testBuild()
build.PostProcessors = [][]CoreBuildPostProcessor{
{
{&MockPostProcessor{ArtifactId: "pp"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false)},
{&MockPostProcessor{ArtifactId: "pp"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false)},
},
}
@ -330,10 +331,10 @@ func TestBuild_Run_Artifacts(t *testing.T) {
build = testBuild()
build.PostProcessors = [][]CoreBuildPostProcessor{
{
{&MockPostProcessor{ArtifactId: "pp1"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false)},
{&MockPostProcessor{ArtifactId: "pp1"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false)},
},
{
{&MockPostProcessor{ArtifactId: "pp2"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(true)},
{&MockPostProcessor{ArtifactId: "pp2"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(true)},
},
}
@ -358,12 +359,12 @@ func TestBuild_Run_Artifacts(t *testing.T) {
build = testBuild()
build.PostProcessors = [][]CoreBuildPostProcessor{
{
{&MockPostProcessor{ArtifactId: "pp1a"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false)},
{&MockPostProcessor{ArtifactId: "pp1b"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(true)},
{&MockPostProcessor{ArtifactId: "pp1a"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false)},
{&MockPostProcessor{ArtifactId: "pp1b"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(true)},
},
{
{&MockPostProcessor{ArtifactId: "pp2a"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false)},
{&MockPostProcessor{ArtifactId: "pp2b"}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false)},
{&MockPostProcessor{ArtifactId: "pp2a"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false)},
{&MockPostProcessor{ArtifactId: "pp2b"}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false)},
},
}
@ -389,7 +390,7 @@ func TestBuild_Run_Artifacts(t *testing.T) {
build.PostProcessors = [][]CoreBuildPostProcessor{
{
{
&MockPostProcessor{ArtifactId: "pp", Keep: true, ForceOverride: true}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false),
&MockPostProcessor{ArtifactId: "pp", Keep: true, ForceOverride: true}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false),
},
},
}
@ -417,7 +418,7 @@ func TestBuild_Run_Artifacts(t *testing.T) {
build.PostProcessors = [][]CoreBuildPostProcessor{
{
{
&MockPostProcessor{ArtifactId: "pp", Keep: true, ForceOverride: false}, "pp", "testPPName", make(map[string]interface{}), boolPointer(false),
&MockPostProcessor{ArtifactId: "pp", Keep: true, ForceOverride: false}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), boolPointer(false),
},
},
}
@ -444,7 +445,7 @@ func TestBuild_Run_Artifacts(t *testing.T) {
build.PostProcessors = [][]CoreBuildPostProcessor{
{
{
&MockPostProcessor{ArtifactId: "pp", Keep: true, ForceOverride: false}, "pp", "testPPName", make(map[string]interface{}), nil,
&MockPostProcessor{ArtifactId: "pp", Keep: true, ForceOverride: false}, "pp", "testPPName", cty.Value{}, make(map[string]interface{}), nil,
},
},
}

70
packer/telemetry.go
Unescape View File

@ -5,6 +5,7 @@ package packer
import (
"context"
"fmt"
"log"
"os"
"path/filepath"
@ -14,6 +15,7 @@ import (
checkpoint "github.com/hashicorp/go-checkpoint"
"github.com/hashicorp/packer-plugin-sdk/pathing"
packerVersion "github.com/hashicorp/packer/version"
"github.com/zclconf/go-cty/cty"
)
const TelemetryVersion string = "beta/packer/5"
@ -157,22 +159,66 @@ func flattenConfigKeys(options interface{}) []string {
var flatten func(string, interface{}) []string
flatten = func(prefix string, options interface{}) (strOpts []string) {
if m, ok := options.(map[string]interface{}); ok {
for k, v := range m {
if prefix != "" {
k = prefix + "/" + k
}
if n, ok := v.(map[string]interface{}); ok {
strOpts = append(strOpts, flatten(k, n)...)
} else {
strOpts = append(strOpts, k)
}
}
switch opt := options.(type) {
case map[string]interface{}:
return flattenJSON(prefix, options)
case cty.Value:
return flattenHCL(prefix, opt)
default:
return nil
}
return
}
flattened := flatten("", options)
sort.Strings(flattened)
return flattened
}
func flattenJSON(prefix string, options interface{}) (strOpts []string) {
if m, ok := options.(map[string]interface{}); ok {
for k, v := range m {
if prefix != "" {
k = prefix + "/" + k
}
if n, ok := v.(map[string]interface{}); ok {
strOpts = append(strOpts, flattenJSON(k, n)...)
} else {
strOpts = append(strOpts, k)
}
}
}
return
}
func flattenHCL(prefix string, v cty.Value) (args []string) {
if v.IsNull() {
return []string{}
}
t := v.Type()
switch {
case t.IsObjectType(), t.IsMapType():
if !v.IsKnown() {
return []string{}
}
it := v.ElementIterator()
for it.Next() {
key, val := it.Element()
keyStr := key.AsString()
if val.IsNull() {
continue
}
if prefix != "" {
keyStr = fmt.Sprintf("%s/%s", prefix, keyStr)
}
if val.Type().IsObjectType() || val.Type().IsMapType() {
args = append(args, flattenHCL(keyStr, val)...)
} else {
args = append(args, keyStr)
}
}
}
return args
}

Write Preview
Loading…
Cancel
Save