aherman
/
packer
mirror of https://github.com/hashicorp/packer
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Backport Build Metadata Phase 1 Release (#12910)

Browse Source 
* Packer tracks Version and Plugins Metadata (#12860)

* HPR-1537: Packer Core sends metadata to HCP Packer in UpdateBuild API (#12891)

* Use latest version of hcp-sdk-go

Remove loggers and move AddMetadataToBuild method to Version

* Update hcp-sdk-go
pull/12922/head
Devashish 2 years ago committed by GitHub
parent 134ea09518
commit 4560d79a41
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 179 additions and 28 deletions
Show Stats Download Patch File Download Diff File

2
go.mod
Unescape View File

@ -22,7 +22,7 @@ require (
github.com/hashicorp/go-uuid v1.0.3
github.com/hashicorp/go-version v1.6.0
github.com/hashicorp/hcl/v2 v2.19.1
github.com/hashicorp/hcp-sdk-go v0.85.0
github.com/hashicorp/hcp-sdk-go v0.91.0
github.com/hashicorp/packer-plugin-amazon v1.2.1
github.com/hashicorp/packer-plugin-sdk v0.5.2
github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869

4
go.sum
Unescape View File

@ -299,8 +299,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/hcl/v2 v2.19.1 h1://i05Jqznmb2EXqa39Nsvyan2o5XyMowW5fnCKW5RPI=
github.com/hashicorp/hcl/v2 v2.19.1/go.mod h1:ThLC89FV4p9MPW804KVbe/cEXoQ8NZEh+JtMeeGErHE=
github.com/hashicorp/hcp-sdk-go v0.85.0 h1:RDXpIf4gIGfz6je8Qq0FGg+8kwj7KboQTOxnklWyVJk=
github.com/hashicorp/hcp-sdk-go v0.85.0/go.mod h1:vQ4fzdL1AmhIAbCw+4zmFe5Hbpajj3NvRWkJoVuxmAk=
github.com/hashicorp/hcp-sdk-go v0.91.0 h1:XFJuB/KWP7kh6u3c/ruUy/lSA272vRrHJ1kzusMx9zw=
github.com/hashicorp/hcp-sdk-go v0.91.0/go.mod h1:vQ4fzdL1AmhIAbCw+4zmFe5Hbpajj3NvRWkJoVuxmAk=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc=
github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM=

3
hcl2template/types.build.hcp_packer_registry_test.go
Unescape View File

@ -57,6 +57,7 @@ func Test_ParseHCPPackerRegistryBlock(t *testing.T) {
Builder: emptyMockBuilder,
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},
BuilderType: "virtualbox-iso",
},
},
false,
@ -110,6 +111,7 @@ func Test_ParseHCPPackerRegistryBlock(t *testing.T) {
Builder: emptyMockBuilder,
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},
BuilderType: "virtualbox-iso",
},
},
false,
@ -241,6 +243,7 @@ func Test_ParseHCPPackerRegistryBlock(t *testing.T) {
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},
Prepared: true,
BuilderType: "null",
},
},
false,

34
hcl2template/types.build_test.go
Unescape View File

@ -283,6 +283,7 @@ func TestParse_build(t *testing.T) {
[]packersdk.Build{
&packer.CoreBuild{
Type: "virtualbox-iso.ubuntu-1204",
BuilderType: "virtualbox-iso",
Prepared: true,
Builder: emptyMockBuilder,
Provisioners: []packer.CoreBuildProvisioner{},
@ -317,6 +318,7 @@ func TestParse_build(t *testing.T) {
},
&packer.CoreBuild{
Type: "amazon-ebs.aws-ubuntu-16.04",
BuilderType: "amazon-ebs",
Prepared: true,
Builder: emptyMockBuilder,
Provisioners: []packer.CoreBuildProvisioner{},
@ -397,9 +399,10 @@ func TestParse_build(t *testing.T) {
false, false,
[]packersdk.Build{
&packer.CoreBuild{
Type: "virtualbox-iso.ubuntu-1204",
Prepared: true,
Builder: emptyMockBuilder,
Type: "virtualbox-iso.ubuntu-1204",
BuilderType: "virtualbox-iso",
Prepared: true,
Builder: emptyMockBuilder,
Provisioners: []packer.CoreBuildProvisioner{
{
PType: "shell",
@ -427,9 +430,10 @@ func TestParse_build(t *testing.T) {
PostProcessors: [][]packer.CoreBuildPostProcessor{},
},
&packer.CoreBuild{
Type: "amazon-ebs.aws-ubuntu-16.04",
Prepared: true,
Builder: emptyMockBuilder,
Type: "amazon-ebs.aws-ubuntu-16.04",
BuilderType: "amazon-ebs",
Prepared: true,
Builder: emptyMockBuilder,
Provisioners: []packer.CoreBuildProvisioner{
{
PType: "file",
@ -486,9 +490,10 @@ func TestParse_build(t *testing.T) {
false, false,
[]packersdk.Build{
&packer.CoreBuild{
Type: "virtualbox-iso.ubuntu-1204",
Prepared: true,
Builder: emptyMockBuilder,
Type: "virtualbox-iso.ubuntu-1204",
BuilderType: "virtualbox-iso",
Prepared: true,
Builder: emptyMockBuilder,
Provisioners: []packer.CoreBuildProvisioner{
{
PType: "shell",
@ -550,6 +555,7 @@ func TestParse_build(t *testing.T) {
&packer.CoreBuild{
BuildName: "build-name",
Type: "virtualbox-iso.ubuntu-1204",
BuilderType: "virtualbox-iso",
Prepared: true,
Builder: emptyMockBuilder,
Provisioners: []packer.CoreBuildProvisioner{},
@ -604,6 +610,7 @@ func TestParse_build(t *testing.T) {
&packer.CoreBuild{
BuildName: "test-build",
Type: "virtualbox-iso.ubuntu-1204",
BuilderType: "virtualbox-iso",
Prepared: true,
Builder: emptyMockBuilder,
Provisioners: []packer.CoreBuildProvisioner{},
@ -659,10 +666,11 @@ func TestParse_build(t *testing.T) {
false, false,
[]packersdk.Build{
&packer.CoreBuild{
BuildName: "build-name-test",
Type: "virtualbox-iso.ubuntu-1204",
Prepared: true,
Builder: emptyMockBuilder,
BuildName: "build-name-test",
Type: "virtualbox-iso.ubuntu-1204",
BuilderType: "virtualbox-iso",
Prepared: true,
Builder: emptyMockBuilder,
Provisioners: []packer.CoreBuildProvisioner{
{
PName: "build-name-test",

2
hcl2template/types.datasource_test.go
Unescape View File

@ -57,6 +57,7 @@ func TestParse_datasource(t *testing.T) {
[]packersdk.Build{
&packer.CoreBuild{
Type: "null.test",
BuilderType: "null",
Builder: &null.Builder{},
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},
@ -134,6 +135,7 @@ func TestParse_datasource(t *testing.T) {
[]packersdk.Build{
&packer.CoreBuild{
Type: "null.test",
BuilderType: "null",
Builder: &null.Builder{},
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},

1
hcl2template/types.packer_config.go
Unescape View File

@ -622,6 +622,7 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packersdk.Bu
decoded, _ := decodeHCL2Spec(srcUsage.Body, cfg.EvalContext(BuildContext, nil), builder)
pcb.HCLConfig = decoded
pcb.BuilderType = srcUsage.Type
// If the builder has provided a list of to-be-generated variables that
// should be made accessible to provisioners, pass that list into

10
hcl2template/types.packer_config_test.go
Unescape View File

@ -208,8 +208,9 @@ func TestParser_complete(t *testing.T) {
false, false,
[]packersdk.Build{
&packer.CoreBuild{
Type: "virtualbox-iso.ubuntu-1204",
Prepared: true,
Type: "virtualbox-iso.ubuntu-1204",
BuilderType: "virtualbox-iso",
Prepared: true,
Builder: &MockBuilder{
Config: MockConfig{
NestedMockConfig: NestedMockConfig{
@ -319,8 +320,9 @@ func TestParser_complete(t *testing.T) {
},
},
&packer.CoreBuild{
Type: "amazon-ebs.ubuntu-1604",
Prepared: true,
Type: "amazon-ebs.ubuntu-1604",
BuilderType: "amazon-ebs",
Prepared: true,
Builder: &MockBuilder{
Config: MockConfig{
NestedMockConfig: NestedMockConfig{

1
hcl2template/types.source_test.go
Unescape View File

@ -55,6 +55,7 @@ func TestParse_source(t *testing.T) {
[]packersdk.Build{
&packer.CoreBuild{
Type: "null.test",
BuilderType: "null",
Builder: &null.Builder{},
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},

13
hcl2template/types.variables_test.go
Unescape View File

@ -129,6 +129,7 @@ func TestParse_variables(t *testing.T) {
[]packersdk.Build{
&packer.CoreBuild{
Type: "null.test",
BuilderType: "null",
Builder: &null.Builder{},
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},
@ -293,6 +294,7 @@ func TestParse_variables(t *testing.T) {
[]packersdk.Build{
&packer.CoreBuild{
Type: "null",
BuilderType: "null",
Builder: &null.Builder{},
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},
@ -379,6 +381,7 @@ func TestParse_variables(t *testing.T) {
[]packersdk.Build{
&packer.CoreBuild{
Type: "null.test",
BuilderType: "null",
Builder: &null.Builder{},
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},
@ -442,6 +445,7 @@ func TestParse_variables(t *testing.T) {
[]packersdk.Build{
&packer.CoreBuild{
Type: "null.test",
BuilderType: "null",
Builder: &null.Builder{},
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},
@ -483,6 +487,7 @@ func TestParse_variables(t *testing.T) {
[]packersdk.Build{
&packer.CoreBuild{
Type: "null.test",
BuilderType: "null",
Builder: &null.Builder{},
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},
@ -538,9 +543,10 @@ func TestParse_variables(t *testing.T) {
},
false, false,
[]packersdk.Build{&packer.CoreBuild{
Type: "null.null-builder",
Prepared: true,
Builder: &null.Builder{},
Type: "null.null-builder",
BuilderType: "null",
Prepared: true,
Builder: &null.Builder{},
Provisioners: []packer.CoreBuildProvisioner{
{
PType: "shell",
@ -627,6 +633,7 @@ func TestParse_variables(t *testing.T) {
[]packersdk.Build{
&packer.CoreBuild{
Type: "null.test",
BuilderType: "null",
Builder: &null.Builder{},
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},

2
internal/hcp/api/service_build.go
Unescape View File

@ -58,6 +58,7 @@ func (c *Client) UpdateBuild(
buildLabels map[string]string,
buildStatus hcpPackerModels.HashicorpCloudPacker20230101BuildStatus,
artifacts []*hcpPackerModels.HashicorpCloudPacker20230101ArtifactCreateBody,
metadata *hcpPackerModels.HashicorpCloudPacker20230101BuildMetadata,
) (string, error) {
params := hcpPackerAPI.NewPackerServiceUpdateBuildParamsWithContext(ctx)
@ -76,6 +77,7 @@ func (c *Client) UpdateBuild(
Platform: platform,
SourceExternalIdentifier: sourceExternalIdentifier,
Status: &buildStatus,
Metadata: metadata,
}
resp, err := c.Packer.PackerServiceUpdateBuild(params, nil)

18
internal/hcp/registry/hcl.go
Unescape View File

@ -70,6 +70,12 @@ func (h *HCLRegistry) StartBuild(ctx context.Context, build sdkpacker.Build) err
if ok {
name = cb.Type
}
metadata := cb.GetMetadata()
err := h.bucket.Version.AddMetadataToBuild(ctx, name, metadata)
if err != nil {
return err
}
return h.bucket.startBuild(ctx, name)
}
@ -80,12 +86,18 @@ func (h *HCLRegistry) CompleteBuild(
artifacts []sdkpacker.Artifact,
buildErr error,
) ([]sdkpacker.Artifact, error) {
name := build.Name()
buildName := build.Name()
cb, ok := build.(*packer.CoreBuild)
if ok {
name = cb.Type
buildName = cb.Type
}
buildMetadata := cb.GetMetadata()
err := h.bucket.Version.AddMetadataToBuild(ctx, buildName, buildMetadata)
if err != nil {
return nil, err
}
return h.bucket.completeBuild(ctx, name, artifacts, buildErr)
return h.bucket.completeBuild(ctx, buildName, artifacts, buildErr)
}
// VersionStatusSummary prints a status report in the UI if the version is not yet done

17
internal/hcp/registry/json.go
Unescape View File

@ -83,7 +83,14 @@ func (h *JSONRegistry) PopulateVersion(ctx context.Context) error {
// StartBuild is invoked when one build for the configuration is starting to be processed
func (h *JSONRegistry) StartBuild(ctx context.Context, build sdkpacker.Build) error {
return h.bucket.startBuild(ctx, build.Name())
name := build.Name()
metadata := build.(*packer.CoreBuild).GetMetadata()
err := h.bucket.Version.AddMetadataToBuild(ctx, name, metadata)
if err != nil {
return err
}
return h.bucket.startBuild(ctx, name)
}
// CompleteBuild is invoked when one build for the configuration has finished
@ -93,7 +100,13 @@ func (h *JSONRegistry) CompleteBuild(
artifacts []sdkpacker.Artifact,
buildErr error,
) ([]sdkpacker.Artifact, error) {
return h.bucket.completeBuild(ctx, build.Name(), artifacts, buildErr)
buildName := build.Name()
buildMetadata := build.(*packer.CoreBuild).GetMetadata()
err := h.bucket.Version.AddMetadataToBuild(ctx, buildName, buildMetadata)
if err != nil {
return nil, err
}
return h.bucket.completeBuild(ctx, buildName, artifacts, buildErr)
}
// VersionStatusSummary prints a status report in the UI if the version is not yet done

3
internal/hcp/registry/types.bucket.go
Unescape View File

@ -213,6 +213,7 @@ func (bucket *Bucket) UpdateBuildStatus(
nil,
status,
nil,
&buildToUpdate.Metadata,
)
if err != nil {
return err
@ -285,6 +286,7 @@ func (bucket *Bucket) markBuildComplete(ctx context.Context, name string) error
buildToUpdate.Labels,
status,
artifacts,
&buildToUpdate.Metadata,
)
if err != nil {
return err
@ -535,6 +537,7 @@ func (bucket *Bucket) HeartbeatBuild(ctx context.Context, build string) (func(),
nil,
hcpPackerModels.HashicorpCloudPacker20230101BuildStatusBUILDRUNNING,
nil,
nil,
)
if err != nil {
log.Printf("[ERROR] failed to send heartbeat for build %q: %s", build, err)

1
internal/hcp/registry/types.builds.go
Unescape View File

@ -19,6 +19,7 @@ type Build struct {
Labels map[string]string
Artifacts map[string]packerSDKRegistry.Image
Status hcpPackerModels.HashicorpCloudPacker20230101BuildStatus
Metadata hcpPackerModels.HashicorpCloudPacker20230101BuildMetadata
}
// NewBuildFromCloudPackerBuild converts a HashicorpCloudPackerBuild to a local build that can be tracked and

28
internal/hcp/registry/types.version.go
Unescape View File

@ -4,6 +4,7 @@
package registry
import (
"context"
"errors"
"fmt"
"math/rand"
@ -16,6 +17,7 @@ import (
sdkpacker "github.com/hashicorp/packer-plugin-sdk/packer"
packerSDKRegistry "github.com/hashicorp/packer-plugin-sdk/packer/registry/image"
"github.com/hashicorp/packer/internal/hcp/env"
"github.com/hashicorp/packer/packer"
"github.com/oklog/ulid"
)
@ -174,3 +176,29 @@ func (version *Version) statusSummary(ui sdkpacker.Ui) {
ui.Say(buf.String())
}
// AddMetadataToBuild adds metadata to a build in the HCP Packer registry.
func (version *Version) AddMetadataToBuild(
ctx context.Context, buildName string, metadata packer.BuildMetadata,
) error {
buildToUpdate, err := version.Build(buildName)
if err != nil {
return err
}
packerMetadata := make(map[string]interface{})
packerMetadata["version"] = metadata.PackerVersion
var pluginsMetadata []map[string]interface{}
for _, plugin := range metadata.Plugins {
pluginMetadata := map[string]interface{}{
"version": plugin.Description.Version,
"name": plugin.Name,
}
pluginsMetadata = append(pluginsMetadata, pluginMetadata)
}
packerMetadata["plugins"] = pluginsMetadata
buildToUpdate.Metadata.Packer = packerMetadata
return nil
}

40
packer/build.go
Unescape View File

@ -52,6 +52,46 @@ type CoreBuild struct {
prepareCalled bool
}
type BuildMetadata struct {
PackerVersion string
Plugins map[string]PluginDetails
}
func (b *CoreBuild) getPluginsMetadata() map[string]PluginDetails {
resp := map[string]PluginDetails{}
builderPlugin, builderPluginOk := PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentBuilder, b.BuilderType)]
if builderPluginOk {
resp[builderPlugin.Name] = builderPlugin
}
for _, pp := range b.PostProcessors {
for _, p := range pp {
postprocessorsPlugin, postprocessorsPluginOk := PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentPostProcessor, p.PType)]
if postprocessorsPluginOk {
resp[postprocessorsPlugin.Name] = postprocessorsPlugin
}
}
}
for _, pv := range b.Provisioners {
provisionerPlugin, provisionerPluginOk := PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentProvisioner, pv.PType)]
if provisionerPluginOk {
resp[provisionerPlugin.Name] = provisionerPlugin
}
}
return resp
}
func (b *CoreBuild) GetMetadata() BuildMetadata {
metadata := BuildMetadata{
PackerVersion: version.FormattedVersion(),
Plugins: b.getPluginsMetadata(),
}
return metadata
}
// CoreBuildPostProcessor Keeps track of the post-processor and the
// configuration of the post-processor used within a build.
type CoreBuildPostProcessor struct {

28
packer/plugin.go
Unescape View File

@ -264,6 +264,11 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error
}
pluginPrefix := pluginName + "-"
pluginDetails := PluginDetails{
Name: pluginName,
Description: desc,
PluginPath: pluginPath,
}
for _, builderName := range desc.Builders {
builderName := builderName // copy to avoid pointer overwrite issue
@ -274,6 +279,8 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error
c.Builders.Set(key, func() (packersdk.Builder, error) {
return c.Client(pluginPath, "start", "builder", builderName).Builder()
})
PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentBuilder, key)] = pluginDetails
}
if len(desc.Builders) > 0 {
@ -289,6 +296,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error
c.PostProcessors.Set(key, func() (packersdk.PostProcessor, error) {
return c.Client(pluginPath, "start", "post-processor", postProcessorName).PostProcessor()
})
PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentPostProcessor, key)] = pluginDetails
}
if len(desc.PostProcessors) > 0 {
@ -304,6 +312,8 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error
c.Provisioners.Set(key, func() (packersdk.Provisioner, error) {
return c.Client(pluginPath, "start", "provisioner", provisionerName).Provisioner()
})
PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentProvisioner, key)] = pluginDetails
}
if len(desc.Provisioners) > 0 {
log.Printf("found external %v provisioner from %s plugin", desc.Provisioners, pluginName)
@ -318,6 +328,7 @@ func (c *PluginConfig) DiscoverMultiPlugin(pluginName, pluginPath string) error
c.DataSources.Set(key, func() (packersdk.Datasource, error) {
return c.Client(pluginPath, "start", "datasource", datasourceName).Datasource()
})
PluginsDetailsStorage[fmt.Sprintf("%q-%q", PluginComponentDataSource, key)] = pluginDetails
}
if len(desc.Datasources) > 0 {
log.Printf("found external %v datasource from %s plugin", desc.Datasources, pluginName)
@ -423,3 +434,20 @@ func (c *PluginConfig) discoverInstalledComponents(path string) error {
return nil
}
type PluginComponentType string
const (
PluginComponentBuilder PluginComponentType = "builder"
PluginComponentPostProcessor PluginComponentType = "post-processor"
PluginComponentProvisioner PluginComponentType = "provisioner"
PluginComponentDataSource PluginComponentType = "data-source"
)
type PluginDetails struct {
Name string
Description pluginsdk.SetDescription
PluginPath string
}
var PluginsDetailsStorage = map[string]PluginDetails{}

Write Preview
Loading…
Cancel
Save