From 738238272734f36d3f7cf7d1b24ec826bdb21bbc Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Thu, 4 May 2017 16:29:21 -0700 Subject: [PATCH 1/7] Add telemetry reporting through checkpoint Will report builders/provisioner/post-processor types used per build, and whether or not the build passed. Will also report any panics we see. You may opt out of this reporting by setting the environment variable `CHECKPOINT_DISABLE`. --- main.go | 21 ++- packer/build.go | 11 +- packer/core.go | 1 + packer/provisioner.go | 9 +- packer/telemetry.go | 133 ++++++++++++++++++ panic.go | 5 + .../aws-sdk-go/private/endpoints/endpoints.go | 70 +++++++++ .../private/endpoints/endpoints.json | 78 ++++++++++ .../private/endpoints/endpoints_map.go | 91 ++++++++++++ .../hashicorp/go-checkpoint/README.md | 4 +- .../hashicorp/go-checkpoint/checkpoint.go | 105 ++++++++++++++ vendor/vendor.json | 5 +- 12 files changed, 520 insertions(+), 13 deletions(-) create mode 100644 packer/telemetry.go create mode 100644 vendor/github.com/aws/aws-sdk-go/private/endpoints/endpoints.go create mode 100644 vendor/github.com/aws/aws-sdk-go/private/endpoints/endpoints.json create mode 100644 vendor/github.com/aws/aws-sdk-go/private/endpoints/endpoints_map.go diff --git a/main.go b/main.go index 6bca18d2d..43ae94f4a 100644 --- a/main.go +++ b/main.go @@ -73,6 +73,14 @@ func realMain() int { outR, outW := io.Pipe() go copyOutput(outR, doneCh) + // Enable checkpoint for panic reporting + config, err := loadConfig() + if err == nil { + if !config.DisableCheckpoint { + packer.CheckpointReporter.Enable(config.DisableCheckpointSignature) + } + } + // Create the configuration for panicwrap and wrap our executable wrapConfig.Handler = panicHandler(logTempFile) wrapConfig.Writer = io.MultiWriter(logTempFile, logWriter) @@ -117,9 +125,11 @@ func wrappedMain() int { log.Printf("Packer Target OS/Arch: %s %s", runtime.GOOS, runtime.GOARCH) log.Printf("Built with Go Version: %s", runtime.Version()) + inPlugin := os.Getenv(plugin.MagicCookieKey) == plugin.MagicCookieValue + // Prepare stdin for plugin usage by switching it to a pipe // But do not switch to pipe in plugin - if os.Getenv(plugin.MagicCookieKey) != plugin.MagicCookieValue { + if !inPlugin { setupStdin() } @@ -132,6 +142,9 @@ func wrappedMain() int { // Fire off the checkpoint. go runCheckpoint(config) + if !config.DisableCheckpoint { + packer.CheckpointReporter.Enable(config.DisableCheckpointSignature) + } cacheDir := os.Getenv("PACKER_CACHE_DIR") if cacheDir == "" { @@ -196,6 +209,12 @@ func wrappedMain() int { } exitCode, err := cli.Run() + if !inPlugin { + if err := packer.CheckpointReporter.Finalize(cli.Subcommand(), exitCode, err); err != nil { + log.Printf("Error finalizing telemetry report. This is safe to ignore. %s", err.Error()) + } + } + if err != nil { fmt.Fprintf(os.Stderr, "Error executing CLI: %s\n", err) return 1 diff --git a/packer/build.go b/packer/build.go index d14024b77..40f6d38ab 100644 --- a/packer/build.go +++ b/packer/build.go @@ -115,6 +115,7 @@ type coreBuildPostProcessor struct { // Keeps track of the provisioner and the configuration of the provisioner // within the build. type coreBuildProvisioner struct { + pType string provisioner Provisioner config []interface{} } @@ -193,17 +194,13 @@ func (b *coreBuild) Run(originalUi Ui, cache Cache) ([]Artifact, error) { // Add a hook for the provisioners if we have provisioners if len(b.provisioners) > 0 { - provisioners := make([]Provisioner, len(b.provisioners)) - for i, p := range b.provisioners { - provisioners[i] = p.provisioner - } if _, ok := hooks[HookProvision]; !ok { hooks[HookProvision] = make([]Hook, 0, 1) } hooks[HookProvision] = append(hooks[HookProvision], &ProvisionHook{ - Provisioners: provisioners, + Provisioners: b.provisioners, }) } @@ -217,7 +214,9 @@ func (b *coreBuild) Run(originalUi Ui, cache Cache) ([]Artifact, error) { } log.Printf("Running builder: %s", b.builderType) + ts := CheckpointReporter.AddSpan(b.builderType, "builder") builderArtifact, err := b.builder.Run(builderUi, hook, cache) + ts.End(err) if err != nil { return nil, err } @@ -242,7 +241,9 @@ PostProcessorRunSeqLoop: } builderUi.Say(fmt.Sprintf("Running post-processor: %s", corePP.processorType)) + ts := CheckpointReporter.AddSpan(corePP.processorType, "post-processor") artifact, keep, err := corePP.processor.PostProcess(ppUi, priorArtifact) + ts.End(err) if err != nil { errors = append(errors, fmt.Errorf("Post-processor failed: %s", err)) continue PostProcessorRunSeqLoop diff --git a/packer/core.go b/packer/core.go index 4371fbbc4..7da3e4510 100644 --- a/packer/core.go +++ b/packer/core.go @@ -154,6 +154,7 @@ func (c *Core) Build(n string) (Build, error) { } provisioners = append(provisioners, coreBuildProvisioner{ + pType: rawP.Type, provisioner: provisioner, config: config, }) diff --git a/packer/provisioner.go b/packer/provisioner.go index f4f3fce11..1fa2d66dc 100644 --- a/packer/provisioner.go +++ b/packer/provisioner.go @@ -30,7 +30,7 @@ type Provisioner interface { type ProvisionHook struct { // The provisioners to run as part of the hook. These should already // be prepared (by calling Prepare) at some earlier stage. - Provisioners []Provisioner + Provisioners []coreBuildProvisioner lock sync.Mutex runningProvisioner Provisioner @@ -59,10 +59,13 @@ func (h *ProvisionHook) Run(name string, ui Ui, comm Communicator, data interfac for _, p := range h.Provisioners { h.lock.Lock() - h.runningProvisioner = p + h.runningProvisioner = p.provisioner h.lock.Unlock() - if err := p.Provision(ui, comm); err != nil { + ts := CheckpointReporter.AddSpan(p.pType, "provisioner") + err := p.provisioner.Provision(ui, comm) + ts.End(err) + if err != nil { return err } } diff --git a/packer/telemetry.go b/packer/telemetry.go new file mode 100644 index 000000000..9ff22ad22 --- /dev/null +++ b/packer/telemetry.go @@ -0,0 +1,133 @@ +package packer + +import ( + "fmt" + "log" + "os" + "path/filepath" + "time" + + checkpoint "github.com/hashicorp/go-checkpoint" + packerVersion "github.com/hashicorp/packer/version" +) + +const TelemetryVersion string = "beta/packer/4" +const TelemetryPanicVersion string = "beta/packer_panic/4" + +var CheckpointReporter CheckpointTelemetry + +func init() { + CheckpointReporter.startTime = time.Now().UTC() +} + +type PackerReport struct { + Spans []*TelemetrySpan `json:"spans"` + ExitCode int `json:"exit_code"` + Error string `json:"error"` + Command string `json:"command"` +} + +type CheckpointTelemetry struct { + enabled bool + spans []*TelemetrySpan + signatureFile string + startTime time.Time +} + +func (c *CheckpointTelemetry) Enable(disableSignature bool) { + configDir, err := ConfigDir() + if err != nil { + log.Printf("[ERR] Checkpoint telemetry setup error: %s", err) + return + } + + signatureFile := "" + if disableSignature { + log.Printf("[INFO] Checkpoint telemetry signature disabled") + } else { + signatureFile = filepath.Join(configDir, "checkpoint_signature") + } + + c.signatureFile = signatureFile + c.enabled = true +} + +func (c *CheckpointTelemetry) baseParams(prefix string) *checkpoint.ReportParams { + version := packerVersion.Version + if packerVersion.VersionPrerelease != "" { + version += fmt.Sprintf("-%s", packerVersion.VersionPrerelease) + } + + return &checkpoint.ReportParams{ + Product: "packer", + SchemaVersion: prefix, + StartTime: c.startTime, + Version: version, + RunID: os.Getenv("PACKER_RUN_UUID"), + SignatureFile: c.signatureFile, + } +} + +func (c *CheckpointTelemetry) log(m string, args ...interface{}) { +} + +func (c *CheckpointTelemetry) ReportPanic(m string) error { + if !c.enabled { + return nil + } + log.Printf("[TELEMETRY] Add panic: %s", m) + panicParams := c.baseParams(TelemetryPanicVersion) + panicParams.Payload = m + panicParams.EndTime = time.Now().UTC() + return checkpoint.Report(panicParams) +} + +func (c *CheckpointTelemetry) AddSpan(name, pluginType string) *TelemetrySpan { + log.Printf("[TELEMETRY] Starting %s %s", pluginType, name) + ts := &TelemetrySpan{ + Name: name, + Type: pluginType, + StartTime: time.Now().UTC(), + } + c.spans = append(c.spans, ts) + return ts +} + +func (c *CheckpointTelemetry) Finalize(command string, errCode int, err error) error { + if !c.enabled { + return nil + } + + log.Printf("[TELEMETRY] finalize: %#v", c) + params := c.baseParams(TelemetryVersion) + params.EndTime = time.Now().UTC() + + extra := &PackerReport{ + Spans: c.spans, + ExitCode: errCode, + Command: command, + } + if err != nil { + extra.Error = err.Error() + } + params.Payload = extra + + return checkpoint.Report(params) +} + +type TelemetrySpan struct { + Name string `json:"name"` + Type string `json:"type"` + StartTime time.Time `json:"start_time"` + EndTime time.Time `json:"end_time"` + Error string `json:"error"` +} + +func (s *TelemetrySpan) End(err error) { + s.EndTime = time.Now().UTC() + log.Printf("[TELEMETRY] ending %s", s.Name) + if err != nil { + s.Error = err.Error() + log.Printf("[TELEMETRY] ERROR: %s", err.Error()) + } +} diff --git a/panic.go b/panic.go index 480d04324..5d58c5685 100644 --- a/panic.go +++ b/panic.go @@ -6,6 +6,7 @@ import ( "os" "strings" + "github.com/hashicorp/packer/packer" "github.com/mitchellh/panicwrap" ) @@ -34,6 +35,10 @@ func panicHandler(logF *os.File) panicwrap.HandlerFunc { // shown in case anything below fails. fmt.Fprintf(os.Stderr, fmt.Sprintf("%s\n", m)) + if err := packer.CheckpointReporter.ReportPanic(m); err != nil { + fmt.Fprintf(os.Stderr, "Failed to report panic. This is safe to ignore: %s", err) + } + // Create the crash log file where we'll write the logs f, err := os.Create("crash.log") if err != nil { diff --git a/vendor/github.com/aws/aws-sdk-go/private/endpoints/endpoints.go b/vendor/github.com/aws/aws-sdk-go/private/endpoints/endpoints.go new file mode 100644 index 000000000..b4ad7405c --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go/private/endpoints/endpoints.go @@ -0,0 +1,70 @@ +// Package endpoints validates regional endpoints for services. +package endpoints + +//go:generate go run ../model/cli/gen-endpoints/main.go endpoints.json endpoints_map.go +//go:generate gofmt -s -w endpoints_map.go + +import ( + "fmt" + "regexp" + "strings" +) + +// NormalizeEndpoint takes and endpoint and service API information to return a +// normalized endpoint and signing region. If the endpoint is not an empty string +// the service name and region will be used to look up the service's API endpoint. +// If the endpoint is provided the scheme will be added if it is not present. +func NormalizeEndpoint(endpoint, serviceName, region string, disableSSL, useDualStack bool) (normEndpoint, signingRegion string) { + if endpoint == "" { + return EndpointForRegion(serviceName, region, disableSSL, useDualStack) + } + + return AddScheme(endpoint, disableSSL), "" +} + +// EndpointForRegion returns an endpoint and its signing region for a service and region. +// if the service and region pair are not found endpoint and signingRegion will be empty. +func EndpointForRegion(svcName, region string, disableSSL, useDualStack bool) (endpoint, signingRegion string) { + dualStackField := "" + if useDualStack { + dualStackField = "/dualstack" + } + + derivedKeys := []string{ + region + "/" + svcName + dualStackField, + region + "/*" + dualStackField, + "*/" + svcName + dualStackField, + "*/*" + dualStackField, + } + + for _, key := range derivedKeys { + if val, ok := endpointsMap.Endpoints[key]; ok { + ep := val.Endpoint + ep = strings.Replace(ep, "{region}", region, -1) + ep = strings.Replace(ep, "{service}", svcName, -1) + + endpoint = ep + signingRegion = val.SigningRegion + break + } + } + + return AddScheme(endpoint, disableSSL), signingRegion +} + +// Regular expression to determine if the endpoint string is prefixed with a scheme. +var schemeRE = regexp.MustCompile("^([^:]+)://") + +// AddScheme adds the HTTP or HTTPS schemes to a endpoint URL if there is no +// scheme. If disableSSL is true HTTP will be added instead of the default HTTPS. +func AddScheme(endpoint string, disableSSL bool) string { + if endpoint != "" && !schemeRE.MatchString(endpoint) { + scheme := "https" + if disableSSL { + scheme = "http" + } + endpoint = fmt.Sprintf("%s://%s", scheme, endpoint) + } + + return endpoint +} diff --git a/vendor/github.com/aws/aws-sdk-go/private/endpoints/endpoints.json b/vendor/github.com/aws/aws-sdk-go/private/endpoints/endpoints.json new file mode 100644 index 000000000..c5bf3c7c3 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go/private/endpoints/endpoints.json @@ -0,0 +1,78 @@ +{ + "version": 2, + "endpoints": { + "*/*": { + "endpoint": "{service}.{region}.amazonaws.com" + }, + "cn-north-1/*": { + "endpoint": "{service}.{region}.amazonaws.com.cn", + "signatureVersion": "v4" + }, + "cn-north-1/ec2metadata": { + "endpoint": "http://169.254.169.254/latest" + }, + "us-gov-west-1/iam": { + "endpoint": "iam.us-gov.amazonaws.com" + }, + "us-gov-west-1/sts": { + "endpoint": "sts.us-gov-west-1.amazonaws.com" + }, + "us-gov-west-1/s3": { + "endpoint": "s3-{region}.amazonaws.com" + }, + "us-gov-west-1/ec2metadata": { + "endpoint": "http://169.254.169.254/latest" + }, + "*/cloudfront": { + "endpoint": "cloudfront.amazonaws.com", + "signingRegion": "us-east-1" + }, + "*/cloudsearchdomain": { + "endpoint": "", + "signingRegion": "us-east-1" + }, + "*/data.iot": { + "endpoint": "", + "signingRegion": "us-east-1" + }, + "*/ec2metadata": { + "endpoint": "http://169.254.169.254/latest" + }, + "*/iam": { + "endpoint": "iam.amazonaws.com", + "signingRegion": "us-east-1" + }, + "*/importexport": { + "endpoint": "importexport.amazonaws.com", + "signingRegion": "us-east-1" + }, + "*/route53": { + "endpoint": "route53.amazonaws.com", + "signingRegion": "us-east-1" + }, + "*/sts": { + "endpoint": "sts.amazonaws.com", + "signingRegion": "us-east-1" + }, + "*/waf": { + "endpoint": "waf.amazonaws.com", + "signingRegion": "us-east-1" + }, + "us-east-1/sdb": { + "endpoint": "sdb.amazonaws.com", + "signingRegion": "us-east-1" + }, + "*/s3": { + "endpoint": "s3-{region}.amazonaws.com" + }, + "*/s3/dualstack": { + "endpoint": "s3.dualstack.{region}.amazonaws.com" + }, + "us-east-1/s3": { + "endpoint": "s3.amazonaws.com" + }, + "eu-central-1/s3": { + "endpoint": "{service}.{region}.amazonaws.com" + } + } +} diff --git a/vendor/github.com/aws/aws-sdk-go/private/endpoints/endpoints_map.go b/vendor/github.com/aws/aws-sdk-go/private/endpoints/endpoints_map.go new file mode 100644 index 000000000..a81d158c3 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go/private/endpoints/endpoints_map.go @@ -0,0 +1,91 @@ +package endpoints + +// THIS FILE IS AUTOMATICALLY GENERATED. DO NOT EDIT. + +type endpointStruct struct { + Version int + Endpoints map[string]endpointEntry +} + +type endpointEntry struct { + Endpoint string + SigningRegion string +} + +var endpointsMap = endpointStruct{ + Version: 2, + Endpoints: map[string]endpointEntry{ + "*/*": { + Endpoint: "{service}.{region}.amazonaws.com", + }, + "*/cloudfront": { + Endpoint: "cloudfront.amazonaws.com", + SigningRegion: "us-east-1", + }, + "*/cloudsearchdomain": { + Endpoint: "", + SigningRegion: "us-east-1", + }, + "*/data.iot": { + Endpoint: "", + SigningRegion: "us-east-1", + }, + "*/ec2metadata": { + Endpoint: "http://169.254.169.254/latest", + }, + "*/iam": { + Endpoint: "iam.amazonaws.com", + SigningRegion: "us-east-1", + }, + "*/importexport": { + Endpoint: "importexport.amazonaws.com", + SigningRegion: "us-east-1", + }, + "*/route53": { + Endpoint: "route53.amazonaws.com", + SigningRegion: "us-east-1", + }, + "*/s3": { + Endpoint: "s3-{region}.amazonaws.com", + }, + "*/s3/dualstack": { + Endpoint: "s3.dualstack.{region}.amazonaws.com", + }, + "*/sts": { + Endpoint: "sts.amazonaws.com", + SigningRegion: "us-east-1", + }, + "*/waf": { + Endpoint: "waf.amazonaws.com", + SigningRegion: "us-east-1", + }, + "cn-north-1/*": { + Endpoint: "{service}.{region}.amazonaws.com.cn", + }, + "cn-north-1/ec2metadata": { + Endpoint: "http://169.254.169.254/latest", + }, + "eu-central-1/s3": { + Endpoint: "{service}.{region}.amazonaws.com", + }, + "us-east-1/s3": { + Endpoint: "s3.amazonaws.com", + }, + "us-east-1/sdb": { + Endpoint: "sdb.amazonaws.com", + SigningRegion: "us-east-1", + }, + "us-gov-west-1/ec2metadata": { + Endpoint: "http://169.254.169.254/latest", + }, + "us-gov-west-1/iam": { + Endpoint: "iam.us-gov.amazonaws.com", + }, + "us-gov-west-1/s3": { + Endpoint: "s3-{region}.amazonaws.com", + }, + "us-gov-west-1/sts": { + Endpoint: "sts.us-gov-west-1.amazonaws.com", + }, + }, +} diff --git a/vendor/github.com/hashicorp/go-checkpoint/README.md b/vendor/github.com/hashicorp/go-checkpoint/README.md index ab8ebc0d3..e717b6ad3 100644 --- a/vendor/github.com/hashicorp/go-checkpoint/README.md +++ b/vendor/github.com/hashicorp/go-checkpoint/README.md @@ -1,7 +1,7 @@ # Go Checkpoint Client [Checkpoint](http://checkpoint.hashicorp.com) is an internal service at -Hashicorp that we use to check version information, broadcoast security +Hashicorp that we use to check version information, broadcast security bulletins, etc. We understand that software making remote calls over the internet @@ -10,7 +10,7 @@ disabled in all of our software that includes it. You can view the source of this client to see that we're not sending any private information. Each Hashicorp application has it's specific configuration option -to disable chekpoint calls, but the `CHECKPOINT_DISABLE` makes +to disable checkpoint calls, but the `CHECKPOINT_DISABLE` makes the underlying checkpoint component itself disabled. For example in the case of packer: ``` diff --git a/vendor/github.com/hashicorp/go-checkpoint/checkpoint.go b/vendor/github.com/hashicorp/go-checkpoint/checkpoint.go index abea934bf..8dff68f80 100644 --- a/vendor/github.com/hashicorp/go-checkpoint/checkpoint.go +++ b/vendor/github.com/hashicorp/go-checkpoint/checkpoint.go @@ -3,6 +3,7 @@ package checkpoint import ( + "bytes" "crypto/rand" "encoding/binary" "encoding/json" @@ -20,10 +21,113 @@ import ( "time" "github.com/hashicorp/go-cleanhttp" + uuid "github.com/hashicorp/go-uuid" ) var magicBytes [4]byte = [4]byte{0x35, 0x77, 0x69, 0xFB} +type ReportParams struct { + // Signature is some random signature that should be stored and used + // as a cookie-like value. This ensures that alerts aren't repeated. + // If the signature is changed, repeat alerts may be sent down. The + // signature should NOT be anything identifiable to a user (such as + // a MAC address). It should be random. + // + // If SignatureFile is given, then the signature will be read from this + // file. If the file doesn't exist, then a random signature will + // automatically be generated and stored here. SignatureFile will be + // ignored if Signature is given. + Signature string `json:"signature"` + SignatureFile string `json:"-"` + + StartTime time.Time `json:"start_time"` + EndTime time.Time `json:"end_time"` + Version string `json:"version"` + Product string `json:"product"` + Payload interface{} `json:"payload,omitempty"` + RunID string `json:"run_id"` + OS string `json:"os"` + Arch string `json:"arch"` + Args []string `json:"args"` + SchemaVersion string `json:"schema_version"` +} + +func (i *ReportParams) signature() string { + signature := i.Signature + if i.Signature == "" && i.SignatureFile != "" { + var err error + signature, err = checkSignature(i.SignatureFile) + if err != nil { + return "" + } + } + return signature +} + +func Report(r *ReportParams) error { + if disabled := os.Getenv("CHECKPOINT_DISABLE"); disabled != "" { + return nil + } + + if r.RunID == "" { + uuid, err := uuid.GenerateUUID() + if err != nil { + return err + } + r.RunID = uuid + } + if r.Arch == "" { + r.Arch = runtime.GOARCH + } + if r.OS == "" { + r.OS = runtime.GOOS + } + if len(r.Args) == 0 { + r.Args = os.Args + } + if r.Signature == "" { + r.Signature = r.signature() + } + + b, err := json.Marshal(r) + if err != nil { + return err + } + + // file logging while debugging + file, err := os.OpenFile("telemetry.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0755) + if err != nil { + return err + } + defer file.Close() + file.Write(b) + file.WriteString("\n") + + u := &url.URL{ + Scheme: "https", + Host: "checkpoint-api.hashicorp.com", + Path: fmt.Sprintf("/v1/telemetry/%s", r.Product), + } + + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(b)) + if err != nil { + return err + } + req.Header.Add("Accept", "application/json") + req.Header.Add("User-Agent", "HashiCorp/go-checkpoint") + + client := cleanhttp.DefaultClient() + resp, err := client.Do(req) + if err != nil { + return err + } + if resp.StatusCode != 201 { + return fmt.Errorf("Unknown status: %d", resp.StatusCode) + } + + return nil +} + // CheckParams are the parameters for configuring a check request. type CheckParams struct { // Product and version are used to lookup the correct product and @@ -116,6 +220,7 @@ func Check(p *CheckParams) (*CheckResponse, error) { p.OS = runtime.GOOS } + // TODO: race here if we try to write this file twice // If we're given a SignatureFile, then attempt to read that. signature := p.Signature if p.Signature == "" && p.SignatureFile != "" { diff --git a/vendor/vendor.json b/vendor/vendor.json index 26e73a706..dce822a05 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -546,9 +546,10 @@ "revision": "7554cd9344cec97297fa6649b055a8c98c2a1e55" }, { - "checksumSHA1": "nd3S1qkFv7zZxA9be0bw4nT0pe0=", + "checksumSHA1": "rPdTLJsefFK1hgWuVXcM9wmnSMI=", "path": "github.com/hashicorp/go-checkpoint", - "revision": "e4b2dc34c0f698ee04750bf2035d8b9384233e1b" + "revision": "194925eac2c1f69fcac1693d3f02f1337c341763", + "revisionTime": "2017-06-15T06:56:40Z" }, { "checksumSHA1": "fSe5y1UgTDeYlnFfUcDA1zzcw+U=", From 35332e44980338bae773139d5d1d01d09938947d Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Thu, 15 Jun 2017 01:15:32 -0700 Subject: [PATCH 2/7] fix panic with deadlines. update panicwrap dep --- main.go | 1 + packer/telemetry.go | 12 +++- .../hashicorp/go-checkpoint/checkpoint.go | 5 +- .../mitchellh/panicwrap/panicwrap.go | 56 ++++++++++++++++++- vendor/vendor.json | 7 ++- 5 files changed, 71 insertions(+), 10 deletions(-) diff --git a/main.go b/main.go index 43ae94f4a..47dc7e506 100644 --- a/main.go +++ b/main.go @@ -85,6 +85,7 @@ func realMain() int { wrapConfig.Handler = panicHandler(logTempFile) wrapConfig.Writer = io.MultiWriter(logTempFile, logWriter) wrapConfig.Stdout = outW + wrapConfig.DetectDuration = 500 * time.Millisecond exitStatus, err := panicwrap.Wrap(&wrapConfig) if err != nil { fmt.Fprintf(os.Stderr, "Couldn't start Packer: %s", err) diff --git a/packer/telemetry.go b/packer/telemetry.go index 9ff22ad22..ef7a4852b 100644 --- a/packer/telemetry.go +++ b/packer/telemetry.go @@ -1,6 +1,7 @@ package packer import ( + "context" "fmt" "log" "os" @@ -79,7 +80,11 @@ func (c *CheckpointTelemetry) ReportPanic(m string) error { panicParams := c.baseParams(TelemetryPanicVersion) panicParams.Payload = m panicParams.EndTime = time.Now().UTC() - return checkpoint.Report(panicParams) + + ctx, cancel := context.WithTimeout(context.Background(), 4500*time.Millisecond) + defer cancel() + + return checkpoint.Report(ctx, panicParams) } func (c *CheckpointTelemetry) AddSpan(name, pluginType string) *TelemetrySpan { @@ -112,7 +117,10 @@ func (c *CheckpointTelemetry) Finalize(command string, errCode int, err error) e } params.Payload = extra - return checkpoint.Report(params) + ctx, cancel := context.WithTimeout(context.Background(), 450*time.Millisecond) + defer cancel() + + return checkpoint.Report(ctx, params) } type TelemetrySpan struct { diff --git a/vendor/github.com/hashicorp/go-checkpoint/checkpoint.go b/vendor/github.com/hashicorp/go-checkpoint/checkpoint.go index 8dff68f80..2fc442d1f 100644 --- a/vendor/github.com/hashicorp/go-checkpoint/checkpoint.go +++ b/vendor/github.com/hashicorp/go-checkpoint/checkpoint.go @@ -4,6 +4,7 @@ package checkpoint import ( "bytes" + "context" "crypto/rand" "encoding/binary" "encoding/json" @@ -64,7 +65,7 @@ func (i *ReportParams) signature() string { return signature } -func Report(r *ReportParams) error { +func Report(ctx context.Context, r *ReportParams) error { if disabled := os.Getenv("CHECKPOINT_DISABLE"); disabled != "" { return nil } @@ -117,7 +118,7 @@ func Report(r *ReportParams) error { req.Header.Add("User-Agent", "HashiCorp/go-checkpoint") client := cleanhttp.DefaultClient() - resp, err := client.Do(req) + resp, err := client.Do(req.WithContext(ctx)) if err != nil { return err } diff --git a/vendor/github.com/mitchellh/panicwrap/panicwrap.go b/vendor/github.com/mitchellh/panicwrap/panicwrap.go index 11eafe712..028d69bfe 100644 --- a/vendor/github.com/mitchellh/panicwrap/panicwrap.go +++ b/vendor/github.com/mitchellh/panicwrap/panicwrap.go @@ -12,13 +12,16 @@ package panicwrap import ( "bytes" "errors" - "github.com/kardianos/osext" "io" "os" "os/exec" "os/signal" + "runtime" + "sync/atomic" "syscall" "time" + + "github.com/kardianos/osext" ) const ( @@ -61,6 +64,17 @@ type WrapConfig struct { // The writer to send stdout to. If this is nil, then it defaults to // os.Stdout. Stdout io.Writer + + // Catch and igore these signals in the parent process, let the child + // handle them gracefully. + IgnoreSignals []os.Signal + + // Catch these signals in the parent process and manually forward + // them to the child process. Some signals such as SIGINT are usually + // sent to the entire process group so setting it isn't necessary. Other + // signals like SIGTERM are only sent to the parent process and need + // to be forwarded. This defaults to empty. + ForwardSignals []os.Signal } // BasicWrap calls Wrap with the given handler function, using defaults @@ -145,6 +159,13 @@ func Wrap(c *WrapConfig) (int, error) { cmd.Stdin = os.Stdin cmd.Stdout = stdout_w cmd.Stderr = stderr_w + + // Windows doesn't support this, but on other platforms pass in + // the original file descriptors so they can be used. + if runtime.GOOS != "windows" { + cmd.ExtraFiles = []*os.File{os.Stdin, os.Stdout, os.Stderr} + } + if err := cmd.Start(); err != nil { return 1, err } @@ -152,13 +173,23 @@ func Wrap(c *WrapConfig) (int, error) { // Listen to signals and capture them forever. We allow the child // process to handle them in some way. sigCh := make(chan os.Signal) - signal.Notify(sigCh, os.Interrupt) + fwdSigCh := make(chan os.Signal) + if len(c.IgnoreSignals) == 0 { + c.IgnoreSignals = []os.Signal{os.Interrupt} + } + signal.Notify(sigCh, c.IgnoreSignals...) + signal.Notify(fwdSigCh, c.ForwardSignals...) go func() { defer signal.Stop(sigCh) + defer signal.Stop(fwdSigCh) for { select { case <-doneCh: return + case s := <-fwdSigCh: + if cmd.Process != nil { + cmd.Process.Signal(s) + } case <-sigCh: } } @@ -200,7 +231,17 @@ func Wrap(c *WrapConfig) (int, error) { // // Wrapped is very cheap and can be used early to short-circuit some pre-wrap // logic your application may have. +// +// If the given configuration is nil, then this will return a cached +// value of Wrapped. This is useful because Wrapped is usually called early +// to verify a process hasn't been wrapped before wrapping. After this, +// the value of Wrapped hardly changes and is process-global, so other +// libraries can check with Wrapped(nil). func Wrapped(c *WrapConfig) bool { + if c == nil { + return wrapCache.Load().(bool) + } + if c.CookieKey == "" { c.CookieKey = DEFAULT_COOKIE_KEY } @@ -211,7 +252,16 @@ func Wrapped(c *WrapConfig) bool { // If the cookie key/value match our environment, then we are the // child, so just exit now and tell the caller that we're the child - return os.Getenv(c.CookieKey) == c.CookieValue + result := os.Getenv(c.CookieKey) == c.CookieValue + wrapCache.Store(result) + return result +} + +// wrapCache is the cached value for Wrapped when called with nil +var wrapCache atomic.Value + +func init() { + wrapCache.Store(false) } // trackPanic monitors the given reader for a panic. If a panic is detected, diff --git a/vendor/vendor.json b/vendor/vendor.json index dce822a05..05ab00d06 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -546,7 +546,7 @@ "revision": "7554cd9344cec97297fa6649b055a8c98c2a1e55" }, { - "checksumSHA1": "rPdTLJsefFK1hgWuVXcM9wmnSMI=", + "checksumSHA1": "kBuCrFoNYcM0PcdbrOJQwec3Heg=", "path": "github.com/hashicorp/go-checkpoint", "revision": "194925eac2c1f69fcac1693d3f02f1337c341763", "revisionTime": "2017-06-15T06:56:40Z" @@ -719,9 +719,10 @@ "revisionTime": "2017-03-16T18:53:39Z" }, { - "checksumSHA1": "VBo7ciCNRr7wNVFmBTW8sm4PQ14=", + "checksumSHA1": "m2L8ohfZiFRsMW3iynaH/TWgnSY=", "path": "github.com/mitchellh/panicwrap", - "revision": "a1e50bc201f387747a45ffff020f1af2d8759e88" + "revision": "fce601fe55579125e1b3cb0b992287e7290f7b83", + "revisionTime": "2017-01-06T18:23:40Z" }, { "checksumSHA1": "h+ODp7a8Vj8XMUsORLbhtQMWOO4=", From 2d581dfc7bccf0a48b0cf3c12e7e9c9316e35bcc Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Thu, 15 Jun 2017 14:08:17 -0700 Subject: [PATCH 3/7] fix tests. --- packer/build.go | 9 ++++++++- packer/build_test.go | 2 +- packer/provisioner.go | 11 ++++++----- packer/provisioner_test.go | 9 ++++++--- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/packer/build.go b/packer/build.go index 40f6d38ab..996c9a331 100644 --- a/packer/build.go +++ b/packer/build.go @@ -194,13 +194,20 @@ func (b *coreBuild) Run(originalUi Ui, cache Cache) ([]Artifact, error) { // Add a hook for the provisioners if we have provisioners if len(b.provisioners) > 0 { + provisioners := make([]Provisioner, len(b.provisioners)) + provisionerTypes := make([]string, len(b.provisioners)) + for i, p := range b.provisioners { + provisioners[i] = p.provisioner + provisionerTypes[i] = p.pType + } if _, ok := hooks[HookProvision]; !ok { hooks[HookProvision] = make([]Hook, 0, 1) } hooks[HookProvision] = append(hooks[HookProvision], &ProvisionHook{ - Provisioners: b.provisioners, + Provisioners: provisioners, + ProvisionerTypes: provisionerTypes, }) } diff --git a/packer/build_test.go b/packer/build_test.go index af5b62ebd..8fe7bc5af 100644 --- a/packer/build_test.go +++ b/packer/build_test.go @@ -15,7 +15,7 @@ func testBuild() *coreBuild { "foo": {&MockHook{}}, }, provisioners: []coreBuildProvisioner{ - {&MockProvisioner{}, []interface{}{42}}, + {"mock-provisioner", &MockProvisioner{}, []interface{}{42}}, }, postProcessors: [][]coreBuildPostProcessor{ { diff --git a/packer/provisioner.go b/packer/provisioner.go index 1fa2d66dc..639e8b1a8 100644 --- a/packer/provisioner.go +++ b/packer/provisioner.go @@ -30,7 +30,8 @@ type Provisioner interface { type ProvisionHook struct { // The provisioners to run as part of the hook. These should already // be prepared (by calling Prepare) at some earlier stage. - Provisioners []coreBuildProvisioner + Provisioners []Provisioner + ProvisionerTypes []string lock sync.Mutex runningProvisioner Provisioner @@ -57,13 +58,13 @@ func (h *ProvisionHook) Run(name string, ui Ui, comm Communicator, data interfac h.runningProvisioner = nil }() - for _, p := range h.Provisioners { + for i, p := range h.Provisioners { h.lock.Lock() - h.runningProvisioner = p.provisioner + h.runningProvisioner = p h.lock.Unlock() - ts := CheckpointReporter.AddSpan(p.pType, "provisioner") - err := p.provisioner.Provision(ui, comm) + ts := CheckpointReporter.AddSpan(h.ProvisionerTypes[i], "provisioner") + err := p.Provision(ui, comm) ts.End(err) if err != nil { return err diff --git a/packer/provisioner_test.go b/packer/provisioner_test.go index 7251d6f05..f303bc7d2 100644 --- a/packer/provisioner_test.go +++ b/packer/provisioner_test.go @@ -23,7 +23,8 @@ func TestProvisionHook(t *testing.T) { var data interface{} = nil hook := &ProvisionHook{ - Provisioners: []Provisioner{pA, pB}, + Provisioners: []Provisioner{pA, pB}, + ProvisionerTypes: []string{"", ""}, } hook.Run("foo", ui, comm, data) @@ -46,7 +47,8 @@ func TestProvisionHook_nilComm(t *testing.T) { var data interface{} = nil hook := &ProvisionHook{ - Provisioners: []Provisioner{pA, pB}, + Provisioners: []Provisioner{pA, pB}, + ProvisionerTypes: []string{"", ""}, } err := hook.Run("foo", ui, comm, data) @@ -72,7 +74,8 @@ func TestProvisionHook_cancel(t *testing.T) { } hook := &ProvisionHook{ - Provisioners: []Provisioner{p}, + Provisioners: []Provisioner{p}, + ProvisionerTypes: []string{""}, } finished := make(chan struct{}) From 3898556caaffa7c794974b61de2d196a99978fd8 Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Fri, 16 Jun 2017 17:09:42 -0700 Subject: [PATCH 4/7] log per eng-003 --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 47dc7e506..5996c69e7 100644 --- a/main.go +++ b/main.go @@ -212,7 +212,7 @@ func wrappedMain() int { exitCode, err := cli.Run() if !inPlugin { if err := packer.CheckpointReporter.Finalize(cli.Subcommand(), exitCode, err); err != nil { - log.Printf("Error finalizing telemetry report. This is safe to ignore. %s", err.Error()) + log.Printf("[WARN] (telemetry) Error finalizing report. This is safe to ignore. %s", err.Error()) } } From f534b85ee9bbc7188dd180b8c35e58bed2763ce7 Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Fri, 16 Jun 2017 17:23:41 -0700 Subject: [PATCH 5/7] get chris's nits. --- main.go | 11 +++++++---- packer/telemetry.go | 6 +----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/main.go b/main.go index 5996c69e7..dfec3064e 100644 --- a/main.go +++ b/main.go @@ -75,10 +75,13 @@ func realMain() int { // Enable checkpoint for panic reporting config, err := loadConfig() - if err == nil { - if !config.DisableCheckpoint { - packer.CheckpointReporter.Enable(config.DisableCheckpointSignature) - } + if err != nil { + fmt.Fprintf(os.Stderr, "Couldn't load config: %s", err) + return 1 + } + + if !config.DisableCheckpoint { + packer.CheckpointReporter.Enable(config.DisableCheckpointSignature) } // Create the configuration for panicwrap and wrap our executable diff --git a/packer/telemetry.go b/packer/telemetry.go index ef7a4852b..3faaecc72 100644 --- a/packer/telemetry.go +++ b/packer/telemetry.go @@ -2,7 +2,6 @@ package packer import ( "context" - "fmt" "log" "os" "path/filepath" @@ -56,7 +55,7 @@ func (c *CheckpointTelemetry) Enable(disableSignature bool) { func (c *CheckpointTelemetry) baseParams(prefix string) *checkpoint.ReportParams { version := packerVersion.Version if packerVersion.VersionPrerelease != "" { - version += fmt.Sprintf("-%s", packerVersion.VersionPrerelease) + version += "-" + packerVersion.VersionPrerelease } return &checkpoint.ReportParams{ @@ -69,9 +68,6 @@ func (c *CheckpointTelemetry) baseParams(prefix string) *checkpoint.ReportParams } } -func (c *CheckpointTelemetry) log(m string, args ...interface{}) { -} - func (c *CheckpointTelemetry) ReportPanic(m string) error { if !c.enabled { return nil From 6ff34c87880c8c568e0385368f067979fa4855c2 Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Fri, 16 Jun 2017 17:56:41 -0700 Subject: [PATCH 6/7] update go-checkpoint --- .../hashicorp/go-checkpoint/checkpoint.go | 59 ++++++++++--------- vendor/vendor.json | 6 +- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/vendor/github.com/hashicorp/go-checkpoint/checkpoint.go b/vendor/github.com/hashicorp/go-checkpoint/checkpoint.go index 2fc442d1f..3ef0c0819 100644 --- a/vendor/github.com/hashicorp/go-checkpoint/checkpoint.go +++ b/vendor/github.com/hashicorp/go-checkpoint/checkpoint.go @@ -27,6 +27,7 @@ import ( var magicBytes [4]byte = [4]byte{0x35, 0x77, 0x69, 0xFB} +// ReportParams are the parameters for configuring a telemetry report. type ReportParams struct { // Signature is some random signature that should be stored and used // as a cookie-like value. This ensures that alerts aren't repeated. @@ -43,14 +44,14 @@ type ReportParams struct { StartTime time.Time `json:"start_time"` EndTime time.Time `json:"end_time"` - Version string `json:"version"` - Product string `json:"product"` - Payload interface{} `json:"payload,omitempty"` - RunID string `json:"run_id"` - OS string `json:"os"` Arch string `json:"arch"` Args []string `json:"args"` + OS string `json:"os"` + Payload interface{} `json:"payload,omitempty"` + Product string `json:"product"` + RunID string `json:"run_id"` SchemaVersion string `json:"schema_version"` + Version string `json:"version"` } func (i *ReportParams) signature() string { @@ -65,15 +66,36 @@ func (i *ReportParams) signature() string { return signature } +// Report sends telemetry information to checkpoint func Report(ctx context.Context, r *ReportParams) error { if disabled := os.Getenv("CHECKPOINT_DISABLE"); disabled != "" { return nil } + req, err := ReportRequest(r) + if err != nil { + return err + } + + client := cleanhttp.DefaultClient() + resp, err := client.Do(req.WithContext(ctx)) + if err != nil { + return err + } + if resp.StatusCode != 201 { + return fmt.Errorf("Unknown status: %d", resp.StatusCode) + } + + return nil +} + +// ReportRequest creates a request object for making a report +func ReportRequest(r *ReportParams) (*http.Request, error) { + // Populate some fields automatically if we can if r.RunID == "" { uuid, err := uuid.GenerateUUID() if err != nil { - return err + return nil, err } r.RunID = uuid } @@ -92,17 +114,8 @@ func Report(ctx context.Context, r *ReportParams) error { b, err := json.Marshal(r) if err != nil { - return err - } - - // file logging while debugging - file, err := os.OpenFile("telemetry.log", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0755) - if err != nil { - return err + return nil, err } - defer file.Close() - file.Write(b) - file.WriteString("\n") u := &url.URL{ Scheme: "https", @@ -112,21 +125,12 @@ func Report(ctx context.Context, r *ReportParams) error { req, err := http.NewRequest("POST", u.String(), bytes.NewReader(b)) if err != nil { - return err + return nil, err } req.Header.Add("Accept", "application/json") req.Header.Add("User-Agent", "HashiCorp/go-checkpoint") - client := cleanhttp.DefaultClient() - resp, err := client.Do(req.WithContext(ctx)) - if err != nil { - return err - } - if resp.StatusCode != 201 { - return fmt.Errorf("Unknown status: %d", resp.StatusCode) - } - - return nil + return req, nil } // CheckParams are the parameters for configuring a check request. @@ -221,7 +225,6 @@ func Check(p *CheckParams) (*CheckResponse, error) { p.OS = runtime.GOOS } - // TODO: race here if we try to write this file twice // If we're given a SignatureFile, then attempt to read that. signature := p.Signature if p.Signature == "" && p.SignatureFile != "" { diff --git a/vendor/vendor.json b/vendor/vendor.json index 05ab00d06..f3bbea787 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -546,10 +546,10 @@ "revision": "7554cd9344cec97297fa6649b055a8c98c2a1e55" }, { - "checksumSHA1": "kBuCrFoNYcM0PcdbrOJQwec3Heg=", + "checksumSHA1": "EPwsEGG/9t4sCexmFYnlZpE548A=", "path": "github.com/hashicorp/go-checkpoint", - "revision": "194925eac2c1f69fcac1693d3f02f1337c341763", - "revisionTime": "2017-06-15T06:56:40Z" + "revision": "04fd58160a0619a814172a795aa173fa64be731c", + "revisionTime": "2017-06-17T00:44:57Z" }, { "checksumSHA1": "fSe5y1UgTDeYlnFfUcDA1zzcw+U=", From beb338b78bd50062ce03f4fd536d80e811456ef5 Mon Sep 17 00:00:00 2001 From: Matthew Hooker Date: Mon, 19 Jun 2017 13:08:53 -0700 Subject: [PATCH 7/7] remove unneeded logs --- packer/telemetry.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/packer/telemetry.go b/packer/telemetry.go index 3faaecc72..540406b48 100644 --- a/packer/telemetry.go +++ b/packer/telemetry.go @@ -72,7 +72,6 @@ func (c *CheckpointTelemetry) ReportPanic(m string) error { if !c.enabled { return nil } - log.Printf("[TELEMETRY] Add panic: %s", m) panicParams := c.baseParams(TelemetryPanicVersion) panicParams.Payload = m panicParams.EndTime = time.Now().UTC() @@ -99,7 +98,6 @@ func (c *CheckpointTelemetry) Finalize(command string, errCode int, err error) e return nil } - log.Printf("[TELEMETRY] finalize: %#v", c) params := c.baseParams(TelemetryVersion) params.EndTime = time.Now().UTC()