Merge pull request #6610 from hashicorp/filter_logs

Filter logs
pull/6619/head
Megan Marsh 8 years ago committed by GitHub
commit 1f79b430ee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -67,7 +67,8 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
return nil, errs
}
log.Println(common.ScrubConfig(b.config, b.config.AlicloudAccessKey, b.config.AlicloudSecretKey))
packer.LogSecretFilter.Set(b.config.AlicloudAccessKey, b.config.AlicloudSecretKey)
log.Println(b.config)
return nil, nil
}

@ -176,7 +176,8 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
return warns, errs
}
log.Println(common.ScrubConfig(b.config, b.config.AccessKey, b.config.SecretKey, b.config.Token))
packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token)
log.Println(b.config)
return warns, nil
}

@ -95,7 +95,9 @@ WaitLoop:
"Password (since debug is enabled): %s", s.Comm.WinRMPassword))
}
// store so that we can access this later during provisioning
commonhelper.SetSharedState("winrm_password", s.Comm.WinRMPassword, s.BuildName)
packer.LogSecretFilter.Set(s.Comm.WinRMPassword)
return multistep.ActionContinue
}

@ -81,7 +81,8 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
return nil, errs
}
log.Println(common.ScrubConfig(b.config, b.config.AccessKey, b.config.SecretKey, b.config.Token))
packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token)
log.Println(b.config)
return nil, nil
}
@ -260,7 +261,6 @@ func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packe
// Run!
b.runner = common.NewRunner(steps, b.config.PackerConfig, ui)
b.runner.Run(state)
// If there was an error, return that
if rawErr, ok := state.GetOk("error"); ok {
return nil, rawErr.(error)

@ -96,7 +96,8 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
return nil, errs
}
log.Println(common.ScrubConfig(b.config, b.config.AccessKey, b.config.SecretKey, b.config.Token))
packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token)
log.Println(b.config)
return nil, nil
}

@ -81,7 +81,8 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
return nil, errs
}
log.Println(common.ScrubConfig(b.config, b.config.AccessKey, b.config.SecretKey, b.config.Token))
packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token)
log.Println(b.config)
return nil, nil
}

@ -166,8 +166,8 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
if errs != nil && len(errs.Errors) > 0 {
return nil, errs
}
log.Println(common.ScrubConfig(b.config, b.config.AccessKey, b.config.SecretKey, b.config.Token))
packer.LogSecretFilter.Set(b.config.AccessKey, b.config.SecretKey, b.config.Token)
log.Println(b.config)
return nil, nil
}

@ -363,6 +363,7 @@ func setRuntimeValues(c *Config) {
c.tmpAdminPassword = tempName.AdminPassword
// store so that we can access this later during provisioning
commonhelper.SetSharedState("winrm_password", c.tmpAdminPassword, c.PackerConfig.PackerBuildName)
packer.LogSecretFilter.Set(c.tmpAdminPassword)
c.tmpCertificatePassword = tempName.CertificatePassword
if c.TempComputeName == "" {

@ -5,6 +5,7 @@ import (
commonhelper "github.com/hashicorp/packer/helper/common"
"github.com/hashicorp/packer/helper/multistep"
"github.com/hashicorp/packer/packer"
)
type StepSaveWinRMPassword struct {
@ -15,6 +16,7 @@ type StepSaveWinRMPassword struct {
func (s *StepSaveWinRMPassword) Run(_ context.Context, state multistep.StateBag) multistep.StepAction {
// store so that we can access this later during provisioning
commonhelper.SetSharedState("winrm_password", s.Password, s.BuildName)
packer.LogSecretFilter.Set(s.Password)
return multistep.ActionContinue
}

@ -138,6 +138,6 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) {
return nil, nil, errs
}
common.ScrubConfig(c, c.APIToken)
packer.LogSecretFilter.Set(c.APIToken)
return c, nil, nil
}

@ -33,6 +33,7 @@ func (s *StepCreateWindowsPassword) Run(_ context.Context, state multistep.State
if c.Comm.WinRMPassword != "" {
state.Put("winrm_password", c.Comm.WinRMPassword)
packer.LogSecretFilter.Set(c.Comm.WinRMPassword)
return multistep.ActionContinue
}
@ -114,6 +115,7 @@ func (s *StepCreateWindowsPassword) Run(_ context.Context, state multistep.State
state.Put("winrm_password", data.password)
commonhelper.SetSharedState("winrm_password", data.password, c.PackerConfig.PackerBuildName)
packer.LogSecretFilter.Set(data.password)
return multistep.ActionContinue
}

@ -109,7 +109,6 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) {
if errs != nil && len(errs.Errors) > 0 {
return nil, nil, errs
}
common.ScrubConfig(c, c.Token)
packer.LogSecretFilter.Set(c.Token)
return &c, nil, nil
}

@ -57,7 +57,8 @@ func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
b.config.InstanceName = b.config.ImageName
}
log.Println(common.ScrubConfig(b.config, b.config.Password))
packer.LogSecretFilter.Set(b.config.Password)
log.Println(b.config)
return nil, nil
}

@ -53,7 +53,7 @@ func (s *stepGetDefaultCredentials) Run(ctx context.Context, state multistep.Sta
// store so that we can access this later during provisioning
commonhelper.SetSharedState("winrm_password", s.Comm.WinRMPassword, s.BuildName)
packer.LogSecretFilter.Set(s.Comm.WinRMPassword)
return multistep.ActionContinue
}

@ -122,7 +122,7 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) {
if errs != nil && len(errs.Errors) > 0 {
return nil, nil, errs
}
common.ScrubConfig(c, c.PBUsername)
packer.LogSecretFilter.Set(c.PBUsername)
return &c, nil, nil
}

@ -119,6 +119,6 @@ func NewConfig(raws ...interface{}) (*Config, []string, error) {
return nil, nil, errs
}
common.ScrubConfig(c, c.Token)
packer.LogSecretFilter.Set(c.Token)
return c, nil, nil
}

@ -20,19 +20,6 @@ const PackerKeyEnv = "PACKER_KEY_INTERVAL"
// shorter delay (e.g. 10ms) can be used on a workstation. See PackerKeyEnv.
const PackerKeyDefault = 100 * time.Millisecond
// ScrubConfig is a helper that returns a string representation of
// any struct with the given values stripped out.
func ScrubConfig(target interface{}, values ...string) string {
conf := fmt.Sprintf("Config: %+v", target)
for _, value := range values {
if value == "" {
continue
}
conf = strings.Replace(conf, value, "<Filtered>", -1)
}
return conf
}
// ChooseString returns the first non-empty value.
func ChooseString(vals ...string) string {
for _, el := range vals {

@ -310,20 +310,3 @@ func TestFileExistsLocally(t *testing.T) {
}
}
}
func TestScrubConfig(t *testing.T) {
type Inner struct {
Baz string
}
type Local struct {
Foo string
Bar string
Inner
}
c := Local{"foo", "bar", Inner{"bar"}}
expect := "Config: {Foo:foo Bar:<Filtered> Inner:{Baz:<Filtered>}}"
conf := ScrubConfig(c, c.Bar)
if conf != expect {
t.Fatalf("got %s, expected %s", conf, expect)
}
}

@ -4,10 +4,11 @@ package common
// are sent by packer, properly tagged already so mapstructure can load
// them. Embed this structure into your configuration class to get it.
type PackerConfig struct {
PackerBuildName string `mapstructure:"packer_build_name"`
PackerBuilderType string `mapstructure:"packer_builder_type"`
PackerDebug bool `mapstructure:"packer_debug"`
PackerForce bool `mapstructure:"packer_force"`
PackerOnError string `mapstructure:"packer_on_error"`
PackerUserVars map[string]string `mapstructure:"packer_user_variables"`
PackerBuildName string `mapstructure:"packer_build_name"`
PackerBuilderType string `mapstructure:"packer_builder_type"`
PackerDebug bool `mapstructure:"packer_debug"`
PackerForce bool `mapstructure:"packer_force"`
PackerOnError string `mapstructure:"packer_on_error"`
PackerUserVars map[string]string `mapstructure:"packer_user_variables"`
PackerSensitiveVars []string `mapstructure:"packer_sensitive_variables"`
}

@ -70,12 +70,7 @@ func Run(ui packer.Ui, config *Config) (bool, error) {
// buffers and for reading the final exit status.
flattenedCmd := strings.Join(interpolatedCmds, " ")
cmd := &packer.RemoteCmd{Command: flattenedCmd}
sanitized := flattenedCmd
if len(getWinRMPassword(config.PackerBuildName)) > 0 {
sanitized = strings.Replace(flattenedCmd,
getWinRMPassword(config.PackerBuildName), "*****", -1)
}
log.Printf("[INFO] (shell-local): starting local command: %s", sanitized)
log.Printf("[INFO] (shell-local): starting local command: %s", flattenedCmd)
if err := cmd.StartWithUi(comm, ui); err != nil {
return false, fmt.Errorf(
"Error executing script: %s\n\n"+
@ -204,5 +199,6 @@ func createFlattenedEnvVars(config *Config) (string, error) {
func getWinRMPassword(buildName string) string {
winRMPass, _ := commonhelper.RetrieveSharedState("winrm_password", buildName)
packer.LogSecretFilter.Set(winRMPass)
return winRMPass
}

@ -108,10 +108,11 @@ func Decode(target interface{}, config *DecodeOpts, raws ...interface{}) error {
// detecting things like user variables from the raw configuration params.
func DetectContext(raws ...interface{}) (*interpolate.Context, error) {
var s struct {
BuildName string `mapstructure:"packer_build_name"`
BuildType string `mapstructure:"packer_builder_type"`
TemplatePath string `mapstructure:"packer_template_path"`
Vars map[string]string `mapstructure:"packer_user_variables"`
BuildName string `mapstructure:"packer_build_name"`
BuildType string `mapstructure:"packer_builder_type"`
TemplatePath string `mapstructure:"packer_template_path"`
Vars map[string]string `mapstructure:"packer_user_variables"`
SensitiveVars []string `mapstructure:"packer_sensitive_variables"`
}
for _, r := range raws {
@ -121,10 +122,11 @@ func DetectContext(raws ...interface{}) (*interpolate.Context, error) {
}
return &interpolate.Context{
BuildName: s.BuildName,
BuildType: s.BuildType,
TemplatePath: s.TemplatePath,
UserVariables: s.Vars,
BuildName: s.BuildName,
BuildType: s.BuildType,
TemplatePath: s.TemplatePath,
UserVariables: s.Vars,
SensitiveVariables: s.SensitiveVars,
}, nil
}

@ -55,6 +55,10 @@ func realMain() int {
logWriter = ioutil.Discard
}
packer.LogSecretFilter.SetOutput(logWriter)
//packer.LogSecrets.
// Disable logging here
log.SetOutput(ioutil.Discard)
@ -87,7 +91,7 @@ func realMain() int {
// Create the configuration for panicwrap and wrap our executable
wrapConfig.Handler = panicHandler(logTempFile)
wrapConfig.Writer = io.MultiWriter(logTempFile, logWriter)
wrapConfig.Writer = io.MultiWriter(logTempFile, &packer.LogSecretFilter)
wrapConfig.Stdout = outW
wrapConfig.DetectDuration = 500 * time.Millisecond
wrapConfig.ForwardSignals = []os.Signal{syscall.SIGTERM}
@ -125,7 +129,8 @@ func wrappedMain() int {
runtime.GOMAXPROCS(runtime.NumCPU())
}
log.SetOutput(os.Stderr)
packer.LogSecretFilter.SetOutput(os.Stderr)
log.SetOutput(&packer.LogSecretFilter)
log.Printf("[INFO] Packer version: %s", version.FormattedVersion())
log.Printf("Packer Target OS/Arch: %s %s", runtime.GOOS, runtime.GOARCH)

@ -19,15 +19,17 @@ type Core struct {
variables map[string]string
builds map[string]*template.Builder
version string
secrets []string
}
// CoreConfig is the structure for initializing a new Core. Once a CoreConfig
// is used to initialize a Core, it shouldn't be re-used or modified again.
type CoreConfig struct {
Components ComponentFinder
Template *template.Template
Variables map[string]string
Version string
Components ComponentFinder
Template *template.Template
Variables map[string]string
SensitiveVariables []string
Version string
}
// The function type used to lookup Builder implementations.
@ -60,12 +62,16 @@ func NewCore(c *CoreConfig) (*Core, error) {
variables: c.Variables,
version: c.Version,
}
if err := result.validate(); err != nil {
return nil, err
}
if err := result.init(); err != nil {
return nil, err
}
for _, secret := range result.secrets {
LogSecretFilter.Set(secret)
}
// Go through and interpolate all the build names. We should be able
// to do this at this point with the variables.
@ -302,6 +308,16 @@ func (c *Core) init() error {
c.variables[k] = def
}
for _, v := range c.Template.SensitiveVariables {
def, err := interpolate.Render(v.Default, ctx)
if err != nil {
return fmt.Errorf(
"error interpolating default value for '%#v': %s",
v, err)
}
c.secrets = append(c.secrets, def)
}
// Interpolate the push configuration
if _, err := interpolate.RenderInterface(&c.Template.Push, c.Context()); err != nil {
return fmt.Errorf("Error interpolating 'push': %s", err)

@ -516,12 +516,67 @@ func TestCoreValidate(t *testing.T) {
Variables: tc.Vars,
Version: "1.0.0",
})
if (err != nil) != tc.Err {
t.Fatalf("err: %s\n\n%s", tc.File, err)
}
}
}
func TestSensitiveVars(t *testing.T) {
cases := []struct {
File string
Vars map[string]string
SensitiveVars []string
Expected string
Err bool
}{
// hardcoded
{
"sensitive-variables.json",
map[string]string{"foo": "bar"},
[]string{"foo"},
"bar",
false,
},
// interpolated
{
"sensitive-variables.json",
map[string]string{"foo": "{{build_name}}"},
[]string{"foo"},
"test",
false,
},
}
for _, tc := range cases {
f, err := os.Open(fixtureDir(tc.File))
if err != nil {
t.Fatalf("err: %s", err)
}
tpl, err := template.Parse(f)
f.Close()
if err != nil {
t.Fatalf("err: %s\n\n%s", tc.File, err)
}
_, err = NewCore(&CoreConfig{
Template: tpl,
Variables: tc.Vars,
Version: "1.0.0",
})
if (err != nil) != tc.Err {
t.Fatalf("err: %s\n\n%s", tc.File, err)
}
filtered := LogSecretFilter.get()
if filtered[0] != tc.Expected && len(filtered) != 1 {
t.Fatalf("not filtering sensitive vars; filtered is %#v", filtered)
}
}
}
func testComponentFinder() *ComponentFinder {
builderFactory := func(n string) (Builder, error) { return new(MockBuilder), nil }
ppFactory := func(n string) (PostProcessor, error) { return new(MockPostProcessor), nil }

@ -0,0 +1,51 @@
package packer
import (
"bytes"
"io"
"sync"
)
type secretFilter struct {
s map[string]struct{}
m sync.Mutex
w io.Writer
}
func (l *secretFilter) Set(secrets ...string) {
l.m.Lock()
defer l.m.Unlock()
for _, s := range secrets {
l.s[s] = struct{}{}
}
}
func (l *secretFilter) SetOutput(output io.Writer) {
l.m.Lock()
defer l.m.Unlock()
l.w = output
}
func (l *secretFilter) Write(p []byte) (n int, err error) {
for s := range l.s {
if s != "" {
p = bytes.Replace(p, []byte(s), []byte("<sensitive>"), -1)
}
}
return l.w.Write(p)
}
func (l *secretFilter) get() (s []string) {
l.m.Lock()
defer l.m.Unlock()
for k := range l.s {
s = append(s, k)
}
return
}
var LogSecretFilter secretFilter
func init() {
LogSecretFilter.s = make(map[string]struct{})
}

@ -0,0 +1,12 @@
{
"variables": {
"foo": "bar"
},
"sensitive-variables": [
"foo"
],
"builders": [{
"type": "test",
"value": "{{build_name}}"
}]
}

@ -113,7 +113,8 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
return errs
}
log.Println(common.ScrubConfig(p.config, p.config.AlicloudAccessKey, p.config.AlicloudSecretKey))
packer.LogSecretFilter.Set(p.config.AlicloudAccessKey, p.config.AlicloudSecretKey)
log.Println(p.config)
return nil
}

@ -92,7 +92,8 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
return errs
}
log.Println(common.ScrubConfig(p.config, p.config.AccessKey, p.config.SecretKey, p.config.Token))
packer.LogSecretFilter.Set(p.config.AccessKey, p.config.SecretKey, p.config.Token)
log.Println(p.config)
return nil
}

@ -555,6 +555,7 @@ func newSigner(privKeyFile string) (*signer, error) {
func getWinRMPassword(buildName string) string {
winRMPass, _ := commonhelper.RetrieveSharedState("winrm_password", buildName)
packer.LogSecretFilter.Set(winRMPass)
return winRMPass
}

@ -481,6 +481,7 @@ func (p *Provisioner) createCommandTextNonPrivileged() (command string, err erro
func getWinRMPassword(buildName string) string {
winRMPass, _ := commonhelper.RetrieveSharedState("winrm_password", buildName)
packer.LogSecretFilter.Set(winRMPass)
return winRMPass
}

@ -18,6 +18,9 @@ type Context struct {
// "user" function reads from.
UserVariables map[string]string
// SensitiveVariables is a list of variables to sanitize.
SensitiveVariables []string
// EnableEnv enables the env function
EnableEnv bool

@ -23,11 +23,12 @@ type rawTemplate struct {
MinVersion string `mapstructure:"min_packer_version"`
Description string
Builders []map[string]interface{}
Push map[string]interface{}
PostProcessors []interface{} `mapstructure:"post-processors"`
Provisioners []map[string]interface{}
Variables map[string]interface{}
Builders []map[string]interface{}
Push map[string]interface{}
PostProcessors []interface{} `mapstructure:"post-processors"`
Provisioners []map[string]interface{}
Variables map[string]interface{}
SensitiveVariables []string `mapstructure:"sensitive-variables"`
RawContents []byte
}
@ -60,6 +61,12 @@ func (r *rawTemplate) Template() (*Template, error) {
continue
}
for _, sVar := range r.SensitiveVariables {
if sVar == k {
result.SensitiveVariables = append(result.SensitiveVariables, &v)
}
}
result.Variables[k] = &v
}

@ -18,11 +18,12 @@ type Template struct {
Description string
MinVersion string
Variables map[string]*Variable
Builders map[string]*Builder
Provisioners []*Provisioner
PostProcessors [][]*PostProcessor
Push Push
Variables map[string]*Variable
SensitiveVariables []*Variable
Builders map[string]*Builder
Provisioners []*Provisioner
PostProcessors [][]*PostProcessor
Push Push
// RawContents is just the raw data for this template
RawContents []byte

@ -206,6 +206,32 @@ Results in the following variables:
| aws\_access\_key | foo |
| aws\_secret\_key | baz |
# Sensitive Variables
If you use the environment to set a variable that is sensitive, you probably
don't want that variable printed to the Packer logs. You can make sure that
sensitive variables won't get printed to the logs by adding them to the
"sensitive-variables" list within the Packer template:
``` json
{
"variables": {
"my_secret": "{{env `MY_SECRET`}}",
"not_a_secret": "plaintext",
"foo": "bar"
},
"sensitive-variables": ["my_secret", "foo"],
...
}
```
The above snippet of code will function exactly the same as if you did not set
"sensitive-variables", except that the Packer UI and logs will replace all
instances of "bar" and of whatever the value of "my_secret" is with
`<sensitive>`. This allows you to be confident that you are not printing
secrets in plaintext to our logs by accident.
# Recipes
## Making a provisioner step conditional on the value of a variable

Loading…
Cancel
Save