Merge pull request #7183 from hashicorp/skip_post_processor

allow to use `-except` on post-processors

command/build_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/hashicorp/packer/builder/file"
 	"github.com/hashicorp/packer/packer"
+	shell_local "github.com/hashicorp/packer/post-processor/shell-local"
 )
 
 func TestBuildOnlyFileCommaFlags(t *testing.T) {
@@ -26,12 +27,13 @@ func TestBuildOnlyFileCommaFlags(t *testing.T) {
 		fatalCommand(t, c.Meta)
 	}
 
-	if !fileExists("chocolate.txt") {
-		t.Error("Expected to find chocolate.txt")
-	}
-	if !fileExists("vanilla.txt") {
-		t.Error("Expected to find vanilla.txt")
+	for _, f := range []string{"chocolate.txt", "vanilla.txt",
+		"apple.txt", "peach.txt", "pear.txt"} {
+		if !fileExists(f) {
+			t.Errorf("Expected to find %s", f)
+		}
 	}
+
 	if fileExists("cherry.txt") {
 		t.Error("Expected NOT to find cherry.txt")
 	}
@@ -56,14 +58,10 @@ func TestBuildStdin(t *testing.T) {
 		fatalCommand(t, c.Meta)
 	}
 
-	if !fileExists("chocolate.txt") {
-		t.Error("Expected to find chocolate.txt")
-	}
-	if !fileExists("vanilla.txt") {
-		t.Error("Expected to find vanilla.txt")
-	}
-	if !fileExists("cherry.txt") {
-		t.Error("Expected to find cherry.txt")
+	for _, f := range []string{"vanilla.txt", "cherry.txt", "chocolate.txt"} {
+		if !fileExists(f) {
+			t.Errorf("Expected to find %s", f)
+		}
 	}
 }
 
@@ -75,6 +73,9 @@ func TestBuildOnlyFileMultipleFlags(t *testing.T) {
 	args := []string{
 		"-only=chocolate",
 		"-only=cherry",
+		"-only=apple", // ignored
+		"-only=peach", // ignored
+		"-only=pear",  // ignored
 		filepath.Join(testFixture("build-only"), "template.json"),
 	}
 
@@ -84,14 +85,16 @@ func TestBuildOnlyFileMultipleFlags(t *testing.T) {
 		fatalCommand(t, c.Meta)
 	}
 
-	if !fileExists("chocolate.txt") {
-		t.Error("Expected to find chocolate.txt")
+	for _, f := range []string{"vanilla.txt"} {
+		if fileExists(f) {
+			t.Errorf("Expected NOT to find %s", f)
+		}
 	}
-	if fileExists("vanilla.txt") {
-		t.Error("Expected NOT to find vanilla.txt")
-	}
-	if !fileExists("cherry.txt") {
-		t.Error("Expected to find cherry.txt")
+	for _, f := range []string{"chocolate.txt", "cherry.txt",
+		"apple.txt", "peach.txt", "pear.txt"} {
+		if !fileExists(f) {
+			t.Errorf("Expected to find %s", f)
+		}
 	}
 }
 
@@ -101,7 +104,7 @@ func TestBuildExceptFileCommaFlags(t *testing.T) {
 	}
 
 	args := []string{
-		"-except=chocolate",
+		"-except=chocolate,apple",
 		filepath.Join(testFixture("build-only"), "template.json"),
 	}
 
@@ -111,14 +114,15 @@ func TestBuildExceptFileCommaFlags(t *testing.T) {
 		fatalCommand(t, c.Meta)
 	}
 
-	if fileExists("chocolate.txt") {
-		t.Error("Expected NOT to find chocolate.txt")
-	}
-	if !fileExists("vanilla.txt") {
-		t.Error("Expected to find vanilla.txt")
+	for _, f := range []string{"chocolate.txt", "apple.txt", "peach.txt"} {
+		if fileExists(f) {
+			t.Errorf("Expected NOT to find %s", f)
+		}
 	}
-	if !fileExists("cherry.txt") {
-		t.Error("Expected to find cherry.txt")
+	for _, f := range []string{"vanilla.txt", "cherry.txt", "pear.txt"} {
+		if !fileExists(f) {
+			t.Errorf("Expected to find %s", f)
+		}
 	}
 }
 
@@ -137,6 +141,9 @@ func testCoreConfigBuilder(t *testing.T) *packer.CoreConfig {
 		Builder: func(n string) (packer.Builder, error) {
 			return &file.Builder{}, nil
 		},
+		PostProcessor: func(n string) (packer.PostProcessor, error) {
+			return &shell_local.PostProcessor{}, nil
+		},
 	}
 	return &packer.CoreConfig{
 		Components: components,
@@ -159,4 +166,7 @@ func cleanup() {
 	os.RemoveAll("chocolate.txt")
 	os.RemoveAll("vanilla.txt")
 	os.RemoveAll("cherry.txt")
+	os.RemoveAll("apple.txt")
+	os.RemoveAll("peach.txt")
+	os.RemoveAll("pear.txt")
 }

command/meta.go

@@ -6,8 +6,8 @@ import (
 	"fmt"
 	"io"
 
-	"github.com/hashicorp/packer/helper/flag-kv"
-	"github.com/hashicorp/packer/helper/flag-slice"
+	kvflag "github.com/hashicorp/packer/helper/flag-kv"
+	sliceflag "github.com/hashicorp/packer/helper/flag-slice"
 	"github.com/hashicorp/packer/packer"
 	"github.com/hashicorp/packer/template"
 )
@@ -31,9 +31,7 @@ type Meta struct {
 	Version    string
 
 	// These are set by command-line flags
-	flagBuildExcept []string
-	flagBuildOnly   []string
-	flagVars        map[string]string
+	flagVars map[string]string
 }
 
 // Core returns the core for the given template given the configured
@@ -59,7 +57,7 @@ func (m *Meta) BuildNames(c *packer.Core) []string {
 	// TODO: test
 
 	// Filter the "only"
-	if len(m.flagBuildOnly) > 0 {
+	if len(m.CoreConfig.Only) > 0 {
 		// Build a set of all the available names
 		nameSet := make(map[string]struct{})
 		for _, n := range c.BuildNames() {
@@ -67,8 +65,8 @@ func (m *Meta) BuildNames(c *packer.Core) []string {
 		}
 
 		// Build our result set which we pre-allocate some sane number
-		result := make([]string, 0, len(m.flagBuildOnly))
-		for _, n := range m.flagBuildOnly {
+		result := make([]string, 0, len(m.CoreConfig.Only))
+		for _, n := range m.CoreConfig.Only {
 			if _, ok := nameSet[n]; ok {
 				result = append(result, n)
 			}
@@ -78,10 +76,10 @@ func (m *Meta) BuildNames(c *packer.Core) []string {
 	}
 
 	// Filter the "except"
-	if len(m.flagBuildExcept) > 0 {
+	if len(m.CoreConfig.Except) > 0 {
 		// Build a set of the things we don't want
 		nameSet := make(map[string]struct{})
-		for _, n := range m.flagBuildExcept {
+		for _, n := range m.CoreConfig.Except {
 			nameSet[n] = struct{}{}
 		}
 
@@ -111,8 +109,8 @@ func (m *Meta) FlagSet(n string, fs FlagSetFlags) *flag.FlagSet {
 	// FlagSetBuildFilter tells us to enable the settings for selecting
 	// builds we care about.
 	if fs&FlagSetBuildFilter != 0 {
-		f.Var((*sliceflag.StringFlag)(&m.flagBuildExcept), "except", "")
-		f.Var((*sliceflag.StringFlag)(&m.flagBuildOnly), "only", "")
+		f.Var((*sliceflag.StringFlag)(&m.CoreConfig.Except), "except", "")
+		f.Var((*sliceflag.StringFlag)(&m.CoreConfig.Only), "only", "")
 	}
 
 	// FlagSetVars tells us what variables to use

command/test-fixtures/build-only/template.json

@@ -1,22 +1,43 @@
 {
     "builders": [
         {
-            "name":"chocolate",
-            "type":"file",
-            "content":"chocolate",
-            "target":"chocolate.txt"
+            "name": "chocolate",
+            "type": "file",
+            "content": "chocolate",
+            "target": "chocolate.txt"
         },
         {
-            "name":"vanilla",
-            "type":"file",
-            "content":"vanilla",
-            "target":"vanilla.txt"
+            "name": "vanilla",
+            "type": "file",
+            "content": "vanilla",
+            "target": "vanilla.txt"
         },
         {
-            "name":"cherry",
-            "type":"file",
-            "content":"cherry",
-            "target":"cherry.txt"
+            "name": "cherry",
+            "type": "file",
+            "content": "cherry",
+            "target": "cherry.txt"
         }
+    ],
+    "post-processors": [
+        [
+            {
+                "name": "apple",
+                "type": "shell-local",
+                "inline": [ "touch apple.txt" ]
+            },
+            {
+                "name": "peach",
+                "type": "shell-local",
+                "inline": [ "touch peach.txt" ]
+            }
+        ],
+        [
+            {
+                "name": "pear",
+                "type": "shell-local",
+                "inline": [ "touch pear.txt" ]
+            }
+        ]
     ]
-}
+}

contrib/zsh-completion/_packer

@@ -14,7 +14,7 @@ _packer () {
     '-force[Force a build to continue if artifacts exist, deletes existing artifacts.]'
     '-machine-readable[Produce machine-readable output.]'
     '-color=[(false) Disable color output. (Default: color)]'
-    '-except=[(foo,bar,baz) Build all builds other than these.]'
+    '-except=[(foo,bar,baz) Run all builds and post-procesors other than these.]'
     '-on-error=[(cleanup,abort,ask) If the build fails do: clean up (default), abort, or ask.]'
     '-only=[(foo,bar,baz) Only build the given builds by name.]'
     '-parallel=[(false) Disable parallelization. (Default: parallel)]'

packer/core.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"sort"
 
-	"github.com/hashicorp/go-multierror"
-	"github.com/hashicorp/go-version"
+	multierror "github.com/hashicorp/go-multierror"
+	version "github.com/hashicorp/go-version"
 	"github.com/hashicorp/packer/template"
 	"github.com/hashicorp/packer/template/interpolate"
 )
@@ -20,6 +20,9 @@ type Core struct {
 	builds     map[string]*template.Builder
 	version    string
 	secrets    []string
+
+	except []string
+	only   []string
 }
 
 // CoreConfig is the structure for initializing a new Core. Once a CoreConfig
@@ -30,6 +33,10 @@ type CoreConfig struct {
 	Variables          map[string]string
 	SensitiveVariables []string
 	Version            string
+
+	// These are set by command-line flags
+	Except []string
+	Only   []string
 }
 
 // The function type used to lookup Builder implementations.
@@ -61,6 +68,8 @@ func NewCore(c *CoreConfig) (*Core, error) {
 		components: c.Components,
 		variables:  c.Variables,
 		version:    c.Version,
+		only:       c.Only,
+		except:     c.Except,
 	}
 
 	if err := result.validate(); err != nil {
@@ -126,7 +135,7 @@ func (c *Core) Build(n string) (Build, error) {
 	provisioners := make([]coreBuildProvisioner, 0, len(c.Template.Provisioners))
 	for _, rawP := range c.Template.Provisioners {
 		// If we're skipping this, then ignore it
-		if rawP.Skip(rawName) {
+		if rawP.OnlyExcept.Skip(rawName) {
 			continue
 		}
 
@@ -172,9 +181,13 @@ func (c *Core) Build(n string) (Build, error) {
 		current := make([]coreBuildPostProcessor, 0, len(rawPs))
 		for _, rawP := range rawPs {
 			// If we skip, ignore
-			if rawP.Skip(rawName) {
+			rawP.OnlyExcept.Except = append(rawP.OnlyExcept.Except, c.except...)
+			if rawP.OnlyExcept.Skip(rawName) {
 				continue
 			}
+			if rawP.OnlyExcept.Skip(rawP.Name) {
+				break
+			}
 
 			// Get the post-processor
 			postProcessor, err := c.components.PostProcessor(rawP.Type)

template/parse.go

@@ -11,7 +11,7 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/hashicorp/go-multierror"
+	multierror "github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/packer/packer/tmp"
 	"github.com/mitchellh/mapstructure"
 )
@@ -149,6 +149,7 @@ func (r *rawTemplate) Template() (*Template, error) {
 			delete(c, "only")
 			delete(c, "keep_input_artifact")
 			delete(c, "type")
+			delete(c, "name")
 			if len(c) > 0 {
 				pp.Config = c
 			}

template/template.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/hashicorp/go-multierror"
+	multierror "github.com/hashicorp/go-multierror"
 )
 
 // Template represents the parsed template that is used to configure
@@ -40,6 +40,7 @@ type Builder struct {
 type PostProcessor struct {
 	OnlyExcept `mapstructure:",squash"`
 
+	Name              string
 	Type              string
 	KeepInputArtifact bool `mapstructure:"keep_input_artifact"`
 	Config            map[string]interface{}

website/source/docs/commands/build.html.md

@@ -26,10 +26,12 @@ artifacts that are created will be outputted at the end of the build.
     will stop between each step, waiting for keyboard input before continuing.
     This will allow the user to inspect state and so on.
 
--   `-except=foo,bar,baz` - Builds all the builds except those with the given
-    comma-separated names. Build names by default are the names of their
-    builders, unless a specific `name` attribute is specified within the
-    configuration.
+-   `-except=foo,bar,baz` - Run all the builds and post-processors except those
+    with the given comma-separated names. Build and post-processor names by
+    default are their type, unless a specific `name` attribute is specified
+    within the configuration. Any post-processor following a skipped
+    post-processor will not run. Because post-processors can be nested in
+    arrays a differ post-processor chain can still run.
 
 -   `-force` - Forces a builder to run when artifacts from a previous build
     prevent a build from running. The exact behavior of a forced build is left
@@ -44,9 +46,10 @@ artifacts that are created will be outputted at the end of the build.
     presents a prompt and waits for you to decide to clean up, abort, or retry
     the failed step.
 
--   `-only=foo,bar,baz` - Only build the builds with the given comma-separated
-    names. Build names by default are the names of their builders, unless a
-    specific `name` attribute is specified within the configuration.
+-   `-only=foo,bar,baz` - Only run the builds with the given comma-separated
+    names. Build names by default are their type, unless a specific `name`
+    attribute is specified within the configuration. `-only` does not apply to
+    post-processors.
 
 -   `-parallel=false` - Disable parallelization of multiple builders (on by
     default).

website/source/docs/templates/provisioners.html.md

@@ -81,6 +81,7 @@ The values within `only` or `except` are *build names*, not builder types. If
 you recall, build names by default are just their builder type, but if you
 specify a custom `name` parameter, then you should use that as the value
 instead of the type.
+Values within `except` could also be a *post-processor* name.
 
 ## Build-Specific Overrides