From 92e75f838a63dd6d02691c51b3e9f0cc739bd092 Mon Sep 17 00:00:00 2001 From: Brett Wandel Date: Sat, 13 Apr 2019 22:24:35 +1000 Subject: [PATCH] added the ability to limit number of builds running in parallel --- command/build.go | 20 ++++++++++++++++---- command/build_test.go | 10 +++++----- contrib/zsh-completion/_packer | 2 +- website/source/docs/commands/build.html.md | 2 +- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/command/build.go b/command/build.go index 9072b9ea4..e9daa1074 100644 --- a/command/build.go +++ b/command/build.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "log" + "math" "os" "os/signal" "strconv" @@ -15,6 +16,7 @@ import ( "github.com/hashicorp/packer/helper/enumflag" "github.com/hashicorp/packer/packer" "github.com/hashicorp/packer/template" + "golang.org/x/sync/semaphore" "github.com/posener/complete" ) @@ -24,7 +26,8 @@ type BuildCommand struct { } func (c *BuildCommand) Run(args []string) int { - var cfgColor, cfgDebug, cfgForce, cfgTimestamp, cfgParallel bool + var cfgColor, cfgDebug, cfgForce, cfgTimestamp bool + var cfgParallel int64 var cfgOnError string flags := c.Meta.FlagSet("build", FlagSetBuildFilter|FlagSetVars) flags.Usage = func() { c.Ui.Say(c.Help()) } @@ -34,7 +37,7 @@ func (c *BuildCommand) Run(args []string) int { flags.BoolVar(&cfgTimestamp, "timestamp-ui", false, "") flagOnError := enumflag.New(&cfgOnError, "cleanup", "abort", "ask") flags.Var(flagOnError, "on-error", "") - flags.BoolVar(&cfgParallel, "parallel", true, "") + flags.Int64Var(&cfgParallel, "parallel", 0, "") if err := flags.Parse(args); err != nil { return 1 } @@ -150,10 +153,18 @@ func (c *BuildCommand) Run(args []string) int { }{m: make(map[string][]packer.Artifact)} errors := make(map[string]error) // ctx := context.Background() + if cfgParallel < 1 { + cfgParallel = math.MaxInt64 + } + limitParallel := semaphore.NewWeighted(cfgParallel) for _, b := range builds { // Increment the waitgroup so we wait for this item to finish properly wg.Add(1) buildCtx, cancelCtx := context.WithCancel(context.Background()) + if err := limitParallel.Acquire(buildCtx, 1); err != nil { + log.Printf("Stopping build: failed to acquire semaphore %s", err) + return 1 + } // Handle interrupts for this build sigCh := make(chan os.Signal, 1) @@ -173,6 +184,7 @@ func (c *BuildCommand) Run(args []string) int { // Run the build in a goroutine go func(b packer.Build) { defer wg.Done() + defer limitParallel.Release(1) name := b.Name() log.Printf("Starting build run: %s", name) @@ -195,7 +207,7 @@ func (c *BuildCommand) Run(args []string) int { wg.Wait() } - if !cfgParallel { + if cfgParallel == 1 { log.Printf("Parallelization disabled, waiting for build to finish: %s", b.Name()) wg.Wait() } @@ -308,7 +320,7 @@ Options: -force Force a build to continue if artifacts exist, deletes existing artifacts. -machine-readable Produce machine-readable output. -on-error=[cleanup|abort|ask] If the build fails do: clean up (default), abort, or ask. - -parallel=false Disable parallelization. (Default: parallel) + -parallel=count Number of builds to run in parallel. (Default: 0) -timestamp-ui Enable prefixing of each ui output with an RFC3339 timestamp. -var 'key=value' Variable for templates, can be used multiple times. -var-file=path JSON file containing user variables. diff --git a/command/build_test.go b/command/build_test.go index 8f3661b95..12282769b 100644 --- a/command/build_test.go +++ b/command/build_test.go @@ -17,7 +17,7 @@ func TestBuildOnlyFileCommaFlags(t *testing.T) { } args := []string{ - "-parallel=false", + "-parallel=1", "-only=chocolate,vanilla", filepath.Join(testFixture("build-only"), "template.json"), } @@ -59,7 +59,7 @@ func TestBuildStdin(t *testing.T) { defer func() { os.Stdin = stdin }() defer cleanup() - if code := c.Run([]string{"-parallel=false", "-"}); code != 0 { + if code := c.Run([]string{"-parallel=1", "-"}); code != 0 { fatalCommand(t, c.Meta) } @@ -77,7 +77,7 @@ func TestBuildOnlyFileMultipleFlags(t *testing.T) { } args := []string{ - "-parallel=false", + "-parallel=1", "-only=chocolate", "-only=cherry", "-only=apple", // ignored @@ -111,7 +111,7 @@ func TestBuildEverything(t *testing.T) { } args := []string{ - "-parallel=false", + "-parallel=1", `-except=`, filepath.Join(testFixture("build-only"), "template.json"), } @@ -136,7 +136,7 @@ func TestBuildExceptFileCommaFlags(t *testing.T) { } args := []string{ - "-parallel=false", + "-parallel=1", "-except=chocolate,vanilla", filepath.Join(testFixture("build-only"), "template.json"), } diff --git a/contrib/zsh-completion/_packer b/contrib/zsh-completion/_packer index f0ae47fd9..4a79f3cd6 100644 --- a/contrib/zsh-completion/_packer +++ b/contrib/zsh-completion/_packer @@ -17,7 +17,7 @@ _packer () { '-except=[(foo,bar,baz) Run all builds and post-procesors other than these.]' '-on-error=[(cleanup,abort,ask) If the build fails do: clean up (default), abort, or ask.]' '-only=[(foo,bar,baz) Only build the given builds by name.]' - '-parallel=[(false) Disable parallelization. (Default: parallel)]' + '-parallel=[(0) Number of builds to run in parallel. (Default: 0)]' '-var[("key=value") Variable for templates, can be used multiple times.]' '-var-file=[(path) JSON file containing user variables.]' '(-)*:files:_files -g "*.json"' diff --git a/website/source/docs/commands/build.html.md b/website/source/docs/commands/build.html.md index 7e559b974..a1d416245 100644 --- a/website/source/docs/commands/build.html.md +++ b/website/source/docs/commands/build.html.md @@ -52,7 +52,7 @@ artifacts that are created will be outputted at the end of the build. attribute is specified within the configuration. `-only` does not apply to post-processors. -- `-parallel=false` - Disable parallelization of multiple builders (on by +- `-parallel=1` - Limit the number of builds to run in parallel (no limit by default). - `-timestamp-ui` - Enable prefixing of each ui output with an RFC3339