From cd74430e0c474d7cc37dd3aa785401ce1c96ddf5 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Fri, 30 Aug 2024 14:38:32 -0400 Subject: [PATCH] command: add use-sequential options for commands For all the commands that call Initialise, we introduce a new flag: UseSequential. This disables DAG scheduling for evaluating datasources and locals as a fallback to the newly introduced DAG scheduling approach. `hcl2_upgrade` is a special case here, as the template is always JSON, there cannot be any datasource, so the DAG in this case becomes meaningless, and is not integrated in this code path. --- command/build.go | 5 ++++- command/cli.go | 10 ++++++++++ command/console.go | 11 +++++++---- command/hcl2_upgrade.go | 7 ++++++- command/inspect.go | 7 +++++-- command/validate.go | 2 ++ 6 files changed, 34 insertions(+), 8 deletions(-) diff --git a/command/build.go b/command/build.go index a7503eeea..58b548b00 100644 --- a/command/build.go +++ b/command/build.go @@ -102,7 +102,9 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int return ret } - diags = packerStarter.Initialize(packer.InitializeOptions{}) + diags = packerStarter.Initialize(packer.InitializeOptions{ + UseSequential: cla.UseSequential, + }) ret = writeDiags(c.Ui, nil, diags) if ret != 0 { return ret @@ -439,6 +441,7 @@ Options: -var-file=path JSON or HCL2 file containing user variables, can be used multiple times. -warn-on-undeclared-var Display warnings for user variable files containing undeclared variables. -ignore-prerelease-plugins Disable the loading of prerelease plugin binaries (x.y.z-dev). + -use-sequential-evaluation Fallback to using a sequential approach for local/datasource evaluation. ` return strings.TrimSpace(helpText) diff --git a/command/cli.go b/command/cli.go index 4a6b32dd7..2ad922229 100644 --- a/command/cli.go +++ b/command/cli.go @@ -75,6 +75,12 @@ type MetaArgs struct { // WarnOnUndeclared does not have a common default, as the default varies per sub-command usage. // Refer to individual command FlagSets for usage. WarnOnUndeclaredVar bool + // UseSequential specifies to use a sequential/phased approach for + // evaluating datasources/locals instead of a DAG. + // + // This allows users to fall-back to using the approach used by Packer + // before the introduction of a DAG in case they run in an impasse/bug. + UseSequential bool } func (ba *BuildArgs) AddFlagSets(flags *flag.FlagSet) { @@ -90,6 +96,7 @@ func (ba *BuildArgs) AddFlagSets(flags *flag.FlagSet) { flags.Var(flagOnError, "on-error", "") flags.BoolVar(&ba.MetaArgs.WarnOnUndeclaredVar, "warn-on-undeclared-var", false, "Show warnings for variable files containing undeclared variables.") + flags.BoolVar(&ba.MetaArgs.UseSequential, "use-sequential-evaluation", false, "Fallback to using a sequential approach for local/datasource evaluation.") flags.BoolVar(&ba.ReleaseOnly, "ignore-prerelease-plugins", false, "Disable the loading of prerelease plugin binaries (x.y.z-dev).") @@ -156,6 +163,7 @@ type ConsoleArgs struct { func (fa *FixArgs) AddFlagSets(flags *flag.FlagSet) { flags.BoolVar(&fa.Validate, "validate", true, "") + flags.BoolVar(&fa.MetaArgs.UseSequential, "use-sequential-evaluation", false, "Fallback to using a sequential approach for local/datasource evaluation.") fa.MetaArgs.AddFlagSets(flags) } @@ -171,6 +179,7 @@ func (va *ValidateArgs) AddFlagSets(flags *flag.FlagSet) { flags.BoolVar(&va.NoWarnUndeclaredVar, "no-warn-undeclared-var", false, "Ignore warnings for variable files containing undeclared variables.") flags.BoolVar(&va.EvaluateDatasources, "evaluate-datasources", false, "evaluate datasources for validation (HCL2 only, may incur costs)") flags.BoolVar(&va.ReleaseOnly, "ignore-prerelease-plugins", false, "Disable the loading of prerelease plugin binaries (x.y.z-dev).") + flags.BoolVar(&va.MetaArgs.UseSequential, "use-sequential-evaluation", false, "Fallback to using a sequential approach for local/datasource evaluation.") va.MetaArgs.AddFlagSets(flags) } @@ -184,6 +193,7 @@ type ValidateArgs struct { } func (va *InspectArgs) AddFlagSets(flags *flag.FlagSet) { + flags.BoolVar(&va.MetaArgs.UseSequential, "use-sequential-evaluation", false, "Fallback to using a sequential approach for local/datasource evaluation.") va.MetaArgs.AddFlagSets(flags) } diff --git a/command/console.go b/command/console.go index da24a0615..eac4f4bd2 100644 --- a/command/console.go +++ b/command/console.go @@ -63,7 +63,9 @@ func (c *ConsoleCommand) RunContext(ctx context.Context, cla *ConsoleArgs) int { return ret } - _ = packerStarter.Initialize(packer.InitializeOptions{}) + _ = packerStarter.Initialize(packer.InitializeOptions{ + UseSequential: cla.UseSequential, + }) // Determine if stdin is a pipe. If so, we evaluate directly. if c.StdinPiped() { @@ -83,9 +85,10 @@ Usage: packer console [options] [TEMPLATE] interpolation. Options: - -var 'key=value' Variable for templates, can be used multiple times. - -var-file=path JSON or HCL2 file containing user variables. - -config-type Set to 'hcl2' to run in HCL2 mode when no file is passed. Defaults to json. + -var 'key=value' Variable for templates, can be used multiple times. + -var-file=path JSON or HCL2 file containing user variables. + -config-type Set to 'hcl2' to run in HCL2 mode when no file is passed. Defaults to json. + -use-sequential-evaluation Fallback to using a sequential approach for local/datasource evaluation. ` return strings.TrimSpace(helpText) diff --git a/command/hcl2_upgrade.go b/command/hcl2_upgrade.go index 08ffa767e..56edf175d 100644 --- a/command/hcl2_upgrade.go +++ b/command/hcl2_upgrade.go @@ -181,7 +181,12 @@ func (c *HCL2UpgradeCommand) RunContext(_ context.Context, cla *HCL2UpgradeArgs) } core := hdl.(*packer.Core) - if err := core.Initialize(packer.InitializeOptions{}); err != nil { + if err := core.Initialize(packer.InitializeOptions{ + // Note: this is always true here as the DAG is only usable for + // HCL2 configs, so since the command only works on JSON templates, + // we can safely use the phased approach, which changes nothing. + UseSequential: true, + }); err != nil { c.Ui.Error(fmt.Sprintf("Ignoring following initialization error: %v", err)) } tpl := core.Template diff --git a/command/inspect.go b/command/inspect.go index 7e5ed3166..2c371d6c4 100644 --- a/command/inspect.go +++ b/command/inspect.go @@ -49,7 +49,9 @@ func (c *InspectCommand) RunContext(ctx context.Context, cla *InspectArgs) int { } // here we ignore init diags to allow unknown variables to be used - _ = packerStarter.Initialize(packer.InitializeOptions{}) + _ = packerStarter.Initialize(packer.InitializeOptions{ + UseSequential: cla.UseSequential, + }) return packerStarter.InspectConfig(packer.InspectConfigOptions{ Ui: c.Ui, @@ -66,7 +68,8 @@ Usage: packer inspect TEMPLATE Options: - -machine-readable Machine-readable output + -machine-readable Machine-readable output + -use-sequential-evaluation Fallback to using a sequential approach for local/datasource evaluation. ` return strings.TrimSpace(helpText) diff --git a/command/validate.go b/command/validate.go index 47d88a901..f3f6df378 100644 --- a/command/validate.go +++ b/command/validate.go @@ -78,6 +78,7 @@ func (c *ValidateCommand) RunContext(ctx context.Context, cla *ValidateArgs) int diags = packerStarter.Initialize(packer.InitializeOptions{ SkipDatasourcesExecution: !cla.EvaluateDatasources, + UseSequential: cla.UseSequential, }) ret = writeDiags(c.Ui, nil, diags) if ret != 0 { @@ -124,6 +125,7 @@ Options: -no-warn-undeclared-var Disable warnings for user variable files containing undeclared variables. -evaluate-datasources Evaluate data sources during validation (HCL2 only, may incur costs); Defaults to false. -ignore-prerelease-plugins Disable the loading of prerelease plugin binaries (x.y.z-dev). + -use-sequential-evaluation Fallback to using a sequential approach for local/datasource evaluation. ` return strings.TrimSpace(helpText)