command: add use-sequential options for commands

For all the commands that call Initialise, we introduce a new flag:
UseSequential.

This disables DAG scheduling for evaluating datasources and locals as a
fallback to the newly introduced DAG scheduling approach.

`hcl2_upgrade` is a special case here, as the template is always JSON,
there cannot be any datasource, so the DAG in this case becomes
meaningless, and is not integrated in this code path.

command/build.go

@@ -102,7 +102,9 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
 		return ret
 	}
 
-	diags = packerStarter.Initialize(packer.InitializeOptions{})
+	diags = packerStarter.Initialize(packer.InitializeOptions{
+		UseSequential: cla.UseSequential,
+	})
 	ret = writeDiags(c.Ui, nil, diags)
 	if ret != 0 {
 		return ret
@@ -439,6 +441,7 @@ Options:
   -var-file=path                JSON or HCL2 file containing user variables, can be used multiple times.
   -warn-on-undeclared-var       Display warnings for user variable files containing undeclared variables.
   -ignore-prerelease-plugins    Disable the loading of prerelease plugin binaries (x.y.z-dev).
+  -use-sequential-evaluation    Fallback to using a sequential approach for local/datasource evaluation.
 `
 
 	return strings.TrimSpace(helpText)

command/cli.go

@@ -75,6 +75,12 @@ type MetaArgs struct {
 	// WarnOnUndeclared does not have a common default, as the default varies per sub-command usage.
 	// Refer to individual command FlagSets for usage.
 	WarnOnUndeclaredVar bool
+	// UseSequential specifies to use a sequential/phased approach for
+	// evaluating datasources/locals instead of a DAG.
+	//
+	// This allows users to fall-back to using the approach used by Packer
+	// before the introduction of a DAG in case they run in an impasse/bug.
+	UseSequential bool
 }
 
 func (ba *BuildArgs) AddFlagSets(flags *flag.FlagSet) {
@@ -90,6 +96,7 @@ func (ba *BuildArgs) AddFlagSets(flags *flag.FlagSet) {
 	flags.Var(flagOnError, "on-error", "")
 
 	flags.BoolVar(&ba.MetaArgs.WarnOnUndeclaredVar, "warn-on-undeclared-var", false, "Show warnings for variable files containing undeclared variables.")
+	flags.BoolVar(&ba.MetaArgs.UseSequential, "use-sequential-evaluation", false, "Fallback to using a sequential approach for local/datasource evaluation.")
 
 	flags.BoolVar(&ba.ReleaseOnly, "ignore-prerelease-plugins", false, "Disable the loading of prerelease plugin binaries (x.y.z-dev).")
 
@@ -156,6 +163,7 @@ type ConsoleArgs struct {
 
 func (fa *FixArgs) AddFlagSets(flags *flag.FlagSet) {
 	flags.BoolVar(&fa.Validate, "validate", true, "")
+	flags.BoolVar(&fa.MetaArgs.UseSequential, "use-sequential-evaluation", false, "Fallback to using a sequential approach for local/datasource evaluation.")
 
 	fa.MetaArgs.AddFlagSets(flags)
 }
@@ -171,6 +179,7 @@ func (va *ValidateArgs) AddFlagSets(flags *flag.FlagSet) {
 	flags.BoolVar(&va.NoWarnUndeclaredVar, "no-warn-undeclared-var", false, "Ignore warnings for variable files containing undeclared variables.")
 	flags.BoolVar(&va.EvaluateDatasources, "evaluate-datasources", false, "evaluate datasources for validation (HCL2 only, may incur costs)")
 	flags.BoolVar(&va.ReleaseOnly, "ignore-prerelease-plugins", false, "Disable the loading of prerelease plugin binaries (x.y.z-dev).")
+	flags.BoolVar(&va.MetaArgs.UseSequential, "use-sequential-evaluation", false, "Fallback to using a sequential approach for local/datasource evaluation.")
 
 	va.MetaArgs.AddFlagSets(flags)
 }
@@ -184,6 +193,7 @@ type ValidateArgs struct {
 }
 
 func (va *InspectArgs) AddFlagSets(flags *flag.FlagSet) {
+	flags.BoolVar(&va.MetaArgs.UseSequential, "use-sequential-evaluation", false, "Fallback to using a sequential approach for local/datasource evaluation.")
 	va.MetaArgs.AddFlagSets(flags)
 }
 

command/console.go

@@ -63,7 +63,9 @@ func (c *ConsoleCommand) RunContext(ctx context.Context, cla *ConsoleArgs) int {
 		return ret
 	}
 
-	_ = packerStarter.Initialize(packer.InitializeOptions{})
+	_ = packerStarter.Initialize(packer.InitializeOptions{
+		UseSequential: cla.UseSequential,
+	})
 
 	// Determine if stdin is a pipe. If so, we evaluate directly.
 	if c.StdinPiped() {
@@ -83,9 +85,10 @@ Usage: packer console [options] [TEMPLATE]
   interpolation.
 
 Options:
-  -var 'key=value'       Variable for templates, can be used multiple times.
-  -var-file=path         JSON or HCL2 file containing user variables.
-  -config-type           Set to 'hcl2' to run in HCL2 mode when no file is passed. Defaults to json.
+  -var 'key=value'              Variable for templates, can be used multiple times.
+  -var-file=path                JSON or HCL2 file containing user variables.
+  -config-type                  Set to 'hcl2' to run in HCL2 mode when no file is passed. Defaults to json.
+  -use-sequential-evaluation    Fallback to using a sequential approach for local/datasource evaluation.
 `
 
 	return strings.TrimSpace(helpText)

command/hcl2_upgrade.go

@@ -181,7 +181,12 @@ func (c *HCL2UpgradeCommand) RunContext(_ context.Context, cla *HCL2UpgradeArgs)
 	}
 
 	core := hdl.(*packer.Core)
-	if err := core.Initialize(packer.InitializeOptions{}); err != nil {
+	if err := core.Initialize(packer.InitializeOptions{
+		// Note: this is always true here as the DAG is only usable for
+		// HCL2 configs, so since the command only works on JSON templates,
+		// we can safely use the phased approach, which changes nothing.
+		UseSequential: true,
+	}); err != nil {
 		c.Ui.Error(fmt.Sprintf("Ignoring following initialization error: %v", err))
 	}
 	tpl := core.Template

command/inspect.go

@@ -49,7 +49,9 @@ func (c *InspectCommand) RunContext(ctx context.Context, cla *InspectArgs) int {
 	}
 
 	// here we ignore init diags to allow unknown variables to be used
-	_ = packerStarter.Initialize(packer.InitializeOptions{})
+	_ = packerStarter.Initialize(packer.InitializeOptions{
+		UseSequential: cla.UseSequential,
+	})
 
 	return packerStarter.InspectConfig(packer.InspectConfigOptions{
 		Ui: c.Ui,
@@ -66,7 +68,8 @@ Usage: packer inspect TEMPLATE
 
 Options:
 
-  -machine-readable  Machine-readable output
+  -machine-readable             Machine-readable output
+  -use-sequential-evaluation    Fallback to using a sequential approach for local/datasource evaluation.
 `
 
 	return strings.TrimSpace(helpText)

command/validate.go

@@ -78,6 +78,7 @@ func (c *ValidateCommand) RunContext(ctx context.Context, cla *ValidateArgs) int
 
 	diags = packerStarter.Initialize(packer.InitializeOptions{
 		SkipDatasourcesExecution: !cla.EvaluateDatasources,
+		UseSequential:            cla.UseSequential,
 	})
 	ret = writeDiags(c.Ui, nil, diags)
 	if ret != 0 {
@@ -124,6 +125,7 @@ Options:
   -no-warn-undeclared-var       Disable warnings for user variable files containing undeclared variables.
   -evaluate-datasources         Evaluate data sources during validation (HCL2 only, may incur costs); Defaults to false. 
   -ignore-prerelease-plugins    Disable the loading of prerelease plugin binaries (x.y.z-dev).
+  -use-sequential-evaluation    Fallback to using a sequential approach for local/datasource evaluation.
 `
 
 	return strings.TrimSpace(helpText)