From 56e08c2eff838505dca556c9ad8d0862f7240ee8 Mon Sep 17 00:00:00 2001 From: Lucas Bajolet Date: Fri, 30 Aug 2024 14:10:27 -0400 Subject: [PATCH] hcl2template: add DAG-based eval for local/data As we have finished setting-up the codebase for it, this commit adds the logic that uses the internal DAG package, and is able to orchestrate evaluation of datasources and locals in a non-phased way. Instead, this code acts by first detecting the dependencies for those components, builds a graph from them, with edges representing the dependency links between them, and finally walking on the graph breadth-first to evaluate those components. This can act as a drop-in replacement for the current phased logic, but both should be supported until we are confident that the approach works, and that there are little to no bugs left to squash. --- hcl2template/parser.go | 177 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) diff --git a/hcl2template/parser.go b/hcl2template/parser.go index 72d5c1634..1c3430dea 100644 --- a/hcl2template/parser.go +++ b/hcl2template/parser.go @@ -7,12 +7,14 @@ import ( "fmt" "os" "path/filepath" + "reflect" "github.com/hashicorp/go-version" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/ext/dynblock" "github.com/hashicorp/hcl/v2/hclparse" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" + "github.com/hashicorp/packer/internal/dag" "github.com/hashicorp/packer/packer" "github.com/zclconf/go-cty/cty" ) @@ -295,6 +297,181 @@ func filterVarsFromLogs(inputOrLocal Variables) { } } +func (cfg *PackerConfig) detectBuildPrereqDependencies() hcl.Diagnostics { + var diags hcl.Diagnostics + + for _, ds := range cfg.Datasources { + dependencies := GetVarsByType(ds.block, "data") + dependencies = append(dependencies, GetVarsByType(ds.block, "local")...) + + for _, dep := range dependencies { + // If something is locally aliased as `local` or `data`, we'll falsely + // report it as a local variable, which is not necessarily what we + // want to process here, so we continue. + // + // Note: this is kinda brittle, we should understand scopes to accurately + // mark something from an expression as a reference to a local variable. + // No real good solution for this now, besides maybe forbidding something + // to be locally aliased as `local`. + if len(dep) < 2 { + continue + } + rs, err := NewRefStringFromDep(dep) + if err != nil { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to process datasource dependency", + Detail: fmt.Sprintf("An error occurred while processing a dependency for data source %s: %s", + ds.Name(), err), + }) + continue + } + + err = ds.RegisterDependency(rs) + if err != nil { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to register datasource dependency", + Detail: fmt.Sprintf("An error occurred while registering %q as a dependency for data source %s: %s", + rs, ds.Name(), err), + }) + } + } + + } + + for _, loc := range cfg.LocalBlocks { + dependencies := FilterTraversalsByType(loc.Expr.Variables(), "data") + dependencies = append(dependencies, FilterTraversalsByType(loc.Expr.Variables(), "local")...) + + for _, dep := range dependencies { + // If something is locally aliased as `local` or `data`, we'll falsely + // report it as a local variable, which is not necessarily what we + // want to process here, so we continue. + // + // Note: this is kinda brittle, we should understand scopes to accurately + // mark something from an expression as a reference to a local variable. + // No real good solution for this now, besides maybe forbidding something + // to be locally aliased as `local`. + if len(dep) < 2 { + continue + } + rs, err := NewRefStringFromDep(dep) + if err != nil { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to process local dependency", + Detail: fmt.Sprintf("An error occurred while processing a dependency for local variable %s: %s", + loc.Name, err), + }) + continue + } + + err = loc.RegisterDependency(rs) + if err != nil { + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to register local dependency", + Detail: fmt.Sprintf("An error occurred while registering %q as a dependency for local variable %s: %s", + rs, loc.Name, err), + }) + } + } + } + + return diags +} + +func (cfg *PackerConfig) buildPrereqsDAG() (*dag.AcyclicGraph, error) { + retGraph := dag.AcyclicGraph{} + + verticesMap := map[string]dag.Vertex{} + + // Do a first pass to create all the vertices + for _, ds := range cfg.Datasources { + v := retGraph.Add(&ds) + verticesMap[fmt.Sprintf("data.%s", ds.Name())] = v + } + for _, local := range cfg.LocalBlocks { + v := retGraph.Add(local) + verticesMap[fmt.Sprintf("local.%s", local.Name)] = v + } + + // Connect the vertices together + // + // Vertices that don't have dependencies will be connected to the + // root vertex of the graph + for _, ds := range cfg.Datasources { + dsName := fmt.Sprintf("data.%s", ds.Name()) + + for _, dep := range ds.Dependencies { + retGraph.Connect( + dag.BasicEdge(verticesMap[dsName], + verticesMap[dep.String()])) + } + } + for _, loc := range cfg.LocalBlocks { + locName := fmt.Sprintf("local.%s", loc.Name) + + for _, dep := range loc.dependencies { + retGraph.Connect( + dag.BasicEdge(verticesMap[locName], + verticesMap[dep.String()])) + } + } + + return &retGraph, nil +} + +func (cfg *PackerConfig) evaluateBuildPrereqs(skipDatasources bool) hcl.Diagnostics { + diags := cfg.detectBuildPrereqDependencies() + if diags.HasErrors() { + return diags + } + + graph, err := cfg.buildPrereqsDAG() + if err != nil { + return diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "failed to prepare execution graph", + Detail: fmt.Sprintf("An error occurred while building the graph for datasources/locals: %s", err), + }) + } + + walkFunc := func(v dag.Vertex) hcl.Diagnostics { + var diags hcl.Diagnostics + + switch bl := v.(type) { + case *DatasourceBlock: + diags = cfg.evaluateDatasource(*bl, skipDatasources) + case *LocalBlock: + diags = cfg.evaluateLocalVariable(bl) + default: + diags = diags.Append(&hcl.Diagnostic{ + Severity: hcl.DiagError, + Summary: "unsupported DAG node type", + Detail: fmt.Sprintf("A node of type %q was added to the DAG, but cannot be "+ + "evaluated as it is unsupported. "+ + "This is a Packer bug, please report it so we can investigate.", + reflect.TypeOf(v).String()), + }) + } + + // ("unsupported node of type %q") + if diags.HasErrors() { + return diags + } + + return nil + } + + if cfg.LocalVariables == nil { + cfg.LocalVariables = Variables{} + } + + return diags.Extend(graph.Walk(walkFunc)) +} + func (cfg *PackerConfig) Initialize(opts packer.InitializeOptions) hcl.Diagnostics { diags := cfg.InputVariables.ValidateValues() diags = append(diags, cfg.evaluateDatasources(opts.SkipDatasourcesExecution)...)