From 95019e3d0225caaf027d889a922bcae68de5cb24 Mon Sep 17 00:00:00 2001 From: James Bardin Date: Fri, 22 Jul 2022 10:20:17 -0400 Subject: [PATCH] Implement breadth-first walks and add tests Make DAG walks test-able, and add tests for more complex graph ordering. We also add breadth-first for comparison, though it's not used currently in Terraform. --- internal/dag/dag.go | 137 +++++++++++++++++++++++++-------------- internal/dag/dag_test.go | 119 ++++++++++++++++++++++++++-------- 2 files changed, 183 insertions(+), 73 deletions(-) diff --git a/internal/dag/dag.go b/internal/dag/dag.go index 6da10df51b..fd086f6845 100644 --- a/internal/dag/dag.go +++ b/internal/dag/dag.go @@ -2,6 +2,7 @@ package dag import ( "fmt" + "sort" "strings" "github.com/hashicorp/terraform/internal/tfdiags" @@ -178,24 +179,72 @@ type vertexAtDepth struct { Depth int } +type walkType uint64 + +const ( + depthFirst walkType = 1 << iota + breadthFirst + downOrder + upOrder +) + // DepthFirstWalk does a depth-first walk of the graph starting from // the vertices in start. -// The algorithm used here does not do a complete topological sort. To ensure -// correct overall ordering run TransitiveReduction first. func (g *AcyclicGraph) DepthFirstWalk(start Set, f DepthWalkFunc) error { + return g.walk(depthFirst|downOrder, false, start, f) +} + +// ReverseDepthFirstWalk does a depth-first walk _up_ the graph starting from +// the vertices in start. +func (g *AcyclicGraph) ReverseDepthFirstWalk(start Set, f DepthWalkFunc) error { + return g.walk(depthFirst|upOrder, false, start, f) +} + +// BreadthFirstWalk does a breadth-first walk of the graph starting from +// the vertices in start. +func (g *AcyclicGraph) BreadthFirstWalk(start Set, f DepthWalkFunc) error { + return g.walk(breadthFirst|downOrder, false, start, f) +} + +// ReverseBreadthFirstWalk does a breadth-first walk _up_ the graph starting from +// the vertices in start. +func (g *AcyclicGraph) ReverseBreadthFirstWalk(start Set, f DepthWalkFunc) error { + return g.walk(breadthFirst|upOrder, false, start, f) +} + +// Setting test to true will walk sets of vertices in sorted order for +// deterministic testing. +func (g *AcyclicGraph) walk(order walkType, test bool, start Set, f DepthWalkFunc) error { seen := make(map[Vertex]struct{}) - frontier := make([]*vertexAtDepth, 0, len(start)) + frontier := make([]vertexAtDepth, 0, len(start)) for _, v := range start { - frontier = append(frontier, &vertexAtDepth{ + frontier = append(frontier, vertexAtDepth{ Vertex: v, Depth: 0, }) } + + if test { + testSortFrontier(frontier) + } + for len(frontier) > 0 { // Pop the current vertex - n := len(frontier) - current := frontier[n-1] - frontier = frontier[:n-1] + var current vertexAtDepth + + switch { + case order&depthFirst != 0: + // depth first, the frontier is used like a stack + n := len(frontier) + current = frontier[n-1] + frontier = frontier[:n-1] + case order&breadthFirst != 0: + // breadth first, the frontier is used like a queue + current = frontier[0] + frontier = frontier[1:] + default: + panic(fmt.Sprint("invalid visit order", order)) + } // Check if we've seen this already and return... if _, ok := seen[current.Vertex]; ok { @@ -208,54 +257,48 @@ func (g *AcyclicGraph) DepthFirstWalk(start Set, f DepthWalkFunc) error { return err } - for _, v := range g.downEdgesNoCopy(current.Vertex) { - frontier = append(frontier, &vertexAtDepth{ - Vertex: v, - Depth: current.Depth + 1, - }) + var edges Set + switch { + case order&downOrder != 0: + edges = g.downEdgesNoCopy(current.Vertex) + case order&upOrder != 0: + edges = g.upEdgesNoCopy(current.Vertex) + default: + panic(fmt.Sprint("invalid walk order", order)) } - } + if test { + frontier = testAppendNextSorted(frontier, edges, current.Depth+1) + } else { + frontier = appendNext(frontier, edges, current.Depth+1) + } + } return nil } -// ReverseDepthFirstWalk does a depth-first walk _up_ the graph starting from -// the vertices in start. -// The algorithm used here does not do a complete topological sort. To ensure -// correct overall ordering run TransitiveReduction first. -func (g *AcyclicGraph) ReverseDepthFirstWalk(start Set, f DepthWalkFunc) error { - seen := make(map[Vertex]struct{}) - frontier := make([]*vertexAtDepth, 0, len(start)) - for _, v := range start { - frontier = append(frontier, &vertexAtDepth{ +func appendNext(frontier []vertexAtDepth, next Set, depth int) []vertexAtDepth { + for _, v := range next { + frontier = append(frontier, vertexAtDepth{ Vertex: v, - Depth: 0, + Depth: depth, }) } - for len(frontier) > 0 { - // Pop the current vertex - n := len(frontier) - current := frontier[n-1] - frontier = frontier[:n-1] - - // Check if we've seen this already and return... - if _, ok := seen[current.Vertex]; ok { - continue - } - seen[current.Vertex] = struct{}{} - - for _, t := range g.upEdgesNoCopy(current.Vertex) { - frontier = append(frontier, &vertexAtDepth{ - Vertex: t, - Depth: current.Depth + 1, - }) - } + return frontier +} - // Visit the current node - if err := f(current.Vertex, current.Depth); err != nil { - return err - } +func testAppendNextSorted(frontier []vertexAtDepth, edges Set, depth int) []vertexAtDepth { + var newEdges []vertexAtDepth + for _, v := range edges { + newEdges = append(newEdges, vertexAtDepth{ + Vertex: v, + Depth: depth, + }) } - - return nil + testSortFrontier(newEdges) + return append(frontier, newEdges...) +} +func testSortFrontier(f []vertexAtDepth) { + sort.Slice(f, func(i, j int) bool { + return VertexName(f[i].Vertex) < VertexName(f[j].Vertex) + }) } diff --git a/internal/dag/dag_test.go b/internal/dag/dag_test.go index 75cfb86ff0..6e9cf5ac43 100644 --- a/internal/dag/dag_test.go +++ b/internal/dag/dag_test.go @@ -414,34 +414,101 @@ func BenchmarkDAG(b *testing.B) { } } -func TestAcyclicGraph_ReverseDepthFirstWalk_WithRemoval(t *testing.T) { - var g AcyclicGraph - g.Add(1) - g.Add(2) - g.Add(3) - g.Connect(BasicEdge(3, 2)) - g.Connect(BasicEdge(2, 1)) +func TestAcyclicGraphWalkOrder(t *testing.T) { + /* Sample dependency graph, + all edges pointing downwards. + 1 2 + / \ / \ + 3 4 5 + / \ / + 6 7 + / | \ + 8 9 10 + \ | / + 11 + */ - var visits []Vertex - var lock sync.Mutex - root := make(Set) - root.Add(1) - - err := g.ReverseDepthFirstWalk(root, func(v Vertex, d int) error { - lock.Lock() - defer lock.Unlock() - visits = append(visits, v) - g.Remove(v) - return nil - }) - if err != nil { - t.Fatalf("err: %s", err) - } - - expected := []Vertex{1, 2, 3} - if !reflect.DeepEqual(visits, expected) { - t.Fatalf("expected: %#v, got: %#v", expected, visits) + var g AcyclicGraph + for i := 0; i <= 11; i++ { + g.Add(i) } + g.Connect(BasicEdge(1, 3)) + g.Connect(BasicEdge(1, 4)) + g.Connect(BasicEdge(2, 4)) + g.Connect(BasicEdge(2, 5)) + g.Connect(BasicEdge(3, 6)) + g.Connect(BasicEdge(4, 7)) + g.Connect(BasicEdge(5, 7)) + g.Connect(BasicEdge(7, 8)) + g.Connect(BasicEdge(7, 9)) + g.Connect(BasicEdge(7, 10)) + g.Connect(BasicEdge(8, 11)) + g.Connect(BasicEdge(9, 11)) + g.Connect(BasicEdge(10, 11)) + + start := make(Set) + start.Add(2) + start.Add(1) + reverse := make(Set) + reverse.Add(11) + reverse.Add(6) + + t.Run("DepthFirst", func(t *testing.T) { + var visits []vertexAtDepth + g.walk(depthFirst|downOrder, true, start, func(v Vertex, d int) error { + visits = append(visits, vertexAtDepth{v, d}) + return nil + + }) + expect := []vertexAtDepth{ + {2, 0}, {5, 1}, {7, 2}, {9, 3}, {11, 4}, {8, 3}, {10, 3}, {4, 1}, {1, 0}, {3, 1}, {6, 2}, + } + if !reflect.DeepEqual(visits, expect) { + t.Errorf("expected visits:\n%v\ngot:\n%v\n", expect, visits) + } + }) + t.Run("ReverseDepthFirst", func(t *testing.T) { + var visits []vertexAtDepth + g.walk(depthFirst|upOrder, true, reverse, func(v Vertex, d int) error { + visits = append(visits, vertexAtDepth{v, d}) + return nil + + }) + expect := []vertexAtDepth{ + {6, 0}, {3, 1}, {1, 2}, {11, 0}, {9, 1}, {7, 2}, {5, 3}, {2, 4}, {4, 3}, {8, 1}, {10, 1}, + } + if !reflect.DeepEqual(visits, expect) { + t.Errorf("expected visits:\n%v\ngot:\n%v\n", expect, visits) + } + }) + t.Run("BreadthFirst", func(t *testing.T) { + var visits []vertexAtDepth + g.walk(breadthFirst|downOrder, true, start, func(v Vertex, d int) error { + visits = append(visits, vertexAtDepth{v, d}) + return nil + + }) + expect := []vertexAtDepth{ + {1, 0}, {2, 0}, {3, 1}, {4, 1}, {5, 1}, {6, 2}, {7, 2}, {10, 3}, {8, 3}, {9, 3}, {11, 4}, + } + if !reflect.DeepEqual(visits, expect) { + t.Errorf("expected visits:\n%v\ngot:\n%v\n", expect, visits) + } + }) + t.Run("ReverseBreadthFirst", func(t *testing.T) { + var visits []vertexAtDepth + g.walk(breadthFirst|upOrder, true, reverse, func(v Vertex, d int) error { + visits = append(visits, vertexAtDepth{v, d}) + return nil + + }) + expect := []vertexAtDepth{ + {11, 0}, {6, 0}, {10, 1}, {8, 1}, {9, 1}, {3, 1}, {7, 2}, {1, 2}, {4, 3}, {5, 3}, {2, 4}, + } + if !reflect.DeepEqual(visits, expect) { + t.Errorf("expected visits:\n%v\ngot:\n%v\n", expect, visits) + } + }) } const testGraphTransReductionStr = `