Add graph transformation recording

The external api provided here is simply
dag.Graph.SetDebugWriter(io.Writer). When a writer is provided to a
Graph, it will immediately encode itself to the stream, and subsequently
encode any additional transformations to the graph. This will allow
easier logging of graph transformations without writing complete graphs
to the logs at every step. Since the marshalGraph can also be dot
encoded, this will allow translation from the JSON logs to dot graphs.

dag/dag.go

@@ -103,6 +103,8 @@ func (g *AcyclicGraph) TransitiveReduction() {
 	// v such that the edge (u,v) exists (v is a direct descendant of u).
 	//
 	// For each v-prime reachable from v, remove the edge (u, v-prime).
+	defer g.debug.BeginReduction().End()
+
 	for _, u := range g.Vertices() {
 		uTargets := g.DownEdges(u)
 		vs := AsVertexList(g.DownEdges(u))
@@ -165,6 +167,8 @@ func (g *AcyclicGraph) Cycles() [][]Vertex {
 // This will walk nodes in parallel if it can. Because the walk is done
 // in parallel, the error returned will be a multierror.
 func (g *AcyclicGraph) Walk(cb WalkFunc) error {
+	defer g.debug.BeginWalk().End()
+
 	// Cache the vertices since we use it multiple times
 	vertices := g.Vertices()
 
@@ -274,6 +278,8 @@ type vertexAtDepth struct {
 // the vertices in start. This is not exported now but it would make sense
 // to export this publicly at some point.
 func (g *AcyclicGraph) DepthFirstWalk(start []Vertex, f DepthWalkFunc) error {
+	defer g.debug.BeginDepthFirstWalk().End()
+
 	seen := make(map[Vertex]struct{})
 	frontier := make([]*vertexAtDepth, len(start))
 	for i, v := range start {
@@ -316,6 +322,8 @@ func (g *AcyclicGraph) DepthFirstWalk(start []Vertex, f DepthWalkFunc) error {
 // reverseDepthFirstWalk does a depth-first walk _up_ the graph starting from
 // the vertices in start.
 func (g *AcyclicGraph) ReverseDepthFirstWalk(start []Vertex, f DepthWalkFunc) error {
+	defer g.debug.BeginReverseDepthFirstWalk().End()
+
 	seen := make(map[Vertex]struct{})
 	frontier := make([]*vertexAtDepth, len(start))
 	for i, v := range start {

dag/graph.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"sort"
 	"sync"
 )
@@ -15,6 +16,9 @@ type Graph struct {
 	downEdges map[interface{}]*Set
 	upEdges   map[interface{}]*Set
 	once      sync.Once
+
+	// JSON encoder for recording debug information
+	debug *encoder
 }
 
 // Subgrapher allows a Vertex to be a Graph itself, by returning a Grapher.
@@ -106,6 +110,7 @@ func (g *Graph) HasEdge(e Edge) bool {
 func (g *Graph) Add(v Vertex) Vertex {
 	g.once.Do(g.init)
 	g.vertices.Add(v)
+	g.debug.Add(v)
 	return v
 }
 
@@ -114,6 +119,7 @@ func (g *Graph) Add(v Vertex) Vertex {
 func (g *Graph) Remove(v Vertex) Vertex {
 	// Delete the vertex itself
 	g.vertices.Delete(v)
+	g.debug.Remove(v)
 
 	// Delete the edges to non-existent things
 	for _, target := range g.DownEdges(v).List() {
@@ -135,6 +141,8 @@ func (g *Graph) Replace(original, replacement Vertex) bool {
 		return false
 	}
 
+	defer g.debug.BeginReplace().End()
+
 	// If they're the same, then don't do anything
 	if original == replacement {
 		return true
@@ -158,6 +166,7 @@ func (g *Graph) Replace(original, replacement Vertex) bool {
 // RemoveEdge removes an edge from the graph.
 func (g *Graph) RemoveEdge(edge Edge) {
 	g.once.Do(g.init)
+	g.debug.RemoveEdge(edge)
 
 	// Delete the edge from the set
 	g.edges.Delete(edge)
@@ -189,6 +198,7 @@ func (g *Graph) UpEdges(v Vertex) *Set {
 // value of the edge itself.
 func (g *Graph) Connect(edge Edge) {
 	g.once.Do(g.init)
+	g.debug.Connect(edge)
 
 	source := edge.Source()
 	target := edge.Target()
@@ -301,20 +311,30 @@ func (g *Graph) String() string {
 	return buf.String()
 }
 
+func (g *Graph) init() {
+	g.vertices = new(Set)
+	g.edges = new(Set)
+	g.downEdges = make(map[interface{}]*Set)
+	g.upEdges = make(map[interface{}]*Set)
+}
+
+// Dot returns a dot-formatted representation of the Graph.
 func (g *Graph) Dot(opts *DotOpts) []byte {
 	return newMarshalGraph("", g).Dot(opts)
 }
 
+// MarshalJSON returns a JSON representation of the entire Graph.
 func (g *Graph) MarshalJSON() ([]byte, error) {
-	dg := newMarshalGraph("", g)
+	dg := newMarshalGraph("root", g)
 	return json.MarshalIndent(dg, "", "  ")
 }
 
-func (g *Graph) init() {
-	g.vertices = new(Set)
-	g.edges = new(Set)
-	g.downEdges = make(map[interface{}]*Set)
-	g.upEdges = make(map[interface{}]*Set)
+// SetDebugWriter sets the io.Writer where the Graph will record debug
+// information. After this is set, the graph will immediately encode itself to
+// the stream, and continue to record all subsequent operations.
+func (g *Graph) SetDebugWriter(w io.Writer) {
+	g.debug = &encoder{w}
+	g.debug.Encode(newMarshalGraph("root", g))
 }
 
 // VertexName returns the name of a vertex.

dag/marshal.go

@@ -1,7 +1,10 @@
 package dag
 
 import (
+	"encoding/json"
 	"fmt"
+	"io"
+	"log"
 	"reflect"
 	"sort"
 	"strconv"
@@ -9,6 +12,10 @@ import (
 
 // the marshal* structs are for serialization of the graph data.
 type marshalGraph struct {
+	// Type is always "Graph", for identification as a top level object in the
+	// JSON stream.
+	Type string
+
 	// Each marshal structure requires a unique ID so that it can be referenced
 	// by other structures.
 	ID string `json:",omitempty"`
@@ -33,6 +40,36 @@ type marshalGraph struct {
 	Cycles [][]*marshalVertex `json:",omitempty"`
 }
 
+// The add, remove, connect, removeEdge methods mirror the basic Graph
+// manipulations to reconstruct a marshalGraph from a debug log.
+func (g *marshalGraph) add(v *marshalVertex) {
+	g.Vertices = append(g.Vertices, v)
+	sort.Sort(vertices(g.Vertices))
+}
+
+func (g *marshalGraph) remove(v *marshalVertex) {
+	for i, existing := range g.Vertices {
+		if v.ID == existing.ID {
+			g.Vertices = append(g.Vertices[:i], g.Vertices[i+1:]...)
+			return
+		}
+	}
+}
+
+func (g *marshalGraph) connect(e *marshalEdge) {
+	g.Edges = append(g.Edges, e)
+	sort.Sort(edges(g.Edges))
+}
+
+func (g *marshalGraph) removeEdge(e *marshalEdge) {
+	for i, existing := range g.Edges {
+		if e.Source == existing.Source && e.Target == existing.Target {
+			g.Edges = append(g.Edges[:i], g.Edges[i+1:]...)
+			return
+		}
+	}
+}
+
 func (g *marshalGraph) vertexByID(id string) *marshalVertex {
 	for _, v := range g.Vertices {
 		if id == v.ID {
@@ -57,6 +94,27 @@ type marshalVertex struct {
 	graphNodeDotter bool
 }
 
+func newMarshalVertex(v Vertex) *marshalVertex {
+	return &marshalVertex{
+		ID:              marshalVertexID(v),
+		Name:            VertexName(v),
+		Attrs:           make(map[string]string),
+		graphNodeDotter: isDotter(v),
+	}
+}
+
+func isDotter(v Vertex) bool {
+	dn, isDotter := v.(GraphNodeDotter)
+	dotOpts := &DotOpts{
+		Verbose:    true,
+		DrawCycles: true,
+	}
+	if isDotter && dn.DotNode("fake", dotOpts) == nil {
+		isDotter = false
+	}
+	return isDotter
+}
+
 // vertices is a sort.Interface implementation for sorting vertices by ID
 type vertices []*marshalVertex
 
@@ -75,6 +133,15 @@ type marshalEdge struct {
 	Attrs map[string]string `json:",omitempty"`
 }
 
+func newMarshalEdge(e Edge) *marshalEdge {
+	return &marshalEdge{
+		Name:   fmt.Sprintf("%s|%s", VertexName(e.Source()), VertexName(e.Target())),
+		Source: marshalVertexID(e.Source()),
+		Target: marshalVertexID(e.Target()),
+		Attrs:  make(map[string]string),
+	}
+}
+
 // edges is a sort.Interface implementation for sorting edges by Source ID
 type edges []*marshalEdge
 
@@ -84,69 +151,42 @@ func (e edges) Swap(i, j int)      { e[i], e[j] = e[j], e[i] }
 
 // build a marshalGraph structure from a *Graph
 func newMarshalGraph(name string, g *Graph) *marshalGraph {
-	dg := &marshalGraph{
+	mg := &marshalGraph{
+		Type:  "Graph",
 		Name:  name,
 		Attrs: make(map[string]string),
 	}
 
 	for _, v := range g.Vertices() {
-		// We only care about nodes that yield non-empty Dot strings.
-		dn, isDotter := v.(GraphNodeDotter)
-		dotOpts := &DotOpts{
-			Verbose:    true,
-			DrawCycles: true,
-		}
-		if isDotter && dn.DotNode("fake", dotOpts) == nil {
-			isDotter = false
-		}
-
 		id := marshalVertexID(v)
 		if sg, ok := marshalSubgrapher(v); ok {
-
-			sdg := newMarshalGraph(VertexName(v), sg)
-			sdg.ID = id
-			dg.Subgraphs = append(dg.Subgraphs, sdg)
-		}
-
-		dv := &marshalVertex{
-			ID:              id,
-			Name:            VertexName(v),
-			Attrs:           make(map[string]string),
-			graphNodeDotter: isDotter,
+			smg := newMarshalGraph(VertexName(v), sg)
+			smg.ID = id
+			mg.Subgraphs = append(mg.Subgraphs, smg)
 		}
 
-		dg.Vertices = append(dg.Vertices, dv)
+		mv := newMarshalVertex(v)
+		mg.Vertices = append(mg.Vertices, mv)
 	}
 
-	sort.Sort(vertices(dg.Vertices))
+	sort.Sort(vertices(mg.Vertices))
 
 	for _, e := range g.Edges() {
-		de := &marshalEdge{
-			Name:   fmt.Sprintf("%s|%s", VertexName(e.Source()), VertexName(e.Target())),
-			Source: marshalVertexID(e.Source()),
-			Target: marshalVertexID(e.Target()),
-			Attrs:  make(map[string]string),
-		}
-		dg.Edges = append(dg.Edges, de)
+		mg.Edges = append(mg.Edges, newMarshalEdge(e))
 	}
 
-	sort.Sort(edges(dg.Edges))
+	sort.Sort(edges(mg.Edges))
 
 	for _, c := range (&AcyclicGraph{*g}).Cycles() {
 		var cycle []*marshalVertex
 		for _, v := range c {
-			dv := &marshalVertex{
-				ID:    marshalVertexID(v),
-				Name:  VertexName(v),
-				Attrs: make(map[string]string),
-			}
-
-			cycle = append(cycle, dv)
+			mv := newMarshalVertex(v)
+			cycle = append(cycle, mv)
 		}
-		dg.Cycles = append(dg.Cycles, cycle)
+		mg.Cycles = append(mg.Cycles, cycle)
 	}
 
-	return dg
+	return mg
 }
 
 // Attempt to return a unique ID for any vertex.
@@ -189,3 +229,241 @@ func marshalSubgrapher(v Vertex) (*Graph, bool) {
 
 	return nil, false
 }
+
+// ender provides a way to call any End* method expression via an End method
+type ender func()
+
+func (e ender) End() { e() }
+
+// encoder provides methods to write debug data to an io.Writer, and is a noop
+// when no writer is present
+type encoder struct {
+	w io.Writer
+}
+
+// Encode is analogous to json.Encoder.Encode
+func (e *encoder) Encode(i interface{}) {
+	if e == nil || e.w == nil {
+		return
+	}
+
+	js, err := json.Marshal(i)
+	if err != nil {
+		log.Println("[ERROR] dag:", err)
+		return
+	}
+	js = append(js, '\n')
+
+	_, err = e.w.Write(js)
+	if err != nil {
+		log.Println("[ERROR] dag:", err)
+		return
+	}
+}
+
+func (e *encoder) Add(v Vertex) {
+	e.Encode(marshalTransform{
+		Type:      "Transform",
+		AddVertex: newMarshalVertex(v),
+	})
+}
+
+// Remove records the removal of Vertex v.
+func (e *encoder) Remove(v Vertex) {
+	e.Encode(marshalTransform{
+		Type:         "Transform",
+		RemoveVertex: newMarshalVertex(v),
+	})
+}
+
+func (e *encoder) Connect(edge Edge) {
+	e.Encode(marshalTransform{
+		Type:    "Transform",
+		AddEdge: newMarshalEdge(edge),
+	})
+}
+
+func (e *encoder) RemoveEdge(edge Edge) {
+	e.Encode(marshalTransform{
+		Type:       "Transform",
+		RemoveEdge: newMarshalEdge(edge),
+	})
+}
+
+// BeginReplace marks the start of a replace operation, and returns the encoder
+// to chain the EndReplace call.
+func (e *encoder) BeginReplace() ender {
+	e.Encode(marshalOperation{
+		Type:  "Operation",
+		Begin: newString("Replace"),
+	})
+	return e.EndReplace
+}
+
+func (e *encoder) EndReplace() {
+	e.Encode(marshalOperation{
+		Type: "Operation",
+		End:  newString("Replace"),
+	})
+}
+
+// BeginReduction marks the start of a replace operation, and returns the encoder
+// to chain the EndReduction call.
+func (e *encoder) BeginReduction() ender {
+	e.Encode(marshalOperation{
+		Type:  "Operation",
+		Begin: newString("Reduction"),
+	})
+	return e.EndReduction
+}
+
+func (e *encoder) EndReduction() {
+	e.Encode(marshalOperation{
+		Type: "Operation",
+		End:  newString("Reduction"),
+	})
+}
+
+// BeginDepthFirstWalk marks the start of a replace operation, and returns the
+// encoder to chain the EndDepthFirstWalk call.
+func (e *encoder) BeginDepthFirstWalk() ender {
+	e.Encode(marshalOperation{
+		Type:  "Operation",
+		Begin: newString("DepthFirstWalk"),
+	})
+	return e.EndDepthFirstWalk
+}
+
+func (e *encoder) EndDepthFirstWalk() {
+	e.Encode(marshalOperation{
+		Type: "Operation",
+		End:  newString("DepthFirstWalk"),
+	})
+}
+
+// BeginReverseDepthFirstWalk marks the start of a replace operation, and
+// returns the encoder to chain the EndReverseDepthFirstWalk call.
+func (e *encoder) BeginReverseDepthFirstWalk() ender {
+	e.Encode(marshalOperation{
+		Type:  "Operation",
+		Begin: newString("ReverseDepthFirstWalk"),
+	})
+	return e.EndReverseDepthFirstWalk
+}
+
+func (e *encoder) EndReverseDepthFirstWalk() {
+	e.Encode(marshalOperation{
+		Type: "Operation",
+		End:  newString("ReverseDepthFirstWalk"),
+	})
+}
+
+// BeginWalk marks the start of a replace operation, and returns the encoder
+// to chain the EndWalk call.
+func (e *encoder) BeginWalk() ender {
+	e.Encode(marshalOperation{
+		Type:  "Operation",
+		Begin: newString("Walk"),
+	})
+	return e.EndWalk
+}
+
+func (e *encoder) EndWalk() {
+	e.Encode(marshalOperation{
+		Type: "Operation",
+		End:  newString("Walk"),
+	})
+}
+
+// structure for recording graph transformations
+type marshalTransform struct {
+	// Type: "Transform"
+	Type         string
+	AddEdge      *marshalEdge   `json:",omitempty"`
+	RemoveEdge   *marshalEdge   `json:",omitempty"`
+	AddVertex    *marshalVertex `json:",omitempty"`
+	RemoveVertex *marshalVertex `json:",omitempty"`
+}
+
+func (t marshalTransform) Transform(g *marshalGraph) {
+	switch {
+	case t.AddEdge != nil:
+		g.connect(t.AddEdge)
+	case t.RemoveEdge != nil:
+		g.removeEdge(t.RemoveEdge)
+	case t.AddVertex != nil:
+		g.add(t.AddVertex)
+	case t.RemoveVertex != nil:
+		g.remove(t.RemoveVertex)
+	}
+}
+
+// this structure allows us to decode any object in the json stream for
+// inspection, then re-decode it into a proper struct if needed.
+type streamDecode struct {
+	Type string
+	Map  map[string]interface{}
+	JSON []byte
+}
+
+func (s *streamDecode) UnmarshalJSON(d []byte) error {
+	s.JSON = d
+	err := json.Unmarshal(d, &s.Map)
+	if err != nil {
+		return err
+	}
+
+	if t, ok := s.Map["Type"]; ok {
+		s.Type, _ = t.(string)
+	}
+	return nil
+}
+
+// structure for recording the beginning and end of any multi-step
+// transformations. These are informational, and not required to reproduce the
+// graph state.
+type marshalOperation struct {
+	Type  string
+	Begin *string `json:",omitempty"`
+	End   *string `json:",omitempty"`
+}
+
+func newBool(b bool) *bool { return &b }
+
+func newString(s string) *string { return &s }
+
+// decodeGraph decodes a marshalGraph from an encoded graph stream.
+func decodeGraph(r io.Reader) (*marshalGraph, error) {
+	dec := json.NewDecoder(r)
+
+	// a stream should always start with a graph
+	g := &marshalGraph{}
+
+	err := dec.Decode(g)
+	if err != nil {
+		return nil, err
+	}
+
+	// now replay any operations that occurred on the original graph
+	for dec.More() {
+		s := &streamDecode{}
+		err := dec.Decode(s)
+		if err != nil {
+			return g, err
+		}
+
+		// the only Type we're concerned with here is Transform to complete the
+		// Graph
+		if s.Type != "Transform" {
+			continue
+		}
+
+		t := &marshalTransform{}
+		err = json.Unmarshal(s.JSON, t)
+		if err != nil {
+			return g, err
+		}
+		t.Transform(g)
+	}
+	return g, nil
+}

dag/marshal_test.go

@@ -0,0 +1,175 @@
+package dag
+
+import (
+	"bytes"
+	"encoding/json"
+	"strings"
+	"testing"
+)
+
+func TestGraphDot_empty(t *testing.T) {
+	var g Graph
+	g.Add(1)
+	g.Add(2)
+	g.Add(3)
+
+	actual := strings.TrimSpace(string(g.Dot(nil)))
+	expected := strings.TrimSpace(testGraphDotEmptyStr)
+	if actual != expected {
+		t.Fatalf("bad: %s", actual)
+	}
+}
+
+func TestGraphDot_basic(t *testing.T) {
+	var g Graph
+	g.Add(1)
+	g.Add(2)
+	g.Add(3)
+	g.Connect(BasicEdge(1, 3))
+
+	actual := strings.TrimSpace(string(g.Dot(nil)))
+	expected := strings.TrimSpace(testGraphDotBasicStr)
+	if actual != expected {
+		t.Fatalf("bad: %s", actual)
+	}
+}
+
+const testGraphDotBasicStr = `digraph {
+	compound = "true"
+	newrank = "true"
+	subgraph "root" {
+		"[root] 1" -> "[root] 3"
+	}
+}
+`
+
+const testGraphDotEmptyStr = `digraph {
+	compound = "true"
+	newrank = "true"
+	subgraph "root" {
+	}
+}`
+
+func TestGraphJSON_empty(t *testing.T) {
+	var g Graph
+	g.Add(1)
+	g.Add(2)
+	g.Add(3)
+
+	js, err := g.MarshalJSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	actual := strings.TrimSpace(string(js))
+	expected := strings.TrimSpace(testGraphJSONEmptyStr)
+	if actual != expected {
+		t.Fatalf("bad: %s", actual)
+	}
+}
+
+func TestGraphJSON_basic(t *testing.T) {
+	var g Graph
+	g.Add(1)
+	g.Add(2)
+	g.Add(3)
+	g.Connect(BasicEdge(1, 3))
+
+	js, err := g.MarshalJSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+	actual := strings.TrimSpace(string(js))
+	expected := strings.TrimSpace(testGraphJSONBasicStr)
+	if actual != expected {
+		t.Fatalf("bad: %s", actual)
+	}
+}
+
+// record some graph transformations, and make sure we get the same graph when
+// they're replayed
+func TestGraphJSON_basicRecord(t *testing.T) {
+	var g Graph
+	var buf bytes.Buffer
+	g.SetDebugWriter(&buf)
+
+	g.Add(1)
+	g.Add(2)
+	g.Add(3)
+	g.Connect(BasicEdge(1, 2))
+	g.Connect(BasicEdge(1, 3))
+	g.Connect(BasicEdge(2, 3))
+	(&AcyclicGraph{g}).TransitiveReduction()
+
+	recorded := buf.Bytes()
+	// the Walk doesn't happen in a determined order, so just count operations
+	// for now to make sure we wrote stuff out.
+	if len(bytes.Split(recorded, []byte{'\n'})) != 17 {
+		t.Fatalf("bad: %s", recorded)
+	}
+
+	original, err := g.MarshalJSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// replay the logs, and marshal the graph back out again
+	m, err := decodeGraph(bytes.NewReader(buf.Bytes()))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	replayed, err := json.MarshalIndent(m, "", "  ")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !bytes.Equal(original, replayed) {
+		t.Fatalf("\noriginal: %s\nreplayed: %s", original, replayed)
+	}
+}
+
+const testGraphJSONEmptyStr = `{
+  "Type": "Graph",
+  "Name": "root",
+  "Vertices": [
+    {
+      "ID": "1",
+      "Name": "1"
+    },
+    {
+      "ID": "2",
+      "Name": "2"
+    },
+    {
+      "ID": "3",
+      "Name": "3"
+    }
+  ]
+}`
+
+const testGraphJSONBasicStr = `{
+  "Type": "Graph",
+  "Name": "root",
+  "Vertices": [
+    {
+      "ID": "1",
+      "Name": "1"
+    },
+    {
+      "ID": "2",
+      "Name": "2"
+    },
+    {
+      "ID": "3",
+      "Name": "3"
+    }
+  ],
+  "Edges": [
+    {
+      "Name": "1|3",
+      "Source": "1",
+      "Target": "3"
+    }
+  ]
+}`