testharness: actually run the testers

Currently there is only a stub tester available, but the basic framework
around testers is working and ready to support real testers.

command/test.go

@@ -2,6 +2,7 @@ package command
 
 import (
 	"fmt"
+	"os"
 	"sort"
 	"strings"
 	"time"
@@ -162,13 +163,74 @@ func (c *TestCommand) Run(args []string) int {
 
 			endTime := time.Now()
 
-			fmt.Printf("\n%d resources created in %s\n\n", hook.ApplyCount(), endTime.Sub(startTime))
+			fmt.Printf("\nTotal created: %d in %s\n\n", hook.ApplyCount(), endTime.Sub(startTime))
 		}
 
 		if created {
 			fmt.Print("## Verify\n\n")
 
-			fmt.Println("")
+			subject := testharness.NewSubject(mod, stateMgr.State(), scenario.Variables)
+
+			itemCh := make(chan testharness.CheckItem)
+			logCh := make(chan string)
+			cs := testharness.NewCheckStream(itemCh, logCh)
+			testharness.TestStream(subject, spec, cs)
+
+			logOpen := false
+			var successes, failures, errors, skips int
+			for {
+				select {
+				case item, ok := <-itemCh:
+					if ok {
+						if logOpen {
+							// End the open log block before we produce more items
+							fmt.Fprint(os.Stderr, "```\n\n")
+							os.Stderr.Sync()
+							logOpen = false
+						}
+
+						check := "[x]"
+						if item.Result != testharness.Success {
+							check = "[ ]"
+						}
+						exclam := ""
+						switch item.Result {
+						case testharness.Success:
+							successes++
+						case testharness.Failure:
+							failures++
+						case testharness.Error:
+							errors++
+							exclam = "**(ERROR)** "
+						case testharness.Skipped:
+							skips++
+							exclam = "(SKIPPED) "
+						}
+						fmt.Printf("* %s %s%s\n", check, exclam, item.Caption)
+					} else {
+						itemCh = nil
+					}
+				case msg, ok := <-logCh:
+					if ok {
+						if !logOpen {
+							os.Stdout.Sync()
+							// Open a log block before we print our message
+							fmt.Fprint(os.Stderr, "\n```\n")
+							logOpen = true
+						}
+						fmt.Fprintln(os.Stderr, msg)
+					} else {
+						logCh = nil
+					}
+				}
+
+				if itemCh == nil && logCh == nil {
+					break
+				}
+			}
+
+			total := successes + failures + skips + errors
+			fmt.Printf("\nTotal assertions: %d (%d passed, %d failed, %d skipped, %d errored)\n\n", total, successes, failures, skips, errors)
 		}
 
 		{
@@ -213,7 +275,7 @@ func (c *TestCommand) Run(args []string) int {
 
 			endTime := time.Now()
 
-			fmt.Printf("\n%d resources destroyed in %s\n\n", hook.ApplyCount(), endTime.Sub(startTime))
+			fmt.Printf("\nTotal destroyed: %d in %s\n\n", hook.ApplyCount(), endTime.Sub(startTime))
 		}
 	}
 
@@ -266,7 +328,7 @@ func (h *testCommandHook) PostRefresh(info *terraform.InstanceInfo, s *terraform
 	startTime := h.startTimes["r"+info.ResourceAddress().String()]
 	endTime := time.Now()
 	delta := endTime.Sub(startTime)
-	fmt.Printf("* [x] Read %s (%s)\n", info.ResourceAddress(), delta)
+	fmt.Printf("* [x] %s is read (%s)\n", info.ResourceAddress(), delta)
 	return terraform.HookActionContinue, nil
 }
 
@@ -283,15 +345,15 @@ func (h *testCommandHook) PostApply(info *terraform.InstanceInfo, s *terraform.I
 
 	check := "[x]"
 	if err != nil {
-		check = "[ ]"
+		check = "[ ] **(ERROR)**"
 	}
 
-	verb := "Create"
+	verb := "created"
 	if s == nil || s.ID == "" {
-		verb = "Destroy"
+		verb = "destroyed"
 	}
 
-	fmt.Printf("* %s %s %s (%s)\n", check, verb, info.ResourceAddress(), delta)
+	fmt.Printf("* %s %s is %s (%s)\n", check, info.ResourceAddress(), verb, delta)
 
 	return terraform.HookActionContinue, nil
 }

testharness/checklist.go

@@ -105,13 +105,15 @@ func (s CheckStream) Substream() (CheckStream, Waiter) {
 		for {
 			select {
 			case item, ok := <-proxyItemCh:
-				s.Write(item)
-				if !ok {
+				if ok {
+					s.Write(item)
+				} else {
 					proxyItemCh = nil
 				}
 			case msg, ok := <-proxyLogCh:
-				s.Log(msg)
-				if !ok {
+				if ok {
+					s.Log(msg)
+				} else {
 					proxyLogCh = nil
 				}
 			}

testharness/context.go

@@ -3,6 +3,7 @@ package testharness
 import (
 	"fmt"
 
+	lua "github.com/yuin/gopher-lua"
 	"github.com/zclconf/go-cty/cty"
 )
 
@@ -12,6 +13,7 @@ import (
 // A Context is immutable, but derived contexts can be created using the
 // methods of this type.
 type Context struct {
+	lstate   *lua.LState
 	name     string
 	resource cty.Value
 	output   cty.Value
@@ -19,12 +21,11 @@ type Context struct {
 	each     map[string]cty.Value
 }
 
-var RootContext *Context
-
-func init() {
-	RootContext = &Context{
-		name: "",
-		each: map[string]cty.Value{},
+func rootContext(lstate *lua.LState) *Context {
+	return &Context{
+		lstate: lstate,
+		name:   "",
+		each:   map[string]cty.Value{},
 	}
 }
 

testharness/loader.go

@@ -140,7 +140,7 @@ func loadSpec(r io.Reader, filename string, L *lua.LState) (*Spec, tfdiags.Diagn
 		Diags: builderDiags,
 	}
 	testersB := testersBuilder{
-		Context: RootContext,
+		Context: rootContext(L),
 		Diags:   builderDiags,
 	}
 	topEnv.RawSet(lua.LString("scenario"), L.NewFunction(scenariosB.luaScenarioFunc))

testharness/spec.go

@@ -24,3 +24,8 @@ func (s *Spec) Scenarios() map[string]*Scenario {
 func (s *Spec) Scenario(name string) *Scenario {
 	return s.scenarios[name]
 }
+
+// Tester implementation
+func (s *Spec) test(subject *Subject, cs CheckStream) {
+	s.testers.test(subject, cs)
+}

testharness/subject.go

@@ -14,3 +14,11 @@ type Subject struct {
 	state     *terraform.State
 	variables map[string]cty.Value
 }
+
+func NewSubject(config *module.Tree, state *terraform.State, variables map[string]cty.Value) *Subject {
+	return &Subject{
+		config:    config,
+		state:     state,
+		variables: variables,
+	}
+}

testharness/tester.go

@@ -5,25 +5,91 @@ import (
 	lua "github.com/yuin/gopher-lua"
 )
 
+// Test applies the given tester to the given subject and returns a checklist
+// of results. A Spec is a tester.
+//
+// This function blocks until all of the tests have completed, and so it is
+// not possible to receive progress logs. To run tests asynchronously with
+// progress logs, use TestStream.
+func Test(subject *Subject, tester Tester) Checklist {
+	var ret Checklist
+	itemCh := make(chan CheckItem)
+	cs := NewCheckStream(itemCh, nil)
+	TestStream(subject, tester, cs)
+	for {
+		item, more := <-itemCh
+		if !more {
+			break
+		}
+		ret = append(ret, item)
+	}
+	return ret
+}
+
+// TestStream is like Test except that it appends its results to the given
+// CheckStream rather than returning a Checklist.
+//
+// This function returns immediately and then runs its tests in a separate
+// goroutine. The CheckStream is closed once the tests are complete.
+func TestStream(subject *Subject, tester Tester, cs CheckStream) {
+	go func() {
+		subCs, closed := cs.Substream()
+		tester.test(subject, subCs)
+		closed.Wait()
+		cs.Close()
+	}()
+}
+
 // Tester is an interface implemented by objects that can run tests.
 type Tester interface {
-	Test() // TODO: Flesh out the arguments for this
+	test(subject *Subject, cs CheckStream)
 }
 
 // Testers is a slice of Tester.
 type Testers []Tester
 
+// Tester implementation
+func (ts Testers) test(subject *Subject, cs CheckStream) {
+	for _, tester := range ts {
+		subCs, closed := cs.Substream()
+		tester.test(subject, subCs)
+		closed.Wait()
+	}
+	cs.Close()
+}
+
 // describe represents a single "describe" call in a test specification.
 //
 // describe implements Tester.
 type describe struct {
 	Described contextSetter
 	BodyFn    *lua.LFunction
+	Context   *Context
 
 	DefRange tfdiags.SourceRange
 }
 
-func (t *describe) Test() {
-	// TODO: Implement this once we figure out what the Tester interface
-	// really contains.
+// Tester implementation
+func (t *describe) test(subject *Subject, cs CheckStream) {
+	defer cs.Close()
+
+	childContexts, diags := t.Described.AppendContexts(t.Context, subject, nil)
+	if diags.HasErrors() {
+		cs.Write(CheckItem{
+			Result:  Error,
+			Caption: t.Context.Name(),
+			Diags:   diags,
+		})
+		return
+	}
+
+	for _, childContext := range childContexts {
+		// TODO: run our BodyFn to collect our child testers
+		// TODO: run child testers, each in a sub-stream of cs
+		// TODO: wait for sub-stream to close before returning
+		cs.Write(CheckItem{
+			Result:  Skipped,
+			Caption: childContext.Name(),
+		})
+	}
 }

testharness/testers_builder.go

@@ -67,6 +67,7 @@ func (b *testersBuilder) luaDescribeFunc(L *lua.LState) int {
 	desc := &describe{
 		Described: described,
 		BodyFn:    bodyFn,
+		Context:   b.Context,
 	}
 	if rng := callingRange(L, 1); rng != nil {
 		desc.DefRange = tfdiags.SourceRangeFromHCL(*rng)