From 1c7bde3778f3d5653b2988be1dfd3ff4f293f72f Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Sat, 18 Nov 2017 17:54:14 -0800 Subject: [PATCH] testharness: actually run the testers Currently there is only a stub tester available, but the basic framework around testers is working and ready to support real testers. --- command/test.go | 78 ++++++++++++++++++++++++++++++---- testharness/checklist.go | 10 +++-- testharness/context.go | 13 +++--- testharness/loader.go | 2 +- testharness/spec.go | 5 +++ testharness/subject.go | 8 ++++ testharness/tester.go | 74 ++++++++++++++++++++++++++++++-- testharness/testers_builder.go | 1 + 8 files changed, 168 insertions(+), 23 deletions(-) diff --git a/command/test.go b/command/test.go index 26f028010f..d49fc1d882 100644 --- a/command/test.go +++ b/command/test.go @@ -2,6 +2,7 @@ package command import ( "fmt" + "os" "sort" "strings" "time" @@ -162,13 +163,74 @@ func (c *TestCommand) Run(args []string) int { endTime := time.Now() - fmt.Printf("\n%d resources created in %s\n\n", hook.ApplyCount(), endTime.Sub(startTime)) + fmt.Printf("\nTotal created: %d in %s\n\n", hook.ApplyCount(), endTime.Sub(startTime)) } if created { fmt.Print("## Verify\n\n") - fmt.Println("") + subject := testharness.NewSubject(mod, stateMgr.State(), scenario.Variables) + + itemCh := make(chan testharness.CheckItem) + logCh := make(chan string) + cs := testharness.NewCheckStream(itemCh, logCh) + testharness.TestStream(subject, spec, cs) + + logOpen := false + var successes, failures, errors, skips int + for { + select { + case item, ok := <-itemCh: + if ok { + if logOpen { + // End the open log block before we produce more items + fmt.Fprint(os.Stderr, "```\n\n") + os.Stderr.Sync() + logOpen = false + } + + check := "[x]" + if item.Result != testharness.Success { + check = "[ ]" + } + exclam := "" + switch item.Result { + case testharness.Success: + successes++ + case testharness.Failure: + failures++ + case testharness.Error: + errors++ + exclam = "**(ERROR)** " + case testharness.Skipped: + skips++ + exclam = "(SKIPPED) " + } + fmt.Printf("* %s %s%s\n", check, exclam, item.Caption) + } else { + itemCh = nil + } + case msg, ok := <-logCh: + if ok { + if !logOpen { + os.Stdout.Sync() + // Open a log block before we print our message + fmt.Fprint(os.Stderr, "\n```\n") + logOpen = true + } + fmt.Fprintln(os.Stderr, msg) + } else { + logCh = nil + } + } + + if itemCh == nil && logCh == nil { + break + } + } + + total := successes + failures + skips + errors + fmt.Printf("\nTotal assertions: %d (%d passed, %d failed, %d skipped, %d errored)\n\n", total, successes, failures, skips, errors) } { @@ -213,7 +275,7 @@ func (c *TestCommand) Run(args []string) int { endTime := time.Now() - fmt.Printf("\n%d resources destroyed in %s\n\n", hook.ApplyCount(), endTime.Sub(startTime)) + fmt.Printf("\nTotal destroyed: %d in %s\n\n", hook.ApplyCount(), endTime.Sub(startTime)) } } @@ -266,7 +328,7 @@ func (h *testCommandHook) PostRefresh(info *terraform.InstanceInfo, s *terraform startTime := h.startTimes["r"+info.ResourceAddress().String()] endTime := time.Now() delta := endTime.Sub(startTime) - fmt.Printf("* [x] Read %s (%s)\n", info.ResourceAddress(), delta) + fmt.Printf("* [x] %s is read (%s)\n", info.ResourceAddress(), delta) return terraform.HookActionContinue, nil } @@ -283,15 +345,15 @@ func (h *testCommandHook) PostApply(info *terraform.InstanceInfo, s *terraform.I check := "[x]" if err != nil { - check = "[ ]" + check = "[ ] **(ERROR)**" } - verb := "Create" + verb := "created" if s == nil || s.ID == "" { - verb = "Destroy" + verb = "destroyed" } - fmt.Printf("* %s %s %s (%s)\n", check, verb, info.ResourceAddress(), delta) + fmt.Printf("* %s %s is %s (%s)\n", check, info.ResourceAddress(), verb, delta) return terraform.HookActionContinue, nil } diff --git a/testharness/checklist.go b/testharness/checklist.go index f12760cbfb..eefbe45358 100644 --- a/testharness/checklist.go +++ b/testharness/checklist.go @@ -105,13 +105,15 @@ func (s CheckStream) Substream() (CheckStream, Waiter) { for { select { case item, ok := <-proxyItemCh: - s.Write(item) - if !ok { + if ok { + s.Write(item) + } else { proxyItemCh = nil } case msg, ok := <-proxyLogCh: - s.Log(msg) - if !ok { + if ok { + s.Log(msg) + } else { proxyLogCh = nil } } diff --git a/testharness/context.go b/testharness/context.go index cf63f1ec8c..4b6d6bf947 100644 --- a/testharness/context.go +++ b/testharness/context.go @@ -3,6 +3,7 @@ package testharness import ( "fmt" + lua "github.com/yuin/gopher-lua" "github.com/zclconf/go-cty/cty" ) @@ -12,6 +13,7 @@ import ( // A Context is immutable, but derived contexts can be created using the // methods of this type. type Context struct { + lstate *lua.LState name string resource cty.Value output cty.Value @@ -19,12 +21,11 @@ type Context struct { each map[string]cty.Value } -var RootContext *Context - -func init() { - RootContext = &Context{ - name: "", - each: map[string]cty.Value{}, +func rootContext(lstate *lua.LState) *Context { + return &Context{ + lstate: lstate, + name: "", + each: map[string]cty.Value{}, } } diff --git a/testharness/loader.go b/testharness/loader.go index c7f39a4ad7..a6fe3875da 100644 --- a/testharness/loader.go +++ b/testharness/loader.go @@ -140,7 +140,7 @@ func loadSpec(r io.Reader, filename string, L *lua.LState) (*Spec, tfdiags.Diagn Diags: builderDiags, } testersB := testersBuilder{ - Context: RootContext, + Context: rootContext(L), Diags: builderDiags, } topEnv.RawSet(lua.LString("scenario"), L.NewFunction(scenariosB.luaScenarioFunc)) diff --git a/testharness/spec.go b/testharness/spec.go index a9296ff80c..f8d798406f 100644 --- a/testharness/spec.go +++ b/testharness/spec.go @@ -24,3 +24,8 @@ func (s *Spec) Scenarios() map[string]*Scenario { func (s *Spec) Scenario(name string) *Scenario { return s.scenarios[name] } + +// Tester implementation +func (s *Spec) test(subject *Subject, cs CheckStream) { + s.testers.test(subject, cs) +} diff --git a/testharness/subject.go b/testharness/subject.go index ac7621dba8..90105040f9 100644 --- a/testharness/subject.go +++ b/testharness/subject.go @@ -14,3 +14,11 @@ type Subject struct { state *terraform.State variables map[string]cty.Value } + +func NewSubject(config *module.Tree, state *terraform.State, variables map[string]cty.Value) *Subject { + return &Subject{ + config: config, + state: state, + variables: variables, + } +} diff --git a/testharness/tester.go b/testharness/tester.go index 201a51e95f..1b2fe802c9 100644 --- a/testharness/tester.go +++ b/testharness/tester.go @@ -5,25 +5,91 @@ import ( lua "github.com/yuin/gopher-lua" ) +// Test applies the given tester to the given subject and returns a checklist +// of results. A Spec is a tester. +// +// This function blocks until all of the tests have completed, and so it is +// not possible to receive progress logs. To run tests asynchronously with +// progress logs, use TestStream. +func Test(subject *Subject, tester Tester) Checklist { + var ret Checklist + itemCh := make(chan CheckItem) + cs := NewCheckStream(itemCh, nil) + TestStream(subject, tester, cs) + for { + item, more := <-itemCh + if !more { + break + } + ret = append(ret, item) + } + return ret +} + +// TestStream is like Test except that it appends its results to the given +// CheckStream rather than returning a Checklist. +// +// This function returns immediately and then runs its tests in a separate +// goroutine. The CheckStream is closed once the tests are complete. +func TestStream(subject *Subject, tester Tester, cs CheckStream) { + go func() { + subCs, closed := cs.Substream() + tester.test(subject, subCs) + closed.Wait() + cs.Close() + }() +} + // Tester is an interface implemented by objects that can run tests. type Tester interface { - Test() // TODO: Flesh out the arguments for this + test(subject *Subject, cs CheckStream) } // Testers is a slice of Tester. type Testers []Tester +// Tester implementation +func (ts Testers) test(subject *Subject, cs CheckStream) { + for _, tester := range ts { + subCs, closed := cs.Substream() + tester.test(subject, subCs) + closed.Wait() + } + cs.Close() +} + // describe represents a single "describe" call in a test specification. // // describe implements Tester. type describe struct { Described contextSetter BodyFn *lua.LFunction + Context *Context DefRange tfdiags.SourceRange } -func (t *describe) Test() { - // TODO: Implement this once we figure out what the Tester interface - // really contains. +// Tester implementation +func (t *describe) test(subject *Subject, cs CheckStream) { + defer cs.Close() + + childContexts, diags := t.Described.AppendContexts(t.Context, subject, nil) + if diags.HasErrors() { + cs.Write(CheckItem{ + Result: Error, + Caption: t.Context.Name(), + Diags: diags, + }) + return + } + + for _, childContext := range childContexts { + // TODO: run our BodyFn to collect our child testers + // TODO: run child testers, each in a sub-stream of cs + // TODO: wait for sub-stream to close before returning + cs.Write(CheckItem{ + Result: Skipped, + Caption: childContext.Name(), + }) + } } diff --git a/testharness/testers_builder.go b/testharness/testers_builder.go index 32360344e2..095bf91d9d 100644 --- a/testharness/testers_builder.go +++ b/testharness/testers_builder.go @@ -67,6 +67,7 @@ func (b *testersBuilder) luaDescribeFunc(L *lua.LState) int { desc := &describe{ Described: described, BodyFn: bodyFn, + Context: b.Context, } if rng := callingRange(L, 1); rng != nil { desc.DefRange = tfdiags.SourceRangeFromHCL(*rng)