From fee61a44b48df5fbce2491e4cee606d73f173356 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Fri, 7 Jul 2017 18:46:24 -0700 Subject: [PATCH 1/4] command/e2etest: end-to-end testing harness Previously we had no automated testing of whether we can produce a Terraform executable that actually works. Our various functional tests have good coverage of specific Terraform features and whole operations, but we lacked end-to-end testing of actual usage of the generated binary, without any stubbing. This package is intended as a vehicle for such end-to-end testing. When run normally under "go test" it will produce a build of the main Terraform binary and make it available for tests to execute. The harness exposes a flag for whether tests are allowed to reach out to external network services, controlled with our standard TF_ACC environment variable, so that basic local tests can be safely run as part of "make test" while more elaborate tests can be run easily when desired. It also provides a separate mode of operation where the included script make-archive.sh can be used to produce a self-contained test archive that can be copied to another system to run the tests there. This is intended to allow testing of cross-compiled binaries, by shipping them over to the target OS and architecture to run without requiring a full Go compiler installation on the target system. The goal here is not to test again functionality that's already well-covered by our existing tests, but rather to test chains of normal operations against the build binary that are not otherwise tested together. --- command/e2etest/.gitignore | 1 + command/e2etest/doc.go | 29 ++++ command/e2etest/main_test.go | 289 ++++++++++++++++++++++++++++++++ command/e2etest/make-archive.sh | 47 ++++++ 4 files changed, 366 insertions(+) create mode 100644 command/e2etest/.gitignore create mode 100644 command/e2etest/doc.go create mode 100644 command/e2etest/main_test.go create mode 100755 command/e2etest/make-archive.sh diff --git a/command/e2etest/.gitignore b/command/e2etest/.gitignore new file mode 100644 index 0000000000..a007feab07 --- /dev/null +++ b/command/e2etest/.gitignore @@ -0,0 +1 @@ +build/* diff --git a/command/e2etest/doc.go b/command/e2etest/doc.go new file mode 100644 index 0000000000..147cc48bcd --- /dev/null +++ b/command/e2etest/doc.go @@ -0,0 +1,29 @@ +// Package e2etest contains a small number of tests that run against a real +// Terraform binary, compiled on the fly at the start of the test run. +// +// These tests help ensure that key end-to-end Terraform use-cases are working +// for a real binary, whereas other tests always have at least _some_ amount +// of test stubbing. +// +// The goal of this package is not to duplicate the functional testing done +// in other packages but rather to fully exercise a few important workflows +// in a realistic way. +// +// These tests can be used in two ways. The simplest way is to just run them +// with "go test" as normal: +// +// go test -v github.com/hashicorp/terraform/command/e2etest +// +// This will compile on the fly a Terraform binary and run the tests against +// it. +// +// Alternatively, the make-archive.sh script can be used to produce a +// self-contained zip file that can be shipped to another machine to run +// the tests there without needing a locally-installed Go compiler. This +// is primarily useful for testing cross-compiled builds. For more information, +// see the commentary in make-archive.sh. +// +// The TF_ACC environment variable must be set for the tests to reach out +// to external network services. Since these are end-to-end tests, only a +// few very basic tests can execute without this environment variable set. +package e2etest diff --git a/command/e2etest/main_test.go b/command/e2etest/main_test.go new file mode 100644 index 0000000000..33a3279762 --- /dev/null +++ b/command/e2etest/main_test.go @@ -0,0 +1,289 @@ +package e2etest + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "testing" + + tfcore "github.com/hashicorp/terraform/terraform" +) + +var terraformBin string + +func TestMain(m *testing.M) { + teardown := setup() + code := m.Run() + teardown() + os.Exit(code) +} + +func setup() func() { + if terraformBin != "" { + // this is pre-set when we're running in a binary produced from + // the make-archive.sh script, since that builds a ready-to-go + // binary into the archive. However, we do need to turn it into + // an absolute path so that we can find it when we change the + // working directory during tests. + var err error + terraformBin, err = filepath.Abs(terraformBin) + if err != nil { + panic(fmt.Sprintf("failed to find absolute path of terraform executable: %s", err)) + } + return func() {} + } + + tmpFile, err := ioutil.TempFile("", "terraform") + if err != nil { + panic(err) + } + tmpFilename := tmpFile.Name() + if err = tmpFile.Close(); err != nil { + panic(err) + } + + cmd := exec.Command( + "go", "build", + "-o", tmpFilename, + "github.com/hashicorp/terraform", + ) + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + + err = cmd.Run() + if err != nil { + // The go compiler will have already produced some error messages + // on stderr by the time we get here. + panic(fmt.Sprintf("failed to build terraform executable: %s", err)) + } + + // Make the executable available for use in tests + terraformBin = tmpFilename + + return func() { + os.Remove(tmpFilename) + } +} + +func canAccessNetwork() bool { + // We re-use the flag normally used for acceptance tests since that's + // established as a way to opt-in to reaching out to real systems that + // may suffer transient errors. + return os.Getenv("TF_ACC") != "" +} + +func skipIfCannotAccessNetwork(t *testing.T) { + if !canAccessNetwork() { + t.Skip("network access not allowed; use TF_ACC=1 to enable") + } +} + +// Type terraform represents the combination of a compiled Terraform binary +// and a temporary working directory to run it in. +// +// This is the main harness for tests in this package. +type terraform struct { + bin string + dir string +} + +// newTerraform prepares a temporary directory containing the files from the +// given fixture and returns an instance of type terraform that can run +// the generated Terraform binary in that directory. +// +// If the temporary directory cannot be created, a fixture of the given name +// cannot be found, or if an error occurs while _copying_ the fixture files, +// this function will panic. Tests should be written to assume that this +// function always succeeds. +func newTerraform(fixtureName string) *terraform { + tmpDir, err := ioutil.TempDir("", "terraform-e2etest") + if err != nil { + panic(err) + } + + // For our purposes here we do a very simplistic file copy that doesn't + // attempt to preserve file permissions, attributes, alternate data + // streams, etc. Since we only have to deal with our own fixtures in + // the test-fixtures subdir, we know we don't need to deal with anything + // of this nature. + srcDir := filepath.Join("test-fixtures", fixtureName) + err = filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if path == srcDir { + // nothing to do at the root + return nil + } + + srcFn := path + + path, err = filepath.Rel(srcDir, path) + if err != nil { + return err + } + + dstFn := filepath.Join(tmpDir, path) + + if info.IsDir() { + return os.Mkdir(dstFn, os.ModePerm) + } + + src, err := os.Open(srcFn) + if err != nil { + return err + } + dst, err := os.OpenFile(dstFn, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm) + if err != nil { + return err + } + + _, err = io.Copy(dst, src) + if err != nil { + return err + } + + if err := src.Close(); err != nil { + return err + } + if err := dst.Close(); err != nil { + return err + } + + return nil + }) + if err != nil { + panic(err) + } + + return &terraform{ + bin: terraformBin, + dir: tmpDir, + } +} + +// Cmd returns an exec.Cmd pre-configured to run the generated Terraform +// binary with the given arguments in the temporary working directory. +// +// The returned object can be mutated by the caller to customize how the +// process will be run, before calling Run. +func (t *terraform) Cmd(args ...string) *exec.Cmd { + cmd := exec.Command(t.bin, args...) + cmd.Dir = t.dir + cmd.Env = os.Environ() + + // Disable checkpoint since we don't want to harass that service when + // our tests run. (This does, of course, mean we can't actually do + // end-to-end testing of our Checkpoint interactions.) + cmd.Env = append(cmd.Env, "CHECKPOINT_DISABLE=1") + + return cmd +} + +// Run executes the generated Terraform binary with the given arguments +// and returns the bytes that it wrote to both stdout and stderr. +// +// This is a simple way to run Terraform for non-interactive commands +// that don't need any special environment variables. For more complex +// situations, use Cmd and customize the command before running it. +func (t *terraform) Run(args ...string) (stdout, stderr string, err error) { + cmd := t.Cmd(args...) + cmd.Stdin = nil + cmd.Stdout = &bytes.Buffer{} + cmd.Stderr = &bytes.Buffer{} + err = cmd.Run() + stdout = cmd.Stdout.(*bytes.Buffer).String() + stderr = cmd.Stderr.(*bytes.Buffer).String() + return +} + +// Path returns a file path within the temporary working directory by +// appending the given arguments as path segments. +func (t *terraform) Path(parts ...string) string { + args := make([]string, len(parts)+1) + args[0] = t.dir + args = append(args, parts...) + return filepath.Join(args...) +} + +// OpenFile is a helper for easily opening a file from the working directory +// for reading. +func (t *terraform) OpenFile(path ...string) (*os.File, error) { + flatPath := t.Path(path...) + return os.Open(flatPath) +} + +// ReadFile is a helper for easily reading a whole file from the working +// directory. +func (t *terraform) ReadFile(path ...string) ([]byte, error) { + flatPath := t.Path(path...) + return ioutil.ReadFile(flatPath) +} + +// FileExists is a helper for easily testing whether a particular file +// exists in the working directory. +func (t *terraform) FileExists(path ...string) bool { + flatPath := t.Path(path...) + _, err := os.Stat(flatPath) + return !os.IsNotExist(err) +} + +// LocalState is a helper for easily reading the local backend's state file +// terraform.tfstate from the working directory. +func (t *terraform) LocalState() (*tfcore.State, error) { + f, err := t.OpenFile("terraform.tfstate") + if err != nil { + return nil, err + } + defer f.Close() + return tfcore.ReadState(f) +} + +// Plan is a helper for easily reading a plan file from the working directory. +func (t *terraform) Plan(path ...string) (*tfcore.Plan, error) { + f, err := t.OpenFile(path...) + if err != nil { + return nil, err + } + defer f.Close() + return tfcore.ReadPlan(f) +} + +// SetLocalState is a helper for easily writing to the file the local backend +// uses for state in the working directory. This does not go through the +// actual local backend code, so processing such as management of serials +// does not apply and the given state will simply be written verbatim. +func (t *terraform) SetLocalState(state *tfcore.State) error { + path := t.Path("terraform.tfstate") + f, err := os.OpenFile(path, os.O_TRUNC|os.O_CREATE|os.O_WRONLY, os.ModePerm) + if err != nil { + return err + } + defer func() { + err := f.Close() + if err != nil { + panic(fmt.Sprintf("failed to close state file after writing: %s", err)) + } + }() + + return tfcore.WriteState(state, f) +} + +// Close cleans up the temporary resources associated with the object, +// including its working directory. It is not valid to call Cmd or Run +// after Close returns. +// +// This method does _not_ stop any running child processes. It's the +// caller's responsibility to also terminate those _before_ closing the +// underlying terraform object. +// +// This function is designed to run under "defer", so it doesn't actually +// do any error handling and will leave dangling temporary files on disk +// if any errors occur while cleaning up. +func (t *terraform) Close() { + os.RemoveAll(t.dir) +} diff --git a/command/e2etest/make-archive.sh b/command/e2etest/make-archive.sh new file mode 100755 index 0000000000..fa8e25ee6b --- /dev/null +++ b/command/e2etest/make-archive.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# For normal use this package can just be tested with "go test" as standard, +# but this script is an alternative to allow the tests to be run somewhere +# other than where they are built. + +# The primary use for this is cross-compilation, where e.g. we can produce an +# archive that can be extracted on a Windows system to run the e2e tests there: +# $ GOOS=windows GOARCH=amd64 ./make-archive.sh +# +# This will produce a zip file build/terraform-s2stest_windows_amd64.zip which +# can be shipped off to a Windows amd64 system, extracted to some directory, +# and then executed as follows: +# set TF_ACC=1 +# ./e2etest.exe +# Since the test archive includes both the test fixtures and the compiled +# terraform executable along with this test program, the result is +# self-contained and does not require a local Go compiler on the target system. + +set +euo pipefail + +# Always run from the directory where this script lives +cd "$( dirname "${BASH_SOURCE[0]}" )" + +GOOS="$(go env GOOS)" +GOARCH="$(go env GOARCH)" +GOEXE="$(go env GOEXE)" +OUTDIR="build/${GOOS}_${GOARCH}" +OUTFILE="terraform-e2etest_${GOOS}_${GOARCH}.zip" + +mkdir -p "$OUTDIR" + +# We need the test fixtures available when we run the tests. +cp -r test-fixtures "$OUTDIR/test-fixtures" + +# Bundle a copy of our binary so the target system doesn't need the go +# compiler installed. +go build -o "$OUTDIR/terraform$GOEXE" github.com/hashicorp/terraform + +# Build the test program +go test -o "$OUTDIR/e2etest$GOEXE" -c -ldflags "-X github.com/hashicorp/terraform/command/e2etest.terraformBin=./terraform$GOEXE" github.com/hashicorp/terraform/command/e2etest + +# Now bundle it all together for easy shipping! +cd "$OUTDIR" +zip -r "../$OUTFILE" * + +echo "e2etest archive created at build/$OUTFILE" From 0e0b0d125a824624eba23b17a421056073aa8015 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Fri, 7 Jul 2017 18:47:56 -0700 Subject: [PATCH 2/4] command/e2etest: "terraform version" test This basic test is here primarily because it's one of the few that can run without reaching out to external services, and so it means our usual test runs will catch situations where the main executable build is somehow broken. The version command itself is not very interesting to test, but it's convenient in that its behavior is very predictable and self-contained. --- command/e2etest/test-fixtures/empty/.exists | 0 command/e2etest/version_test.go | 35 +++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 command/e2etest/test-fixtures/empty/.exists create mode 100644 command/e2etest/version_test.go diff --git a/command/e2etest/test-fixtures/empty/.exists b/command/e2etest/test-fixtures/empty/.exists new file mode 100644 index 0000000000..e69de29bb2 diff --git a/command/e2etest/version_test.go b/command/e2etest/version_test.go new file mode 100644 index 0000000000..d8610e98bd --- /dev/null +++ b/command/e2etest/version_test.go @@ -0,0 +1,35 @@ +package e2etest + +import ( + "fmt" + "strings" + "testing" + + tfcore "github.com/hashicorp/terraform/terraform" +) + +func TestVersion(t *testing.T) { + // Along with testing the "version" command in particular, this serves + // as a good smoke test for whether the Terraform binary can even be + // compiled and run, since it doesn't require any external network access + // to do its job. + + t.Parallel() + + tf := newTerraform("empty") + defer tf.Close() + + stdout, stderr, err := tf.Run("version") + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + if stderr != "" { + t.Errorf("unexpected stderr output:\n%s", stderr) + } + + wantVersion := fmt.Sprintf("Terraform %s", tfcore.VersionString()) + if strings.Contains(stdout, wantVersion) { + t.Errorf("output does not contain our current version %q:\n%s", wantVersion, stdout) + } +} From 52df81ee497e06b8db1b8ec7b762ce4af544c3b3 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Fri, 7 Jul 2017 18:49:41 -0700 Subject: [PATCH 3/4] command/e2etest: test that we can install provider plugins We already have good tests for the business logic around provider installation, but the existing tests all stub out the main repository server. This test completes that coverage by verifying that the installer is able to run against the real repository and install an official release of the template provider. --- command/e2etest/init_test.go | 43 +++++++++++++++++++ .../test-fixtures/template-provider/main.tf | 7 +++ 2 files changed, 50 insertions(+) create mode 100644 command/e2etest/init_test.go create mode 100644 command/e2etest/test-fixtures/template-provider/main.tf diff --git a/command/e2etest/init_test.go b/command/e2etest/init_test.go new file mode 100644 index 0000000000..7a5ec2ab23 --- /dev/null +++ b/command/e2etest/init_test.go @@ -0,0 +1,43 @@ +package e2etest + +import ( + "strings" + "testing" +) + +func TestInitProviders(t *testing.T) { + t.Parallel() + + // This test reaches out to releases.hashicorp.com to download the + // template provider, so it can only run if network access is allowed. + // We intentionally don't try to stub this here, because there's already + // a stubbed version of this in the "command" package and so the goal here + // is to test the interaction with the real repository. + skipIfCannotAccessNetwork(t) + + tf := newTerraform("template-provider") + defer tf.Close() + + stdout, stderr, err := tf.Run("init") + if err != nil { + t.Errorf("unexpected error: %s", err) + } + + if stderr != "" { + t.Errorf("unexpected stderr output:\n%s", stderr) + } + + if !strings.Contains(stdout, "Terraform has been successfully initialized!") { + t.Errorf("success message is missing from output:\n%s", stdout) + } + + if !strings.Contains(stdout, "- Downloading plugin for provider \"template\"") { + t.Errorf("provider download message is missing from output:\n%s", stdout) + t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)") + } + + if !strings.Contains(stdout, "* provider.template: version = ") { + t.Errorf("provider pinning recommendation is missing from output:\n%s", stdout) + } + +} diff --git a/command/e2etest/test-fixtures/template-provider/main.tf b/command/e2etest/test-fixtures/template-provider/main.tf new file mode 100644 index 0000000000..31af45150a --- /dev/null +++ b/command/e2etest/test-fixtures/template-provider/main.tf @@ -0,0 +1,7 @@ +provider "template" { + +} + +data "template_file" "test" { + template = "Hello World" +} From 23f9c8785e3b16f7b73580da680f1d5d19d773ad Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Fri, 7 Jul 2017 18:54:29 -0700 Subject: [PATCH 4/4] command/e2etest: an initial test for the primary workflow This e2etest runs an init, plan, apply, destroy sequence against a test configuration using the real template and null providers downloaded from the official repository. This test _does_ trample a bit on the scope of some already-existing tests, but this is mainly just to check our assumptions about how Terraform behaves to ensure that we can reach our main conclusion here: that the main Terraform workflow commands interact correctly with each other in real use and we can complete the full workflow. --- command/e2etest/primary_test.go | 123 ++++++++++++++++++ .../test-fixtures/full-workflow-null/main.tf | 22 ++++ 2 files changed, 145 insertions(+) create mode 100644 command/e2etest/primary_test.go create mode 100644 command/e2etest/test-fixtures/full-workflow-null/main.tf diff --git a/command/e2etest/primary_test.go b/command/e2etest/primary_test.go new file mode 100644 index 0000000000..f491683c9e --- /dev/null +++ b/command/e2etest/primary_test.go @@ -0,0 +1,123 @@ +package e2etest + +import ( + "reflect" + "sort" + "strings" + "testing" + + "github.com/davecgh/go-spew/spew" +) + +// The tests in this file are for the "primary workflow", which includes +// variants of the following sequence, with different details: +// terraform init +// terraform plan +// terraform apply +// terraform destroy + +func TestPrimarySeparatePlan(t *testing.T) { + t.Parallel() + + // This test reaches out to releases.hashicorp.com to download the + // template and null providers, so it can only run if network access is + // allowed. + skipIfCannotAccessNetwork(t) + + tf := newTerraform("full-workflow-null") + defer tf.Close() + + //// INIT + stdout, stderr, err := tf.Run("init") + if err != nil { + t.Fatalf("unexpected init error: %s\nstderr:\n%s", err, stderr) + } + + // Make sure we actually downloaded the plugins, rather than picking up + // copies that might be already installed globally on the system. + if !strings.Contains(stdout, "- Downloading plugin for provider \"template\"") { + t.Errorf("template provider download message is missing from init output:\n%s", stdout) + t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)") + } + if !strings.Contains(stdout, "- Downloading plugin for provider \"null\"") { + t.Errorf("null provider download message is missing from init output:\n%s", stdout) + t.Logf("(this can happen if you have a copy of the plugin in one of the global plugin search dirs)") + } + + //// PLAN + stdout, stderr, err = tf.Run("plan", "-out=tfplan") + if err != nil { + t.Fatalf("unexpected plan error: %s\nstderr:\n%s", err, stderr) + } + + if !strings.Contains(stdout, "1 to add, 0 to change, 0 to destroy") { + t.Errorf("incorrect plan tally; want 1 to add:\n%s", stdout) + } + + plan, err := tf.Plan("tfplan") + if err != nil { + t.Fatalf("failed to read plan file: %s", err) + } + + stateResources := plan.State.RootModule().Resources + diffResources := plan.Diff.RootModule().Resources + + if len(stateResources) != 1 || stateResources["data.template_file.test"] == nil { + t.Errorf("incorrect state in plan; want just data.template_file.test to have been rendered, but have:\n%s", spew.Sdump(stateResources)) + } + if len(diffResources) != 1 || diffResources["null_resource.test"] == nil { + t.Errorf("incorrect diff in plan; want just null_resource.test to have been rendered, but have:\n%s", spew.Sdump(diffResources)) + } + + //// APPLY + stdout, stderr, err = tf.Run("apply", "tfplan") + if err != nil { + t.Fatalf("unexpected apply error: %s\nstderr:\n%s", err, stderr) + } + + if !strings.Contains(stdout, "Resources: 1 added, 0 changed, 0 destroyed") { + t.Errorf("incorrect apply tally; want 1 added:\n%s", stdout) + } + + state, err := tf.LocalState() + if err != nil { + t.Fatalf("failed to read state file: %s", err) + } + + stateResources = state.RootModule().Resources + var gotResources []string + for n := range stateResources { + gotResources = append(gotResources, n) + } + sort.Strings(gotResources) + + wantResources := []string{ + "data.template_file.test", + "null_resource.test", + } + + if !reflect.DeepEqual(gotResources, wantResources) { + t.Errorf("wrong resources in state\ngot: %#v\nwant: %#v", gotResources, wantResources) + } + + //// DESTROY + stdout, stderr, err = tf.Run("destroy", "-force") + if err != nil { + t.Fatalf("unexpected destroy error: %s\nstderr:\n%s", err, stderr) + } + + if !strings.Contains(stdout, "Resources: 2 destroyed") { + t.Errorf("incorrect destroy tally; want 2 destroyed:\n%s", stdout) + } + + state, err = tf.LocalState() + if err != nil { + t.Fatalf("failed to read state file after destroy: %s", err) + } + + stateResources = state.RootModule().Resources + if len(stateResources) != 0 { + t.Errorf("wrong resources in state after destroy; want none, but still have:%s", spew.Sdump(stateResources)) + } + +} diff --git a/command/e2etest/test-fixtures/full-workflow-null/main.tf b/command/e2etest/test-fixtures/full-workflow-null/main.tf new file mode 100644 index 0000000000..1c3fc36e10 --- /dev/null +++ b/command/e2etest/test-fixtures/full-workflow-null/main.tf @@ -0,0 +1,22 @@ + +variable "name" { + default = "world" +} + +data "template_file" "test" { + template = "Hello, $${name}" + + vars = { + name = "${var.name}" + } +} + +resource "null_resource" "test" { + triggers = { + greeting = "${data.template_file.test.rendered}" + } +} + +output "greeting" { + value = "${null_resource.test.triggers["greeting"]}" +}