provider/template: Add a 'dir' resource to template entire directories

When TerraForm is used to configure and deploy infrastructure
applications that require dozens templated files, such as Kubernetes, it
becomes extremely burdensome to template them individually: each of them
requires a data source block as well as an upload/export (file
provisioner, AWS S3, ...).

Instead, this commit introduces a mean to template an entire folder of
files (recursively), that can then be treated as a whole by any provider
or provisioner that support directory inputs (such as the
file provisioner, the archive provider, ...).

This does not intend to make TerraForm a full-fledged templating system
as the templating grammar and capabilities are left unchanged. This only
aims at improving the user-experience of the existing templating
provider by significantly reducing the overhead when several files are
to be generated - without forcing the users to rely on external tools
when these templates stay simple and that their generation in TerraForm
is justified.

builtin/providers/template/provider.go

@@ -20,6 +20,7 @@ func Provider() terraform.ResourceProvider {
 				"template_cloudinit_config",
 				dataSourceCloudinitConfig(),
 			),
+			"template_dir": resourceDir(),
 		},
 	}
 }

builtin/providers/template/resource_template_dir.go

@@ -0,0 +1,225 @@
+package template
+
+import (
+	"archive/tar"
+	"bytes"
+	"crypto/sha1"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+
+	"github.com/hashicorp/terraform/helper/pathorcontents"
+	"github.com/hashicorp/terraform/helper/schema"
+)
+
+func resourceDir() *schema.Resource {
+	return &schema.Resource{
+		Create: resourceTemplateDirCreate,
+		Read:   resourceTemplateDirRead,
+		Delete: resourceTemplateDirDelete,
+
+		Schema: map[string]*schema.Schema{
+			"source_dir": {
+				Type:        schema.TypeString,
+				Description: "Path to the directory where the files to template reside",
+				Required:    true,
+				ForceNew:    true,
+			},
+			"vars": {
+				Type:         schema.TypeMap,
+				Optional:     true,
+				Default:      make(map[string]interface{}),
+				Description:  "Variables to substitute",
+				ValidateFunc: validateVarsAttribute,
+				ForceNew:     true,
+			},
+			"destination_dir": {
+				Type:        schema.TypeString,
+				Description: "Path to the directory where the templated files will be written",
+				Required:    true,
+				ForceNew:    true,
+			},
+		},
+	}
+}
+
+func resourceTemplateDirRead(d *schema.ResourceData, meta interface{}) error {
+	sourceDir := d.Get("source_dir").(string)
+	destinationDir := d.Get("destination_dir").(string)
+
+	// If the output doesn't exist, mark the resource for creation.
+	if _, err := os.Stat(destinationDir); os.IsNotExist(err) {
+		d.SetId("")
+		return nil
+	}
+
+	// If the combined hash of the input and output directories is different from
+	// the stored one, mark the resource for re-creation.
+	//
+	// The output directory is technically enough for the general case, but by
+	// hashing the input directory as well, we make development much easier: when
+	// a developer modifies one of the input files, the generation is
+	// re-triggered.
+	hash, err := generateID(sourceDir, destinationDir)
+	if err != nil {
+		return err
+	}
+	if hash != d.Id() {
+		d.SetId("")
+		return nil
+	}
+
+	return nil
+}
+
+func resourceTemplateDirCreate(d *schema.ResourceData, meta interface{}) error {
+	sourceDir := d.Get("source_dir").(string)
+	destinationDir := d.Get("destination_dir").(string)
+	vars := d.Get("vars").(map[string]interface{})
+
+	// Always delete the output first, otherwise files that got deleted from the
+	// input directory might still be present in the output afterwards.
+	if err := resourceTemplateDirDelete(d, meta); err != nil {
+		return err
+	}
+
+	// Recursively crawl the input files/directories and generate the output ones.
+	err := filepath.Walk(sourceDir, func(p string, f os.FileInfo, err error) error {
+		if f.IsDir() {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+
+		relPath, _ := filepath.Rel(sourceDir, p)
+		return generateDirFile(p, path.Join(destinationDir, relPath), f, vars)
+	})
+	if err != nil {
+		return err
+	}
+
+	// Compute ID.
+	hash, err := generateID(sourceDir, destinationDir)
+	if err != nil {
+		return err
+	}
+	d.SetId(hash)
+
+	return nil
+}
+
+func resourceTemplateDirDelete(d *schema.ResourceData, _ interface{}) error {
+	d.SetId("")
+
+	destinationDir := d.Get("destination_dir").(string)
+	if _, err := os.Stat(destinationDir); os.IsNotExist(err) {
+		return nil
+	}
+
+	if err := os.RemoveAll(destinationDir); err != nil {
+		return fmt.Errorf("could not delete directory %q: %s", destinationDir, err)
+	}
+
+	return nil
+}
+
+func generateDirFile(sourceDir, destinationDir string, f os.FileInfo, vars map[string]interface{}) error {
+	inputContent, _, err := pathorcontents.Read(sourceDir)
+	if err != nil {
+		return err
+	}
+
+	outputContent, err := execute(inputContent, vars)
+	if err != nil {
+		return templateRenderError(fmt.Errorf("failed to render %v: %v", sourceDir, err))
+	}
+
+	outputDir := path.Dir(destinationDir)
+	if _, err := os.Stat(outputDir); err != nil {
+		if err := os.MkdirAll(outputDir, 0777); err != nil {
+			return err
+		}
+	}
+
+	err = ioutil.WriteFile(destinationDir, []byte(outputContent), f.Mode())
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func generateID(sourceDir, destinationDir string) (string, error) {
+	inputHash, err := generateDirHash(sourceDir)
+	if err != nil {
+		return "", err
+	}
+	outputHash, err := generateDirHash(destinationDir)
+	if err != nil {
+		return "", err
+	}
+	checksum := sha1.Sum([]byte(inputHash + outputHash))
+	return hex.EncodeToString(checksum[:]), nil
+}
+
+func generateDirHash(directoryPath string) (string, error) {
+	tarData, err := tarDir(directoryPath)
+	if err != nil {
+		return "", fmt.Errorf("could not generate output checksum: %s", err)
+	}
+
+	checksum := sha1.Sum(tarData)
+	return hex.EncodeToString(checksum[:]), nil
+}
+
+func tarDir(directoryPath string) ([]byte, error) {
+	buf := new(bytes.Buffer)
+	tw := tar.NewWriter(buf)
+
+	writeFile := func(p string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		var header *tar.Header
+		var file *os.File
+
+		header, err = tar.FileInfoHeader(f, f.Name())
+		if err != nil {
+			return err
+		}
+		relPath, _ := filepath.Rel(directoryPath, p)
+		header.Name = relPath
+
+		if err := tw.WriteHeader(header); err != nil {
+			return err
+		}
+
+		if f.IsDir() {
+			return nil
+		}
+
+		file, err = os.Open(p)
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		_, err = io.Copy(tw, file)
+		return err
+	}
+
+	if err := filepath.Walk(directoryPath, writeFile); err != nil {
+		return []byte{}, err
+	}
+	if err := tw.Flush(); err != nil {
+		return []byte{}, err
+	}
+
+	return buf.Bytes(), nil
+}

builtin/providers/template/resource_template_dir_test.go

@@ -0,0 +1,104 @@
+package template
+
+import (
+	"fmt"
+	"testing"
+
+	"errors"
+	r "github.com/hashicorp/terraform/helper/resource"
+	"github.com/hashicorp/terraform/terraform"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+const templateDirRenderingConfig = `
+resource "template_dir" "dir" {
+	source_dir = "%s"
+  destination_dir = "%s"
+  vars = %s
+}`
+
+type testTemplate struct {
+	template string
+	want     string
+}
+
+func testTemplateDirWriteFiles(files map[string]testTemplate) (in, out string, err error) {
+	in, err = ioutil.TempDir(os.TempDir(), "terraform_template_dir")
+	if err != nil {
+		return
+	}
+
+	for name, file := range files {
+		path := filepath.Join(in, name)
+
+		err = os.MkdirAll(filepath.Dir(path), 0777)
+		if err != nil {
+			return
+		}
+
+		err = ioutil.WriteFile(path, []byte(file.template), 0777)
+		if err != nil {
+			return
+		}
+	}
+
+	out = fmt.Sprintf("%s.out", in)
+	return
+}
+
+func TestTemplateDirRendering(t *testing.T) {
+	var cases = []struct {
+		vars  string
+		files map[string]testTemplate
+	}{
+		{
+			files: map[string]testTemplate{
+				"foo.txt":           {"${bar}", "bar"},
+				"nested/monkey.txt": {"ooh-ooh-ooh-eee-eee", "ooh-ooh-ooh-eee-eee"},
+				"maths.txt":         {"${1+2+3}", "6"},
+			},
+			vars: `{bar = "bar"}`,
+		},
+	}
+
+	for _, tt := range cases {
+		// Write the desired templates in a temporary directory.
+		in, out, err := testTemplateDirWriteFiles(tt.files)
+		if err != nil {
+			t.Skipf("could not write templates to temporary directory: %s", err)
+			continue
+		}
+		defer os.RemoveAll(in)
+		defer os.RemoveAll(out)
+
+		// Run test case.
+		r.UnitTest(t, r.TestCase{
+			Providers: testProviders,
+			Steps: []r.TestStep{
+				{
+					Config: fmt.Sprintf(templateDirRenderingConfig, in, out, tt.vars),
+					Check: func(s *terraform.State) error {
+						for name, file := range tt.files {
+							content, err := ioutil.ReadFile(filepath.Join(out, name))
+							if err != nil {
+								return fmt.Errorf("template:\n%s\nvars:\n%s\ngot:\n%s\nwant:\n%s\n", file.template, tt.vars, err, file.want)
+							}
+							if string(content) != file.want {
+								return fmt.Errorf("template:\n%s\nvars:\n%s\ngot:\n%s\nwant:\n%s\n", file.template, tt.vars, content, file.want)
+							}
+						}
+						return nil
+					},
+				},
+			},
+			CheckDestroy: func(*terraform.State) error {
+				if _, err := os.Stat(out); os.IsNotExist(err) {
+					return nil
+				}
+				return errors.New("template_dir did not get destroyed")
+			},
+		})
+	}
+}

website/source/docs/providers/template/r/dir.html.md

@@ -0,0 +1,58 @@
+---
+layout: "template"
+page_title: "Template: template_dir"
+sidebar_current: "docs-template-resource-dir"
+description: |-
+  Renders templates from a directory.
+---
+
+# template_dir
+
+Renders templates from a directory.
+
+## Example Usage
+```hcl
+data "template_directory" "init" {
+  source_dir      = "${path.cwd}/templates"
+  destination_dir = "${path.cwd}/templates.generated"
+  
+  vars {
+    consul_address = "${aws_instance.consul.private_ip}"
+  }
+}
+```
+
+## Argument Reference
+
+The following arguments are supported:
+
+* `source_path` - (Required) Path to the directory where the files to template reside.
+
+* `destination_path` - (Required) Path to the directory where the templated files will be written.
+
+* `vars` - (Optional) Variables for interpolation within the template. Note
+  that variables must all be primitives. Direct references to lists or maps
+  will cause a validation error.
+
+NOTE: Any required parent directories are created automatically. Additionally, any external modification to either the files in the source or destination directories will trigger the resource to be re-created.
+
+## Template Syntax
+
+The syntax of the template files is the same as
+[standard interpolation syntax](/docs/configuration/interpolation.html),
+but you only have access to the variables defined in the `vars` section.
+
+To access interpolations that are normally available to Terraform
+configuration (such as other variables, resource attributes, module
+outputs, etc.) you'll have to expose them via `vars` as shown below:
+
+```hcl
+resource "template_dir" "init" {
+  # ...
+
+  vars {
+    foo  = "${var.foo}"
+    attr = "${aws_instance.foo.private_ip}"
+  }
+}
+```

website/source/layouts/template.erb

@@ -21,6 +21,15 @@
             </li>
           </ul>
         </li>
+
+        <li<%= sidebar_current("docs-template-resource") %>>
+          <a href="#">Resources</a>
+          <ul class="nav nav-visible">
+            <li<%= sidebar_current("docs-template-resource-dir") %>>
+              <a href="/docs/providers/template/r/dir.html">template_dir</a>
+            </li>
+          </ul>
+        </li>
       </ul>
     </div>
   <% end %>