datasource: add file datasource

The file datasource is meant to be used to pre-create a file that can
then be used elsewhere in the build process by its path.

This is useful for example when building a configuration file from a
template, so then the resulting file can be referenced by components
which only accept file paths.
file_datasource
Lucas Bajolet 3 years ago
parent 6096a38778
commit 5ce9b1867b

@ -15,6 +15,7 @@ import (
filebuilder "github.com/hashicorp/packer/builder/file"
nullbuilder "github.com/hashicorp/packer/builder/null"
filedatasource "github.com/hashicorp/packer/datasource/file"
hcppackerartifactdatasource "github.com/hashicorp/packer/datasource/hcp-packer-artifact"
hcppackerimagedatasource "github.com/hashicorp/packer/datasource/hcp-packer-image"
hcppackeriterationdatasource "github.com/hashicorp/packer/datasource/hcp-packer-iteration"
@ -65,6 +66,7 @@ var PostProcessors = map[string]packersdk.PostProcessor{
}
var Datasources = map[string]packersdk.Datasource{
"file": new(filedatasource.Datasource),
"hcp-packer-artifact": new(hcppackerartifactdatasource.Datasource),
"hcp-packer-image": new(hcppackerimagedatasource.Datasource),
"hcp-packer-iteration": new(hcppackeriterationdatasource.Datasource),

@ -0,0 +1,131 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
//go:generate packer-sdc struct-markdown
//go:generate packer-sdc mapstructure-to-hcl2 -type DatasourceOutput,Config
package file
import (
"fmt"
"log"
"os"
"path/filepath"
"github.com/hashicorp/hcl/v2/hcldec"
"github.com/hashicorp/packer-plugin-sdk/common"
"github.com/hashicorp/packer-plugin-sdk/hcl2helper"
"github.com/hashicorp/packer-plugin-sdk/template/config"
"github.com/zclconf/go-cty/cty"
)
type Config struct {
common.PackerConfig `mapstructure:",squash"`
// The contents of the file to create
//
// This is useful especially for files that involve templating so that
// Packer can dynamically create files and expose them for later importing
// as attributes in another entity.
//
// If no contents are specified, the resulting file will be empty.
Contents string `mapstructure:"contents" required:"false"`
// The file or directory to write the contents to.
Destination string `mapstructure:"destination" required:"false"`
}
type Datasource struct {
config Config
}
type DatasourceOutput struct {
// The path of the file created
Path string `mapstructure:"path"`
}
func (d *Datasource) ConfigSpec() hcldec.ObjectSpec {
return d.config.FlatMapstructure().HCL2Spec()
}
func (d *Datasource) Configure(raws ...interface{}) error {
err := config.Decode(&d.config, nil, raws...)
if err != nil {
return err
}
return nil
}
func (d *Datasource) OutputSpec() hcldec.ObjectSpec {
return (&DatasourceOutput{}).FlatMapstructure().HCL2Spec()
}
func (d *Datasource) Execute() (cty.Value, error) {
nulVal := cty.NullVal(cty.EmptyObject)
dest, err := d.createTempOutputFile()
if err != nil {
return nulVal, fmt.Errorf("failed to create output file: %s", err)
}
defer dest.Close()
log.Printf("[INFO] data/file - Writing to %q", dest.Name())
written, err := dest.Write([]byte(d.config.Contents))
if err != nil {
defer os.Remove(d.config.Destination)
return nulVal, fmt.Errorf("failed to write contents to %q: %s", d.config.Destination, err)
}
if written != len(d.config.Contents) {
defer os.Remove(d.config.Destination)
return nulVal, fmt.Errorf(
"failed to write contents to %q: expected to write %d bytes, but wrote %d instead",
d.config.Destination,
len(d.config.Contents),
written)
}
output := DatasourceOutput{
Path: dest.Name(),
}
return hcl2helper.HCL2ValueFromConfig(output, d.OutputSpec()), nil
}
func (d *Datasource) createTempOutputFile() (*os.File, error) {
// If we did not get a destination, we'll create a temp file in the
// system's temporary directory
if d.config.Destination == "" {
return os.CreateTemp("", "")
}
// First try to stat the destination, to determine if it already exists and its type
st, statErr := os.Stat(d.config.Destination)
if statErr == nil {
if st.IsDir() {
return os.CreateTemp(d.config.Destination, "")
}
return os.OpenFile(d.config.Destination, os.O_TRUNC|os.O_RDWR, 0644)
}
outDir := filepath.Dir(d.config.Destination)
// In case the destination does not exist, we'll get the dirpath,
// and create it if it doesn't already exist
err := os.MkdirAll(outDir, 0755)
if err != nil {
return nil, fmt.Errorf("failed to create destination directory %q: %s", outDir, err)
}
// Check if the destination is a directory after the previous step.
//
// This happens if the path specified ends with a `/`, in which case the
// destination is a directory, and we must create a temporary file in
// this destination directory.
destStat, statErr := os.Stat(d.config.Destination)
if statErr == nil && destStat.IsDir() {
return os.CreateTemp(d.config.Destination, "")
}
return os.Create(d.config.Destination)
}

@ -0,0 +1,72 @@
// Code generated by "packer-sdc mapstructure-to-hcl2"; DO NOT EDIT.
package file
import (
"github.com/hashicorp/hcl/v2/hcldec"
"github.com/zclconf/go-cty/cty"
)
// FlatConfig is an auto-generated flat version of Config.
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
type FlatConfig struct {
PackerBuildName *string `mapstructure:"packer_build_name" cty:"packer_build_name" hcl:"packer_build_name"`
PackerBuilderType *string `mapstructure:"packer_builder_type" cty:"packer_builder_type" hcl:"packer_builder_type"`
PackerCoreVersion *string `mapstructure:"packer_core_version" cty:"packer_core_version" hcl:"packer_core_version"`
PackerDebug *bool `mapstructure:"packer_debug" cty:"packer_debug" hcl:"packer_debug"`
PackerForce *bool `mapstructure:"packer_force" cty:"packer_force" hcl:"packer_force"`
PackerOnError *string `mapstructure:"packer_on_error" cty:"packer_on_error" hcl:"packer_on_error"`
PackerUserVars map[string]string `mapstructure:"packer_user_variables" cty:"packer_user_variables" hcl:"packer_user_variables"`
PackerSensitiveVars []string `mapstructure:"packer_sensitive_variables" cty:"packer_sensitive_variables" hcl:"packer_sensitive_variables"`
Contents *string `mapstructure:"contents" required:"false" cty:"contents" hcl:"contents"`
Destination *string `mapstructure:"destination" required:"false" cty:"destination" hcl:"destination"`
}
// FlatMapstructure returns a new FlatConfig.
// FlatConfig is an auto-generated flat version of Config.
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
func (*Config) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
return new(FlatConfig)
}
// HCL2Spec returns the hcl spec of a Config.
// This spec is used by HCL to read the fields of Config.
// The decoded values from this spec will then be applied to a FlatConfig.
func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec {
s := map[string]hcldec.Spec{
"packer_build_name": &hcldec.AttrSpec{Name: "packer_build_name", Type: cty.String, Required: false},
"packer_builder_type": &hcldec.AttrSpec{Name: "packer_builder_type", Type: cty.String, Required: false},
"packer_core_version": &hcldec.AttrSpec{Name: "packer_core_version", Type: cty.String, Required: false},
"packer_debug": &hcldec.AttrSpec{Name: "packer_debug", Type: cty.Bool, Required: false},
"packer_force": &hcldec.AttrSpec{Name: "packer_force", Type: cty.Bool, Required: false},
"packer_on_error": &hcldec.AttrSpec{Name: "packer_on_error", Type: cty.String, Required: false},
"packer_user_variables": &hcldec.AttrSpec{Name: "packer_user_variables", Type: cty.Map(cty.String), Required: false},
"packer_sensitive_variables": &hcldec.AttrSpec{Name: "packer_sensitive_variables", Type: cty.List(cty.String), Required: false},
"contents": &hcldec.AttrSpec{Name: "contents", Type: cty.String, Required: false},
"destination": &hcldec.AttrSpec{Name: "destination", Type: cty.String, Required: false},
}
return s
}
// FlatDatasourceOutput is an auto-generated flat version of DatasourceOutput.
// Where the contents of a field with a `mapstructure:,squash` tag are bubbled up.
type FlatDatasourceOutput struct {
Path *string `mapstructure:"path" cty:"path" hcl:"path"`
}
// FlatMapstructure returns a new FlatDatasourceOutput.
// FlatDatasourceOutput is an auto-generated flat version of DatasourceOutput.
// Where the contents a fields with a `mapstructure:,squash` tag are bubbled up.
func (*DatasourceOutput) FlatMapstructure() interface{ HCL2Spec() map[string]hcldec.Spec } {
return new(FlatDatasourceOutput)
}
// HCL2Spec returns the hcl spec of a DatasourceOutput.
// This spec is used by HCL to read the fields of DatasourceOutput.
// The decoded values from this spec will then be applied to a FlatDatasourceOutput.
func (*FlatDatasourceOutput) HCL2Spec() map[string]hcldec.Spec {
s := map[string]hcldec.Spec{
"path": &hcldec.AttrSpec{Name: "path", Type: cty.String, Required: false},
}
return s
}

@ -0,0 +1,180 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
package file
import (
"fmt"
"os"
"os/exec"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/hashicorp/packer-plugin-sdk/acctest"
)
func TestFileDataSource(t *testing.T) {
tests := []struct {
name string
template string
createOutput bool
expectError bool
expectOutput string
}{
{
"Success - write empty file",
basicEmptyFileWrite,
false,
false,
"",
},
{
"Fail - write empty file, pre-existing output",
basicEmptyFileWrite,
true,
true,
"",
},
{
"Success - write empty file, pre-existing output",
basicEmptyFileWriteForce,
true,
false,
"",
},
{
"Success - write template to output",
basicFileWithTemplateContents,
false,
false,
"contents are 12345\n",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
testCase := &acctest.PluginTestCase{
Name: tt.name,
Setup: func() error {
return nil
},
Teardown: func() error {
return nil
},
Template: tt.template,
Type: "http",
Check: func(buildCommand *exec.Cmd, logfile string) error {
if buildCommand.ProcessState != nil {
if buildCommand.ProcessState.ExitCode() != 0 && !tt.expectError {
return fmt.Errorf("Bad exit code. Logfile: %s", logfile)
}
if tt.expectError && buildCommand.ProcessState.ExitCode() == 0 {
return fmt.Errorf("Expected an error but succeeded.")
}
}
if tt.expectError {
return nil
}
outFile, err := os.ReadFile("output")
if err != nil {
return fmt.Errorf("failed to read output file: %s", err)
}
diff := cmp.Diff(string(outFile), tt.expectOutput)
if diff != "" {
return fmt.Errorf("diff found in output: %s", diff)
}
return nil
},
}
os.RemoveAll("output")
if tt.createOutput {
err := os.WriteFile("output", []byte{}, 0644)
if err != nil {
t.Fatalf("failed to pre-create output file: %s", err)
}
}
acctest.TestPlugin(t, testCase)
os.RemoveAll("output")
})
}
}
var basicEmptyFileWrite string = `
source "null" "test" {
communicator = "none"
}
data "file" "empty" {
destination = "output"
}
build {
sources = [
"source.null.test"
]
provisioner "shell-local" {
inline = [
"set -ex",
"test -f ${data.file.empty.path}",
]
}
}
`
var basicEmptyFileWriteForce string = `
source "null" "test" {
communicator = "none"
}
data "file" "empty" {
destination = "output"
force = true
}
build {
sources = [
"source.null.test"
]
provisioner "shell-local" {
inline = [
"set -ex",
"test -f ${data.file.empty.path}",
]
}
}
`
var basicFileWithTemplateContents string = `
source "null" "test" {
communicator = "none"
}
data "file" "empty" {
contents = templatefile("test-fixtures/template.pkrtpl.hcl", {
"value" = "12345",
})
destination = "output"
}
build {
sources = [
"source.null.test"
]
provisioner "shell-local" {
inline = [
"set -ex",
"test -f ${data.file.empty.path}",
]
}
}
`

@ -0,0 +1,185 @@
package main
import (
"fmt"
"os"
"regexp"
"github.com/hashicorp/packer/packer_test/common/check"
)
var outputFileRegexp = regexp.MustCompile("data/file - Writing to \"([^\"]+)\"")
func cleanupOutputFile(stderr string) error {
matches := outputFileRegexp.FindStringSubmatch(stderr)
if len(matches) != 2 {
return fmt.Errorf("cannot match file datasource from packer output")
}
filePath := matches[1]
return os.Remove(filePath)
}
// TestWithNothing checks that in its simplest form, the datasource succeeds and writes an empty file to TMPDIR
func (ts *FileDatasourceTestSuite) TestWithNothing() {
pd := ts.MakePluginDir()
defer pd.Cleanup()
cmd := ts.PackerCommand().UsePluginDir(pd).
SetArgs("build", "./templates/file_simplest.pkr.hcl")
cmd.Assert(check.MustSucceed(),
check.Grep("data/file - Writing to", check.GrepStderr))
_, stderr, _ := cmd.Output()
err := cleanupOutputFile(stderr)
if err != nil {
ts.T().Logf("failed to find file to cleanup from stderr, will need some manual action")
}
}
// TestWithContents checks that the datasource writes what is expected to the output file, in TMPDIR
func (ts *FileDatasourceTestSuite) TestWithContents() {
pd := ts.MakePluginDir()
defer pd.Cleanup()
cmd := ts.PackerCommand().UsePluginDir(pd).
SetArgs("build", "./templates/file_with_contents.pkr.hcl")
cmd.Assert(check.MustSucceed(),
check.Grep("data/file - Writing to", check.GrepStderr),
check.Grep("file contents: Hello there!"))
_, stderr, _ := cmd.Output()
err := cleanupOutputFile(stderr)
if err != nil {
ts.T().Logf("failed to find file to cleanup from stderr, will need some manual action")
}
}
// TestWithFileDestination checks that we can specify a file directory, with its hierarchy existing in the first place
func (ts *FileDatasourceTestSuite) TestWithFileDestination() {
pd := ts.MakePluginDir()
defer pd.Cleanup()
// Create full hierarchy for output directory
err := os.MkdirAll("out_dir/subdir", 0755)
if err != nil {
ts.T().Fatalf("failed to create output directory: %s", err)
}
defer os.RemoveAll("out_dir")
// No need to clean output file, since the directory is cleaned-up automatically
ts.PackerCommand().UsePluginDir(pd).
SetArgs("build", "./templates/local_destination.pkr.hcl").
Assert(check.MustSucceed(),
check.Grep("data/file - Writing to", check.GrepStderr),
check.Grep("file contents: Hello there!"),
check.FileExists("out_dir/subdir/out.txt", false))
}
// TestWithFileDestinationAlreadyExists checks that we can specify a file output, even if it exists, and the output is strictly the contents of the file
func (ts *FileDatasourceTestSuite) TestWithFileDestinationAlreadyExists() {
pd := ts.MakePluginDir()
defer pd.Cleanup()
// Create full hierarchy for output directory
err := os.MkdirAll("out_dir/subdir", 0755)
if err != nil {
ts.T().Fatalf("failed to create output directory: %s", err)
}
defer os.RemoveAll("out_dir")
err = os.WriteFile("out_dir/subdir/out.txt", []byte("Hello there!\n"), 0644)
if err != nil {
ts.T().Fatalf("failed to write output file 'out_dir/subdir/out.txt' before test: %s", err)
}
// No need to clean output file, since the directory is cleaned-up automatically
ts.PackerCommand().UsePluginDir(pd).
SetArgs("build", "./templates/local_destination.pkr.hcl").
Assert(check.MustSucceed(),
check.Grep("data/file - Writing to", check.GrepStderr),
check.Grep("file contents: Hello there!"),
check.MkPipeCheck("only one occurrence in contents of output file",
check.PipeGrep("Hello there!"), check.LineCount()).
SetStream(check.OnlyStdout).
SetTester(check.IntCompare(check.Eq, 1)),
check.FileExists("out_dir/subdir/out.txt", false))
}
// TestWithFileDestination checks that we can specify a destination directory, with it existing in the first place
func (ts *FileDatasourceTestSuite) TestWithDirectoryDestination() {
pd := ts.MakePluginDir()
defer pd.Cleanup()
err := os.MkdirAll("out_dir/subdir", 0755)
if err != nil {
ts.T().Fatalf("failed to create output directory: %s", err)
}
// Cleanup output directory
defer os.RemoveAll("out_dir")
ts.PackerCommand().UsePluginDir(pd).
SetArgs("build", "./templates/local_dir_destination.pkr.hcl").
Assert(check.MustSucceed(),
check.Grep("data/file - Writing to", check.GrepStderr),
check.Grep("file contents: Hello there!"),
check.FileExists("out_dir/subdir", true))
}
// TestWithFileDestinationNoPreCreate checks that we can specify a destination directory, without it existing in the first place
func (ts *FileDatasourceTestSuite) TestWithDirectoryDestinationNoPreCreate() {
pd := ts.MakePluginDir()
defer pd.Cleanup()
ts.PackerCommand().UsePluginDir(pd).
SetArgs("build", "./templates/local_dir_destination.pkr.hcl").
Assert(check.MustSucceed(),
check.Grep("data/file - Writing to", check.GrepStderr),
check.Grep("file contents: Hello there!"),
check.FileExists("out_dir/subdir", true))
// Cleanup output directory
os.RemoveAll("out_dir")
}
// TestWithTempDirNotWritable checks that the datasource fails if the temporary directory is not writable, and we did not provide a Destination.
//
// NOTE: this one fails to execute completely since Packer needs TMPDIR to be writable for logs, and changing this may include more work.
// Leaving it here still if that changes, to be sure we don't have an unexpected crash if that changes.
func (ts *FileDatasourceTestSuite) TestWithTempDirNotWritable() {
pd := ts.MakePluginDir()
defer pd.Cleanup()
tempName := "fake_temp"
err := os.Mkdir(tempName, 0555)
if err != nil {
ts.T().Fatalf("failed to create temporary tmpdir %q: %s", tempName, err)
}
defer os.RemoveAll(tempName)
ts.PackerCommand().UsePluginDir(pd).
SetArgs("build", "./templates/file_with_contents.pkr.hcl").
AddEnv("TMPDIR", tempName).
Assert(check.MustFail())
}
// TestWithDestDirNotWritable checks that the datasource fails if the destination directory is not writable, and a destination is provided.
func (ts *FileDatasourceTestSuite) TestWithDestDirNotWritable() {
pd := ts.MakePluginDir()
defer pd.Cleanup()
err := os.MkdirAll("out_dir", 0555)
if err != nil {
ts.T().Fatalf("failed to create output directory: %s", err)
}
defer func() {
err := os.RemoveAll("out_dir")
if err != nil {
ts.T().Logf("failed to remove out_dir: %s", err)
}
}()
ts.PackerCommand().UsePluginDir(pd).
SetArgs("build", "./templates/local_destination.pkr.hcl").
Assert(check.MustFail())
}

@ -0,0 +1,23 @@
package main
import (
"testing"
"github.com/hashicorp/packer/packer_test/common"
"github.com/stretchr/testify/suite"
)
type FileDatasourceTestSuite struct {
*common.PackerTestSuite
}
func Test_FileDatasourceTestSuite(t *testing.T) {
baseSuite, cleanup := common.InitBaseSuite(t)
defer cleanup()
ts := &FileDatasourceTestSuite{
baseSuite,
}
suite.Run(t, ts)
}

@ -0,0 +1,10 @@
data "file" "test" {
}
source "null" "test" {
communicator = "none"
}
build {
sources = ["null.test"]
}

@ -0,0 +1,11 @@
data "file" "test" {
contents = "Hello there!"
}
source "null" "test" {
communicator = "none"
}
build {
sources = ["null.test"]
}

@ -0,0 +1,12 @@
data "file" "test" {
contents = "Hello there!"
destination = "./out_dir/subdir/out.txt"
}
source "null" "test" {
communicator = "none"
}
build {
sources = ["null.test"]
}

@ -0,0 +1,12 @@
data "file" "test" {
contents = "Hello there!"
destination = "./out_dir/subdir/"
}
source "null" "test" {
communicator = "none"
}
build {
sources = ["null.test"]
}

@ -0,0 +1,45 @@
---
description: |
The File Data Source writes contents to a file so it can be used
later during Packer builds
page_title: File - Data Sources
---
<BadgesHeader>
<PluginBadge type="official" />
</BadgesHeader>
# File Data Source
Type: `file`
The `file` data source writes the specified contents to a file, creating it in the process if it doesn't exist.
This is particularly useful if you have a file to dynamically generate (with [`templatefile`](/packer/docs/templates/hcl_templates/functions/file/templatefile) for example), and the component you
rely on only accepts files, and not the generated string.
Using this data source, you can use those functions and have the output written to a temporary file that you can
reference in a component afterwards.
Note: being a datasource, the created file is not wiped-out after the build finishes. By default Packer will output
the file into the system's `TEMPDIR` (typically `/tmp` on UNIX systems, or `` on Windows).
You can also change this by specifying a `destination` for the data source.
## Basic Example
```hcl
data "file" "example" {
contents = "this is an example"
}
```
## Configuration Reference
### Not Required:
@include 'datasource/file/Config-not-required.mdx'
## Datasource outputs
The outputs for this datasource are as follows:
@include 'datasource/file/DatasourceOutput.mdx'

@ -0,0 +1,13 @@
<!-- Code generated from the comments of the Config struct in datasource/file/data.go; DO NOT EDIT MANUALLY -->
- `contents` (string) - The contents of the file to create
This is useful especially for files that involve templating so that
Packer can dynamically create files and expose them for later importing
as attributes in another entity.
If no contents are specified, the resulting file will be empty.
- `destination` (string) - The file or directory to write the contents to.
<!-- End of code generated from the comments of the Config struct in datasource/file/data.go; -->

@ -0,0 +1,5 @@
<!-- Code generated from the comments of the DatasourceOutput struct in datasource/file/data.go; DO NOT EDIT MANUALLY -->
- `path` (string) - The path of the file created
<!-- End of code generated from the comments of the DatasourceOutput struct in datasource/file/data.go; -->

@ -762,6 +762,10 @@
{
"title": "HTTP",
"path": "datasources/http"
},
{
"title": "File",
"path": "datasources/file"
}
]
},

Loading…
Cancel
Save