From 47bd0e31ede43ffc279fe205baf8698a0d8da9f3 Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 13 Feb 2019 15:50:15 -0800 Subject: [PATCH] config: Backport file hashing functions from 0.12 As an aid to provider developers wanting to write acceptance tests that can apply to both Terraform 0.12 and 0.11 at once, here we backport the new file-based hashing functions that were added in Terraform 0.12. In Terraform 0.11 and earlier, immediately passing the result of file(..) into a hash function was safe because HIL tolerated non-UTF8 strings, but other uses of non-UTF8 data from file(..) would often lead to corruption and so this function requires UTF-8 validity as of Terraform 0.12. The new file-based hashing functions allow hashing of raw binary data from files without first buffering it in memory as a string. We are backporting these only so that provider acceptance tests can be written to work in both Terraform 0.11 and 0.12 at the same time. Users may also find these functions useful if they too are trying to write a module that should work in both 0.11 and 0.12. --- config/interpolate_funcs.go | 151 +++++++++++------- config/interpolate_funcs_test.go | 46 ++++++ .../docs/configuration/interpolation.html.md | 9 ++ 3 files changed, 144 insertions(+), 62 deletions(-) diff --git a/config/interpolate_funcs.go b/config/interpolate_funcs.go index b94fca8896..421edb041d 100644 --- a/config/interpolate_funcs.go +++ b/config/interpolate_funcs.go @@ -61,68 +61,74 @@ func listVariableValueToStringSlice(values []ast.Variable) ([]string, error) { // Funcs is the mapping of built-in functions for configuration. func Funcs() map[string]ast.Function { return map[string]ast.Function{ - "abs": interpolationFuncAbs(), - "basename": interpolationFuncBasename(), - "base64decode": interpolationFuncBase64Decode(), - "base64encode": interpolationFuncBase64Encode(), - "base64gzip": interpolationFuncBase64Gzip(), - "base64sha256": interpolationFuncBase64Sha256(), - "base64sha512": interpolationFuncBase64Sha512(), - "bcrypt": interpolationFuncBcrypt(), - "ceil": interpolationFuncCeil(), - "chomp": interpolationFuncChomp(), - "cidrhost": interpolationFuncCidrHost(), - "cidrnetmask": interpolationFuncCidrNetmask(), - "cidrsubnet": interpolationFuncCidrSubnet(), - "coalesce": interpolationFuncCoalesce(), - "coalescelist": interpolationFuncCoalesceList(), - "compact": interpolationFuncCompact(), - "concat": interpolationFuncConcat(), - "contains": interpolationFuncContains(), - "dirname": interpolationFuncDirname(), - "distinct": interpolationFuncDistinct(), - "element": interpolationFuncElement(), - "chunklist": interpolationFuncChunklist(), - "file": interpolationFuncFile(), - "matchkeys": interpolationFuncMatchKeys(), - "flatten": interpolationFuncFlatten(), - "floor": interpolationFuncFloor(), - "format": interpolationFuncFormat(), - "formatlist": interpolationFuncFormatList(), - "indent": interpolationFuncIndent(), - "index": interpolationFuncIndex(), - "join": interpolationFuncJoin(), - "jsonencode": interpolationFuncJSONEncode(), - "length": interpolationFuncLength(), - "list": interpolationFuncList(), - "log": interpolationFuncLog(), - "lower": interpolationFuncLower(), - "map": interpolationFuncMap(), - "max": interpolationFuncMax(), - "md5": interpolationFuncMd5(), - "merge": interpolationFuncMerge(), - "min": interpolationFuncMin(), - "pathexpand": interpolationFuncPathExpand(), - "pow": interpolationFuncPow(), - "uuid": interpolationFuncUUID(), - "replace": interpolationFuncReplace(), - "rsadecrypt": interpolationFuncRsaDecrypt(), - "sha1": interpolationFuncSha1(), - "sha256": interpolationFuncSha256(), - "sha512": interpolationFuncSha512(), - "signum": interpolationFuncSignum(), - "slice": interpolationFuncSlice(), - "sort": interpolationFuncSort(), - "split": interpolationFuncSplit(), - "substr": interpolationFuncSubstr(), - "timestamp": interpolationFuncTimestamp(), - "timeadd": interpolationFuncTimeAdd(), - "title": interpolationFuncTitle(), - "transpose": interpolationFuncTranspose(), - "trimspace": interpolationFuncTrimSpace(), - "upper": interpolationFuncUpper(), - "urlencode": interpolationFuncURLEncode(), - "zipmap": interpolationFuncZipMap(), + "abs": interpolationFuncAbs(), + "basename": interpolationFuncBasename(), + "base64decode": interpolationFuncBase64Decode(), + "base64encode": interpolationFuncBase64Encode(), + "base64gzip": interpolationFuncBase64Gzip(), + "base64sha256": interpolationFuncBase64Sha256(), + "base64sha512": interpolationFuncBase64Sha512(), + "bcrypt": interpolationFuncBcrypt(), + "ceil": interpolationFuncCeil(), + "chomp": interpolationFuncChomp(), + "cidrhost": interpolationFuncCidrHost(), + "cidrnetmask": interpolationFuncCidrNetmask(), + "cidrsubnet": interpolationFuncCidrSubnet(), + "coalesce": interpolationFuncCoalesce(), + "coalescelist": interpolationFuncCoalesceList(), + "compact": interpolationFuncCompact(), + "concat": interpolationFuncConcat(), + "contains": interpolationFuncContains(), + "dirname": interpolationFuncDirname(), + "distinct": interpolationFuncDistinct(), + "element": interpolationFuncElement(), + "chunklist": interpolationFuncChunklist(), + "file": interpolationFuncFile(), + "filebase64sha256": interpolationFuncMakeFileHash(interpolationFuncBase64Sha256()), + "filebase64sha512": interpolationFuncMakeFileHash(interpolationFuncBase64Sha512()), + "filemd5": interpolationFuncMakeFileHash(interpolationFuncMd5()), + "filesha1": interpolationFuncMakeFileHash(interpolationFuncSha1()), + "filesha256": interpolationFuncMakeFileHash(interpolationFuncSha256()), + "filesha512": interpolationFuncMakeFileHash(interpolationFuncSha512()), + "matchkeys": interpolationFuncMatchKeys(), + "flatten": interpolationFuncFlatten(), + "floor": interpolationFuncFloor(), + "format": interpolationFuncFormat(), + "formatlist": interpolationFuncFormatList(), + "indent": interpolationFuncIndent(), + "index": interpolationFuncIndex(), + "join": interpolationFuncJoin(), + "jsonencode": interpolationFuncJSONEncode(), + "length": interpolationFuncLength(), + "list": interpolationFuncList(), + "log": interpolationFuncLog(), + "lower": interpolationFuncLower(), + "map": interpolationFuncMap(), + "max": interpolationFuncMax(), + "md5": interpolationFuncMd5(), + "merge": interpolationFuncMerge(), + "min": interpolationFuncMin(), + "pathexpand": interpolationFuncPathExpand(), + "pow": interpolationFuncPow(), + "uuid": interpolationFuncUUID(), + "replace": interpolationFuncReplace(), + "rsadecrypt": interpolationFuncRsaDecrypt(), + "sha1": interpolationFuncSha1(), + "sha256": interpolationFuncSha256(), + "sha512": interpolationFuncSha512(), + "signum": interpolationFuncSignum(), + "slice": interpolationFuncSlice(), + "sort": interpolationFuncSort(), + "split": interpolationFuncSplit(), + "substr": interpolationFuncSubstr(), + "timestamp": interpolationFuncTimestamp(), + "timeadd": interpolationFuncTimeAdd(), + "title": interpolationFuncTitle(), + "transpose": interpolationFuncTranspose(), + "trimspace": interpolationFuncTrimSpace(), + "upper": interpolationFuncUpper(), + "urlencode": interpolationFuncURLEncode(), + "zipmap": interpolationFuncZipMap(), } } @@ -1725,3 +1731,24 @@ func interpolationFuncRsaDecrypt() ast.Function { }, } } + +// interpolationFuncMakeFileHash constructs a function that hashes the contents +// of a file by combining the implementations of the file(...) function and +// a given other function that is assumed to take a single string argument and +// return a hash value. +func interpolationFuncMakeFileHash(hashFunc ast.Function) ast.Function { + fileFunc := interpolationFuncFile() + + return ast.Function{ + ArgTypes: []ast.Type{ast.TypeString}, + ReturnType: ast.TypeString, + Callback: func(args []interface{}) (interface{}, error) { + filename := args[0].(string) + contents, err := fileFunc.Callback([]interface{}{filename}) + if err != nil { + return nil, err + } + return hashFunc.Callback([]interface{}{contents}) + }, + } +} diff --git a/config/interpolate_funcs_test.go b/config/interpolate_funcs_test.go index f63d81c30b..90b913dbd7 100644 --- a/config/interpolate_funcs_test.go +++ b/config/interpolate_funcs_test.go @@ -1223,6 +1223,52 @@ func TestInterpolateFuncFile(t *testing.T) { }) } +func TestInterpolateFuncFileHashFuncs(t *testing.T) { + tf, err := ioutil.TempFile("", "tf") + if err != nil { + t.Fatalf("err: %s", err) + } + path := tf.Name() + tf.Write([]byte("foo")) + tf.Close() + defer os.Remove(path) + + testFunction(t, testFunctionConfig{ + Cases: []testFunctionCase{ + { + fmt.Sprintf(`${filebase64sha256("%s")}`, path), + "LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564=", + false, + }, + { + fmt.Sprintf(`${filebase64sha512("%s")}`, path), + "9/u6bgY2+JDlb7vzKD5STG+jIErimDgtYkdB0NxmODJuKCxBvl5CVNiCB3LFUYosWowMf37aGVlKfrU5RT4e1w==", + false, + }, + { + fmt.Sprintf(`${filemd5("%s")}`, path), + "acbd18db4cc2f85cedef654fccc4a4d8", + false, + }, + { + fmt.Sprintf(`${filesha1("%s")}`, path), + "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", + false, + }, + { + fmt.Sprintf(`${filesha256("%s")}`, path), + "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + false, + }, + { + fmt.Sprintf(`${filesha512("%s")}`, path), + "f7fbba6e0636f890e56fbbf3283e524c6fa3204ae298382d624741d0dc6638326e282c41be5e4254d8820772c5518a2c5a8c0c7f7eda19594a7eb539453e1ed7", + false, + }, + }, + }) +} + func TestInterpolateFuncFormat(t *testing.T) { testFunction(t, testFunctionConfig{ Cases: []testFunctionCase{ diff --git a/website/docs/configuration/interpolation.html.md b/website/docs/configuration/interpolation.html.md index b4b4b1607c..3534b6b414 100644 --- a/website/docs/configuration/interpolation.html.md +++ b/website/docs/configuration/interpolation.html.md @@ -432,6 +432,15 @@ The supported built-in functions are: of the key used to encrypt their initial password, you might use: `zipmap(aws_iam_user.users.*.name, aws_iam_user_login_profile.users.*.key_fingerprint)`. +The hashing functions `base64sha256`, `base64sha512`, `md5`, `sha1`, `sha256`, +and `sha512` all have variants with a `file` prefix, like `filesha1`, which +interpret their first argument as a path to a file on disk rather than as a +literal string. This allows safely creating hashes of binary files that might +otherwise be corrupted in memory if loaded into Terraform strings (which are +assumed to be UTF-8). `filesha1(filename)` is equivalent to `sha1(file(filename))` +in Terraform 0.11 and earlier, but the latter will fail for binary files in +Terraform 0.12 and later. + ## Templates Long strings can be managed using templates.