From 71d14e78fd8001c0415fef1f2ae28528db2e83dd Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Wed, 19 Jun 2024 14:37:51 -0700 Subject: [PATCH] lang: Experimental "ephemeralasnull" function This is another part of the existing ephemeral_values experiment, taking a value of any type that might have ephemeral values in it and returning a value of the same type which has any ephemeral value replaced with a null value. The primary purpose of this is to allow a module to conveniently return an object that would normally contain nested ephemeral values -- such as an instance of a managed resource type that has a write-only attribute -- through an output value that isn't declared as ephemeral. This would then expose all of the non-ephemeral parts of the object but withhold the ephemeral parts. In the case of write-only attributes, it exposes the normal attributes while withholding the write-only ones. The name of this function could potentially change before stabilization, because it's quite long and clunky. I did originally consider "nonephemeral" to match with the existing "nonsensitive", but that didn't feel right because "nonsensitive" removes the sensitive mark while preserving the underlying value while this function removes the mark and the real value at the same time. (It would not be appropriate to have a function that just removes the ephemeral mark while preserving the value, because correct handling of ephemerality is important for correctness while sensitivity is primarily a UI concern so we don't need to be quite so picky about it.) --- internal/command/metadata_functions.go | 2 +- internal/lang/funcs/conversion.go | 54 +++++++++ internal/lang/funcs/conversion_test.go | 146 ++++++++++++++++++++++++- internal/lang/funcs/descriptions.go | 4 + internal/lang/functions.go | 1 + internal/lang/functions_test.go | 13 ++- 6 files changed, 217 insertions(+), 3 deletions(-) diff --git a/internal/command/metadata_functions.go b/internal/command/metadata_functions.go index 0b5ba14f88..ca8367326f 100644 --- a/internal/command/metadata_functions.go +++ b/internal/command/metadata_functions.go @@ -12,7 +12,7 @@ import ( ) var ( - ignoredFunctions = []string{"map", "list", "core::map", "core::list"} + ignoredFunctions = []string{"map", "list", "core::map", "core::list", "ephemeralasnull", "core::ephemeralasnull"} ) // MetadataFunctionsCommand is a Command implementation that prints out information diff --git a/internal/lang/funcs/conversion.go b/internal/lang/funcs/conversion.go index fc858c56ba..6e223089b4 100644 --- a/internal/lang/funcs/conversion.go +++ b/internal/lang/funcs/conversion.go @@ -99,6 +99,60 @@ func MakeToFunc(wantTy cty.Type) function.Function { }) } +// EphemeralAsNullFunc is a cty function that takes a value of any type and +// returns a similar value with any ephemeral-marked values anywhere in the +// structure replaced with a null value of the same type that is not marked +// as ephemeral. +// +// This is intended as a convenience for returning the non-ephemeral parts of +// a partially-ephemeral data structure through an output value that isn't +// ephemeral itself. +var EphemeralAsNullFunc = function.New(&function.Spec{ + Params: []function.Parameter{ + { + Name: "value", + Type: cty.DynamicPseudoType, + AllowDynamicType: true, + AllowUnknown: true, + AllowNull: true, + AllowMarked: true, + }, + }, + Type: func(args []cty.Value) (cty.Type, error) { + // This function always preserves the type of the given argument. + return args[0].Type(), nil + }, + Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { + return cty.Transform(args[0], func(p cty.Path, v cty.Value) (cty.Value, error) { + _, givenMarks := v.Unmark() + if _, isEphemeral := givenMarks[marks.Ephemeral]; isEphemeral { + // We'll strip the ephemeral mark but retain any other marks + // that might be present on the input. + delete(givenMarks, marks.Ephemeral) + if !v.IsKnown() { + // If the source value is unknown then we must leave it + // unknown because its final type might be more precise + // than the associated type constraint and returning a + // typed null could therefore over-promise on what the + // final result type will be. + // We're deliberately constructing a fresh unknown value + // here, rather than returning the one we were given, + // because we need to discard any refinements that the + // unknown value might be carrying that definitely won't + // be honored when we force the final result to be null. + return cty.UnknownVal(v.Type()).WithMarks(givenMarks), nil + } + return cty.NullVal(v.Type()).WithMarks(givenMarks), nil + } + return v, nil + }) + }, +}) + +func EphemeralAsNull(input cty.Value) (cty.Value, error) { + return EphemeralAsNullFunc.Call([]cty.Value{input}) +} + // TypeFunc returns an encapsulated value containing its argument's type. This // value is marked to allow us to limit the use of this function at the moment // to only a few supported use cases. diff --git a/internal/lang/funcs/conversion_test.go b/internal/lang/funcs/conversion_test.go index c8929da098..79ea88404a 100644 --- a/internal/lang/funcs/conversion_test.go +++ b/internal/lang/funcs/conversion_test.go @@ -7,8 +7,11 @@ import ( "fmt" "testing" - "github.com/hashicorp/terraform/internal/lang/marks" + "github.com/google/go-cmp/cmp" + "github.com/zclconf/go-cty-debug/ctydebug" "github.com/zclconf/go-cty/cty" + + "github.com/hashicorp/terraform/internal/lang/marks" ) func TestTo(t *testing.T) { @@ -203,3 +206,144 @@ func TestTo(t *testing.T) { }) } } + +func TestEphemeralAsNull(t *testing.T) { + tests := []struct { + Input cty.Value + Want cty.Value + }{ + // Simple cases + { + cty.StringVal("127.0.0.1:12654").Mark(marks.Ephemeral), + cty.NullVal(cty.String), + }, + { + cty.StringVal("hello"), + cty.StringVal("hello"), + }, + { + // Unknown values stay unknown because an unknown value with + // an imprecise type constraint is allowed to take on a more + // precise type in later phases, but known values (even if null) + // should not. We do know that the final known result definitely + // won't be ephemeral, though. + cty.UnknownVal(cty.String).Mark(marks.Ephemeral), + cty.UnknownVal(cty.String), + }, + { + cty.UnknownVal(cty.String), + cty.UnknownVal(cty.String), + }, + { + // Unknown value refinements should be discarded when unmarking, + // both because we know our final value is going to be null + // anyway and because an ephemeral value is not required to + // have consistent refinements between the plan and apply phases. + cty.UnknownVal(cty.String).RefineNotNull().Mark(marks.Ephemeral), + cty.UnknownVal(cty.String), + }, + { + // Refinements must be preserved for non-ephemeral values, though. + cty.UnknownVal(cty.String).RefineNotNull(), + cty.UnknownVal(cty.String).RefineNotNull(), + }, + + // Should preserve other marks in all cases + { + cty.StringVal("127.0.0.1:12654").Mark(marks.Ephemeral).Mark(marks.Sensitive), + cty.NullVal(cty.String).Mark(marks.Sensitive), + }, + { + cty.StringVal("hello").Mark(marks.Sensitive), + cty.StringVal("hello").Mark(marks.Sensitive), + }, + { + cty.UnknownVal(cty.String).Mark(marks.Ephemeral).Mark(marks.Sensitive), + cty.UnknownVal(cty.String).Mark(marks.Sensitive), + }, + { + cty.UnknownVal(cty.String).Mark(marks.Sensitive), + cty.UnknownVal(cty.String).Mark(marks.Sensitive), + }, + { + cty.UnknownVal(cty.String).RefineNotNull().Mark(marks.Ephemeral).Mark(marks.Sensitive), + cty.UnknownVal(cty.String).Mark(marks.Sensitive), + }, + { + cty.UnknownVal(cty.String).RefineNotNull().Mark(marks.Sensitive), + cty.UnknownVal(cty.String).RefineNotNull().Mark(marks.Sensitive), + }, + + // Nested ephemeral values + { + cty.ListVal([]cty.Value{ + cty.StringVal("127.0.0.1:12654").Mark(marks.Ephemeral), + cty.StringVal("hello"), + }), + cty.ListVal([]cty.Value{ + cty.NullVal(cty.String), + cty.StringVal("hello"), + }), + }, + { + cty.TupleVal([]cty.Value{ + cty.True, + cty.StringVal("127.0.0.1:12654").Mark(marks.Ephemeral), + cty.StringVal("hello"), + }), + cty.TupleVal([]cty.Value{ + cty.True, + cty.NullVal(cty.String), + cty.StringVal("hello"), + }), + }, + { + // Sets can't actually preserve individual element marks, so + // this gets treated as the entire set being ephemeral. + // (That's true of the input value, despite how it's written here, + // not just the result value; cty.SetVal does the simplification + // itself during the construction of the value.) + cty.SetVal([]cty.Value{ + cty.StringVal("127.0.0.1:12654").Mark(marks.Ephemeral), + cty.StringVal("hello"), + }), + cty.NullVal(cty.Set(cty.String)), + }, + { + cty.MapVal(map[string]cty.Value{ + "addr": cty.StringVal("127.0.0.1:12654").Mark(marks.Ephemeral), + "greet": cty.StringVal("hello"), + }), + cty.MapVal(map[string]cty.Value{ + "addr": cty.NullVal(cty.String), + "greet": cty.StringVal("hello"), + }), + }, + { + cty.ObjectVal(map[string]cty.Value{ + "addr": cty.StringVal("127.0.0.1:12654").Mark(marks.Ephemeral), + "greet": cty.StringVal("hello"), + "happy": cty.True, + }), + cty.ObjectVal(map[string]cty.Value{ + "addr": cty.NullVal(cty.String), + "greet": cty.StringVal("hello"), + "happy": cty.True, + }), + }, + } + + for _, test := range tests { + t.Run(test.Input.GoString(), func(t *testing.T) { + got, err := EphemeralAsNull(test.Input) + if err != nil { + // This function is supposed to be infallible + t.Fatal(err) + } + + if diff := cmp.Diff(test.Want, got, ctydebug.CmpOptions); diff != "" { + t.Errorf("wrong result\n%s", diff) + } + }) + } +} diff --git a/internal/lang/funcs/descriptions.go b/internal/lang/funcs/descriptions.go index e21dad10ad..86b70ccb2a 100644 --- a/internal/lang/funcs/descriptions.go +++ b/internal/lang/funcs/descriptions.go @@ -162,6 +162,10 @@ var DescriptionList = map[string]descriptionEntry{ Description: "`endswith` takes two values: a string to check and a suffix string. The function returns true if the first string ends with that exact suffix.", ParamDescription: []string{"", ""}, }, + "ephemeralasnull": { + Description: "`ephemeralasnull` takes a value of any type and returns a similar value of the same type with any ephemeral values replaced with non-ephemeral null values and all non-ephemeral values preserved.", + ParamDescription: []string{""}, + }, "file": { Description: "`file` reads the contents of a file at the given path and returns them as a string.", ParamDescription: []string{""}, diff --git a/internal/lang/functions.go b/internal/lang/functions.go index 0ae17d0836..ba3d6cb406 100644 --- a/internal/lang/functions.go +++ b/internal/lang/functions.go @@ -89,6 +89,7 @@ func (s *Scope) Functions() map[string]function.Function { "distinct": stdlib.DistinctFunc, "element": stdlib.ElementFunc, "endswith": funcs.EndsWithFunc, + "ephemeralasnull": s.experimentalFunction(experiments.EphemeralValues, funcs.EphemeralAsNullFunc), "chunklist": stdlib.ChunklistFunc, "file": funcs.MakeFileFunc(s.BaseDir, false), "fileexists": funcs.MakeFileExistsFunc(s.BaseDir), diff --git a/internal/lang/functions_test.go b/internal/lang/functions_test.go index d6ebdaafec..65c7245141 100644 --- a/internal/lang/functions_test.go +++ b/internal/lang/functions_test.go @@ -363,6 +363,17 @@ func TestFunctions(t *testing.T) { }, }, + "ephemeralasnull": { + // We can't actually test the main behavior of this one here + // because we don't have any ephemeral values in scope, so + // this is just to check that the function is registered. The + // real tests for this function are in package funcs. + { + `ephemeralasnull("not ephemeral")`, + cty.StringVal("not ephemeral"), + }, + }, + "file": { { `file("hello.txt")`, @@ -1231,7 +1242,7 @@ func TestFunctions(t *testing.T) { } experimentalFuncs := map[string]experiments.Experiment{} - experimentalFuncs["defaults"] = experiments.ModuleVariableOptionalAttrs + experimentalFuncs["ephemeralasnull"] = experiments.EphemeralValues // We'll also register a few "external functions" so that we can // verify that registering these works. The functions actually