lang: Experimental "ephemeralasnull" function

This is another part of the existing ephemeral_values experiment, taking
a value of any type that might have ephemeral values in it and returning
a value of the same type which has any ephemeral value replaced with a
null value.

The primary purpose of this is to allow a module to conveniently return an
object that would normally contain nested ephemeral values -- such as an
instance of a managed resource type that has a write-only attribute --
through an output value that isn't declared as ephemeral. This would then
expose all of the non-ephemeral parts of the object but withhold the
ephemeral parts. In the case of write-only attributes, it exposes the
normal attributes while withholding the write-only ones.

The name of this function could potentially change before stabilization,
because it's quite long and clunky. I did originally consider
"nonephemeral" to match with the existing "nonsensitive", but that didn't
feel right because "nonsensitive" removes the sensitive mark while
preserving the underlying value while this function removes the mark and
the real value at the same time. (It would not be appropriate to have a
function that just removes the ephemeral mark while preserving the value,
because correct handling of ephemerality is important for correctness
while sensitivity is primarily a UI concern so we don't need to be quite
so picky about it.)

internal/command/metadata_functions.go

@@ -12,7 +12,7 @@ import (
 )
 
 var (
-	ignoredFunctions = []string{"map", "list", "core::map", "core::list"}
+	ignoredFunctions = []string{"map", "list", "core::map", "core::list", "ephemeralasnull", "core::ephemeralasnull"}
 )
 
 // MetadataFunctionsCommand is a Command implementation that prints out information

internal/lang/funcs/conversion.go

@@ -99,6 +99,60 @@ func MakeToFunc(wantTy cty.Type) function.Function {
 	})
 }
 
+// EphemeralAsNullFunc is a cty function that takes a value of any type and
+// returns a similar value with any ephemeral-marked values anywhere in the
+// structure replaced with a null value of the same type that is not marked
+// as ephemeral.
+//
+// This is intended as a convenience for returning the non-ephemeral parts of
+// a partially-ephemeral data structure through an output value that isn't
+// ephemeral itself.
+var EphemeralAsNullFunc = function.New(&function.Spec{
+	Params: []function.Parameter{
+		{
+			Name:             "value",
+			Type:             cty.DynamicPseudoType,
+			AllowDynamicType: true,
+			AllowUnknown:     true,
+			AllowNull:        true,
+			AllowMarked:      true,
+		},
+	},
+	Type: func(args []cty.Value) (cty.Type, error) {
+		// This function always preserves the type of the given argument.
+		return args[0].Type(), nil
+	},
+	Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
+		return cty.Transform(args[0], func(p cty.Path, v cty.Value) (cty.Value, error) {
+			_, givenMarks := v.Unmark()
+			if _, isEphemeral := givenMarks[marks.Ephemeral]; isEphemeral {
+				// We'll strip the ephemeral mark but retain any other marks
+				// that might be present on the input.
+				delete(givenMarks, marks.Ephemeral)
+				if !v.IsKnown() {
+					// If the source value is unknown then we must leave it
+					// unknown because its final type might be more precise
+					// than the associated type constraint and returning a
+					// typed null could therefore over-promise on what the
+					// final result type will be.
+					// We're deliberately constructing a fresh unknown value
+					// here, rather than returning the one we were given,
+					// because we need to discard any refinements that the
+					// unknown value might be carrying that definitely won't
+					// be honored when we force the final result to be null.
+					return cty.UnknownVal(v.Type()).WithMarks(givenMarks), nil
+				}
+				return cty.NullVal(v.Type()).WithMarks(givenMarks), nil
+			}
+			return v, nil
+		})
+	},
+})
+
+func EphemeralAsNull(input cty.Value) (cty.Value, error) {
+	return EphemeralAsNullFunc.Call([]cty.Value{input})
+}
+
 // TypeFunc returns an encapsulated value containing its argument's type. This
 // value is marked to allow us to limit the use of this function at the moment
 // to only a few supported use cases.

internal/lang/funcs/conversion_test.go

@@ -7,8 +7,11 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/hashicorp/terraform/internal/lang/marks"
+	"github.com/google/go-cmp/cmp"
+	"github.com/zclconf/go-cty-debug/ctydebug"
 	"github.com/zclconf/go-cty/cty"
+
+	"github.com/hashicorp/terraform/internal/lang/marks"
 )
 
 func TestTo(t *testing.T) {
@@ -203,3 +206,144 @@ func TestTo(t *testing.T) {
 		})
 	}
 }
+
+func TestEphemeralAsNull(t *testing.T) {
+	tests := []struct {
+		Input cty.Value
+		Want  cty.Value
+	}{
+		// Simple cases
+		{
+			cty.StringVal("127.0.0.1:12654").Mark(marks.Ephemeral),
+			cty.NullVal(cty.String),
+		},
+		{
+			cty.StringVal("hello"),
+			cty.StringVal("hello"),
+		},
+		{
+			// Unknown values stay unknown because an unknown value with
+			// an imprecise type constraint is allowed to take on a more
+			// precise type in later phases, but known values (even if null)
+			// should not. We do know that the final known result definitely
+			// won't be ephemeral, though.
+			cty.UnknownVal(cty.String).Mark(marks.Ephemeral),
+			cty.UnknownVal(cty.String),
+		},
+		{
+			cty.UnknownVal(cty.String),
+			cty.UnknownVal(cty.String),
+		},
+		{
+			// Unknown value refinements should be discarded when unmarking,
+			// both because we know our final value is going to be null
+			// anyway and because an ephemeral value is not required to
+			// have consistent refinements between the plan and apply phases.
+			cty.UnknownVal(cty.String).RefineNotNull().Mark(marks.Ephemeral),
+			cty.UnknownVal(cty.String),
+		},
+		{
+			// Refinements must be preserved for non-ephemeral values, though.
+			cty.UnknownVal(cty.String).RefineNotNull(),
+			cty.UnknownVal(cty.String).RefineNotNull(),
+		},
+
+		// Should preserve other marks in all cases
+		{
+			cty.StringVal("127.0.0.1:12654").Mark(marks.Ephemeral).Mark(marks.Sensitive),
+			cty.NullVal(cty.String).Mark(marks.Sensitive),
+		},
+		{
+			cty.StringVal("hello").Mark(marks.Sensitive),
+			cty.StringVal("hello").Mark(marks.Sensitive),
+		},
+		{
+			cty.UnknownVal(cty.String).Mark(marks.Ephemeral).Mark(marks.Sensitive),
+			cty.UnknownVal(cty.String).Mark(marks.Sensitive),
+		},
+		{
+			cty.UnknownVal(cty.String).Mark(marks.Sensitive),
+			cty.UnknownVal(cty.String).Mark(marks.Sensitive),
+		},
+		{
+			cty.UnknownVal(cty.String).RefineNotNull().Mark(marks.Ephemeral).Mark(marks.Sensitive),
+			cty.UnknownVal(cty.String).Mark(marks.Sensitive),
+		},
+		{
+			cty.UnknownVal(cty.String).RefineNotNull().Mark(marks.Sensitive),
+			cty.UnknownVal(cty.String).RefineNotNull().Mark(marks.Sensitive),
+		},
+
+		// Nested ephemeral values
+		{
+			cty.ListVal([]cty.Value{
+				cty.StringVal("127.0.0.1:12654").Mark(marks.Ephemeral),
+				cty.StringVal("hello"),
+			}),
+			cty.ListVal([]cty.Value{
+				cty.NullVal(cty.String),
+				cty.StringVal("hello"),
+			}),
+		},
+		{
+			cty.TupleVal([]cty.Value{
+				cty.True,
+				cty.StringVal("127.0.0.1:12654").Mark(marks.Ephemeral),
+				cty.StringVal("hello"),
+			}),
+			cty.TupleVal([]cty.Value{
+				cty.True,
+				cty.NullVal(cty.String),
+				cty.StringVal("hello"),
+			}),
+		},
+		{
+			// Sets can't actually preserve individual element marks, so
+			// this gets treated as the entire set being ephemeral.
+			// (That's true of the input value, despite how it's written here,
+			// not just the result value; cty.SetVal does the simplification
+			// itself during the construction of the value.)
+			cty.SetVal([]cty.Value{
+				cty.StringVal("127.0.0.1:12654").Mark(marks.Ephemeral),
+				cty.StringVal("hello"),
+			}),
+			cty.NullVal(cty.Set(cty.String)),
+		},
+		{
+			cty.MapVal(map[string]cty.Value{
+				"addr":  cty.StringVal("127.0.0.1:12654").Mark(marks.Ephemeral),
+				"greet": cty.StringVal("hello"),
+			}),
+			cty.MapVal(map[string]cty.Value{
+				"addr":  cty.NullVal(cty.String),
+				"greet": cty.StringVal("hello"),
+			}),
+		},
+		{
+			cty.ObjectVal(map[string]cty.Value{
+				"addr":  cty.StringVal("127.0.0.1:12654").Mark(marks.Ephemeral),
+				"greet": cty.StringVal("hello"),
+				"happy": cty.True,
+			}),
+			cty.ObjectVal(map[string]cty.Value{
+				"addr":  cty.NullVal(cty.String),
+				"greet": cty.StringVal("hello"),
+				"happy": cty.True,
+			}),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Input.GoString(), func(t *testing.T) {
+			got, err := EphemeralAsNull(test.Input)
+			if err != nil {
+				// This function is supposed to be infallible
+				t.Fatal(err)
+			}
+
+			if diff := cmp.Diff(test.Want, got, ctydebug.CmpOptions); diff != "" {
+				t.Errorf("wrong result\n%s", diff)
+			}
+		})
+	}
+}

internal/lang/funcs/descriptions.go

@@ -162,6 +162,10 @@ var DescriptionList = map[string]descriptionEntry{
 		Description:      "`endswith` takes two values: a string to check and a suffix string. The function returns true if the first string ends with that exact suffix.",
 		ParamDescription: []string{"", ""},
 	},
+	"ephemeralasnull": {
+		Description:      "`ephemeralasnull` takes a value of any type and returns a similar value of the same type with any ephemeral values replaced with non-ephemeral null values and all non-ephemeral values preserved.",
+		ParamDescription: []string{""},
+	},
 	"file": {
 		Description:      "`file` reads the contents of a file at the given path and returns them as a string.",
 		ParamDescription: []string{""},

internal/lang/functions.go

@@ -89,6 +89,7 @@ func (s *Scope) Functions() map[string]function.Function {
 			"distinct":         stdlib.DistinctFunc,
 			"element":          stdlib.ElementFunc,
 			"endswith":         funcs.EndsWithFunc,
+			"ephemeralasnull":  s.experimentalFunction(experiments.EphemeralValues, funcs.EphemeralAsNullFunc),
 			"chunklist":        stdlib.ChunklistFunc,
 			"file":             funcs.MakeFileFunc(s.BaseDir, false),
 			"fileexists":       funcs.MakeFileExistsFunc(s.BaseDir),

internal/lang/functions_test.go

@@ -363,6 +363,17 @@ func TestFunctions(t *testing.T) {
 			},
 		},
 
+		"ephemeralasnull": {
+			// We can't actually test the main behavior of this one here
+			// because we don't have any ephemeral values in scope, so
+			// this is just to check that the function is registered. The
+			// real tests for this function are in package funcs.
+			{
+				`ephemeralasnull("not ephemeral")`,
+				cty.StringVal("not ephemeral"),
+			},
+		},
+
 		"file": {
 			{
 				`file("hello.txt")`,
@@ -1231,7 +1242,7 @@ func TestFunctions(t *testing.T) {
 	}
 
 	experimentalFuncs := map[string]experiments.Experiment{}
-	experimentalFuncs["defaults"] = experiments.ModuleVariableOptionalAttrs
+	experimentalFuncs["ephemeralasnull"] = experiments.EphemeralValues
 
 	// We'll also register a few "external functions" so that we can
 	// verify that registering these works. The functions actually