move function results hashing to lang

We need to abstract the function results verification to use internally
too, so start by moving it out of the providers code.
pull/37001/head
James Bardin 1 year ago
parent 49e8b56b32
commit d016070564

@ -0,0 +1,129 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package lang
import (
"crypto/sha256"
"fmt"
"io"
"log"
"sync"
"github.com/hashicorp/terraform/internal/addrs"
"github.com/zclconf/go-cty/cty"
)
type priorResult struct {
hash [sha256.Size]byte
// when the result was from a current run, we keep a record of the result
// value to aid in debugging. Results stored in the plan will only have the
// hash to avoid bloating the plan with what could be many very large
// values.
value cty.Value
}
type FunctionResults struct {
mu sync.Mutex
// results stores the prior result from a function call, keyed by
// the hash of the function name and arguments.
results map[[sha256.Size]byte]priorResult
}
// NewFunctionResultsTable initializes a mapping of function calls to prior
// results used to validate function calls. The hashes argument is an
// optional slice of prior result hashes used to preload the cache.
func NewFunctionResultsTable(hashes []FunctionHash) *FunctionResults {
res := &FunctionResults{
results: make(map[[sha256.Size]byte]priorResult),
}
res.insertHashes(hashes)
return res
}
// CheckPrior compares the function call against any cached results, and returns
// an error if the result does not match a prior call. A zero-value provider
// address can be used for internal functions which need this validation.
func (f *FunctionResults) CheckPrior(provider addrs.Provider, name string, args []cty.Value, result cty.Value) error {
argSum := sha256.New()
if !provider.IsZero() {
io.WriteString(argSum, provider.String()+"|")
}
io.WriteString(argSum, name)
for _, arg := range args {
// cty.Values have a Hash method, but it is not collision resistant. We
// are going to rely on the GoString formatting instead, which gives
// detailed results for all values.
io.WriteString(argSum, "|"+arg.GoString())
}
f.mu.Lock()
defer f.mu.Unlock()
argHash := [sha256.Size]byte(argSum.Sum(nil))
resHash := sha256.Sum256([]byte(result.GoString()))
res, ok := f.results[argHash]
if !ok {
f.results[argHash] = priorResult{
hash: resHash,
value: result,
}
return nil
}
if resHash != res.hash {
provPrefix := ""
if !provider.IsZero() {
provPrefix = fmt.Sprintf("provider %s ", provider)
}
// Log the args for debugging in case the hcl context is
// insufficient. The error should be adequate most of the time, and
// could already be quite long, so we don't want to add all
// arguments too.
log.Printf("[ERROR] %sfunction %s returned an inconsistent result with args: %#v\n", provPrefix, name, args)
// The hcl package will add the necessary context around the error in
// the diagnostic, but we add the differing results when we can.
if res.value != cty.NilVal {
return fmt.Errorf("function returned an inconsistent result,\nwas: %#v,\nnow: %#v", res.value, result)
}
return fmt.Errorf("function returned an inconsistent result")
}
return nil
}
// insertHashes insert key-value pairs to the functionResults map. This is used
// to preload stored values before any Verify calls are made.
func (f *FunctionResults) insertHashes(hashes []FunctionHash) {
f.mu.Lock()
defer f.mu.Unlock()
for _, res := range hashes {
f.results[[sha256.Size]byte(res.Key)] = priorResult{
hash: [sha256.Size]byte(res.Result),
}
}
}
// FunctionHash contains the key and result hash values from a prior function
// call.
type FunctionHash struct {
Key []byte
Result []byte
}
// copy the hash values into a struct which can be recorded in the plan.
func (f *FunctionResults) GetHashes() []FunctionHash {
f.mu.Lock()
defer f.mu.Unlock()
var res []FunctionHash
for k, r := range f.results {
res = append(res, FunctionHash{Key: k[:], Result: r.hash[:]})
}
return res
}

@ -1,7 +1,7 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package providers
package lang
import (
"fmt"
@ -162,12 +162,12 @@ func TestFunctionCache(t *testing.T) {
for i, test := range tests {
t.Run(fmt.Sprint(i), func(t *testing.T) {
results := NewFunctionResultsTable(nil)
err := results.checkPrior(test.first.provider, test.first.name, test.first.args, test.first.result)
err := results.CheckPrior(test.first.provider, test.first.name, test.first.args, test.first.result)
if err != nil {
t.Fatal("error on first call!", err)
}
err = results.checkPrior(test.second.provider, test.second.name, test.second.args, test.second.result)
err = results.CheckPrior(test.second.provider, test.second.name, test.second.args, test.second.result)
if err != nil && !test.expectErr {
t.Fatal(err)
@ -177,7 +177,7 @@ func TestFunctionCache(t *testing.T) {
newResults := NewFunctionResultsTable(results.GetHashes())
originalErr := err != nil
reloadedErr := newResults.checkPrior(test.second.provider, test.second.name, test.second.args, test.second.result) != nil
reloadedErr := newResults.CheckPrior(test.second.provider, test.second.name, test.second.args, test.second.result) != nil
if originalErr != reloadedErr {
t.Fatalf("original check returned err:%t, reloaded check returned err:%t", originalErr, reloadedErr)

@ -12,9 +12,9 @@ import (
"github.com/hashicorp/terraform/internal/addrs"
"github.com/hashicorp/terraform/internal/collections"
"github.com/hashicorp/terraform/internal/configs/configschema"
"github.com/hashicorp/terraform/internal/lang"
"github.com/hashicorp/terraform/internal/lang/globalref"
"github.com/hashicorp/terraform/internal/moduletest/mocking"
"github.com/hashicorp/terraform/internal/providers"
"github.com/hashicorp/terraform/internal/states"
)
@ -162,7 +162,7 @@ type Plan struct {
// ProviderFunctionResults stores hashed results from all provider
// function calls, so that calls during apply can be checked for
// consistency.
ProviderFunctionResults []providers.FunctionHash
ProviderFunctionResults []lang.FunctionHash
}
// ProviderAddrs returns a list of all of the provider configuration addresses

@ -15,6 +15,7 @@ import (
"github.com/hashicorp/terraform/internal/addrs"
"github.com/hashicorp/terraform/internal/checks"
"github.com/hashicorp/terraform/internal/collections"
"github.com/hashicorp/terraform/internal/lang"
"github.com/hashicorp/terraform/internal/lang/globalref"
"github.com/hashicorp/terraform/internal/plans"
"github.com/hashicorp/terraform/internal/plans/planproto"
@ -171,7 +172,7 @@ func readTfplan(r io.Reader) (*plans.Plan, error) {
for _, hash := range rawPlan.ProviderFunctionResults {
plan.ProviderFunctionResults = append(plan.ProviderFunctionResults,
providers.FunctionHash{
lang.FunctionHash{
Key: hash.Key,
Result: hash.Result,
},

@ -4,17 +4,14 @@
package providers
import (
"crypto/sha256"
"fmt"
"io"
"log"
"sync"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/function"
"github.com/hashicorp/terraform/internal/addrs"
"github.com/hashicorp/terraform/internal/configs/configschema"
"github.com/hashicorp/terraform/internal/lang"
)
type FunctionDecl struct {
@ -59,7 +56,7 @@ type FunctionParam struct {
//
// The resTable argument is a shared instance of *FunctionResults, used to
// check the result values from each function call.
func (d FunctionDecl) BuildFunction(providerAddr addrs.Provider, name string, resTable *FunctionResults, factory func() (Interface, error)) function.Function {
func (d FunctionDecl) BuildFunction(providerAddr addrs.Provider, name string, resTable *lang.FunctionResults, factory func() (Interface, error)) function.Function {
var params []function.Parameter
var varParam *function.Parameter
@ -123,7 +120,7 @@ func (d FunctionDecl) BuildFunction(providerAddr addrs.Provider, name string, re
}
if resTable != nil {
err = resTable.checkPrior(providerAddr, name, args, resp.Result)
err = resTable.CheckPrior(providerAddr, name, args, resp.Result)
if err != nil {
return cty.UnknownVal(retType), err
}
@ -154,113 +151,3 @@ func (p *FunctionParam) ctyParameter() function.Parameter {
AllowUnknown: p.AllowUnknownValues,
}
}
type priorResult struct {
hash [sha256.Size]byte
// when the result was from a current run, we keep a record of the result
// value to aid in debugging. Results stored in the plan will only have the
// hash to avoid bloating the plan with what could be many very large
// values.
value cty.Value
}
type FunctionResults struct {
mu sync.Mutex
// results stores the prior result from a provider function call, keyed by
// the hash of the function name and arguments.
results map[[sha256.Size]byte]priorResult
}
// NewFunctionResultsTable initializes a mapping of function calls to prior
// results used to validate provider function calls. The hashes argument is an
// optional slice of prior result hashes used to preload the cache.
func NewFunctionResultsTable(hashes []FunctionHash) *FunctionResults {
res := &FunctionResults{
results: make(map[[sha256.Size]byte]priorResult),
}
res.insertHashes(hashes)
return res
}
// checkPrior compares the function call against any cached results, and
// returns an error if the result does not match a prior call.
func (f *FunctionResults) checkPrior(provider addrs.Provider, name string, args []cty.Value, result cty.Value) error {
argSum := sha256.New()
io.WriteString(argSum, provider.String())
io.WriteString(argSum, "|"+name)
for _, arg := range args {
// cty.Values have a Hash method, but it is not collision resistant. We
// are going to rely on the GoString formatting instead, which gives
// detailed results for all values.
io.WriteString(argSum, "|"+arg.GoString())
}
f.mu.Lock()
defer f.mu.Unlock()
argHash := [sha256.Size]byte(argSum.Sum(nil))
resHash := sha256.Sum256([]byte(result.GoString()))
res, ok := f.results[argHash]
if !ok {
f.results[argHash] = priorResult{
hash: resHash,
value: result,
}
return nil
}
if resHash != res.hash {
// Log the args for debugging in case the hcl context is
// insufficient. The error should be adequate most of the time, and
// could already be quite long, so we don't want to add all
// arguments too.
log.Printf("[ERROR] provider %s returned an inconsistent result for function %q with args: %#v\n", provider, name, args)
// The hcl package will add the necessary context around the error in
// the diagnostic, but we add the differing results when we can.
// TODO: maybe we should add a call to action, since this is a bug in
// the provider.
if res.value != cty.NilVal {
return fmt.Errorf("provider function returned an inconsistent result,\nwas: %#v,\nnow: %#v", res.value, result)
}
return fmt.Errorf("provider function returned an inconsistent result")
}
return nil
}
// insertHashes insert key-value pairs to the functionResults map. This is used
// to preload stored values before any Verify calls are made.
func (f *FunctionResults) insertHashes(hashes []FunctionHash) {
f.mu.Lock()
defer f.mu.Unlock()
for _, res := range hashes {
f.results[[sha256.Size]byte(res.Key)] = priorResult{
hash: [sha256.Size]byte(res.Result),
}
}
}
// FunctionHash contains the key and result hash values from a prior function
// call.
type FunctionHash struct {
Key []byte
Result []byte
}
// copy the hash values into a struct which can be recorded in the plan.
func (f *FunctionResults) GetHashes() []FunctionHash {
f.mu.Lock()
defer f.mu.Unlock()
var res []FunctionHash
for k, r := range f.results {
res = append(res, FunctionHash{Key: k[:], Result: r.hash[:]})
}
return res
}

@ -11,8 +11,8 @@ import (
"github.com/hashicorp/terraform/internal/addrs"
"github.com/hashicorp/terraform/internal/collections"
"github.com/hashicorp/terraform/internal/lang"
"github.com/hashicorp/terraform/internal/plans"
"github.com/hashicorp/terraform/internal/providers"
"github.com/hashicorp/terraform/internal/stacks/stackaddrs"
"github.com/hashicorp/terraform/internal/states"
)
@ -69,7 +69,7 @@ type Component struct {
// PlannedFunctionResults is a shared table of results from calling
// provider functions. This is stored and loaded from during the planning
// stage to use during apply operations.
PlannedFunctionResults []providers.FunctionHash
PlannedFunctionResults []lang.FunctionHash
// PlannedInputValues and PlannedInputValueMarks are the values that
// Terraform has planned to use for input variables in this component.

@ -13,11 +13,11 @@ import (
"github.com/hashicorp/terraform/internal/addrs"
"github.com/hashicorp/terraform/internal/collections"
"github.com/hashicorp/terraform/internal/lang"
"github.com/hashicorp/terraform/internal/lang/marks"
"github.com/hashicorp/terraform/internal/plans"
"github.com/hashicorp/terraform/internal/plans/planfile"
"github.com/hashicorp/terraform/internal/plans/planproto"
"github.com/hashicorp/terraform/internal/providers"
"github.com/hashicorp/terraform/internal/stacks/stackaddrs"
"github.com/hashicorp/terraform/internal/stacks/tfstackdata1"
"github.com/hashicorp/terraform/internal/states"
@ -135,7 +135,7 @@ func (l *Loader) AddRaw(rawMsg *anypb.Any) error {
case *tfstackdata1.ProviderFunctionResults:
for _, hash := range msg.ProviderFunctionResults {
l.ret.ProviderFunctionResults = append(l.ret.ProviderFunctionResults, providers.FunctionHash{
l.ret.ProviderFunctionResults = append(l.ret.ProviderFunctionResults, lang.FunctionHash{
Key: hash.Key,
Result: hash.Result,
})
@ -212,9 +212,9 @@ func (l *Loader) AddRaw(rawMsg *anypb.Any) error {
return fmt.Errorf("decoding check results: %w", err)
}
var functionResults []providers.FunctionHash
var functionResults []lang.FunctionHash
for _, hash := range msg.ProviderFunctionResults {
functionResults = append(functionResults, providers.FunctionHash{
functionResults = append(functionResults, lang.FunctionHash{
Key: hash.Key,
Result: hash.Result,
})

@ -12,8 +12,8 @@ import (
"github.com/hashicorp/terraform/internal/addrs"
"github.com/hashicorp/terraform/internal/collections"
"github.com/hashicorp/terraform/internal/lang"
"github.com/hashicorp/terraform/internal/plans"
"github.com/hashicorp/terraform/internal/providers"
"github.com/hashicorp/terraform/internal/stacks/stackaddrs"
)
@ -85,7 +85,7 @@ type Plan struct {
// ProviderFunctionResults is a shared table of results from calling
// provider functions. This is stored and loaded from during the planning
// stage to use during apply operations.
ProviderFunctionResults []providers.FunctionHash
ProviderFunctionResults []lang.FunctionHash
// PlanTimestamp is the time at which the plan was created.
PlanTimestamp time.Time

@ -15,6 +15,7 @@ import (
"github.com/hashicorp/terraform/internal/addrs"
"github.com/hashicorp/terraform/internal/collections"
"github.com/hashicorp/terraform/internal/lang"
"github.com/hashicorp/terraform/internal/lang/marks"
"github.com/hashicorp/terraform/internal/plans"
"github.com/hashicorp/terraform/internal/plans/planfile"
@ -217,7 +218,7 @@ type PlannedChangeComponentInstance struct {
PlannedCheckResults *states.CheckResults
PlannedProviderFunctionResults []providers.FunctionHash
PlannedProviderFunctionResults []lang.FunctionHash
// PlanTimestamp is the timestamp that would be returned from the
// "plantimestamp" function in modules inside this component. We
@ -828,7 +829,7 @@ func (pc *PlannedChangeApplyable) PlannedChangeProto() (*stacks.PlannedChange, e
}
type PlannedChangeProviderFunctionResults struct {
Results []providers.FunctionHash
Results []lang.FunctionHash
}
var _ PlannedChange = (*PlannedChangeProviderFunctionResults)(nil)

@ -25,6 +25,7 @@ import (
"github.com/hashicorp/terraform/internal/collections"
"github.com/hashicorp/terraform/internal/depsfile"
"github.com/hashicorp/terraform/internal/getproviders/providerreqs"
"github.com/hashicorp/terraform/internal/lang"
"github.com/hashicorp/terraform/internal/lang/marks"
"github.com/hashicorp/terraform/internal/plans"
"github.com/hashicorp/terraform/internal/providers"
@ -4216,7 +4217,7 @@ func TestApply_WithProviderFunctions(t *testing.T) {
"value": cty.StringVal("hello, world!"),
},
PlannedCheckResults: &states.CheckResults{},
PlannedProviderFunctionResults: []providers.FunctionHash{
PlannedProviderFunctionResults: []lang.FunctionHash{
{
Key: providerFunctionHashArgs(mustDefaultRootProvider("testing").Provider, "echo", cty.StringVal("hello, world!")),
Result: providerFunctionHashResult(cty.StringVal("hello, world!")),
@ -4246,7 +4247,7 @@ func TestApply_WithProviderFunctions(t *testing.T) {
Schema: stacks_testing_provider.TestingResourceSchema,
},
&stackplan.PlannedChangeProviderFunctionResults{
Results: []providers.FunctionHash{
Results: []lang.FunctionHash{
{
Key: providerFunctionHashArgs(mustDefaultRootProvider("testing").Provider, "echo", cty.StringVal("hello, world!")),
Result: providerFunctionHashResult(cty.StringVal("hello, world!")),

@ -84,7 +84,7 @@ type Main struct {
mainStackConfig *StackConfig
mainStack *Stack
providerTypes map[addrs.Provider]*ProviderType
providerFunctionResults *providers.FunctionResults
providerFunctionResults *lang.FunctionResults
cleanupFuncs []func(context.Context) tfdiags.Diagnostics
}
@ -116,7 +116,7 @@ func NewForValidating(config *stackconfig.Config, opts ValidateOpts) *Main {
},
providerFactories: opts.ProviderFactories,
providerTypes: make(map[addrs.Provider]*ProviderType),
providerFunctionResults: providers.NewFunctionResultsTable(nil),
providerFunctionResults: lang.NewFunctionResultsTable(nil),
}
}
@ -134,7 +134,7 @@ func NewForPlanning(config *stackconfig.Config, prevState *stackstate.State, opt
},
providerFactories: opts.ProviderFactories,
providerTypes: make(map[addrs.Provider]*ProviderType),
providerFunctionResults: providers.NewFunctionResultsTable(nil),
providerFunctionResults: lang.NewFunctionResultsTable(nil),
}
}
@ -148,7 +148,7 @@ func NewForApplying(config *stackconfig.Config, plan *stackplan.Plan, execResult
},
providerFactories: opts.ProviderFactories,
providerTypes: make(map[addrs.Provider]*ProviderType),
providerFunctionResults: providers.NewFunctionResultsTable(plan.ProviderFunctionResults),
providerFunctionResults: lang.NewFunctionResultsTable(plan.ProviderFunctionResults),
}
}
@ -161,7 +161,7 @@ func NewForInspecting(config *stackconfig.Config, state *stackstate.State, opts
},
providerFactories: opts.ProviderFactories,
providerTypes: make(map[addrs.Provider]*ProviderType),
providerFunctionResults: providers.NewFunctionResultsTable(nil),
providerFunctionResults: lang.NewFunctionResultsTable(nil),
testOnlyGlobals: opts.TestOnlyGlobals,
}
}

@ -198,7 +198,7 @@ func (c *Context) ApplyAndEval(plan *plans.Plan, config *configs.Config, opts *A
// We also want to propagate the timestamp from the plan file.
PlanTimeTimestamp: plan.Timestamp,
ProviderFuncResults: providers.NewFunctionResultsTable(plan.ProviderFunctionResults),
ProviderFuncResults: lang.NewFunctionResultsTable(plan.ProviderFunctionResults),
})
diags = diags.Append(walker.NonFatalDiagnostics)
diags = diags.Append(walkDiags)

@ -714,7 +714,7 @@ func (c *Context) planWalk(config *configs.Config, prevRunState *states.State, o
// Initialize the results table to validate provider function calls.
// Hold reference to this so we can store the table data in the plan file.
providerFuncResults := providers.NewFunctionResultsTable(nil)
providerFuncResults := lang.NewFunctionResultsTable(nil)
walker, walkDiags := c.walk(graph, walkOp, &graphWalkOpts{
Config: config,

@ -10,6 +10,7 @@ import (
"github.com/hashicorp/terraform/internal/addrs"
"github.com/hashicorp/terraform/internal/configs"
"github.com/hashicorp/terraform/internal/lang"
"github.com/hashicorp/terraform/internal/providers"
"github.com/hashicorp/terraform/internal/states"
"github.com/hashicorp/terraform/internal/tfdiags"
@ -112,7 +113,7 @@ func (c *Context) Validate(config *configs.Config, opts *ValidateOpts) tfdiags.D
walker, walkDiags := c.walk(graph, walkValidate, &graphWalkOpts{
Config: config,
ProviderFuncResults: providers.NewFunctionResultsTable(nil),
ProviderFuncResults: lang.NewFunctionResultsTable(nil),
ExternalProviderConfigs: opts.ExternalProviders,
})
diags = diags.Append(walker.NonFatalDiagnostics)

@ -11,6 +11,7 @@ import (
"github.com/hashicorp/terraform/internal/checks"
"github.com/hashicorp/terraform/internal/configs"
"github.com/hashicorp/terraform/internal/instances"
"github.com/hashicorp/terraform/internal/lang"
"github.com/hashicorp/terraform/internal/moduletest/mocking"
"github.com/hashicorp/terraform/internal/namedvals"
"github.com/hashicorp/terraform/internal/plans"
@ -73,7 +74,7 @@ type graphWalkOpts struct {
MoveResults refactoring.MoveResults
ProviderFuncResults *providers.FunctionResults
ProviderFuncResults *lang.FunctionResults
// Forget if set to true will cause the plan to forget all resources. This is
// only allowd in the context of a destroy plan.

@ -77,7 +77,7 @@ type BuiltinEvalContext struct {
InputValue UIInput
ProviderCache map[string]providers.Interface
ProviderFuncCache map[string]providers.Interface
ProviderFuncResults *providers.FunctionResults
ProviderFuncResults *lang.FunctionResults
ProviderInputConfig map[string]map[string]cty.Value
ProviderLock *sync.Mutex
ProvisionerCache map[string]provisioners.Interface

@ -14,6 +14,7 @@ import (
"github.com/hashicorp/terraform/internal/configs"
"github.com/hashicorp/terraform/internal/configs/configschema"
"github.com/hashicorp/terraform/internal/instances"
"github.com/hashicorp/terraform/internal/lang"
"github.com/hashicorp/terraform/internal/moduletest/mocking"
"github.com/hashicorp/terraform/internal/namedvals"
"github.com/hashicorp/terraform/internal/plans"
@ -63,7 +64,7 @@ type ContextGraphWalker struct {
contextLock sync.Mutex
providerCache map[string]providers.Interface
providerFuncCache map[string]providers.Interface
providerFuncResults *providers.FunctionResults
providerFuncResults *lang.FunctionResults
providerSchemas map[string]providers.ProviderSchema
providerLock sync.Mutex
provisionerCache map[string]provisioners.Interface

Loading…
Cancel
Save