From 9fe3f7a7b44572da80141bb328e2419db3394b17 Mon Sep 17 00:00:00 2001 From: Brandon Croft Date: Thu, 8 Jun 2023 17:29:32 -0600 Subject: [PATCH] cloud: when saving state, create a pending state version then upload Create a pending state version followed by a separate state upload When this version of the endpoint fails (It is not yet generally available, or when using with Terraform Enterprise) Fall back to the original call with state content included in the request. This strategy will reduce the amount of save failures due to network latency and gateway timeouts. --- go.mod | 6 ++--- go.sum | 14 +++++------ internal/cloud/errored.tfstate | 25 ++++++++++++++++++ internal/cloud/state.go | 42 +++++++++++++++++++++++++++---- internal/cloud/state_test.go | 4 +-- internal/cloud/testing.go | 12 +++++++-- internal/cloud/tfe_client_mock.go | 9 ++++++- 7 files changed, 91 insertions(+), 21 deletions(-) create mode 100644 internal/cloud/errored.tfstate diff --git a/go.mod b/go.mod index 844bc49ca9..4dfc37fa1a 100644 --- a/go.mod +++ b/go.mod @@ -39,8 +39,8 @@ require ( github.com/hashicorp/go-hclog v1.4.0 github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-plugin v1.4.3 - github.com/hashicorp/go-retryablehttp v0.7.2 - github.com/hashicorp/go-tfe v1.26.0 + github.com/hashicorp/go-retryablehttp v0.7.4 + github.com/hashicorp/go-tfe v1.28.0 github.com/hashicorp/go-uuid v1.0.3 github.com/hashicorp/go-version v1.6.0 github.com/hashicorp/hcl v1.0.0 @@ -208,7 +208,7 @@ require ( go.opencensus.io v0.23.0 // indirect golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect golang.org/x/exp/typeparams v0.0.0-20221208152030-732eee02a75a // indirect - golang.org/x/sync v0.1.0 // indirect + golang.org/x/sync v0.3.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index f9290b687f..7966c582df 100644 --- a/go.sum +++ b/go.sum @@ -603,8 +603,8 @@ github.com/hashicorp/go-plugin v1.4.3 h1:DXmvivbWD5qdiBts9TpBC7BYL1Aia5sxbRgQB+v github.com/hashicorp/go-plugin v1.4.3/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= -github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= -github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= +github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= @@ -616,8 +616,8 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-tfe v1.26.0 h1:aacguqCENg6Z7ttfhAxdbbY2vm/jKrntl5sUUY0h6EM= -github.com/hashicorp/go-tfe v1.26.0/go.mod h1:1Y6nsdMuJ14lYdc1VMLl/erlthvMzUsJn+WYWaAdSc4= +github.com/hashicorp/go-tfe v1.28.0 h1:YQNfHz5UPMiOD2idad4GCjzG3R2ExPww741PBPqMOIU= +github.com/hashicorp/go-tfe v1.28.0/go.mod h1:z0182DGE/63AKUaWblUVBIrt+xdSmsuuXg5AoxGqDF4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -931,7 +931,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.588 h1:DYtBXB7sVc3EOW5horg8j55cLZynhsLYhHrvQ/jXKKM= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.588/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= @@ -1167,8 +1167,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/internal/cloud/errored.tfstate b/internal/cloud/errored.tfstate new file mode 100644 index 0000000000..e3ca17d22e --- /dev/null +++ b/internal/cloud/errored.tfstate @@ -0,0 +1,25 @@ +{ + "version": 4, + "terraform_version": "0.14.0", + "serial": 1, + "lineage": "30a4d634-f765-186a-f411-7dfa798a767e", + "outputs": {}, + "resources": [ + { + "mode": "managed", + "type": "null_resource", + "name": "foo", + "provider": "provider[\"registry.terraform.io/hashicorp/null\"]", + "instances": [ + { + "schema_version": 0, + "attributes": { + "id": "yes" + }, + "sensitive_attributes": [] + } + ] + } + ], + "check_results": null +} diff --git a/internal/cloud/state.go b/internal/cloud/state.go index 69bafd01b3..a988ee5092 100644 --- a/internal/cloud/state.go +++ b/internal/cloud/state.go @@ -238,6 +238,7 @@ func (s *State) PersistState(schemas *terraform.Schemas) error { s.readState = s.state.DeepCopy() s.readLineage = s.lineage s.readSerial = s.serial + return nil } @@ -262,15 +263,13 @@ func (s *State) ShouldPersistIntermediateState(info *local.IntermediateStatePers return currentInterval >= wantInterval } -func (s *State) uploadState(lineage string, serial uint64, isForcePush bool, state, jsonState, jsonStateOutputs []byte) error { - ctx := context.Background() - +func (s *State) uploadStateFallback(ctx context.Context, lineage string, serial uint64, isForcePush bool, state, jsonState, jsonStateOutputs []byte) error { options := tfe.StateVersionCreateOptions{ Lineage: tfe.String(lineage), Serial: tfe.Int64(int64(serial)), MD5: tfe.String(fmt.Sprintf("%x", md5.Sum(state))), - State: tfe.String(base64.StdEncoding.EncodeToString(state)), Force: tfe.Bool(isForcePush), + State: tfe.String(base64.StdEncoding.EncodeToString(state)), JSONState: tfe.String(base64.StdEncoding.EncodeToString(jsonState)), JSONStateOutputs: tfe.String(base64.StdEncoding.EncodeToString(jsonStateOutputs)), } @@ -282,13 +281,46 @@ func (s *State) uploadState(lineage string, serial uint64, isForcePush bool, sta options.Run = &tfe.Run{ID: runID} } + // Create the new state. + _, err := s.tfeClient.StateVersions.Create(ctx, s.workspace.ID, options) + return err +} + +func (s *State) uploadState(lineage string, serial uint64, isForcePush bool, state, jsonState, jsonStateOutputs []byte) error { + ctx := context.Background() + + options := tfe.StateVersionUploadOptions{ + StateVersionCreateOptions: tfe.StateVersionCreateOptions{ + Lineage: tfe.String(lineage), + Serial: tfe.Int64(int64(serial)), + MD5: tfe.String(fmt.Sprintf("%x", md5.Sum(state))), + Force: tfe.Bool(isForcePush), + JSONStateOutputs: tfe.String(base64.StdEncoding.EncodeToString(jsonStateOutputs)), + }, + RawState: state, + RawJSONState: jsonState, + } + + // If we have a run ID, make sure to add it to the options + // so the state will be properly associated with the run. + runID := os.Getenv("TFE_RUN_ID") + if runID != "" { + options.StateVersionCreateOptions.Run = &tfe.Run{ID: runID} + } + // The server is allowed to dynamically request a different time interval // than we'd normally use, for example if it's currently under heavy load // and needs clients to backoff for a while. ctx = tfe.ContextWithResponseHeaderHook(ctx, s.readSnapshotIntervalHeader) // Create the new state. - _, err := s.tfeClient.StateVersions.Create(ctx, s.workspace.ID, options) + _, err := s.tfeClient.StateVersions.Upload(ctx, s.workspace.ID, options) + if errors.Is(err, tfe.ErrStateVersionUploadNotSupported) { + // Create the new state with content included in the request (Terraform Enterprise v202306-1 and below) + log.Println("[INFO] Detected that state version upload is not supported. Retrying using compatibility state upload.") + return s.uploadStateFallback(ctx, lineage, serial, isForcePush, state, jsonState, jsonStateOutputs) + } + return err } diff --git a/internal/cloud/state_test.go b/internal/cloud/state_test.go index 14930a274d..fead1b731b 100644 --- a/internal/cloud/state_test.go +++ b/internal/cloud/state_test.go @@ -298,7 +298,6 @@ func TestState_PersistState(t *testing.T) { // since HTTP-level concerns like headers are out of scope for the // mock client we typically use in other tests in this package, which // aim to abstract away HTTP altogether. - var serverURL string // Didn't want to repeat myself here for _, testCase := range []struct { @@ -314,10 +313,9 @@ func TestState_PersistState(t *testing.T) { snapshotsEnabled: false, }, } { - server := testServerWithSnapshotsEnabled(t, serverURL, testCase.snapshotsEnabled) + server := testServerWithSnapshotsEnabled(t, testCase.snapshotsEnabled) defer server.Close() - serverURL = server.URL cfg := &tfe.Config{ Address: server.URL, BasePath: "api", diff --git a/internal/cloud/testing.go b/internal/cloud/testing.go index 8f81f21fc1..119f57d509 100644 --- a/internal/cloud/testing.go +++ b/internal/cloud/testing.go @@ -383,8 +383,9 @@ func testServerWithHandlers(handlers map[string]func(http.ResponseWriter, *http. return httptest.NewServer(mux) } -func testServerWithSnapshotsEnabled(t *testing.T, serverURL string, enabled bool) *httptest.Server { - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +func testServerWithSnapshotsEnabled(t *testing.T, enabled bool) *httptest.Server { + var serverURL string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t.Log(r.Method, r.URL.String()) if r.URL.Path == "/state-json" { @@ -400,6 +401,7 @@ func testServerWithSnapshotsEnabled(t *testing.T, serverURL string, enabled bool w.Write(respBody) return } + if r.URL.Path == "/api/ping" { t.Log("pretending to be Ping") w.WriteHeader(http.StatusNoContent) @@ -409,8 +411,10 @@ func testServerWithSnapshotsEnabled(t *testing.T, serverURL string, enabled bool fakeBody := map[string]any{ "data": map[string]any{ "type": "state-versions", + "id": GenerateID("sv-"), "attributes": map[string]any{ "hosted-state-download-url": serverURL + "/state-json", + "hosted-state-upload-url": serverURL + "/state-json", }, }, } @@ -435,6 +439,8 @@ func testServerWithSnapshotsEnabled(t *testing.T, serverURL string, enabled bool w.Header().Set("x-terraform-snapshot-interval", "300") } w.WriteHeader(http.StatusOK) + case "PUT": + t.Log("pretending to be Archivist") default: t.Fatal("don't know what API operation this was supposed to be") } @@ -442,6 +448,8 @@ func testServerWithSnapshotsEnabled(t *testing.T, serverURL string, enabled bool w.WriteHeader(http.StatusOK) w.Write(fakeBodyRaw) })) + serverURL = server.URL + return server } // testDefaultRequestHandlers is a map of request handlers intended to be used in a request diff --git a/internal/cloud/tfe_client_mock.go b/internal/cloud/tfe_client_mock.go index 83878c7e47..a67ce3f7e3 100644 --- a/internal/cloud/tfe_client_mock.go +++ b/internal/cloud/tfe_client_mock.go @@ -1254,6 +1254,7 @@ func (m *MockStateVersions) Create(ctx context.Context, workspaceID string, opti sv := &tfe.StateVersion{ ID: id, DownloadURL: url, + UploadURL: fmt.Sprintf("/_archivist/upload/%s", id), Serial: *options.Serial, } @@ -1261,7 +1262,6 @@ func (m *MockStateVersions) Create(ctx context.Context, workspaceID string, opti if err != nil { return nil, err } - m.states[sv.DownloadURL] = state m.outputStates[sv.ID] = []byte(*options.JSONStateOutputs) m.stateVersions[sv.ID] = sv @@ -1270,6 +1270,13 @@ func (m *MockStateVersions) Create(ctx context.Context, workspaceID string, opti return sv, nil } +func (m *MockStateVersions) Upload(ctx context.Context, workspaceID string, options tfe.StateVersionUploadOptions) (*tfe.StateVersion, error) { + createOptions := options.StateVersionCreateOptions + createOptions.State = tfe.String(base64.StdEncoding.EncodeToString(options.RawState)) + + return m.Create(ctx, workspaceID, createOptions) +} + func (m *MockStateVersions) Read(ctx context.Context, svID string) (*tfe.StateVersion, error) { return m.ReadWithOptions(ctx, svID, nil) }