From e8cbf3f77d397390fb9714855336ea994c5bffa1 Mon Sep 17 00:00:00 2001 From: Ben Moskovitz Date: Mon, 20 Sep 2021 15:54:13 +1200 Subject: [PATCH 1/2] Improve error message when S3 state checksums don't line up --- internal/backend/remote-state/s3/client.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/backend/remote-state/s3/client.go b/internal/backend/remote-state/s3/client.go index da41dac4f1..2bf6e95456 100644 --- a/internal/backend/remote-state/s3/client.go +++ b/internal/backend/remote-state/s3/client.go @@ -109,7 +109,7 @@ func (c *RemoteClient) Get() (payload *remote.Payload, err error) { continue } - return nil, fmt.Errorf(errBadChecksumFmt, digest) + return nil, fmt.Errorf(errBadChecksumFmt, c.bucketName, c.path, expected, digest, digest) } break @@ -501,8 +501,12 @@ func (c *RemoteClient) logger(operation string) hclog.Logger { const errBadChecksumFmt = `state data in S3 does not have the expected content. +I tried to get the state data in bucket: %s at path: %s, but its MD5 checksum +didn't line up with what I have stored in DynamoDB. The checksum I have stored +is %x, but I calculated a checksum of %x for the state currently stored in S3. + This may be caused by unusually long delays in S3 processing a previous state -update. Please wait for a minute or two and try again. If this problem +update. Please wait for a minute or two and try again. If this problem persists, and neither S3 nor DynamoDB are experiencing an outage, you may need to manually verify the remote state and update the Digest value stored in the DynamoDB table to the following value: %x From 8f516509f95f55fdaa3de51dfa13ba8f5e570d30 Mon Sep 17 00:00:00 2001 From: Graham Davison Date: Mon, 11 Sep 2023 17:12:51 -0700 Subject: [PATCH 2/2] Wraps bad checksum error message in error type and handles empty checksum --- internal/backend/remote-state/s3/client.go | 56 +++++++++++++++---- .../backend/remote-state/s3/client_test.go | 7 ++- 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/internal/backend/remote-state/s3/client.go b/internal/backend/remote-state/s3/client.go index 2bf6e95456..8135880ad1 100644 --- a/internal/backend/remote-state/s3/client.go +++ b/internal/backend/remote-state/s3/client.go @@ -109,7 +109,7 @@ func (c *RemoteClient) Get() (payload *remote.Payload, err error) { continue } - return nil, fmt.Errorf(errBadChecksumFmt, c.bucketName, c.path, expected, digest, digest) + return nil, newBadChecksumError(c.bucketName, c.path, digest, expected) } break @@ -499,18 +499,54 @@ func (c *RemoteClient) logger(operation string) hclog.Logger { return logWithOperation(log, operation) } -const errBadChecksumFmt = `state data in S3 does not have the expected content. +var _ error = badChecksumError{} -I tried to get the state data in bucket: %s at path: %s, but its MD5 checksum -didn't line up with what I have stored in DynamoDB. The checksum I have stored -is %x, but I calculated a checksum of %x for the state currently stored in S3. +type badChecksumError struct { + bucket, key string + digest, expected []byte +} + +func newBadChecksumError(bucket, key string, digest, expected []byte) badChecksumError { + return badChecksumError{ + bucket: bucket, + key: key, + digest: digest, + expected: expected, + } +} + +func (err badChecksumError) Error() string { + return fmt.Sprintf(`state data in S3 does not have the expected content. + +The checksum calculated for the state stored in S3 does not match the checksum +stored in DynamoDB. + +Bucket: %[1]s +Key: %[2]s +Calculated checksum: %[3]x +Stored checksum: %[4]x This may be caused by unusually long delays in S3 processing a previous state -update. Please wait for a minute or two and try again. If this problem -persists, and neither S3 nor DynamoDB are experiencing an outage, you may need -to manually verify the remote state and update the Digest value stored in the -DynamoDB table to the following value: %x -` +update. Please wait for a minute or two and try again. + +%[5]s +`, err.bucket, err.key, err.digest, err.expected, err.resolutionMsg()) +} + +func (err badChecksumError) resolutionMsg() string { + if len(err.digest) > 0 { + return fmt.Sprintf( + `If this problem persists, and neither S3 nor DynamoDB are experiencing an +outage, you may need to manually verify the remote state and update the Digest +value stored in the DynamoDB table to the following value: %x`, + err.expected, + ) + } else { + return `If this problem persists, and neither S3 nor DynamoDB are experiencing an +outage, you may need to manually verify the remote state and remove the Digest +value stored in the DynamoDB table` + } +} const errS3NoSuchBucket = `S3 bucket %q does not exist. diff --git a/internal/backend/remote-state/s3/client_test.go b/internal/backend/remote-state/s3/client_test.go index 1f4a23ecf7..5422ab2e3b 100644 --- a/internal/backend/remote-state/s3/client_test.go +++ b/internal/backend/remote-state/s3/client_test.go @@ -8,7 +8,6 @@ import ( "context" "crypto/md5" "fmt" - "strings" "testing" "time" @@ -300,8 +299,10 @@ func TestRemoteClient_stateChecksum(t *testing.T) { // fetching an empty state through client1 should now error out due to a // mismatched checksum. - if _, err := client1.Get(); !strings.HasPrefix(err.Error(), errBadChecksumFmt[:80]) { + if _, err := client1.Get(); !IsA[badChecksumError](err) { t.Fatalf("expected state checksum error: got %s", err) + } else if bse, ok := As[badChecksumError](err); ok && len(bse.digest) != 0 { + t.Fatalf("expected empty checksum, got %x", bse.digest) } // put the old state in place of the new, without updating the checksum @@ -311,7 +312,7 @@ func TestRemoteClient_stateChecksum(t *testing.T) { // fetching the wrong state through client1 should now error out due to a // mismatched checksum. - if _, err := client1.Get(); !strings.HasPrefix(err.Error(), errBadChecksumFmt[:80]) { + if _, err := client1.Get(); !IsA[badChecksumError](err) { t.Fatalf("expected state checksum error: got %s", err) }