diff --git a/internal/credential/service_list_credentials.go b/internal/credential/service_list_credentials.go new file mode 100644 index 0000000000..5c87c43aa8 --- /dev/null +++ b/internal/credential/service_list_credentials.go @@ -0,0 +1,63 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package credential + +import ( + "context" + "time" + + "github.com/hashicorp/boundary/internal/errors" + "github.com/hashicorp/boundary/internal/pagination" +) + +// CredentialService defines the interface expected +// to list credentials, deleted credential IDs and get +// an estimate count of total credentials. +type CredentialService interface { + EstimatedCredentialCount(context.Context) (int, error) + ListDeletedCredentialIds(context.Context, time.Time) ([]string, time.Time, error) + ListCredentials(context.Context, string, ...Option) ([]Static, time.Time, error) + ListCredentialsRefresh(context.Context, string, time.Time, ...Option) ([]Static, time.Time, error) +} + +// List lists up to page size credentials, filtering out entries that +// do not pass the filter item function. It will automatically request +// more credentials from the database, at page size chunks, to fill the page. +// It returns a new list token used to continue pagination or refresh items. +// Credentials are ordered by create time descending (most recently created first). +func List( + ctx context.Context, + grantsHash []byte, + pageSize int, + filterItemFn pagination.ListFilterFunc[Static], + service CredentialService, + credentialStoreId string, +) (*pagination.ListResponse[Static], error) { + const op = "credential.List" + + switch { + case len(grantsHash) == 0: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grants hash") + case pageSize < 1: + return nil, errors.New(ctx, errors.InvalidParameter, op, "page size must be at least 1") + case filterItemFn == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing filter item callback") + case service == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing service") + case credentialStoreId == "": + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing credential store ID") + } + + listItemsFn := func(ctx context.Context, lastPageItem Static, limit int) ([]Static, time.Time, error) { + opts := []Option{ + WithLimit(limit), + } + if lastPageItem != nil { + opts = append(opts, WithStartPageAfterItem(lastPageItem)) + } + return service.ListCredentials(ctx, credentialStoreId, opts...) + } + + return pagination.List(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, service.EstimatedCredentialCount) +} diff --git a/internal/credential/service_list_credentials_ext_test.go b/internal/credential/service_list_credentials_ext_test.go new file mode 100644 index 0000000000..5bccfa1ce7 --- /dev/null +++ b/internal/credential/service_list_credentials_ext_test.go @@ -0,0 +1,616 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package credential_test + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/hashicorp/boundary/globals" + "github.com/hashicorp/boundary/internal/credential" + "github.com/hashicorp/boundary/internal/credential/static" + "github.com/hashicorp/boundary/internal/credential/static/store" + "github.com/hashicorp/boundary/internal/db" + "github.com/hashicorp/boundary/internal/db/timestamp" + "github.com/hashicorp/boundary/internal/iam" + "github.com/hashicorp/boundary/internal/kms" + "github.com/hashicorp/boundary/internal/listtoken" + "github.com/hashicorp/boundary/internal/types/resource" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/testing/protocmp" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestService_List(t *testing.T) { + // Set database read timeout to avoid duplicates in response + oldReadTimeout := globals.RefreshReadLookbackDuration + globals.RefreshReadLookbackDuration = 0 + t.Cleanup(func() { + globals.RefreshReadLookbackDuration = oldReadTimeout + }) + ctx := context.Background() + conn, _ := db.TestSetup(t, "postgres") + sqlDb, err := conn.SqlDB(ctx) + require.NoError(t, err) + rw := db.New(conn) + wrapper := db.TestWrapper(t) + kms := kms.TestKms(t, conn, wrapper) + _, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper)) + fiveDaysAgo := time.Now().AddDate(0, 0, -5) + credStore := static.TestCredentialStore(t, conn, wrapper, prj.GetPublicId()) + obj, _ := static.TestJsonObject(t) + creds := []credential.Static{ + static.TestJsonCredential(t, conn, wrapper, credStore.GetPublicId(), prj.GetPublicId(), obj), + static.TestUsernamePasswordCredential(t, conn, wrapper, "someuser", "somepassword", credStore.GetPublicId(), prj.GetPublicId()), + static.TestSshPrivateKeyCredential(t, conn, wrapper, "someuser", static.TestSshPrivateKeyPem, credStore.GetPublicId(), prj.GetPublicId()), + static.TestJsonCredential(t, conn, wrapper, credStore.GetPublicId(), prj.GetPublicId(), obj), + static.TestJsonCredential(t, conn, wrapper, credStore.GetPublicId(), prj.GetPublicId(), obj), + } + // since we sort ascending, we need to reverse targets + slices.Reverse(creds) + + // Run analyze to update count estimates + _, err = sqlDb.ExecContext(ctx, "analyze") + require.NoError(t, err) + + repo, err := static.NewRepository(ctx, rw, rw, kms) + require.NoError(t, err) + + cmpOpts := []cmp.Option{ + cmpopts.IgnoreUnexported( + static.JsonCredential{}, + store.JsonCredential{}, + static.UsernamePasswordCredential{}, + store.UsernamePasswordCredential{}, + static.SshPrivateKeyCredential{}, + store.SshPrivateKeyCredential{}, + timestamp.Timestamp{}, + timestamppb.Timestamp{}, + ), + protocmp.Transform(), + protocmp.IgnoreFields( + &store.JsonCredential{}, + "object", + "object_encrypted", + ), + protocmp.IgnoreFields( + &store.UsernamePasswordCredential{}, + "password", + "ct_password", + ), + protocmp.IgnoreFields( + &store.SshPrivateKeyCredential{}, + "private_key", + "private_key_encrypted", + ), + } + + t.Run("List validation", func(t *testing.T) { + t.Parallel() + t.Run("missing grants hash", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + _, err := credential.List(ctx, nil, 1, filterFunc, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing grants hash") + }) + t.Run("zero page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + _, err := credential.List(ctx, []byte("some hash"), 0, filterFunc, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("negative page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + _, err := credential.List(ctx, []byte("some hash"), -1, filterFunc, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("nil filter func", func(t *testing.T) { + t.Parallel() + _, err := credential.List(ctx, []byte("some hash"), 1, nil, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing filter item callback") + }) + t.Run("nil repo", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + _, err := credential.List(ctx, []byte("some hash"), 1, filterFunc, nil, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing service") + }) + t.Run("missing credential store ID", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + _, err := credential.List(ctx, []byte("some hash"), 1, filterFunc, repo, "") + require.ErrorContains(t, err, "missing credential store ID") + }) + }) + t.Run("ListPage validation", func(t *testing.T) { + t.Parallel() + t.Run("missing grants hash", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListPage(ctx, nil, 1, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing grants hash") + }) + t.Run("zero page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListPage(ctx, []byte("some hash"), 0, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("negative page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListPage(ctx, []byte("some hash"), -1, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("nil filter func", func(t *testing.T) { + t.Parallel() + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListPage(ctx, []byte("some hash"), 1, nil, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing filter item callback") + }) + t.Run("nil token", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + _, err = credential.ListPage(ctx, []byte("some hash"), 1, filterFunc, nil, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing token") + }) + t.Run("wrong token type", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListPage(ctx, []byte("some hash"), 1, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "token did not have a pagination token component") + }) + t.Run("nil repo", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListPage(ctx, []byte("some hash"), 1, filterFunc, tok, nil, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing service") + }) + t.Run("missing credential store ID", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListPage(ctx, []byte("some hash"), 1, filterFunc, tok, repo, "") + require.ErrorContains(t, err, "missing credential store ID") + }) + }) + t.Run("ListRefresh validation", func(t *testing.T) { + t.Parallel() + t.Run("missing grants hash", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListRefresh(ctx, nil, 1, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing grants hash") + }) + t.Run("zero page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListRefresh(ctx, []byte("some hash"), 0, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("negative page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListRefresh(ctx, []byte("some hash"), -1, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("nil filter func", func(t *testing.T) { + t.Parallel() + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListRefresh(ctx, []byte("some hash"), 1, nil, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing filter item callback") + }) + t.Run("nil token", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + _, err = credential.ListRefresh(ctx, []byte("some hash"), 1, filterFunc, nil, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing token") + }) + t.Run("nil repo", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListRefresh(ctx, []byte("some hash"), 1, filterFunc, tok, nil, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing service") + }) + t.Run("missing credential store ID", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListRefresh(ctx, []byte("some hash"), 1, filterFunc, tok, repo, "") + require.ErrorContains(t, err, "missing credential store ID") + }) + }) + t.Run("ListRefreshPage validation", func(t *testing.T) { + t.Parallel() + t.Run("missing grants hash", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListRefreshPage(ctx, nil, 1, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing grants hash") + }) + t.Run("zero page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListRefreshPage(ctx, []byte("some hash"), 0, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("negative page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListRefreshPage(ctx, []byte("some hash"), -1, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("nil filter func", func(t *testing.T) { + t.Parallel() + tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListRefreshPage(ctx, []byte("some hash"), 1, nil, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing filter item callback") + }) + t.Run("nil token", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + _, err = credential.ListRefreshPage(ctx, []byte("some hash"), 1, filterFunc, nil, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing token") + }) + t.Run("wrong token type", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListRefreshPage(ctx, []byte("some hash"), 1, filterFunc, tok, repo, credStore.GetPublicId()) + require.ErrorContains(t, err, "token did not have a refresh token component") + }) + t.Run("nil repo", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListRefreshPage(ctx, []byte("some hash"), 1, filterFunc, tok, nil, credStore.GetPublicId()) + require.ErrorContains(t, err, "missing service") + }) + t.Run("missing credential store id", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, c credential.Static) (bool, error) { + return true, nil + } + tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo) + require.NoError(t, err) + _, err = credential.ListRefreshPage(ctx, []byte("some hash"), 1, filterFunc, tok, repo, "") + require.ErrorContains(t, err, "missing credential store ID") + }) + }) + + t.Run("simple pagination", func(t *testing.T) { + filterFunc := func(context.Context, credential.Static) (bool, error) { + return true, nil + } + resp, err := credential.List(ctx, []byte("some hash"), 1, filterFunc, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.NotNil(t, resp.ListToken) + require.Equal(t, resp.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp.CompleteListing) + require.Equal(t, resp.EstimatedItemCount, 5) + require.Empty(t, resp.DeletedIds) + require.Len(t, resp.Items, 1) + require.Empty(t, cmp.Diff(resp.Items[0], creds[0], cmpOpts...)) + + resp2, err := credential.ListPage(ctx, []byte("some hash"), 1, filterFunc, resp.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp2.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp2.CompleteListing) + require.Equal(t, resp2.EstimatedItemCount, 5) + require.Empty(t, resp2.DeletedIds) + require.Len(t, resp2.Items, 1) + require.Empty(t, cmp.Diff(resp2.Items[0], creds[1], cmpOpts...)) + + resp3, err := credential.ListPage(ctx, []byte("some hash"), 1, filterFunc, resp2.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp3.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp3.CompleteListing) + require.Equal(t, resp3.EstimatedItemCount, 5) + require.Empty(t, resp3.DeletedIds) + require.Len(t, resp3.Items, 1) + require.Empty(t, cmp.Diff(resp3.Items[0], creds[2], cmpOpts...)) + + resp4, err := credential.ListPage(ctx, []byte("some hash"), 1, filterFunc, resp3.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp4.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp4.CompleteListing) + require.Equal(t, resp4.EstimatedItemCount, 5) + require.Empty(t, resp4.DeletedIds) + require.Len(t, resp4.Items, 1) + require.Empty(t, cmp.Diff(resp4.Items[0], creds[3], cmpOpts...)) + + resp5, err := credential.ListPage(ctx, []byte("some hash"), 1, filterFunc, resp4.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp5.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp5.CompleteListing) + require.Equal(t, resp5.EstimatedItemCount, 5) + require.Empty(t, resp5.DeletedIds) + require.Len(t, resp5.Items, 1) + require.Empty(t, cmp.Diff(resp5.Items[0], creds[4], cmpOpts...)) + + // Finished initial pagination phase, request refresh + // Expect no results. + resp6, err := credential.ListRefresh(ctx, []byte("some hash"), 1, filterFunc, resp5.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp6.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp6.CompleteListing) + require.Equal(t, resp6.EstimatedItemCount, 5) + require.Empty(t, resp6.DeletedIds) + require.Empty(t, resp6.Items) + + // Create some new credentials + newCred1 := static.TestJsonCredential(t, conn, wrapper, credStore.GetPublicId(), prj.GetPublicId(), obj) + newCred2 := static.TestUsernamePasswordCredential(t, conn, wrapper, "someuser", "somepassword", credStore.GetPublicId(), prj.GetPublicId()) + newCred3 := static.TestSshPrivateKeyCredential(t, conn, wrapper, "someuser", static.TestSshPrivateKeyPem, credStore.GetPublicId(), prj.GetPublicId()) + t.Cleanup(func() { + _, err := repo.DeleteCredential(ctx, credStore.ProjectId, newCred1.GetPublicId()) + require.NoError(t, err) + _, err = repo.DeleteCredential(ctx, credStore.ProjectId, newCred2.GetPublicId()) + require.NoError(t, err) + _, err = repo.DeleteCredential(ctx, credStore.ProjectId, newCred3.GetPublicId()) + require.NoError(t, err) + // Run analyze to update count estimate + _, err = sqlDb.ExecContext(ctx, "analyze") + require.NoError(t, err) + }) + // Run analyze to update count estimate + _, err = sqlDb.ExecContext(ctx, "analyze") + require.NoError(t, err) + + // Refresh again, should get newCred3 + resp7, err := credential.ListRefresh(ctx, []byte("some hash"), 1, filterFunc, resp6.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp7.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp7.CompleteListing) + require.Equal(t, resp7.EstimatedItemCount, 8) + require.Empty(t, resp7.DeletedIds) + require.Len(t, resp7.Items, 1) + require.Empty(t, cmp.Diff(resp7.Items[0], newCred3, cmpOpts...)) + + // Refresh again, should get newCred2 + resp8, err := credential.ListRefreshPage(ctx, []byte("some hash"), 1, filterFunc, resp6.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp8.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp8.CompleteListing) + require.Equal(t, resp8.EstimatedItemCount, 8) + require.Empty(t, resp8.DeletedIds) + require.Len(t, resp8.Items, 1) + require.Empty(t, cmp.Diff(resp8.Items[0], newCred2, cmpOpts...)) + + // Refresh again, should get newCred1 + resp9, err := credential.ListRefreshPage(ctx, []byte("some hash"), 1, filterFunc, resp6.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp9.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp9.CompleteListing) + require.Equal(t, resp9.EstimatedItemCount, 8) + require.Empty(t, resp9.DeletedIds) + require.Len(t, resp9.Items, 1) + require.Empty(t, cmp.Diff(resp9.Items[0], newCred1, cmpOpts...)) + + // Refresh again, should get no results + resp10, err := credential.ListRefresh(ctx, []byte("some hash"), 1, filterFunc, resp6.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp10.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp10.CompleteListing) + require.Equal(t, resp10.EstimatedItemCount, 8) + require.Empty(t, resp10.DeletedIds) + require.Empty(t, resp10.Items) + }) + + t.Run("simple pagination with aggressive filtering", func(t *testing.T) { + filterFunc := func(ctx context.Context, c credential.Static) (bool, error) { + return c.GetPublicId() == creds[1].GetPublicId() || + c.GetPublicId() == creds[len(creds)-1].GetPublicId(), nil + } + resp, err := credential.List(ctx, []byte("some hash"), 1, filterFunc, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.NotNil(t, resp.ListToken) + require.Equal(t, resp.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp.CompleteListing) + require.Equal(t, resp.EstimatedItemCount, 5) + require.Empty(t, resp.DeletedIds) + require.Len(t, resp.Items, 1) + require.Empty(t, cmp.Diff(resp.Items[0], creds[1], cmpOpts...)) + + resp2, err := credential.ListPage(ctx, []byte("some hash"), 1, filterFunc, resp.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.NotNil(t, resp2.ListToken) + require.Equal(t, resp2.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp2.CompleteListing) + require.Equal(t, resp2.EstimatedItemCount, 5) + require.Empty(t, resp2.DeletedIds) + require.Len(t, resp2.Items, 1) + require.Empty(t, cmp.Diff(resp2.Items[0], creds[len(creds)-1], cmpOpts...)) + + // request a refresh, nothing should be returned + resp3, err := credential.ListRefresh(ctx, []byte("some hash"), 1, filterFunc, resp.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp3.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp3.CompleteListing) + require.Equal(t, resp3.EstimatedItemCount, 5) + require.Empty(t, resp3.DeletedIds) + require.Empty(t, resp3.Items) + + newCred1 := static.TestJsonCredential(t, conn, wrapper, credStore.GetPublicId(), prj.GetPublicId(), obj) + newCred2 := static.TestUsernamePasswordCredential(t, conn, wrapper, "someuser", "somepassword", credStore.GetPublicId(), prj.GetPublicId()) + newCred3 := static.TestSshPrivateKeyCredential(t, conn, wrapper, "someuser", static.TestSshPrivateKeyPem, credStore.GetPublicId(), prj.GetPublicId()) + newCred4 := static.TestJsonCredential(t, conn, wrapper, credStore.GetPublicId(), prj.GetPublicId(), obj) + t.Cleanup(func() { + _, err := repo.DeleteCredential(ctx, credStore.ProjectId, newCred1.GetPublicId()) + require.NoError(t, err) + _, err = repo.DeleteCredential(ctx, credStore.ProjectId, newCred2.GetPublicId()) + require.NoError(t, err) + _, err = repo.DeleteCredential(ctx, credStore.ProjectId, newCred3.GetPublicId()) + require.NoError(t, err) + _, err = repo.DeleteCredential(ctx, credStore.ProjectId, newCred4.GetPublicId()) + require.NoError(t, err) + // Run analyze to update count estimate + _, err = sqlDb.ExecContext(ctx, "analyze") + require.NoError(t, err) + }) + // Run analyze to update count estimate + _, err = sqlDb.ExecContext(ctx, "analyze") + require.NoError(t, err) + + filterFunc = func(_ context.Context, c credential.Static) (bool, error) { + return c.GetPublicId() == newCred3.GetPublicId() || + c.GetPublicId() == newCred1.GetPublicId(), nil + } + // Refresh again, should get newCred3 + resp4, err := credential.ListRefresh(ctx, []byte("some hash"), 1, filterFunc, resp3.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp4.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp4.CompleteListing) + require.Equal(t, resp4.EstimatedItemCount, 9) + require.Empty(t, resp4.DeletedIds) + require.Len(t, resp4.Items, 1) + require.Empty(t, cmp.Diff(resp4.Items[0], newCred3, cmpOpts...)) + + // Refresh again, should get newCred1 + resp5, err := credential.ListRefreshPage(ctx, []byte("some hash"), 1, filterFunc, resp4.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp5.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp5.CompleteListing) + require.Equal(t, resp5.EstimatedItemCount, 9) + require.Empty(t, resp5.DeletedIds) + require.Len(t, resp5.Items, 1) + require.Empty(t, cmp.Diff(resp5.Items[0], newCred1, cmpOpts...)) + }) + + t.Run("simple pagination with deletion", func(t *testing.T) { + filterFunc := func(context.Context, credential.Static) (bool, error) { + return true, nil + } + deletedCredentialId := creds[0].GetPublicId() + _, err := repo.DeleteCredential(ctx, prj.GetPublicId(), deletedCredentialId) + require.NoError(t, err) + creds = creds[1:] + + // Run analyze to update count estimate + _, err = sqlDb.ExecContext(ctx, "analyze") + require.NoError(t, err) + + resp, err := credential.List(ctx, []byte("some hash"), 1, filterFunc, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.NotNil(t, resp.ListToken) + require.Equal(t, resp.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp.CompleteListing) + require.Equal(t, resp.EstimatedItemCount, 4) + require.Empty(t, resp.DeletedIds) + require.Len(t, resp.Items, 1) + require.Empty(t, cmp.Diff(resp.Items[0], creds[0], cmpOpts...)) + + // request remaining results + resp2, err := credential.ListPage(ctx, []byte("some hash"), 3, filterFunc, resp.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp2.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp2.CompleteListing) + require.Equal(t, resp2.EstimatedItemCount, 4) + require.Empty(t, resp2.DeletedIds) + require.Len(t, resp2.Items, 3) + require.Empty(t, cmp.Diff(resp2.Items, creds[1:], cmpOpts...)) + + deletedCredentialId = creds[0].GetPublicId() + _, err = repo.DeleteCredential(ctx, prj.GetPublicId(), deletedCredentialId) + require.NoError(t, err) + creds = creds[1:] + + // Run analyze to update count estimate + _, err = sqlDb.ExecContext(ctx, "analyze") + require.NoError(t, err) + + // request a refresh, nothing should be returned except the deleted id + resp3, err := credential.ListRefresh(ctx, []byte("some hash"), 1, filterFunc, resp2.ListToken, repo, credStore.GetPublicId()) + require.NoError(t, err) + require.Equal(t, resp3.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp3.CompleteListing) + require.Equal(t, resp3.EstimatedItemCount, 3) + require.Contains(t, resp3.DeletedIds, deletedCredentialId) + require.Empty(t, resp3.Items) + }) +} diff --git a/internal/credential/service_list_credentials_page.go b/internal/credential/service_list_credentials_page.go new file mode 100644 index 0000000000..e9a686c4cd --- /dev/null +++ b/internal/credential/service_list_credentials_page.go @@ -0,0 +1,67 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package credential + +import ( + "context" + "time" + + "github.com/hashicorp/boundary/internal/errors" + "github.com/hashicorp/boundary/internal/listtoken" + "github.com/hashicorp/boundary/internal/pagination" +) + +// ListPage lists up to page size credentials, filtering out entries that +// do not pass the filter item function. It will automatically request +// more credentials from the database, at page size chunks, to fill the page. +// It will start its paging based on the information in the token. +// It returns a new list token used to continue pagination or refresh items. +// Credentials are ordered by create time descending (most recently created first). +func ListPage( + ctx context.Context, + grantsHash []byte, + pageSize int, + filterItemFn pagination.ListFilterFunc[Static], + tok *listtoken.Token, + service CredentialService, + credentialStoreId string, +) (*pagination.ListResponse[Static], error) { + const op = "credential.ListPage" + + switch { + case len(grantsHash) == 0: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grants hash") + case pageSize < 1: + return nil, errors.New(ctx, errors.InvalidParameter, op, "page size must be at least 1") + case filterItemFn == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing filter item callback") + case tok == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing token") + case service == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing service") + case credentialStoreId == "": + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing credential store ID") + } + if _, ok := tok.Subtype.(*listtoken.PaginationToken); !ok { + return nil, errors.New(ctx, errors.InvalidParameter, op, "token did not have a pagination token component") + } + + listItemsFn := func(ctx context.Context, lastPageItem Static, limit int) ([]Static, time.Time, error) { + opts := []Option{ + WithLimit(limit), + } + if lastPageItem != nil { + opts = append(opts, WithStartPageAfterItem(lastPageItem)) + } else { + lastItem, err := tok.LastItem(ctx) + if err != nil { + return nil, time.Time{}, err + } + opts = append(opts, WithStartPageAfterItem(lastItem)) + } + return service.ListCredentials(ctx, credentialStoreId, opts...) + } + + return pagination.ListPage(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, service.EstimatedCredentialCount, tok) +} diff --git a/internal/credential/service_list_credentials_refresh.go b/internal/credential/service_list_credentials_refresh.go new file mode 100644 index 0000000000..6eeb049207 --- /dev/null +++ b/internal/credential/service_list_credentials_refresh.go @@ -0,0 +1,73 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package credential + +import ( + "context" + "time" + + "github.com/hashicorp/boundary/globals" + "github.com/hashicorp/boundary/internal/errors" + "github.com/hashicorp/boundary/internal/listtoken" + "github.com/hashicorp/boundary/internal/pagination" +) + +// ListRefresh lists up to page size credentials, filtering out entries that +// do not pass the filter item function. It will automatically request +// more credentials from the database, at page size chunks, to fill the page. +// It will start its paging based on the information in the token. +// It returns a new list token used to continue pagination or refresh items. +// Credentials are ordered by update time descending (most recently updated first). +// Credentials may contain items that were already returned during the initial +// pagination phase. It also returns a list of any credentials deleted since the +// start of the initial pagination phase or last response. +func ListRefresh( + ctx context.Context, + grantsHash []byte, + pageSize int, + filterItemFn pagination.ListFilterFunc[Static], + tok *listtoken.Token, + service CredentialService, + credentialStoreId string, +) (*pagination.ListResponse[Static], error) { + const op = "credential.ListRefresh" + + switch { + case len(grantsHash) == 0: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grants hash") + case pageSize < 1: + return nil, errors.New(ctx, errors.InvalidParameter, op, "page size must be at least 1") + case filterItemFn == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing filter item callback") + case tok == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing token") + case service == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing service") + case credentialStoreId == "": + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing credential store ID") + } + rt, ok := tok.Subtype.(*listtoken.StartRefreshToken) + if !ok { + return nil, errors.New(ctx, errors.InvalidParameter, op, "token did not have a start-refresh token component") + } + + listItemsFn := func(ctx context.Context, lastPageItem Static, limit int) ([]Static, time.Time, error) { + opts := []Option{ + WithLimit(limit), + } + if lastPageItem != nil { + opts = append(opts, WithStartPageAfterItem(lastPageItem)) + } + // Add the database read timeout to account for any creations missed due to concurrent + // transactions in the initial pagination phase. + return service.ListCredentialsRefresh(ctx, credentialStoreId, rt.PreviousPhaseUpperBound.Add(-globals.RefreshReadLookbackDuration), opts...) + } + listDeletedIdsFn := func(ctx context.Context, since time.Time) ([]string, time.Time, error) { + // Add the database read timeout to account for any deletions missed due to concurrent + // transactions in previous requests. + return service.ListDeletedCredentialIds(ctx, since.Add(-globals.RefreshReadLookbackDuration)) + } + + return pagination.ListRefresh(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, service.EstimatedCredentialCount, listDeletedIdsFn, tok) +} diff --git a/internal/credential/service_list_credentials_refresh_page.go b/internal/credential/service_list_credentials_refresh_page.go new file mode 100644 index 0000000000..5b6cb907fe --- /dev/null +++ b/internal/credential/service_list_credentials_refresh_page.go @@ -0,0 +1,79 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package credential + +import ( + "context" + "time" + + "github.com/hashicorp/boundary/globals" + "github.com/hashicorp/boundary/internal/errors" + "github.com/hashicorp/boundary/internal/listtoken" + "github.com/hashicorp/boundary/internal/pagination" +) + +// ListRefreshPage lists up to page size credentials, filtering out entries that +// do not pass the filter item function. It will automatically request +// more credentials from the database, at page size chunks, to fill the page. +// It will start its paging based on the information in the token. +// It returns a new list token used to continue pagination or refresh items. +// Credentials are ordered by update time descending (most recently updated first). +// Credentials may contain items that were already returned during the initial +// pagination phase. It also returns a list of any credentials deleted since the +// last response. +func ListRefreshPage( + ctx context.Context, + grantsHash []byte, + pageSize int, + filterItemFn pagination.ListFilterFunc[Static], + tok *listtoken.Token, + service CredentialService, + credentialStoreId string, +) (*pagination.ListResponse[Static], error) { + const op = "credential.ListRefreshPage" + + switch { + case len(grantsHash) == 0: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grants hash") + case pageSize < 1: + return nil, errors.New(ctx, errors.InvalidParameter, op, "page size must be at least 1") + case filterItemFn == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing filter item callback") + case tok == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing token") + case service == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing service") + case credentialStoreId == "": + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing credential store ID") + } + rt, ok := tok.Subtype.(*listtoken.RefreshToken) + if !ok { + return nil, errors.New(ctx, errors.InvalidParameter, op, "token did not have a refresh token component") + } + + listItemsFn := func(ctx context.Context, lastPageItem Static, limit int) ([]Static, time.Time, error) { + opts := []Option{ + WithLimit(limit), + } + if lastPageItem != nil { + opts = append(opts, WithStartPageAfterItem(lastPageItem)) + } else { + lastItem, err := tok.LastItem(ctx) + if err != nil { + return nil, time.Time{}, err + } + opts = append(opts, WithStartPageAfterItem(lastItem)) + } + // Add the database read timeout to account for any creations missed due to concurrent + // transactions in the original list pagination phase. + return service.ListCredentialsRefresh(ctx, credentialStoreId, rt.PhaseLowerBound.Add(-globals.RefreshReadLookbackDuration), opts...) + } + listDeletedIdsFn := func(ctx context.Context, since time.Time) ([]string, time.Time, error) { + // Add the database read timeout to account for any deletes missed due to concurrent + // transactions in the original list pagination phase. + return service.ListDeletedCredentialIds(ctx, since.Add(-globals.RefreshReadLookbackDuration)) + } + + return pagination.ListRefreshPage(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, service.EstimatedCredentialCount, listDeletedIdsFn, tok) +}