From ad5b34da6bd00a7a8a79143a34bafac0c4b69bfd Mon Sep 17 00:00:00 2001 From: Johan Brandhorst-Satzkorn Date: Tue, 2 Jan 2024 17:18:47 -0800 Subject: [PATCH] internal/iam: add scope list pagination --- internal/iam/options_test.go | 24 + internal/iam/query.go | 4 + internal/iam/repository_scope.go | 142 +++++- internal/iam/repository_scope_test.go | 179 ++++++- internal/iam/scope.go | 11 + internal/iam/service_list.go | 41 ++ internal/iam/service_list_ext_test.go | 589 ++++++++++++++++++++++ internal/iam/service_list_page.go | 56 ++ internal/iam/service_list_refresh.go | 61 +++ internal/iam/service_list_refresh_page.go | 68 +++ 10 files changed, 1155 insertions(+), 20 deletions(-) diff --git a/internal/iam/options_test.go b/internal/iam/options_test.go index 663d38d665..7f6b692fc8 100644 --- a/internal/iam/options_test.go +++ b/internal/iam/options_test.go @@ -5,10 +5,27 @@ package iam import ( "testing" + "time" + "github.com/hashicorp/boundary/internal/db/timestamp" + "github.com/hashicorp/boundary/internal/pagination" "github.com/stretchr/testify/assert" ) +type fakeItem struct { + pagination.Item + publicId string + updateTime time.Time +} + +func (p *fakeItem) GetPublicId() string { + return p.publicId +} + +func (p *fakeItem) GetUpdateTime() *timestamp.Timestamp { + return timestamp.New(p.updateTime) +} + // Test_GetOpts provides unit tests for GetOpts and all the options func Test_GetOpts(t *testing.T) { t.Parallel() @@ -82,4 +99,11 @@ func Test_GetOpts(t *testing.T) { testOpts.withPrimaryAuthMethodId = "test" assert.Equal(opts, testOpts) }) + t.Run("WithStartPageAfterItem", func(t *testing.T) { + assert := assert.New(t) + updateTime := time.Now() + opts := getOpts(WithStartPageAfterItem(&fakeItem{nil, "s_1", updateTime})) + assert.Equal(opts.withStartPageAfterItem.GetPublicId(), "s_1") + assert.Equal(opts.withStartPageAfterItem.GetUpdateTime(), timestamp.New(updateTime)) + }) } diff --git a/internal/iam/query.go b/internal/iam/query.go index 4b67883862..1b140d033b 100644 --- a/internal/iam/query.go +++ b/internal/iam/query.go @@ -188,4 +188,8 @@ const ( estimateCountGroups = ` select reltuples::bigint as estimate from pg_class where oid in ('iam_group'::regclass) ` + + estimateCountScopes = ` + select reltuples::bigint as estimate from pg_class where oid in ('iam_scope'::regclass) + ` ) diff --git a/internal/iam/repository_scope.go b/internal/iam/repository_scope.go index 47e5ee8156..e913beff8c 100644 --- a/internal/iam/repository_scope.go +++ b/internal/iam/repository_scope.go @@ -6,11 +6,14 @@ package iam import ( "context" "crypto/rand" + "database/sql" "fmt" "strings" + "time" "github.com/hashicorp/boundary/globals" "github.com/hashicorp/boundary/internal/db" + "github.com/hashicorp/boundary/internal/db/timestamp" "github.com/hashicorp/boundary/internal/errors" "github.com/hashicorp/boundary/internal/kms" "github.com/hashicorp/boundary/internal/oplog" @@ -460,18 +463,98 @@ func (r *Repository) DeleteScope(ctx context.Context, withPublicId string, _ ... return rowsDeleted, nil } -// ListScopes with the parent IDs, supports the WithLimit option. -func (r *Repository) ListScopes(ctx context.Context, withParentIds []string, opt ...Option) ([]*Scope, error) { - const op = "iam.(Repository).ListScopes" +// listScopes lists scopes in the given scopes and supports WithLimit option. +func (r *Repository) listScopes(ctx context.Context, withParentIds []string, opt ...Option) ([]*Scope, time.Time, error) { + const op = "iam.(Repository).listScopes" if len(withParentIds) == 0 { - return nil, errors.New(ctx, errors.InvalidParameter, op, "missing parent id") + return nil, time.Time{}, errors.New(ctx, errors.InvalidParameter, op, "missing parent id") } - var items []*Scope - err := r.list(ctx, &items, "parent_id in (?)", []any{withParentIds}, opt...) - if err != nil { - return nil, errors.Wrap(ctx, err, op) + opts := getOpts(opt...) + + limit := r.defaultLimit + switch { + case opts.withLimit > 0: + // non-zero signals an override of the default limit for the repo. + limit = opts.withLimit + case opts.withLimit < 0: + return nil, time.Time{}, errors.New(ctx, errors.InvalidParameter, op, "limit must be non-negative") } - return items, nil + + var args []any + whereClause := "parent_id in @parent_ids" + args = append(args, sql.Named("parent_ids", withParentIds)) + + if opts.withStartPageAfterItem != nil { + whereClause = fmt.Sprintf("(create_time, public_id) < (@last_item_create_time, @last_item_id) and %s", whereClause) + args = append(args, + sql.Named("last_item_create_time", opts.withStartPageAfterItem.GetCreateTime()), + sql.Named("last_item_id", opts.withStartPageAfterItem.GetPublicId()), + ) + } + dbOpts := []db.Option{db.WithLimit(limit), db.WithOrder("create_time desc, public_id desc")} + return r.queryScopes(ctx, whereClause, args, dbOpts...) +} + +// listScopesRefresh lists scopes in the given scopes and supports the +// WithLimit and WithStartPageAfterItem options. +func (r *Repository) listScopesRefresh(ctx context.Context, updatedAfter time.Time, withParentIds []string, opt ...Option) ([]*Scope, time.Time, error) { + const op = "iam.(Repository).listScopesRefresh" + + switch { + case updatedAfter.IsZero(): + return nil, time.Time{}, errors.New(ctx, errors.InvalidParameter, op, "missing updated after time") + + case len(withParentIds) == 0: + return nil, time.Time{}, errors.New(ctx, errors.InvalidParameter, op, "missing parent id") + } + + opts := getOpts(opt...) + + limit := r.defaultLimit + switch { + case opts.withLimit > 0: + // non-zero signals an override of the default limit for the repo. + limit = opts.withLimit + case opts.withLimit < 0: + return nil, time.Time{}, errors.New(ctx, errors.InvalidParameter, op, "limit must be non-negative") + } + + var args []any + whereClause := "update_time > @updated_after_time and parent_id in @parent_ids" + args = append(args, + sql.Named("updated_after_time", timestamp.New(updatedAfter)), + sql.Named("parent_ids", withParentIds), + ) + if opts.withStartPageAfterItem != nil { + whereClause = fmt.Sprintf("(update_time, public_id) < (@last_item_update_time, @last_item_id) and %s", whereClause) + args = append(args, + sql.Named("last_item_update_time", opts.withStartPageAfterItem.GetUpdateTime()), + sql.Named("last_item_id", opts.withStartPageAfterItem.GetPublicId()), + ) + } + + dbOpts := []db.Option{db.WithLimit(limit), db.WithOrder("update_time desc, public_id desc")} + return r.queryScopes(ctx, whereClause, args, dbOpts...) +} + +func (r *Repository) queryScopes(ctx context.Context, whereClause string, args []any, opt ...db.Option) ([]*Scope, time.Time, error) { + const op = "iam.(Repository).queryScopes" + + var transactionTimestamp time.Time + var ret []*Scope + if _, err := r.writer.DoTx(ctx, db.StdRetryCnt, db.ExpBackoff{}, func(rd db.Reader, w db.Writer) error { + var inRet []*Scope + if err := rd.SearchWhere(ctx, &inRet, whereClause, args, opt...); err != nil { + return errors.Wrap(ctx, err, op) + } + ret = inRet + var err error + transactionTimestamp, err = rd.Now(ctx) + return err + }); err != nil { + return nil, time.Time{}, err + } + return ret, transactionTimestamp, nil } // ListScopesRecursively allows for recursive listing of scopes based on a root scope @@ -503,3 +586,44 @@ func (r *Repository) ListScopesRecursively(ctx context.Context, rootScopeId stri } return scopes, nil } + +// listScopeDeletedIds lists the public IDs of any scopes deleted since the timestamp provided. +func (r *Repository) listScopeDeletedIds(ctx context.Context, since time.Time) ([]string, time.Time, error) { + const op = "iam.(Repository).listScopeDeletedIds" + var deletedScopes []*deletedScope + var transactionTimestamp time.Time + if _, err := r.writer.DoTx(ctx, db.StdRetryCnt, db.ExpBackoff{}, func(r db.Reader, w db.Writer) error { + if err := r.SearchWhere(ctx, &deletedScopes, "delete_time >= ?", []any{since}); err != nil { + return errors.Wrap(ctx, err, op, errors.WithMsg("failed to query deleted scopes")) + } + var err error + transactionTimestamp, err = r.Now(ctx) + if err != nil { + return errors.Wrap(ctx, err, op, errors.WithMsg("failed to query transaction timestamp")) + } + return nil + }); err != nil { + return nil, time.Time{}, err + } + var scopeIds []string + for _, user := range deletedScopes { + scopeIds = append(scopeIds, user.PublicId) + } + return scopeIds, transactionTimestamp, nil +} + +// estimatedScopeCount returns and estimate of the total number of items in the scopes table. +func (r *Repository) estimatedScopeCount(ctx context.Context) (int, error) { + const op = "iam.(Repository).estimatedScopeCount" + rows, err := r.reader.Query(ctx, estimateCountScopes, nil) + if err != nil { + return 0, errors.Wrap(ctx, err, op, errors.WithMsg("failed to query total scopes")) + } + var count int + for rows.Next() { + if err := r.reader.ScanRows(ctx, rows, &count); err != nil { + return 0, errors.Wrap(ctx, err, op, errors.WithMsg("failed to query total scopes")) + } + } + return count, nil +} diff --git a/internal/iam/repository_scope_test.go b/internal/iam/repository_scope_test.go index 27d59c96d1..0054f1d6a5 100644 --- a/internal/iam/repository_scope_test.go +++ b/internal/iam/repository_scope_test.go @@ -496,15 +496,6 @@ func Test_Repository_ListScopes(t *testing.T) { wantCnt int wantErr bool }{ - { - name: "no-limit", - createCnt: repo.defaultLimit + 1, - args: args{ - opt: []Option{WithLimit(-1)}, - }, - wantCnt: repo.defaultLimit + 1, - wantErr: false, - }, { name: "default-limit", createCnt: repo.defaultLimit + 1, @@ -532,15 +523,107 @@ func Test_Repository_ListScopes(t *testing.T) { testOrgs = append(testOrgs, testOrg(t, repo, "", "")) } assert.Equal(tt.createCnt, len(testOrgs)) - got, err := repo.ListScopes(context.Background(), []string{"global"}, tt.args.opt...) + got, ttime, err := repo.listScopes(context.Background(), []string{"global"}, tt.args.opt...) if tt.wantErr { require.Error(err) return } require.NoError(err) assert.Equal(tt.wantCnt, len(got)) + // Transaction timestamp should be within ~10 seconds of now + assert.True(time.Now().Before(ttime.Add(10 * time.Second))) + assert.True(time.Now().After(ttime.Add(-10 * time.Second))) }) } + + t.Run("withStartPageAfter", func(t *testing.T) { + assert, require := assert.New(t), require.New(t) + ctx := context.Background() + + // Create 10 projects in a new org + org := testOrg(t, repo, "", "") + for i := 0; i < 10; i++ { + _ = testProject(t, repo, org.GetPublicId()) + } + + page1, ttime, err := repo.listScopes(ctx, []string{org.GetPublicId()}, WithLimit(2)) + require.NoError(err) + require.Len(page1, 2) + // Transaction timestamp should be within ~10 seconds of now + assert.True(time.Now().Before(ttime.Add(10 * time.Second))) + assert.True(time.Now().After(ttime.Add(-10 * time.Second))) + page2, ttime, err := repo.listScopes(ctx, []string{org.GetPublicId()}, WithLimit(2), WithStartPageAfterItem(page1[1])) + require.NoError(err) + require.Len(page2, 2) + assert.True(time.Now().Before(ttime.Add(10 * time.Second))) + assert.True(time.Now().After(ttime.Add(-10 * time.Second))) + for _, item := range page1 { + assert.NotEqual(item.GetPublicId(), page2[0].GetPublicId()) + assert.NotEqual(item.GetPublicId(), page2[1].GetPublicId()) + } + page3, ttime, err := repo.listScopes(ctx, []string{org.GetPublicId()}, WithLimit(2), WithStartPageAfterItem(page2[1])) + require.NoError(err) + require.Len(page3, 2) + assert.True(time.Now().Before(ttime.Add(10 * time.Second))) + assert.True(time.Now().After(ttime.Add(-10 * time.Second))) + for _, item := range page2 { + assert.NotEqual(item.GetPublicId(), page3[0].GetPublicId()) + assert.NotEqual(item.GetPublicId(), page3[1].GetPublicId()) + } + page4, ttime, err := repo.listScopes(ctx, []string{org.GetPublicId()}, WithLimit(2), WithStartPageAfterItem(page3[1])) + require.NoError(err) + assert.Len(page4, 2) + assert.True(time.Now().Before(ttime.Add(10 * time.Second))) + assert.True(time.Now().After(ttime.Add(-10 * time.Second))) + for _, item := range page3 { + assert.NotEqual(item.GetPublicId(), page4[0].GetPublicId()) + assert.NotEqual(item.GetPublicId(), page4[1].GetPublicId()) + } + page5, ttime, err := repo.listScopes(ctx, []string{org.GetPublicId()}, WithLimit(2), WithStartPageAfterItem(page4[1])) + require.NoError(err) + assert.Len(page5, 2) + assert.True(time.Now().Before(ttime.Add(10 * time.Second))) + assert.True(time.Now().After(ttime.Add(-10 * time.Second))) + for _, item := range page4 { + assert.NotEqual(item.GetPublicId(), page5[0].GetPublicId()) + assert.NotEqual(item.GetPublicId(), page5[1].GetPublicId()) + } + page6, ttime, err := repo.listScopes(ctx, []string{org.GetPublicId()}, WithLimit(2), WithStartPageAfterItem(page5[1])) + require.NoError(err) + assert.Empty(page6) + assert.True(time.Now().Before(ttime.Add(10 * time.Second))) + assert.True(time.Now().After(ttime.Add(-10 * time.Second))) + + // Create 2 new Scopes + newP1 := testProject(t, repo, org.GetPublicId()) + newP2 := testProject(t, repo, org.GetPublicId()) + + // since it will return newest to oldest, we get page1[1] first + page7, ttime, err := repo.listScopesRefresh( + ctx, + time.Now().Add(-1*time.Second), + []string{org.GetPublicId()}, + WithLimit(1), + ) + require.NoError(err) + require.Len(page7, 1) + require.Equal(page7[0].GetPublicId(), newP2.GetPublicId()) + assert.True(time.Now().Before(ttime.Add(10 * time.Second))) + assert.True(time.Now().After(ttime.Add(-10 * time.Second))) + + page8, ttime, err := repo.listScopesRefresh( + context.Background(), + time.Now().Add(-1*time.Second), + []string{org.GetPublicId()}, + WithLimit(1), + WithStartPageAfterItem(page7[0]), + ) + require.NoError(err) + require.Len(page8, 1) + require.Equal(page8[0].GetPublicId(), newP1.GetPublicId()) + assert.True(time.Now().Before(ttime.Add(10 * time.Second))) + assert.True(time.Now().After(ttime.Add(-10 * time.Second))) + }) } func TestRepository_ListScopes_Multiple_Scopes(t *testing.T) { @@ -565,7 +648,7 @@ func TestRepository_ListScopes_Multiple_Scopes(t *testing.T) { // Add global to the mix scopeIds = append(scopeIds, "global") - got, err := repo.ListScopes(context.Background(), scopeIds) + got, _, err := repo.listScopes(context.Background(), scopeIds) require.NoError(t, err) assert.Equal(t, total, len(got)) } @@ -620,3 +703,77 @@ func Test_Repository_ListRecursive(t *testing.T) { }) } } + +func Test_listDeletedIds(t *testing.T) { + t.Parallel() + ctx := context.Background() + conn, _ := db.TestSetup(t, "postgres") + wrapper := db.TestWrapper(t) + repo := TestRepo(t, conn, wrapper) + org := TestOrg(t, repo) + + // Expect no entries at the start + deletedIds, ttime, err := repo.listScopeDeletedIds(ctx, time.Now().AddDate(-1, 0, 0)) + require.NoError(t, err) + require.Empty(t, deletedIds) + + // Transaction time should be within ~10 seconds of now + now := time.Now() + assert.True(t, ttime.Add(-10*time.Second).Before(now)) + assert.True(t, ttime.Add(10*time.Second).After(now)) + + // Delete a scope + p := TestProject(t, repo, org.GetPublicId()) + _, err = repo.DeleteScope(ctx, p.PublicId) + require.NoError(t, err) + + // Expect a single entry + deletedIds, ttime, err = repo.listScopeDeletedIds(ctx, time.Now().AddDate(-1, 0, 0)) + require.NoError(t, err) + require.Equal(t, []string{p.PublicId}, deletedIds) + now = time.Now() + assert.True(t, ttime.Add(-10*time.Second).Before(now)) + assert.True(t, ttime.Add(10*time.Second).After(now)) + + // Try again with the time set to now, expect no entries + deletedIds, ttime, err = repo.listScopeDeletedIds(ctx, time.Now()) + require.NoError(t, err) + require.Empty(t, deletedIds) + now = time.Now() + assert.True(t, ttime.Add(-10*time.Second).Before(now)) + assert.True(t, ttime.Add(10*time.Second).After(now)) +} + +func Test_estimatedScopeCount(t *testing.T) { + t.Parallel() + ctx := context.Background() + conn, _ := db.TestSetup(t, "postgres") + wrapper := db.TestWrapper(t) + repo := TestRepo(t, conn, wrapper) + sqlDb, err := conn.SqlDB(ctx) + require.NoError(t, err) + + // Check total entries at start, expect 1 (global) + numItems, err := repo.estimatedScopeCount(ctx) + require.NoError(t, err) + assert.Equal(t, 1, numItems) + + // Create a scope, expect 2 entries + org := TestOrg(t, repo) + // Run analyze to update estimate + _, err = sqlDb.ExecContext(ctx, "analyze") + require.NoError(t, err) + numItems, err = repo.estimatedScopeCount(ctx) + require.NoError(t, err) + assert.Equal(t, 2, numItems) + + // Delete the scope, expect 1 again + _, err = repo.DeleteScope(ctx, org.PublicId) + require.NoError(t, err) + // Run analyze to update estimate + _, err = sqlDb.ExecContext(ctx, "analyze") + require.NoError(t, err) + numItems, err = repo.estimatedScopeCount(ctx) + require.NoError(t, err) + assert.Equal(t, 1, numItems) +} diff --git a/internal/iam/scope.go b/internal/iam/scope.go index cef27dadca..b182966f2c 100644 --- a/internal/iam/scope.go +++ b/internal/iam/scope.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/hashicorp/boundary/internal/db" + "github.com/hashicorp/boundary/internal/db/timestamp" "github.com/hashicorp/boundary/internal/errors" "github.com/hashicorp/boundary/internal/iam/store" "github.com/hashicorp/boundary/internal/types/action" @@ -214,3 +215,13 @@ func (s *Scope) TableName() string { func (s *Scope) SetTableName(n string) { s.tableName = n } + +type deletedScope struct { + PublicId string `gorm:"primary_key"` + DeleteTime *timestamp.Timestamp +} + +// TableName returns the tablename to override the default gorm table name +func (s *deletedScope) TableName() string { + return "iam_scope_deleted" +} diff --git a/internal/iam/service_list.go b/internal/iam/service_list.go index 567b5658e9..7664944d51 100644 --- a/internal/iam/service_list.go +++ b/internal/iam/service_list.go @@ -133,3 +133,44 @@ func ListGroups( return pagination.List(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, repo.estimatedGroupCount) } + +// ListScopes lists up to page size scopes, filtering out entries that +// do not pass the filter item function. It will automatically request +// more scopes from the database, at page size chunks, to fill the page. +// It returns a new list token used to continue pagination or refresh items. +// Scopes are ordered by create time descending (most recently created first). +func ListScopes( + ctx context.Context, + grantsHash []byte, + pageSize int, + filterItemFn pagination.ListFilterFunc[*Scope], + repo *Repository, + withParentIds []string, +) (*pagination.ListResponse[*Scope], error) { + const op = "iam.ListScopes" + + switch { + case len(grantsHash) == 0: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grants hash") + case pageSize < 1: + return nil, errors.New(ctx, errors.InvalidParameter, op, "page size must be at least 1") + case filterItemFn == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing filter item callback") + case repo == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing repo") + case len(withParentIds) == 0: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing parent ids") + } + + listItemsFn := func(ctx context.Context, lastPageItem *Scope, limit int) ([]*Scope, time.Time, error) { + opts := []Option{ + WithLimit(limit), + } + if lastPageItem != nil { + opts = append(opts, WithStartPageAfterItem(lastPageItem)) + } + return repo.listScopes(ctx, withParentIds, opts...) + } + + return pagination.List(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, repo.estimatedScopeCount) +} diff --git a/internal/iam/service_list_ext_test.go b/internal/iam/service_list_ext_test.go index 54a80e0a19..8e268dfa69 100644 --- a/internal/iam/service_list_ext_test.go +++ b/internal/iam/service_list_ext_test.go @@ -1828,3 +1828,592 @@ func TestService_ListGroups(t *testing.T) { require.Empty(t, resp3.Items) }) } + +func TestService_ListScopes(t *testing.T) { + fiveDaysAgo := time.Now() + // Set database read timeout to avoid duplicates in response + oldReadTimeout := globals.RefreshReadLookbackDuration + globals.RefreshReadLookbackDuration = 0 + t.Cleanup(func() { + globals.RefreshReadLookbackDuration = oldReadTimeout + }) + ctx := context.Background() + conn, _ := db.TestSetup(t, "postgres") + sqlDB, err := conn.SqlDB(context.Background()) + require.NoError(t, err) + rw := db.New(conn) + wrapper := db.TestWrapper(t) + kms := kms.TestKms(t, conn, wrapper) + iamRepo := iam.TestRepo(t, conn, wrapper) + org := iam.TestOrg(t, iamRepo) + + repo, err := iam.NewRepository(ctx, rw, rw, kms) + require.NoError(t, err) + + var allResources []*iam.Scope + for i := 0; i < 5; i++ { + r := iam.TestProject(t, repo, org.GetPublicId()) + allResources = append(allResources, r) + } + + // Reverse since we read items in descending order (newest first) + slices.Reverse(allResources) + + // Run analyze to update postgres estimates + _, err = sqlDB.ExecContext(ctx, "analyze") + require.NoError(t, err) + + cmpIgnoreUnexportedOpts := cmpopts.IgnoreUnexported(iam.Scope{}, store.Scope{}, timestamp.Timestamp{}, timestamppb.Timestamp{}) + + t.Run("List validation", func(t *testing.T) { + t.Parallel() + t.Run("missing grants hash", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + _, err := iam.ListScopes(ctx, nil, 1, filterFunc, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "missing grants hash") + }) + t.Run("zero page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + _, err := iam.ListScopes(ctx, []byte("some hash"), 0, filterFunc, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("negative page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + _, err := iam.ListScopes(ctx, []byte("some hash"), -1, filterFunc, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("nil filter func", func(t *testing.T) { + t.Parallel() + _, err := iam.ListScopes(ctx, []byte("some hash"), 1, nil, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "missing filter item callback") + }) + t.Run("nil repo", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + _, err := iam.ListScopes(ctx, []byte("some hash"), 1, filterFunc, nil, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "missing repo") + }) + t.Run("missing parent ids", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + _, err := iam.ListScopes(ctx, []byte("some hash"), 1, filterFunc, repo, nil) + require.ErrorContains(t, err, "missing parent ids") + }) + }) + t.Run("ListPage validation", func(t *testing.T) { + t.Parallel() + t.Run("missing grants hash", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesPage(ctx, nil, 1, filterFunc, tok, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "missing grants hash") + }) + t.Run("zero page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesPage(ctx, []byte("some hash"), 0, filterFunc, tok, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("negative page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesPage(ctx, []byte("some hash"), -1, filterFunc, tok, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("nil filter func", func(t *testing.T) { + t.Parallel() + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesPage(ctx, []byte("some hash"), 1, nil, tok, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "missing filter item callback") + }) + t.Run("nil token", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + _, err = iam.ListScopesPage(ctx, []byte("some hash"), 1, filterFunc, nil, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "missing token") + }) + t.Run("wrong token type", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesPage(ctx, []byte("some hash"), 1, filterFunc, tok, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "token did not have a pagination token component") + }) + t.Run("nil repo", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesPage(ctx, []byte("some hash"), 1, filterFunc, tok, nil, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "missing repo") + }) + t.Run("missing parent ids", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesPage(ctx, []byte("some hash"), 1, filterFunc, tok, repo, nil) + require.ErrorContains(t, err, "missing parent ids") + }) + t.Run("wrong token resource type", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesPage(ctx, []byte("some hash"), 1, filterFunc, tok, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "token did not have a scope resource type") + }) + }) + t.Run("ListRefresh validation", func(t *testing.T) { + t.Parallel() + t.Run("missing grants hash", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesRefresh(ctx, nil, 1, filterFunc, tok, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "missing grants hash") + }) + t.Run("zero page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesRefresh(ctx, []byte("some hash"), 0, filterFunc, tok, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("negative page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesRefresh(ctx, []byte("some hash"), -1, filterFunc, tok, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("nil filter func", func(t *testing.T) { + t.Parallel() + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesRefresh(ctx, []byte("some hash"), 1, nil, tok, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "missing filter item callback") + }) + t.Run("nil token", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + _, err = iam.ListScopesRefresh(ctx, []byte("some hash"), 1, filterFunc, nil, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "missing token") + }) + t.Run("wrong token type", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesRefresh(ctx, []byte("some hash"), 1, filterFunc, tok, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "token did not have a start-refresh token component") + }) + t.Run("nil repo", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesRefresh(ctx, []byte("some hash"), 1, filterFunc, tok, nil, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "missing repo") + }) + t.Run("missing parent ids", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesRefresh(ctx, []byte("some hash"), 1, filterFunc, tok, repo, nil) + require.ErrorContains(t, err, "missing parent ids") + }) + t.Run("wrong token resource type", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesRefresh(ctx, []byte("some hash"), 1, filterFunc, tok, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "token did not have a scope resource type") + }) + }) + t.Run("ListRefreshPage validation", func(t *testing.T) { + t.Parallel() + t.Run("missing grants hash", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesRefreshPage(ctx, nil, 1, filterFunc, tok, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "missing grants hash") + }) + t.Run("zero page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesRefreshPage(ctx, []byte("some hash"), 0, filterFunc, tok, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("negative page size", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesRefreshPage(ctx, []byte("some hash"), -1, filterFunc, tok, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "page size must be at least 1") + }) + t.Run("nil filter func", func(t *testing.T) { + t.Parallel() + tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesRefreshPage(ctx, []byte("some hash"), 1, nil, tok, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "missing filter item callback") + }) + t.Run("nil token", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + _, err = iam.ListScopesRefreshPage(ctx, []byte("some hash"), 1, filterFunc, nil, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "missing token") + }) + t.Run("wrong token type", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), "some-id", fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesRefreshPage(ctx, []byte("some hash"), 1, filterFunc, tok, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "token did not have a refresh token component") + }) + t.Run("nil repo", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesRefreshPage(ctx, []byte("some hash"), 1, filterFunc, tok, nil, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "missing repo") + }) + t.Run("missing parent ids", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.Scope, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesRefreshPage(ctx, []byte("some hash"), 1, filterFunc, tok, repo, nil) + require.ErrorContains(t, err, "missing parent ids") + }) + t.Run("wrong token resource type", func(t *testing.T) { + t.Parallel() + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo) + require.NoError(t, err) + _, err = iam.ListScopesRefreshPage(ctx, []byte("some hash"), 1, filterFunc, tok, repo, []string{org.GetPublicId()}) + require.ErrorContains(t, err, "token did not have a scope resource type") + }) + }) + + t.Run("simple pagination", func(t *testing.T) { + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + resp, err := iam.ListScopes(ctx, []byte("some hash"), 1, filterFunc, repo, []string{org.GetPublicId()}) + require.NoError(t, err) + require.NotNil(t, resp.ListToken) + require.Equal(t, resp.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp.CompleteListing) + require.Equal(t, resp.EstimatedItemCount, 7) // Includes global and org + require.Empty(t, resp.DeletedIds) + require.Len(t, resp.Items, 1) + require.Empty(t, cmp.Diff(resp.Items[0], allResources[0], cmpIgnoreUnexportedOpts)) + + resp2, err := iam.ListScopesPage(ctx, []byte("some hash"), 1, filterFunc, resp.ListToken, repo, []string{org.GetPublicId()}) + require.NoError(t, err) + require.Equal(t, resp2.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp2.CompleteListing) + require.Equal(t, resp2.EstimatedItemCount, 7) // Includes global and org + require.Empty(t, resp2.DeletedIds) + require.Len(t, resp2.Items, 1) + require.Empty(t, cmp.Diff(resp2.Items[0], allResources[1], cmpIgnoreUnexportedOpts)) + + resp3, err := iam.ListScopesPage(ctx, []byte("some hash"), 1, filterFunc, resp2.ListToken, repo, []string{org.GetPublicId()}) + require.NoError(t, err) + require.Equal(t, resp3.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp3.CompleteListing) + require.Equal(t, resp3.EstimatedItemCount, 7) // Includes global and org + require.Empty(t, resp3.DeletedIds) + require.Len(t, resp3.Items, 1) + require.Empty(t, cmp.Diff(resp3.Items[0], allResources[2], cmpIgnoreUnexportedOpts)) + + resp4, err := iam.ListScopesPage(ctx, []byte("some hash"), 1, filterFunc, resp3.ListToken, repo, []string{org.GetPublicId()}) + require.NoError(t, err) + require.Equal(t, resp4.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp4.CompleteListing) + require.Equal(t, resp4.EstimatedItemCount, 7) // Includes global and org + require.Empty(t, resp4.DeletedIds) + require.Len(t, resp4.Items, 1) + require.Empty(t, cmp.Diff(resp4.Items[0], allResources[3], cmpIgnoreUnexportedOpts)) + + resp5, err := iam.ListScopesPage(ctx, []byte("some hash"), 1, filterFunc, resp4.ListToken, repo, []string{org.GetPublicId()}) + require.NoError(t, err) + require.Equal(t, resp5.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp5.CompleteListing) + require.Equal(t, resp5.EstimatedItemCount, 7) // Includes global and org + require.Empty(t, resp5.DeletedIds) + require.Len(t, resp5.Items, 1) + require.Empty(t, cmp.Diff(resp5.Items[0], allResources[4], cmpIgnoreUnexportedOpts)) + + // Finished initial pagination phase, request refresh + // Expect no results. + resp6, err := iam.ListScopesRefresh(ctx, []byte("some hash"), 1, filterFunc, resp5.ListToken, repo, []string{org.GetPublicId()}) + require.NoError(t, err) + require.Equal(t, resp6.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp6.CompleteListing) + require.Equal(t, resp6.EstimatedItemCount, 7) // Includes global and org + require.Empty(t, resp6.DeletedIds) + require.Empty(t, resp6.Items) + + // Create some new roles + newP1 := iam.TestProject(t, repo, org.GetPublicId()) + newP2 := iam.TestProject(t, repo, org.GetPublicId()) + t.Cleanup(func() { + _, err = repo.DeleteScope(ctx, newP1.GetPublicId()) + require.NoError(t, err) + _, err = repo.DeleteScope(ctx, newP2.GetPublicId()) + require.NoError(t, err) + // Run analyze to update count estimate + _, err = sqlDB.ExecContext(ctx, "analyze") + require.NoError(t, err) + }) + // Run analyze to update count estimate + _, err = sqlDB.ExecContext(ctx, "analyze") + require.NoError(t, err) + + // Refresh again, should get newR2 + resp7, err := iam.ListScopesRefresh(ctx, []byte("some hash"), 1, filterFunc, resp6.ListToken, repo, []string{org.GetPublicId()}) + require.NoError(t, err) + require.Equal(t, resp7.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp7.CompleteListing) + require.Equal(t, resp7.EstimatedItemCount, 9) + require.Empty(t, resp7.DeletedIds) + require.Len(t, resp7.Items, 1) + require.Empty(t, cmp.Diff(resp7.Items[0], newP2, cmpIgnoreUnexportedOpts)) + + // Refresh again, should get newR1 + resp8, err := iam.ListScopesRefreshPage(ctx, []byte("some hash"), 1, filterFunc, resp7.ListToken, repo, []string{org.GetPublicId()}) + require.NoError(t, err) + require.Equal(t, resp8.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp8.CompleteListing) + require.Equal(t, resp8.EstimatedItemCount, 9) + require.Empty(t, resp8.DeletedIds) + require.Len(t, resp8.Items, 1) + require.Empty(t, cmp.Diff(resp8.Items[0], newP1, cmpIgnoreUnexportedOpts)) + + // Refresh again, should get no results + resp9, err := iam.ListScopesRefresh(ctx, []byte("some hash"), 1, filterFunc, resp8.ListToken, repo, []string{org.GetPublicId()}) + require.NoError(t, err) + require.Equal(t, resp9.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp9.CompleteListing) + require.Equal(t, resp9.EstimatedItemCount, 9) + require.Empty(t, resp9.DeletedIds) + require.Empty(t, resp9.Items) + }) + + t.Run("simple pagination with aggressive filtering", func(t *testing.T) { + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return r.GetPublicId() == allResources[1].GetPublicId() || + r.GetPublicId() == allResources[len(allResources)-1].GetPublicId(), nil + } + resp, err := iam.ListScopes(ctx, []byte("some hash"), 1, filterFunc, repo, []string{org.GetPublicId()}) + require.NoError(t, err) + require.NotNil(t, resp.ListToken) + require.Equal(t, resp.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp.CompleteListing) + require.Equal(t, resp.EstimatedItemCount, 7) // Includes global and org + require.Empty(t, resp.DeletedIds) + require.Len(t, resp.Items, 1) + require.Empty(t, cmp.Diff(resp.Items[0], allResources[1], cmpIgnoreUnexportedOpts)) + + resp2, err := iam.ListScopesPage(ctx, []byte("some hash"), 1, filterFunc, resp.ListToken, repo, []string{org.GetPublicId()}) + require.NoError(t, err) + require.NotNil(t, resp2.ListToken) + require.Equal(t, resp2.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp2.CompleteListing) + require.Equal(t, resp2.EstimatedItemCount, 7) // Includes global and org + require.Empty(t, resp2.DeletedIds) + require.Len(t, resp2.Items, 1) + require.Empty(t, cmp.Diff(resp2.Items[0], allResources[len(allResources)-1], cmpIgnoreUnexportedOpts)) + + // request a refresh, nothing should be returned + resp3, err := iam.ListScopesRefresh(ctx, []byte("some hash"), 1, filterFunc, resp.ListToken, repo, []string{org.GetPublicId()}) + require.NoError(t, err) + require.Equal(t, resp3.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp3.CompleteListing) + require.Equal(t, resp3.EstimatedItemCount, 7) // Includes global and org + require.Empty(t, resp3.DeletedIds) + require.Empty(t, resp3.Items) + + // Create some new tokens + newP1 := iam.TestProject(t, repo, org.GetPublicId()) + newP2 := iam.TestProject(t, repo, org.GetPublicId()) + newP3 := iam.TestProject(t, repo, org.GetPublicId()) + newP4 := iam.TestProject(t, repo, org.GetPublicId()) + // Run analyze to update count estimate + _, err = sqlDB.ExecContext(ctx, "analyze") + require.NoError(t, err) + t.Cleanup(func() { + _, err = repo.DeleteScope(ctx, newP1.GetPublicId()) + require.NoError(t, err) + _, err = repo.DeleteScope(ctx, newP2.GetPublicId()) + require.NoError(t, err) + _, err = repo.DeleteScope(ctx, newP3.GetPublicId()) + require.NoError(t, err) + _, err = repo.DeleteScope(ctx, newP4.GetPublicId()) + require.NoError(t, err) + // Run analyze to update count estimate + _, err = sqlDB.ExecContext(ctx, "analyze") + require.NoError(t, err) + }) + + filterFunc = func(_ context.Context, r *iam.Scope) (bool, error) { + return r.GetPublicId() == newP3.GetPublicId() || + r.GetPublicId() == newP1.GetPublicId(), nil + } + // Refresh again, should get newP3 + resp4, err := iam.ListScopesRefresh(ctx, []byte("some hash"), 1, filterFunc, resp3.ListToken, repo, []string{org.GetPublicId()}) + require.NoError(t, err) + require.Equal(t, resp4.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp4.CompleteListing) + require.Equal(t, resp4.EstimatedItemCount, 11) + require.Empty(t, resp4.DeletedIds) + require.Len(t, resp4.Items, 1) + require.Empty(t, cmp.Diff(resp4.Items[0], newP3, cmpIgnoreUnexportedOpts)) + + // Refresh again, should get newP1 + resp5, err := iam.ListScopesRefreshPage(ctx, []byte("some hash"), 1, filterFunc, resp4.ListToken, repo, []string{org.GetPublicId()}) + require.NoError(t, err) + require.Equal(t, resp5.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp5.CompleteListing) + require.Equal(t, resp5.EstimatedItemCount, 11) + require.Empty(t, resp5.DeletedIds) + require.Len(t, resp5.Items, 1) + require.Empty(t, cmp.Diff(resp5.Items[0], newP1, cmpIgnoreUnexportedOpts)) + }) + + t.Run("simple pagination with deletion", func(t *testing.T) { + filterFunc := func(_ context.Context, r *iam.Scope) (bool, error) { + return true, nil + } + deletedScopeId := allResources[0].GetPublicId() + _, err := repo.DeleteScope(ctx, deletedScopeId) + require.NoError(t, err) + allResources = allResources[1:] + + // Run analyze to update count estimate + _, err = sqlDB.ExecContext(ctx, "analyze") + require.NoError(t, err) + + resp, err := iam.ListScopes(ctx, []byte("some hash"), 1, filterFunc, repo, []string{org.GetPublicId()}) + require.NoError(t, err) + require.NotNil(t, resp.ListToken) + require.Equal(t, resp.ListToken.GrantsHash, []byte("some hash")) + require.False(t, resp.CompleteListing) + require.Equal(t, resp.EstimatedItemCount, 6) + require.Empty(t, resp.DeletedIds) + require.Len(t, resp.Items, 1) + require.Empty(t, cmp.Diff(resp.Items[0], allResources[0], cmpIgnoreUnexportedOpts)) + + // request remaining results + resp2, err := iam.ListScopesPage(ctx, []byte("some hash"), 3, filterFunc, resp.ListToken, repo, []string{org.GetPublicId()}) + require.NoError(t, err) + require.Equal(t, resp2.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp2.CompleteListing) + require.Equal(t, resp2.EstimatedItemCount, 6) + require.Empty(t, resp2.DeletedIds) + require.Len(t, resp2.Items, 3) + require.Empty(t, cmp.Diff(resp2.Items, allResources[1:], cmpIgnoreUnexportedOpts)) + + deletedScopeId = allResources[0].GetPublicId() + _, err = repo.DeleteScope(ctx, deletedScopeId) + require.NoError(t, err) + allResources = allResources[1:] + + // Run analyze to update count estimate + _, err = sqlDB.ExecContext(ctx, "analyze") + require.NoError(t, err) + + // request a refresh, nothing should be returned except the deleted id + resp3, err := iam.ListScopesRefresh(ctx, []byte("some hash"), 1, filterFunc, resp2.ListToken, repo, []string{org.GetPublicId()}) + require.NoError(t, err) + require.Equal(t, resp3.ListToken.GrantsHash, []byte("some hash")) + require.True(t, resp3.CompleteListing) + require.Equal(t, resp3.EstimatedItemCount, 5) + require.Contains(t, resp3.DeletedIds, deletedScopeId) + require.Empty(t, resp3.Items) + }) +} diff --git a/internal/iam/service_list_page.go b/internal/iam/service_list_page.go index 7d93ae4ff4..e8daf6f5f1 100644 --- a/internal/iam/service_list_page.go +++ b/internal/iam/service_list_page.go @@ -180,3 +180,59 @@ func ListGroupsPage( return pagination.ListPage(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, repo.estimatedGroupCount, tok) } + +// ListScopesPage lists up to page size scopes, filtering out entries that +// do not pass the filter item function. It will automatically request +// more scopes from the database, at page size chunks, to fill the page. +// It will start its paging based on the information in the token. +// It returns a new list token used to continue pagination or refresh items. +// Scopes are ordered by create time descending (most recently created first). +func ListScopesPage( + ctx context.Context, + grantsHash []byte, + pageSize int, + filterItemFn pagination.ListFilterFunc[*Scope], + tok *listtoken.Token, + repo *Repository, + parentIds []string, +) (*pagination.ListResponse[*Scope], error) { + const op = "iam.ListScopesPage" + + switch { + case len(grantsHash) == 0: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grants hash") + case pageSize < 1: + return nil, errors.New(ctx, errors.InvalidParameter, op, "page size must be at least 1") + case filterItemFn == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing filter item callback") + case tok == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing token") + case repo == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing repo") + case len(parentIds) == 0: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing parent ids") + case tok.ResourceType != resource.Scope: + return nil, errors.New(ctx, errors.InvalidParameter, op, "token did not have a scope resource type") + } + if _, ok := tok.Subtype.(*listtoken.PaginationToken); !ok { + return nil, errors.New(ctx, errors.InvalidParameter, op, "token did not have a pagination token component") + } + + listItemsFn := func(ctx context.Context, lastPageItem *Scope, limit int) ([]*Scope, time.Time, error) { + opts := []Option{ + WithLimit(limit), + } + if lastPageItem != nil { + opts = append(opts, WithStartPageAfterItem(lastPageItem)) + } else { + lastItem, err := tok.LastItem(ctx) + if err != nil { + return nil, time.Time{}, err + } + opts = append(opts, WithStartPageAfterItem(lastItem)) + } + return repo.listScopes(ctx, parentIds, opts...) + } + + return pagination.ListPage(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, repo.estimatedScopeCount, tok) +} diff --git a/internal/iam/service_list_refresh.go b/internal/iam/service_list_refresh.go index 4c60648522..9938439494 100644 --- a/internal/iam/service_list_refresh.go +++ b/internal/iam/service_list_refresh.go @@ -196,3 +196,64 @@ func ListGroupsRefresh( return pagination.ListRefresh(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, repo.estimatedGroupCount, listDeletedIdsFn, tok) } + +// ListScopesRefresh lists up to page size scopes, filtering out entries that +// do not pass the filter item function. It will automatically request +// more scopes from the database, at page size chunks, to fill the page. +// It will start its paging based on the information in the token. +// It returns a new list token used to continue pagination or refresh items. +// Scopes are ordered by update time descending (most recently updated first). +// Scopes may contain items that were already returned during the initial +// pagination phase. It also returns a list of any scopes deleted since the +// start of the initial pagination phase or last response. +func ListScopesRefresh( + ctx context.Context, + grantsHash []byte, + pageSize int, + filterItemFn pagination.ListFilterFunc[*Scope], + tok *listtoken.Token, + repo *Repository, + withParentIds []string, +) (*pagination.ListResponse[*Scope], error) { + const op = "iam.ListScopesRefresh" + + switch { + case len(grantsHash) == 0: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grants hash") + case pageSize < 1: + return nil, errors.New(ctx, errors.InvalidParameter, op, "page size must be at least 1") + case filterItemFn == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing filter item callback") + case tok == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing token") + case repo == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing repo") + case len(withParentIds) == 0: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing parent ids") + case tok.ResourceType != resource.Scope: + return nil, errors.New(ctx, errors.InvalidParameter, op, "token did not have a scope resource type") + } + rt, ok := tok.Subtype.(*listtoken.StartRefreshToken) + if !ok { + return nil, errors.New(ctx, errors.InvalidParameter, op, "token did not have a start-refresh token component") + } + + listItemsFn := func(ctx context.Context, lastPageItem *Scope, limit int) ([]*Scope, time.Time, error) { + opts := []Option{ + WithLimit(limit), + } + if lastPageItem != nil { + opts = append(opts, WithStartPageAfterItem(lastPageItem)) + } + // Add the database read timeout to account for any creations missed due to concurrent + // transactions in the initial pagination phase. + return repo.listScopesRefresh(ctx, rt.PreviousPhaseUpperBound.Add(-globals.RefreshReadLookbackDuration), withParentIds, opts...) + } + listDeletedIdsFn := func(ctx context.Context, since time.Time) ([]string, time.Time, error) { + // Add the database read timeout to account for any deletions missed due to concurrent + // transactions in previous requests. + return repo.listScopeDeletedIds(ctx, since.Add(-globals.RefreshReadLookbackDuration)) + } + + return pagination.ListRefresh(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, repo.estimatedScopeCount, listDeletedIdsFn, tok) +} diff --git a/internal/iam/service_list_refresh_page.go b/internal/iam/service_list_refresh_page.go index be65a97803..1868d8feb3 100644 --- a/internal/iam/service_list_refresh_page.go +++ b/internal/iam/service_list_refresh_page.go @@ -217,3 +217,71 @@ func ListGroupsRefreshPage( return pagination.ListRefreshPage(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, repo.estimatedGroupCount, listDeletedIdsFn, tok) } + +// ListScopesRefreshPage lists up to page size scopes, filtering out entries that +// do not pass the filter item function. It will automatically request +// more scopes from the database, at page size chunks, to fill the page. +// It will start its paging based on the information in the token. +// It returns a new list token used to continue pagination or refresh items. +// Scopes are ordered by update time descending (most recently updated first). +// Scopes may contain items that were already returned during the initial +// pagination phase. It also returns a list of any scopes deleted since the +// last response. +func ListScopesRefreshPage( + ctx context.Context, + grantsHash []byte, + pageSize int, + filterItemFn pagination.ListFilterFunc[*Scope], + tok *listtoken.Token, + repo *Repository, + withParentIds []string, +) (*pagination.ListResponse[*Scope], error) { + const op = "iam.ListScopesRefreshPage" + + switch { + case len(grantsHash) == 0: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grants hash") + case pageSize < 1: + return nil, errors.New(ctx, errors.InvalidParameter, op, "page size must be at least 1") + case filterItemFn == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing filter item callback") + case tok == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing token") + case repo == nil: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing repo") + case len(withParentIds) == 0: + return nil, errors.New(ctx, errors.InvalidParameter, op, "missing parent ids") + case tok.ResourceType != resource.Scope: + return nil, errors.New(ctx, errors.InvalidParameter, op, "token did not have a scope resource type") + } + rt, ok := tok.Subtype.(*listtoken.RefreshToken) + if !ok { + return nil, errors.New(ctx, errors.InvalidParameter, op, "token did not have a refresh token component") + } + + listItemsFn := func(ctx context.Context, lastPageItem *Scope, limit int) ([]*Scope, time.Time, error) { + opts := []Option{ + WithLimit(limit), + } + if lastPageItem != nil { + opts = append(opts, WithStartPageAfterItem(lastPageItem)) + } else { + lastItem, err := tok.LastItem(ctx) + if err != nil { + return nil, time.Time{}, err + } + opts = append(opts, WithStartPageAfterItem(lastItem)) + } + // Add the database read timeout to account for any creations missed due to concurrent + // transactions in the original list pagination phase. + return repo.listScopesRefresh(ctx, rt.PhaseLowerBound.Add(-globals.RefreshReadLookbackDuration), withParentIds, opts...) + } + + listDeletedIdsFn := func(ctx context.Context, since time.Time) ([]string, time.Time, error) { + // Add the database read timeout to account for any deletes missed due to concurrent + // transactions in the original list pagination phase. + return repo.listScopeDeletedIds(ctx, since.Add(-globals.RefreshReadLookbackDuration)) + } + + return pagination.ListRefreshPage(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, repo.estimatedScopeCount, listDeletedIdsFn, tok) +}