aherman
/
boundary
mirror of https://github.com/hashicorp/boundary
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

internal/credential: add pagination service functions

Browse Source 
The new service functions are used for various stages of pagination.
pull/4202/head
Johan Brandhorst-Satzkorn 2 years ago
parent e0092999de
commit fa898fe87b
5 changed files with 898 additions and 0 deletions
Show Stats Download Patch File Download Diff File

63
internal/credential/service_list_credentials.go
Unescape View File

@ -0,0 +1,63 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package credential
import (
"context"
"time"
"github.com/hashicorp/boundary/internal/errors"
"github.com/hashicorp/boundary/internal/pagination"
)
// CredentialService defines the interface expected
// to list credentials, deleted credential IDs and get
// an estimate count of total credentials.
type CredentialService interface {
EstimatedCredentialCount(context.Context) (int, error)
ListDeletedCredentialIds(context.Context, time.Time) ([]string, time.Time, error)
ListCredentials(context.Context, string, ...Option) ([]Static, time.Time, error)
ListCredentialsRefresh(context.Context, string, time.Time, ...Option) ([]Static, time.Time, error)
}
// List lists up to page size credentials, filtering out entries that
// do not pass the filter item function. It will automatically request
// more credentials from the database, at page size chunks, to fill the page.
// It returns a new list token used to continue pagination or refresh items.
// Credentials are ordered by create time descending (most recently created first).
func List(
ctx context.Context,
grantsHash []byte,
pageSize int,
filterItemFn pagination.ListFilterFunc[Static],
service CredentialService,
credentialStoreId string,
) (*pagination.ListResponse[Static], error) {
const op = "credential.List"
switch {
case len(grantsHash) == 0:
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grants hash")
case pageSize < 1:
return nil, errors.New(ctx, errors.InvalidParameter, op, "page size must be at least 1")
case filterItemFn == nil:
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing filter item callback")
case service == nil:
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing service")
case credentialStoreId == "":
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing credential store ID")
}
listItemsFn := func(ctx context.Context, lastPageItem Static, limit int) ([]Static, time.Time, error) {
opts := []Option{
WithLimit(limit),
}
if lastPageItem != nil {
opts = append(opts, WithStartPageAfterItem(lastPageItem))
}
return service.ListCredentials(ctx, credentialStoreId, opts...)
}
return pagination.List(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, service.EstimatedCredentialCount)
}

616
internal/credential/service_list_credentials_ext_test.go
Unescape View File

@ -0,0 +1,616 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package credential_test
import (
"context"
"slices"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/hashicorp/boundary/globals"
"github.com/hashicorp/boundary/internal/credential"
"github.com/hashicorp/boundary/internal/credential/static"
"github.com/hashicorp/boundary/internal/credential/static/store"
"github.com/hashicorp/boundary/internal/db"
"github.com/hashicorp/boundary/internal/db/timestamp"
"github.com/hashicorp/boundary/internal/iam"
"github.com/hashicorp/boundary/internal/kms"
"github.com/hashicorp/boundary/internal/listtoken"
"github.com/hashicorp/boundary/internal/types/resource"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/testing/protocmp"
"google.golang.org/protobuf/types/known/timestamppb"
)
func TestService_List(t *testing.T) {
// Set database read timeout to avoid duplicates in response
oldReadTimeout := globals.RefreshReadLookbackDuration
globals.RefreshReadLookbackDuration = 0
t.Cleanup(func() {
globals.RefreshReadLookbackDuration = oldReadTimeout
})
ctx := context.Background()
conn, _ := db.TestSetup(t, "postgres")
sqlDb, err := conn.SqlDB(ctx)
require.NoError(t, err)
rw := db.New(conn)
wrapper := db.TestWrapper(t)
kms := kms.TestKms(t, conn, wrapper)
_, prj := iam.TestScopes(t, iam.TestRepo(t, conn, wrapper))
fiveDaysAgo := time.Now().AddDate(0, 0, -5)
credStore := static.TestCredentialStore(t, conn, wrapper, prj.GetPublicId())
obj, _ := static.TestJsonObject(t)
creds := []credential.Static{
static.TestJsonCredential(t, conn, wrapper, credStore.GetPublicId(), prj.GetPublicId(), obj),
static.TestUsernamePasswordCredential(t, conn, wrapper, "someuser", "somepassword", credStore.GetPublicId(), prj.GetPublicId()),
static.TestSshPrivateKeyCredential(t, conn, wrapper, "someuser", static.TestSshPrivateKeyPem, credStore.GetPublicId(), prj.GetPublicId()),
static.TestJsonCredential(t, conn, wrapper, credStore.GetPublicId(), prj.GetPublicId(), obj),
static.TestJsonCredential(t, conn, wrapper, credStore.GetPublicId(), prj.GetPublicId(), obj),
}
// since we sort ascending, we need to reverse targets
slices.Reverse(creds)
// Run analyze to update count estimates
_, err = sqlDb.ExecContext(ctx, "analyze")
require.NoError(t, err)
repo, err := static.NewRepository(ctx, rw, rw, kms)
require.NoError(t, err)
cmpOpts := []cmp.Option{
cmpopts.IgnoreUnexported(
static.JsonCredential{},
store.JsonCredential{},
static.UsernamePasswordCredential{},
store.UsernamePasswordCredential{},
static.SshPrivateKeyCredential{},
store.SshPrivateKeyCredential{},
timestamp.Timestamp{},
timestamppb.Timestamp{},
),
protocmp.Transform(),
protocmp.IgnoreFields(
&store.JsonCredential{},
"object",
"object_encrypted",
),
protocmp.IgnoreFields(
&store.UsernamePasswordCredential{},
"password",
"ct_password",
),
protocmp.IgnoreFields(
&store.SshPrivateKeyCredential{},
"private_key",
"private_key_encrypted",
),
}
t.Run("List validation", func(t *testing.T) {
t.Parallel()
t.Run("missing grants hash", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
_, err := credential.List(ctx, nil, 1, filterFunc, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "missing grants hash")
})
t.Run("zero page size", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
_, err := credential.List(ctx, []byte("some hash"), 0, filterFunc, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "page size must be at least 1")
})
t.Run("negative page size", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
_, err := credential.List(ctx, []byte("some hash"), -1, filterFunc, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "page size must be at least 1")
})
t.Run("nil filter func", func(t *testing.T) {
t.Parallel()
_, err := credential.List(ctx, []byte("some hash"), 1, nil, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "missing filter item callback")
})
t.Run("nil repo", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
_, err := credential.List(ctx, []byte("some hash"), 1, filterFunc, nil, credStore.GetPublicId())
require.ErrorContains(t, err, "missing service")
})
t.Run("missing credential store ID", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
_, err := credential.List(ctx, []byte("some hash"), 1, filterFunc, repo, "")
require.ErrorContains(t, err, "missing credential store ID")
})
})
t.Run("ListPage validation", func(t *testing.T) {
t.Parallel()
t.Run("missing grants hash", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), "some-id", fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListPage(ctx, nil, 1, filterFunc, tok, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "missing grants hash")
})
t.Run("zero page size", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), "some-id", fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListPage(ctx, []byte("some hash"), 0, filterFunc, tok, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "page size must be at least 1")
})
t.Run("negative page size", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), "some-id", fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListPage(ctx, []byte("some hash"), -1, filterFunc, tok, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "page size must be at least 1")
})
t.Run("nil filter func", func(t *testing.T) {
t.Parallel()
tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), "some-id", fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListPage(ctx, []byte("some hash"), 1, nil, tok, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "missing filter item callback")
})
t.Run("nil token", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
_, err = credential.ListPage(ctx, []byte("some hash"), 1, filterFunc, nil, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "missing token")
})
t.Run("wrong token type", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListPage(ctx, []byte("some hash"), 1, filterFunc, tok, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "token did not have a pagination token component")
})
t.Run("nil repo", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), "some-id", fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListPage(ctx, []byte("some hash"), 1, filterFunc, tok, nil, credStore.GetPublicId())
require.ErrorContains(t, err, "missing service")
})
t.Run("missing credential store ID", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), "some-id", fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListPage(ctx, []byte("some hash"), 1, filterFunc, tok, repo, "")
require.ErrorContains(t, err, "missing credential store ID")
})
})
t.Run("ListRefresh validation", func(t *testing.T) {
t.Parallel()
t.Run("missing grants hash", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListRefresh(ctx, nil, 1, filterFunc, tok, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "missing grants hash")
})
t.Run("zero page size", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListRefresh(ctx, []byte("some hash"), 0, filterFunc, tok, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "page size must be at least 1")
})
t.Run("negative page size", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListRefresh(ctx, []byte("some hash"), -1, filterFunc, tok, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "page size must be at least 1")
})
t.Run("nil filter func", func(t *testing.T) {
t.Parallel()
tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListRefresh(ctx, []byte("some hash"), 1, nil, tok, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "missing filter item callback")
})
t.Run("nil token", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
_, err = credential.ListRefresh(ctx, []byte("some hash"), 1, filterFunc, nil, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "missing token")
})
t.Run("nil repo", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListRefresh(ctx, []byte("some hash"), 1, filterFunc, tok, nil, credStore.GetPublicId())
require.ErrorContains(t, err, "missing service")
})
t.Run("missing credential store ID", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
tok, err := listtoken.NewStartRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListRefresh(ctx, []byte("some hash"), 1, filterFunc, tok, repo, "")
require.ErrorContains(t, err, "missing credential store ID")
})
})
t.Run("ListRefreshPage validation", func(t *testing.T) {
t.Parallel()
t.Run("missing grants hash", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListRefreshPage(ctx, nil, 1, filterFunc, tok, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "missing grants hash")
})
t.Run("zero page size", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListRefreshPage(ctx, []byte("some hash"), 0, filterFunc, tok, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "page size must be at least 1")
})
t.Run("negative page size", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListRefreshPage(ctx, []byte("some hash"), -1, filterFunc, tok, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "page size must be at least 1")
})
t.Run("nil filter func", func(t *testing.T) {
t.Parallel()
tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListRefreshPage(ctx, []byte("some hash"), 1, nil, tok, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "missing filter item callback")
})
t.Run("nil token", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
_, err = credential.ListRefreshPage(ctx, []byte("some hash"), 1, filterFunc, nil, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "missing token")
})
t.Run("wrong token type", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
tok, err := listtoken.NewPagination(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), "some-id", fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListRefreshPage(ctx, []byte("some hash"), 1, filterFunc, tok, repo, credStore.GetPublicId())
require.ErrorContains(t, err, "token did not have a refresh token component")
})
t.Run("nil repo", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListRefreshPage(ctx, []byte("some hash"), 1, filterFunc, tok, nil, credStore.GetPublicId())
require.ErrorContains(t, err, "missing service")
})
t.Run("missing credential store id", func(t *testing.T) {
t.Parallel()
filterFunc := func(_ context.Context, c credential.Static) (bool, error) {
return true, nil
}
tok, err := listtoken.NewRefresh(ctx, fiveDaysAgo, resource.Target, []byte("some hash"), fiveDaysAgo, fiveDaysAgo, fiveDaysAgo, "some other id", fiveDaysAgo)
require.NoError(t, err)
_, err = credential.ListRefreshPage(ctx, []byte("some hash"), 1, filterFunc, tok, repo, "")
require.ErrorContains(t, err, "missing credential store ID")
})
})
t.Run("simple pagination", func(t *testing.T) {
filterFunc := func(context.Context, credential.Static) (bool, error) {
return true, nil
}
resp, err := credential.List(ctx, []byte("some hash"), 1, filterFunc, repo, credStore.GetPublicId())
require.NoError(t, err)
require.NotNil(t, resp.ListToken)
require.Equal(t, resp.ListToken.GrantsHash, []byte("some hash"))
require.False(t, resp.CompleteListing)
require.Equal(t, resp.EstimatedItemCount, 5)
require.Empty(t, resp.DeletedIds)
require.Len(t, resp.Items, 1)
require.Empty(t, cmp.Diff(resp.Items[0], creds[0], cmpOpts...))
resp2, err := credential.ListPage(ctx, []byte("some hash"), 1, filterFunc, resp.ListToken, repo, credStore.GetPublicId())
require.NoError(t, err)
require.Equal(t, resp2.ListToken.GrantsHash, []byte("some hash"))
require.False(t, resp2.CompleteListing)
require.Equal(t, resp2.EstimatedItemCount, 5)
require.Empty(t, resp2.DeletedIds)
require.Len(t, resp2.Items, 1)
require.Empty(t, cmp.Diff(resp2.Items[0], creds[1], cmpOpts...))
resp3, err := credential.ListPage(ctx, []byte("some hash"), 1, filterFunc, resp2.ListToken, repo, credStore.GetPublicId())
require.NoError(t, err)
require.Equal(t, resp3.ListToken.GrantsHash, []byte("some hash"))
require.False(t, resp3.CompleteListing)
require.Equal(t, resp3.EstimatedItemCount, 5)
require.Empty(t, resp3.DeletedIds)
require.Len(t, resp3.Items, 1)
require.Empty(t, cmp.Diff(resp3.Items[0], creds[2], cmpOpts...))
resp4, err := credential.ListPage(ctx, []byte("some hash"), 1, filterFunc, resp3.ListToken, repo, credStore.GetPublicId())
require.NoError(t, err)
require.Equal(t, resp4.ListToken.GrantsHash, []byte("some hash"))
require.False(t, resp4.CompleteListing)
require.Equal(t, resp4.EstimatedItemCount, 5)
require.Empty(t, resp4.DeletedIds)
require.Len(t, resp4.Items, 1)
require.Empty(t, cmp.Diff(resp4.Items[0], creds[3], cmpOpts...))
resp5, err := credential.ListPage(ctx, []byte("some hash"), 1, filterFunc, resp4.ListToken, repo, credStore.GetPublicId())
require.NoError(t, err)
require.Equal(t, resp5.ListToken.GrantsHash, []byte("some hash"))
require.True(t, resp5.CompleteListing)
require.Equal(t, resp5.EstimatedItemCount, 5)
require.Empty(t, resp5.DeletedIds)
require.Len(t, resp5.Items, 1)
require.Empty(t, cmp.Diff(resp5.Items[0], creds[4], cmpOpts...))
// Finished initial pagination phase, request refresh
// Expect no results.
resp6, err := credential.ListRefresh(ctx, []byte("some hash"), 1, filterFunc, resp5.ListToken, repo, credStore.GetPublicId())
require.NoError(t, err)
require.Equal(t, resp6.ListToken.GrantsHash, []byte("some hash"))
require.True(t, resp6.CompleteListing)
require.Equal(t, resp6.EstimatedItemCount, 5)
require.Empty(t, resp6.DeletedIds)
require.Empty(t, resp6.Items)
// Create some new credentials
newCred1 := static.TestJsonCredential(t, conn, wrapper, credStore.GetPublicId(), prj.GetPublicId(), obj)
newCred2 := static.TestUsernamePasswordCredential(t, conn, wrapper, "someuser", "somepassword", credStore.GetPublicId(), prj.GetPublicId())
newCred3 := static.TestSshPrivateKeyCredential(t, conn, wrapper, "someuser", static.TestSshPrivateKeyPem, credStore.GetPublicId(), prj.GetPublicId())
t.Cleanup(func() {
_, err := repo.DeleteCredential(ctx, credStore.ProjectId, newCred1.GetPublicId())
require.NoError(t, err)
_, err = repo.DeleteCredential(ctx, credStore.ProjectId, newCred2.GetPublicId())
require.NoError(t, err)
_, err = repo.DeleteCredential(ctx, credStore.ProjectId, newCred3.GetPublicId())
require.NoError(t, err)
// Run analyze to update count estimate
_, err = sqlDb.ExecContext(ctx, "analyze")
require.NoError(t, err)
})
// Run analyze to update count estimate
_, err = sqlDb.ExecContext(ctx, "analyze")
require.NoError(t, err)
// Refresh again, should get newCred3
resp7, err := credential.ListRefresh(ctx, []byte("some hash"), 1, filterFunc, resp6.ListToken, repo, credStore.GetPublicId())
require.NoError(t, err)
require.Equal(t, resp7.ListToken.GrantsHash, []byte("some hash"))
require.False(t, resp7.CompleteListing)
require.Equal(t, resp7.EstimatedItemCount, 8)
require.Empty(t, resp7.DeletedIds)
require.Len(t, resp7.Items, 1)
require.Empty(t, cmp.Diff(resp7.Items[0], newCred3, cmpOpts...))
// Refresh again, should get newCred2
resp8, err := credential.ListRefreshPage(ctx, []byte("some hash"), 1, filterFunc, resp6.ListToken, repo, credStore.GetPublicId())
require.NoError(t, err)
require.Equal(t, resp8.ListToken.GrantsHash, []byte("some hash"))
require.False(t, resp8.CompleteListing)
require.Equal(t, resp8.EstimatedItemCount, 8)
require.Empty(t, resp8.DeletedIds)
require.Len(t, resp8.Items, 1)
require.Empty(t, cmp.Diff(resp8.Items[0], newCred2, cmpOpts...))
// Refresh again, should get newCred1
resp9, err := credential.ListRefreshPage(ctx, []byte("some hash"), 1, filterFunc, resp6.ListToken, repo, credStore.GetPublicId())
require.NoError(t, err)
require.Equal(t, resp9.ListToken.GrantsHash, []byte("some hash"))
require.True(t, resp9.CompleteListing)
require.Equal(t, resp9.EstimatedItemCount, 8)
require.Empty(t, resp9.DeletedIds)
require.Len(t, resp9.Items, 1)
require.Empty(t, cmp.Diff(resp9.Items[0], newCred1, cmpOpts...))
// Refresh again, should get no results
resp10, err := credential.ListRefresh(ctx, []byte("some hash"), 1, filterFunc, resp6.ListToken, repo, credStore.GetPublicId())
require.NoError(t, err)
require.Equal(t, resp10.ListToken.GrantsHash, []byte("some hash"))
require.True(t, resp10.CompleteListing)
require.Equal(t, resp10.EstimatedItemCount, 8)
require.Empty(t, resp10.DeletedIds)
require.Empty(t, resp10.Items)
})
t.Run("simple pagination with aggressive filtering", func(t *testing.T) {
filterFunc := func(ctx context.Context, c credential.Static) (bool, error) {
return c.GetPublicId() == creds[1].GetPublicId() ||
c.GetPublicId() == creds[len(creds)-1].GetPublicId(), nil
}
resp, err := credential.List(ctx, []byte("some hash"), 1, filterFunc, repo, credStore.GetPublicId())
require.NoError(t, err)
require.NotNil(t, resp.ListToken)
require.Equal(t, resp.ListToken.GrantsHash, []byte("some hash"))
require.False(t, resp.CompleteListing)
require.Equal(t, resp.EstimatedItemCount, 5)
require.Empty(t, resp.DeletedIds)
require.Len(t, resp.Items, 1)
require.Empty(t, cmp.Diff(resp.Items[0], creds[1], cmpOpts...))
resp2, err := credential.ListPage(ctx, []byte("some hash"), 1, filterFunc, resp.ListToken, repo, credStore.GetPublicId())
require.NoError(t, err)
require.NotNil(t, resp2.ListToken)
require.Equal(t, resp2.ListToken.GrantsHash, []byte("some hash"))
require.True(t, resp2.CompleteListing)
require.Equal(t, resp2.EstimatedItemCount, 5)
require.Empty(t, resp2.DeletedIds)
require.Len(t, resp2.Items, 1)
require.Empty(t, cmp.Diff(resp2.Items[0], creds[len(creds)-1], cmpOpts...))
// request a refresh, nothing should be returned
resp3, err := credential.ListRefresh(ctx, []byte("some hash"), 1, filterFunc, resp.ListToken, repo, credStore.GetPublicId())
require.NoError(t, err)
require.Equal(t, resp3.ListToken.GrantsHash, []byte("some hash"))
require.True(t, resp3.CompleteListing)
require.Equal(t, resp3.EstimatedItemCount, 5)
require.Empty(t, resp3.DeletedIds)
require.Empty(t, resp3.Items)
newCred1 := static.TestJsonCredential(t, conn, wrapper, credStore.GetPublicId(), prj.GetPublicId(), obj)
newCred2 := static.TestUsernamePasswordCredential(t, conn, wrapper, "someuser", "somepassword", credStore.GetPublicId(), prj.GetPublicId())
newCred3 := static.TestSshPrivateKeyCredential(t, conn, wrapper, "someuser", static.TestSshPrivateKeyPem, credStore.GetPublicId(), prj.GetPublicId())
newCred4 := static.TestJsonCredential(t, conn, wrapper, credStore.GetPublicId(), prj.GetPublicId(), obj)
t.Cleanup(func() {
_, err := repo.DeleteCredential(ctx, credStore.ProjectId, newCred1.GetPublicId())
require.NoError(t, err)
_, err = repo.DeleteCredential(ctx, credStore.ProjectId, newCred2.GetPublicId())
require.NoError(t, err)
_, err = repo.DeleteCredential(ctx, credStore.ProjectId, newCred3.GetPublicId())
require.NoError(t, err)
_, err = repo.DeleteCredential(ctx, credStore.ProjectId, newCred4.GetPublicId())
require.NoError(t, err)
// Run analyze to update count estimate
_, err = sqlDb.ExecContext(ctx, "analyze")
require.NoError(t, err)
})
// Run analyze to update count estimate
_, err = sqlDb.ExecContext(ctx, "analyze")
require.NoError(t, err)
filterFunc = func(_ context.Context, c credential.Static) (bool, error) {
return c.GetPublicId() == newCred3.GetPublicId() ||
c.GetPublicId() == newCred1.GetPublicId(), nil
}
// Refresh again, should get newCred3
resp4, err := credential.ListRefresh(ctx, []byte("some hash"), 1, filterFunc, resp3.ListToken, repo, credStore.GetPublicId())
require.NoError(t, err)
require.Equal(t, resp4.ListToken.GrantsHash, []byte("some hash"))
require.False(t, resp4.CompleteListing)
require.Equal(t, resp4.EstimatedItemCount, 9)
require.Empty(t, resp4.DeletedIds)
require.Len(t, resp4.Items, 1)
require.Empty(t, cmp.Diff(resp4.Items[0], newCred3, cmpOpts...))
// Refresh again, should get newCred1
resp5, err := credential.ListRefreshPage(ctx, []byte("some hash"), 1, filterFunc, resp4.ListToken, repo, credStore.GetPublicId())
require.NoError(t, err)
require.Equal(t, resp5.ListToken.GrantsHash, []byte("some hash"))
require.True(t, resp5.CompleteListing)
require.Equal(t, resp5.EstimatedItemCount, 9)
require.Empty(t, resp5.DeletedIds)
require.Len(t, resp5.Items, 1)
require.Empty(t, cmp.Diff(resp5.Items[0], newCred1, cmpOpts...))
})
t.Run("simple pagination with deletion", func(t *testing.T) {
filterFunc := func(context.Context, credential.Static) (bool, error) {
return true, nil
}
deletedCredentialId := creds[0].GetPublicId()
_, err := repo.DeleteCredential(ctx, prj.GetPublicId(), deletedCredentialId)
require.NoError(t, err)
creds = creds[1:]
// Run analyze to update count estimate
_, err = sqlDb.ExecContext(ctx, "analyze")
require.NoError(t, err)
resp, err := credential.List(ctx, []byte("some hash"), 1, filterFunc, repo, credStore.GetPublicId())
require.NoError(t, err)
require.NotNil(t, resp.ListToken)
require.Equal(t, resp.ListToken.GrantsHash, []byte("some hash"))
require.False(t, resp.CompleteListing)
require.Equal(t, resp.EstimatedItemCount, 4)
require.Empty(t, resp.DeletedIds)
require.Len(t, resp.Items, 1)
require.Empty(t, cmp.Diff(resp.Items[0], creds[0], cmpOpts...))
// request remaining results
resp2, err := credential.ListPage(ctx, []byte("some hash"), 3, filterFunc, resp.ListToken, repo, credStore.GetPublicId())
require.NoError(t, err)
require.Equal(t, resp2.ListToken.GrantsHash, []byte("some hash"))
require.True(t, resp2.CompleteListing)
require.Equal(t, resp2.EstimatedItemCount, 4)
require.Empty(t, resp2.DeletedIds)
require.Len(t, resp2.Items, 3)
require.Empty(t, cmp.Diff(resp2.Items, creds[1:], cmpOpts...))
deletedCredentialId = creds[0].GetPublicId()
_, err = repo.DeleteCredential(ctx, prj.GetPublicId(), deletedCredentialId)
require.NoError(t, err)
creds = creds[1:]
// Run analyze to update count estimate
_, err = sqlDb.ExecContext(ctx, "analyze")
require.NoError(t, err)
// request a refresh, nothing should be returned except the deleted id
resp3, err := credential.ListRefresh(ctx, []byte("some hash"), 1, filterFunc, resp2.ListToken, repo, credStore.GetPublicId())
require.NoError(t, err)
require.Equal(t, resp3.ListToken.GrantsHash, []byte("some hash"))
require.True(t, resp3.CompleteListing)
require.Equal(t, resp3.EstimatedItemCount, 3)
require.Contains(t, resp3.DeletedIds, deletedCredentialId)
require.Empty(t, resp3.Items)
})
}

67
internal/credential/service_list_credentials_page.go
Unescape View File

@ -0,0 +1,67 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package credential
import (
"context"
"time"
"github.com/hashicorp/boundary/internal/errors"
"github.com/hashicorp/boundary/internal/listtoken"
"github.com/hashicorp/boundary/internal/pagination"
)
// ListPage lists up to page size credentials, filtering out entries that
// do not pass the filter item function. It will automatically request
// more credentials from the database, at page size chunks, to fill the page.
// It will start its paging based on the information in the token.
// It returns a new list token used to continue pagination or refresh items.
// Credentials are ordered by create time descending (most recently created first).
func ListPage(
ctx context.Context,
grantsHash []byte,
pageSize int,
filterItemFn pagination.ListFilterFunc[Static],
tok *listtoken.Token,
service CredentialService,
credentialStoreId string,
) (*pagination.ListResponse[Static], error) {
const op = "credential.ListPage"
switch {
case len(grantsHash) == 0:
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grants hash")
case pageSize < 1:
return nil, errors.New(ctx, errors.InvalidParameter, op, "page size must be at least 1")
case filterItemFn == nil:
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing filter item callback")
case tok == nil:
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing token")
case service == nil:
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing service")
case credentialStoreId == "":
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing credential store ID")
}
if _, ok := tok.Subtype.(*listtoken.PaginationToken); !ok {
return nil, errors.New(ctx, errors.InvalidParameter, op, "token did not have a pagination token component")
}
listItemsFn := func(ctx context.Context, lastPageItem Static, limit int) ([]Static, time.Time, error) {
opts := []Option{
WithLimit(limit),
}
if lastPageItem != nil {
opts = append(opts, WithStartPageAfterItem(lastPageItem))
} else {
lastItem, err := tok.LastItem(ctx)
if err != nil {
return nil, time.Time{}, err
}
opts = append(opts, WithStartPageAfterItem(lastItem))
}
return service.ListCredentials(ctx, credentialStoreId, opts...)
}
return pagination.ListPage(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, service.EstimatedCredentialCount, tok)
}

73
internal/credential/service_list_credentials_refresh.go
Unescape View File

@ -0,0 +1,73 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package credential
import (
"context"
"time"
"github.com/hashicorp/boundary/globals"
"github.com/hashicorp/boundary/internal/errors"
"github.com/hashicorp/boundary/internal/listtoken"
"github.com/hashicorp/boundary/internal/pagination"
)
// ListRefresh lists up to page size credentials, filtering out entries that
// do not pass the filter item function. It will automatically request
// more credentials from the database, at page size chunks, to fill the page.
// It will start its paging based on the information in the token.
// It returns a new list token used to continue pagination or refresh items.
// Credentials are ordered by update time descending (most recently updated first).
// Credentials may contain items that were already returned during the initial
// pagination phase. It also returns a list of any credentials deleted since the
// start of the initial pagination phase or last response.
func ListRefresh(
ctx context.Context,
grantsHash []byte,
pageSize int,
filterItemFn pagination.ListFilterFunc[Static],
tok *listtoken.Token,
service CredentialService,
credentialStoreId string,
) (*pagination.ListResponse[Static], error) {
const op = "credential.ListRefresh"
switch {
case len(grantsHash) == 0:
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grants hash")
case pageSize < 1:
return nil, errors.New(ctx, errors.InvalidParameter, op, "page size must be at least 1")
case filterItemFn == nil:
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing filter item callback")
case tok == nil:
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing token")
case service == nil:
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing service")
case credentialStoreId == "":
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing credential store ID")
}
rt, ok := tok.Subtype.(*listtoken.StartRefreshToken)
if !ok {
return nil, errors.New(ctx, errors.InvalidParameter, op, "token did not have a start-refresh token component")
}
listItemsFn := func(ctx context.Context, lastPageItem Static, limit int) ([]Static, time.Time, error) {
opts := []Option{
WithLimit(limit),
}
if lastPageItem != nil {
opts = append(opts, WithStartPageAfterItem(lastPageItem))
}
// Add the database read timeout to account for any creations missed due to concurrent
// transactions in the initial pagination phase.
return service.ListCredentialsRefresh(ctx, credentialStoreId, rt.PreviousPhaseUpperBound.Add(-globals.RefreshReadLookbackDuration), opts...)
}
listDeletedIdsFn := func(ctx context.Context, since time.Time) ([]string, time.Time, error) {
// Add the database read timeout to account for any deletions missed due to concurrent
// transactions in previous requests.
return service.ListDeletedCredentialIds(ctx, since.Add(-globals.RefreshReadLookbackDuration))
}
return pagination.ListRefresh(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, service.EstimatedCredentialCount, listDeletedIdsFn, tok)
}

79
internal/credential/service_list_credentials_refresh_page.go
Unescape View File

@ -0,0 +1,79 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: BUSL-1.1
package credential
import (
"context"
"time"
"github.com/hashicorp/boundary/globals"
"github.com/hashicorp/boundary/internal/errors"
"github.com/hashicorp/boundary/internal/listtoken"
"github.com/hashicorp/boundary/internal/pagination"
)
// ListRefreshPage lists up to page size credentials, filtering out entries that
// do not pass the filter item function. It will automatically request
// more credentials from the database, at page size chunks, to fill the page.
// It will start its paging based on the information in the token.
// It returns a new list token used to continue pagination or refresh items.
// Credentials are ordered by update time descending (most recently updated first).
// Credentials may contain items that were already returned during the initial
// pagination phase. It also returns a list of any credentials deleted since the
// last response.
func ListRefreshPage(
ctx context.Context,
grantsHash []byte,
pageSize int,
filterItemFn pagination.ListFilterFunc[Static],
tok *listtoken.Token,
service CredentialService,
credentialStoreId string,
) (*pagination.ListResponse[Static], error) {
const op = "credential.ListRefreshPage"
switch {
case len(grantsHash) == 0:
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grants hash")
case pageSize < 1:
return nil, errors.New(ctx, errors.InvalidParameter, op, "page size must be at least 1")
case filterItemFn == nil:
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing filter item callback")
case tok == nil:
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing token")
case service == nil:
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing service")
case credentialStoreId == "":
return nil, errors.New(ctx, errors.InvalidParameter, op, "missing credential store ID")
}
rt, ok := tok.Subtype.(*listtoken.RefreshToken)
if !ok {
return nil, errors.New(ctx, errors.InvalidParameter, op, "token did not have a refresh token component")
}
listItemsFn := func(ctx context.Context, lastPageItem Static, limit int) ([]Static, time.Time, error) {
opts := []Option{
WithLimit(limit),
}
if lastPageItem != nil {
opts = append(opts, WithStartPageAfterItem(lastPageItem))
} else {
lastItem, err := tok.LastItem(ctx)
if err != nil {
return nil, time.Time{}, err
}
opts = append(opts, WithStartPageAfterItem(lastItem))
}
// Add the database read timeout to account for any creations missed due to concurrent
// transactions in the original list pagination phase.
return service.ListCredentialsRefresh(ctx, credentialStoreId, rt.PhaseLowerBound.Add(-globals.RefreshReadLookbackDuration), opts...)
}
listDeletedIdsFn := func(ctx context.Context, since time.Time) ([]string, time.Time, error) {
// Add the database read timeout to account for any deletes missed due to concurrent
// transactions in the original list pagination phase.
return service.ListDeletedCredentialIds(ctx, since.Add(-globals.RefreshReadLookbackDuration))
}
return pagination.ListRefreshPage(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, service.EstimatedCredentialCount, listDeletedIdsFn, tok)
}
Write Preview
Loading…
Cancel
Save