feat(apptoken): Implement ListRefresh (#6384)

* feat(apptoken): Implement listAppTokensRefresh

* feat(apptokens): Implement ListRefresh

internal/apptoken/apptoken.go

@@ -360,3 +360,15 @@ func (atc *appTokenCipher) decrypt(ctx context.Context, cipher wrapping.Wrapper)
 	}
 	return nil
 }
+
+// deletedAppToken represents a deleted app token record in the app_token_deleted table.
+// These records are trimmed after a 30 day retention period.
+type deletedAppToken struct {
+	PublicId   string `gorm:"primary_key"`
+	DeleteTime *timestamp.Timestamp
+}
+
+// TableName returns the tablename to override the default gorm table name
+func (at *deletedAppToken) TableName() string {
+	return "app_token_deleted"
+}

internal/apptoken/repository.go

@@ -453,6 +453,53 @@ func (r *Repository) listAppTokens(ctx context.Context, withScopeIds []string, o
 	return r.queryAppTokens(ctx, whereClause, args, dbOpts...)
 }
 
+// listAppTokenRefresh lists tokens across all three token subtypes (global, org, proj) that have been
+// updated after the provided time. Cipher information and permissions are not included when listing a token.
+// App Tokens are considered updated when
+//   - update_time is after updatedAfter
+//   - expiration_time is after updatedAfter but before now
+//   - last_approximate_access_time + time_to_stale_seconds is (before now and before expiration_time) and after updatedAfter
+func (r *Repository) listAppTokensRefresh(ctx context.Context, updatedAfter time.Time, withScopeIds []string, opt ...Option) ([]*AppToken, time.Time, error) {
+	const op = "apptoken.(Repository).listAppTokenRefresh"
+
+	switch {
+	case updatedAfter.IsZero():
+		return nil, time.Time{}, errors.New(ctx, errors.InvalidParameter, op, "missing updatedAfter time")
+
+	case len(withScopeIds) == 0:
+		return nil, time.Time{}, errors.New(ctx, errors.InvalidParameter, op, "missing scope ids")
+	}
+
+	opts := getOpts(opt...)
+
+	limit := r.defaultLimit
+	if opts.withLimit != 0 {
+		limit = opts.withLimit
+	}
+
+	args := []any{
+		sql.Named("scope_ids", withScopeIds),
+		sql.Named("updated_after_time", timestamp.New(updatedAfter)),
+	}
+	whereClause := "scope_id in @scope_ids and " +
+		"(update_time > @updated_after_time or " +
+		"(expiration_time > @updated_after_time and expiration_time <= CURRENT_TIMESTAMP) or " +
+		"( (approximate_last_access_time is not null and time_to_stale_seconds is not null) and " +
+		"( (approximate_last_access_time + (time_to_stale_seconds || ' seconds')::interval) <= CURRENT_TIMESTAMP) and " +
+		"( (approximate_last_access_time + (time_to_stale_seconds || ' seconds')::interval) > @updated_after_time) and " +
+		"(expiration_time is null or (approximate_last_access_time + (time_to_stale_seconds || ' seconds')::interval) < expiration_time) ))"
+	if opts.withStartPageAfterItem != nil {
+		whereClause = fmt.Sprintf("(update_time, public_id) < (@last_item_update_time, @last_item_id) and %s", whereClause)
+		args = append(args,
+			sql.Named("last_item_update_time", opts.withStartPageAfterItem.GetUpdateTime()),
+			sql.Named("last_item_id", opts.withStartPageAfterItem.GetPublicId()),
+		)
+	}
+
+	dbOpts := []db.Option{db.WithLimit(limit), db.WithOrder("update_time desc, public_id desc")}
+	return r.queryAppTokens(ctx, whereClause, args, dbOpts...)
+}
+
 func (r *Repository) queryAppTokens(ctx context.Context, whereClause string, args []any, opt ...db.Option) ([]*AppToken, time.Time, error) {
 	const op = "apptoken.(Repository).queryAppTokens"
 
@@ -476,6 +523,31 @@ func (r *Repository) queryAppTokens(ctx context.Context, whereClause string, arg
 	return appTokens, transactionTimestamp, nil
 }
 
+// listDeletedIds lists the public IDs of any app tokens deleted since the timestamp provided.
+func (r *Repository) listDeletedIds(ctx context.Context, since time.Time) ([]string, time.Time, error) {
+	const op = "apptoken.(Repository).listDeletedIds"
+	var deletedAppTokens []*deletedAppToken
+	var transactionTimestamp time.Time
+	if _, err := r.writer.DoTx(ctx, db.StdRetryCnt, db.ExpBackoff{}, func(r db.Reader, _ db.Writer) error {
+		if err := r.SearchWhere(ctx, &deletedAppTokens, "delete_time >= ?", []any{since}); err != nil {
+			return errors.Wrap(ctx, err, op, errors.WithMsg("failed to query deleted app tokens"))
+		}
+		var err error
+		transactionTimestamp, err = r.Now(ctx)
+		if err != nil {
+			return errors.Wrap(ctx, err, op, errors.WithMsg("failed to get transaction timestamp"))
+		}
+		return nil
+	}); err != nil {
+		return nil, time.Time{}, err
+	}
+	var deletedIds []string
+	for _, at := range deletedAppTokens {
+		deletedIds = append(deletedIds, at.PublicId)
+	}
+	return deletedIds, transactionTimestamp, nil
+}
+
 // estimatedCount returns an estimate of the total number of items in the global, org, and project app token tables.
 func (r *Repository) estimatedCount(ctx context.Context) (int, error) {
 	const op = "apptoken.(Repository).estimatedCount"

internal/apptoken/repository_test.go

@@ -6,6 +6,7 @@ package apptoken
 import (
 	"context"
 	"database/sql"
+	"fmt"
 	"testing"
 	"time"
 
@@ -681,9 +682,9 @@ func TestRepository_queryAppTokens(t *testing.T) {
 	orgUser := iam.TestUser(t, iamRepo, org1.PublicId)
 
 	// Create app tokens
-	gToken := TestAppToken(t, repo, globals.GlobalPrefix, []string{"ids=*;type=scope;actions=list,read"}, globalUser, true, globals.GrantScopeIndividual)
-	orgToken := TestAppToken(t, repo, org1.PublicId, []string{"ids=*;type=scope;actions=list,read"}, orgUser, true, globals.GrantScopeIndividual)
-	projToken := TestAppToken(t, repo, proj1.PublicId, []string{"ids=*;type=scope;actions=list,read"}, orgUser, true, globals.GrantScopeIndividual)
+	gToken := TestAppToken(t, repo, globals.GlobalPrefix, globalUser, 0, nil, []string{"ids=*;type=scope;actions=list,read"}, true, globals.GrantScopeIndividual)
+	orgToken := TestAppToken(t, repo, org1.PublicId, orgUser, 0, nil, []string{"ids=*;type=scope;actions=list,read"}, true, globals.GrantScopeIndividual)
+	projToken := TestAppToken(t, repo, proj1.PublicId, orgUser, 0, nil, []string{"ids=*;type=scope;actions=list,read"}, true, globals.GrantScopeIndividual)
 
 	testCases := []struct {
 		name            string
@@ -781,9 +782,9 @@ func TestRepository_listAppTokens(t *testing.T) {
 	orgUser := iam.TestUser(t, iamRepo, org1.PublicId)
 
 	// Create app tokens
-	gToken := TestAppToken(t, repo, globals.GlobalPrefix, []string{"ids=*;type=scope;actions=list,read"}, globalUser, true, globals.GrantScopeIndividual)
-	orgToken := TestAppToken(t, repo, org1.PublicId, []string{"ids=*;type=scope;actions=list,read"}, orgUser, true, globals.GrantScopeIndividual)
-	projToken := TestAppToken(t, repo, proj1.PublicId, []string{"ids=*;type=scope;actions=list,read"}, orgUser, true, globals.GrantScopeIndividual)
+	gToken := TestAppToken(t, repo, globals.GlobalPrefix, globalUser, 0, nil, []string{"ids=*;type=scope;actions=list,read"}, true, globals.GrantScopeIndividual)
+	orgToken := TestAppToken(t, repo, org1.PublicId, orgUser, 0, nil, []string{"ids=*;type=scope;actions=list,read"}, true, globals.GrantScopeIndividual)
+	projToken := TestAppToken(t, repo, proj1.PublicId, orgUser, 0, nil, []string{"ids=*;type=scope;actions=list,read"}, true, globals.GrantScopeIndividual)
 
 	testCases := []struct {
 		name           string
@@ -877,7 +878,7 @@ func TestRepository_listAppTokens(t *testing.T) {
 
 		// Create enough tokens to exceed the limit
 		for range make([]int, 5) {
-			TestAppToken(t, repo, orgExceedLimit.PublicId, []string{"ids=*;type=scope;actions=list,read"}, orgExceedLimitUser, true, "individual")
+			TestAppToken(t, repo, orgExceedLimit.PublicId, orgExceedLimitUser, 0, nil, []string{"ids=*;type=scope;actions=list,read"}, true, "individual")
 		}
 
 		tokens, _, err := repo.listAppTokens(ctx, []string{orgExceedLimit.PublicId}, []Option{WithLimit(10)}...)
@@ -890,6 +891,139 @@ func TestRepository_listAppTokens(t *testing.T) {
 	})
 }
 
+func TestRepository_listAppTokensRefresh(t *testing.T) {
+	ctx := t.Context()
+	conn, _ := db.TestSetup(t, "postgres")
+	wrap := db.TestWrapper(t)
+	repo := TestRepo(t, conn, wrap)
+	iamRepo := iam.TestRepo(t, conn, wrap)
+
+	// Create test data
+	org1, proj1 := iam.TestScopes(t, iamRepo, iam.WithName("org1"), iam.WithDescription("Test Org 1"))
+	globalUser := iam.TestUser(t, iamRepo, globals.GlobalPrefix)
+	orgUser := iam.TestUser(t, iamRepo, org1.PublicId)
+
+	expireInSixSeconds := timestamp.New(timestamp.Now().AsTime().Add(6 * time.Second))
+
+	// Create app tokens
+	allTokens := []*AppToken{}
+	for range 3 {
+		gToken := TestAppToken(t, repo, globals.GlobalPrefix, globalUser, 4, expireInSixSeconds, []string{"ids=*;type=scope;actions=list,read"}, true, globals.GrantScopeIndividual)
+		orgToken := TestAppToken(t, repo, org1.PublicId, orgUser, 4, expireInSixSeconds, []string{"ids=*;type=scope;actions=list,read"}, true, globals.GrantScopeIndividual)
+		projToken := TestAppToken(t, repo, proj1.PublicId, orgUser, 4, expireInSixSeconds, []string{"ids=*;type=scope;actions=list,read"}, true, globals.GrantScopeIndividual)
+
+		allTokens = append(allTokens, gToken, orgToken, projToken)
+	}
+
+	t.Run("list and refresh global, org, and project tokens", func(t *testing.T) {
+		assert, require := assert.New(t), require.New(t)
+
+		tokens, refreshTime, err := repo.listAppTokens(ctx, []string{globals.GlobalPrefix, org1.PublicId, proj1.PublicId}, []Option{WithLimit(10)}...)
+		require.NoError(err)
+		assert.Equal(9, len(tokens))
+		assert.NotZero(refreshTime)
+
+		time.Sleep(1 * time.Second) // ensure time difference for refresh
+
+		// refresh list and see that no tokens are returned since none have been updated
+		tokens, refreshTime, err = repo.listAppTokensRefresh(ctx, refreshTime, []string{globals.GlobalPrefix, org1.PublicId, proj1.PublicId}, []Option{WithLimit(10)}...)
+		require.NoError(err)
+		assert.Equal(0, len(tokens))
+
+		// update token to trigger refresh
+		time.Sleep(1 * time.Second) // ensure time difference for refresh
+		testUpdateAppToken(t, repo, allTokens[0].PublicId, globals.GlobalPrefix, map[string]any{"name": "updated-global-name", "update_time": timestamp.New(timestamp.Now().AsTime())})
+		testUpdateAppToken(t, repo, allTokens[1].PublicId, org1.PublicId, map[string]any{"name": "updated-org-name", "update_time": timestamp.New(timestamp.Now().AsTime())})
+		testUpdateAppToken(t, repo, allTokens[2].PublicId, proj1.PublicId, map[string]any{"name": "updated-proj-name", "update_time": timestamp.New(timestamp.Now().AsTime())})
+
+		// refresh list and see that three updated tokens are returned
+		tokens, refreshTime, err = repo.listAppTokensRefresh(ctx, refreshTime, []string{globals.GlobalPrefix, org1.PublicId, proj1.PublicId}, []Option{WithLimit(10)}...)
+		require.NoError(err)
+		assert.Equal(3, len(tokens))
+		for _, token := range tokens {
+			found := false
+			for _, at := range allTokens[:3] {
+				if token.PublicId == at.PublicId {
+					found = true
+					break
+				}
+			}
+			assert.True(found)
+		}
+
+		// move time forward to trigger last_approximate_access_time + time_to_stale_seconds is (before now and before expiration_time) and after updatedAfter
+		time.Sleep(2 * time.Second)
+
+		// refresh list and see that all nine tokens are returned
+		tokens, refreshTime, err = repo.listAppTokensRefresh(ctx, refreshTime, []string{globals.GlobalPrefix, org1.PublicId, proj1.PublicId}, []Option{WithLimit(10)}...)
+		require.NoError(err)
+		assert.NotNil(refreshTime)
+		assert.Equal(9, len(tokens))
+
+		// move time forward so that expiration_time is after updatedAfter but before now
+		time.Sleep(4 * time.Second)
+
+		// refresh list and see that all nine tokens are returned
+		tokens, refreshTime, err = repo.listAppTokensRefresh(ctx, refreshTime, []string{globals.GlobalPrefix, org1.PublicId, proj1.PublicId}, []Option{WithLimit(10)}...)
+		require.NoError(err)
+		assert.NotNil(refreshTime)
+		assert.Equal(9, len(tokens))
+	})
+
+	t.Run("list refresh with missing updatedAfter time", func(t *testing.T) {
+		t.Parallel()
+		assert, require := assert.New(t), require.New(t)
+
+		emptyTimestamp := timestamp.New(time.Time{})
+		tokens, refreshTime, err := repo.listAppTokensRefresh(ctx, emptyTimestamp.AsTime(), []string{globals.GlobalPrefix, org1.PublicId, proj1.PublicId}, []Option{WithLimit(10)}...)
+		require.Error(err)
+		assert.Contains(err.Error(), "apptoken.(Repository).listAppTokenRefresh: missing updatedAfter time")
+		assert.Nil(tokens)
+		assert.Zero(refreshTime)
+	})
+
+	t.Run("list refresh with missing scope ids", func(t *testing.T) {
+		t.Parallel()
+		assert, require := assert.New(t), require.New(t)
+
+		tokens, refreshTime, err := repo.listAppTokensRefresh(ctx, timestamp.New(time.Now().Add(-1*time.Hour)).AsTime(), []string{}, []Option{WithLimit(10)}...)
+		require.Error(err)
+		assert.Contains(err.Error(), "apptoken.(Repository).listAppTokenRefresh: missing scope ids")
+		assert.Nil(tokens)
+		assert.Zero(refreshTime)
+	})
+}
+
+func TestRepository_listDeletedIds(t *testing.T) {
+	ctx := t.Context()
+	conn, _ := db.TestSetup(t, "postgres")
+	wrap := db.TestWrapper(t)
+	repo := TestRepo(t, conn, wrap)
+	assert, require := assert.New(t), require.New(t)
+
+	// Create test data
+	deletedIds := []string{}
+	sqlDb, err := conn.SqlDB(ctx)
+	require.NoError(err)
+	for i := 0; i < 5; i++ {
+		deletedId := fmt.Sprintf("deleted-id-%d", i)
+		deletedIds = append(deletedIds, deletedId)
+		_, err := sqlDb.ExecContext(ctx, "INSERT INTO app_token_deleted (public_id) VALUES ($1)", deletedId)
+		require.NoError(err)
+	}
+
+	retrievedIds, txnTimestamp, err := repo.listDeletedIds(ctx, time.Now().Add(-1*time.Minute))
+	require.NoError(err)
+	assert.NotNil(txnTimestamp)
+	assert.ElementsMatch(deletedIds, retrievedIds)
+
+	// Test with future timestamp, expect no results
+	retrievedIds, txnTimestamp, err = repo.listDeletedIds(ctx, time.Now().Add(1*time.Minute))
+	require.NoError(err)
+	assert.NotNil(txnTimestamp)
+	assert.Empty(retrievedIds)
+}
+
 func TestRepository_estimatedCount(t *testing.T) {
 	t.Parallel()
 	ctx := t.Context()
@@ -919,9 +1053,9 @@ func TestRepository_estimatedCount(t *testing.T) {
 	org, proj := iam.TestScopes(t, iamRepo, iam.WithName("org1"), iam.WithDescription("Test Org 1"))
 	orgUser := iam.TestUser(t, iamRepo, org.PublicId)
 
-	gToken := TestAppToken(t, repo, globals.GlobalPrefix, []string{"ids=*;type=scope;actions=list,read"}, globalUser, true, globals.GrantScopeIndividual)
-	oToken := TestAppToken(t, repo, org.PublicId, []string{"ids=*;type=scope;actions=list,read"}, orgUser, true, globals.GrantScopeIndividual)
-	pToken := TestAppToken(t, repo, proj.PublicId, []string{"ids=*;type=scope;actions=list,read"}, orgUser, true, globals.GrantScopeIndividual)
+	gToken := TestAppToken(t, repo, globals.GlobalPrefix, globalUser, 0, nil, []string{"ids=*;type=scope;actions=list,read"}, true, globals.GrantScopeIndividual)
+	oToken := TestAppToken(t, repo, org.PublicId, orgUser, 0, nil, []string{"ids=*;type=scope;actions=list,read"}, true, globals.GrantScopeIndividual)
+	pToken := TestAppToken(t, repo, proj.PublicId, orgUser, 0, nil, []string{"ids=*;type=scope;actions=list,read"}, true, globals.GrantScopeIndividual)
 
 	// Run analyze to update estimate
 	_, err = sqlDb.ExecContext(ctx, "analyze")

internal/apptoken/repository_token_grant_test.go

@@ -403,7 +403,7 @@ func TestGrantsForToken(t *testing.T) {
 			}
 
 			// Create a token with the specified grants
-			token := TestAppToken(t, repo, tc.tokenScopeId, tc.grants, tc.u, tc.grantThisScope, tc.grantScope)
+			token := TestAppToken(t, repo, tc.tokenScopeId, tc.u, 0, nil, tc.grants, tc.grantThisScope, tc.grantScope)
 
 			// Fetch the grants for the token
 			gt, err := repo.GrantsForToken(ctx, token.PublicId, tc.rTypes, tc.reqScopeId, opts...)

internal/apptoken/service_list_refresh.go

@@ -0,0 +1,76 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package apptoken
+
+import (
+	"context"
+	"time"
+
+	"github.com/hashicorp/boundary/globals"
+	"github.com/hashicorp/boundary/internal/errors"
+	"github.com/hashicorp/boundary/internal/listtoken"
+	"github.com/hashicorp/boundary/internal/pagination"
+	"github.com/hashicorp/boundary/internal/types/resource"
+)
+
+// ListRefresh lists up to page size app tokens, filtering out entries that
+// do not pass the filter item function. It will automatically request
+// more app tokens from the database, at page size chunks, to fill the page.
+// It will start its paging based on the information in the token.
+// It returns a new list token used to continue pagination or refresh items.
+// App tokens are ordered by update time descending (most recently updated first).
+// App tokens may contain items that were already returned during the initial
+// pagination phase. It also returns a list of any app tokens deleted since the
+// start of the initial pagination phase or last response.
+func ListRefresh(
+	ctx context.Context,
+	grantsHash []byte,
+	pageSize int,
+	filterItemFn pagination.ListFilterFunc[*AppToken],
+	tok *listtoken.Token,
+	repo *Repository,
+	withScopeIds []string,
+) (*pagination.ListResponse[*AppToken], error) {
+	const op = "apptoken.ListRefresh"
+
+	switch {
+	case len(grantsHash) == 0:
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "missing grants hash")
+	case pageSize < 1:
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "page size must be at least 1")
+	case filterItemFn == nil:
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "missing filter item callback")
+	case tok == nil:
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "missing token")
+	case repo == nil:
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "missing repo")
+	case withScopeIds == nil:
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "missing scope ids")
+	case tok.ResourceType != resource.AppToken:
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "token did not have an app token resource type")
+	}
+	rt, ok := tok.Subtype.(*listtoken.StartRefreshToken)
+	if !ok {
+		return nil, errors.New(ctx, errors.InvalidParameter, op, "token did not have a start-refresh token component")
+	}
+
+	listItemsFn := func(ctx context.Context, lastPageItem *AppToken, limit int) ([]*AppToken, time.Time, error) {
+		opts := []Option{
+			WithLimit(limit),
+		}
+		if lastPageItem != nil {
+			opts = append(opts, WithStartPageAfterItem(lastPageItem))
+		}
+		// Add the database read timeout to account for any creations missed due to concurrent
+		// transactions in the initial pagination phase.
+		return repo.listAppTokensRefresh(ctx, rt.PreviousPhaseUpperBound.Add(-globals.RefreshReadLookbackDuration), withScopeIds, opts...)
+	}
+	listDeletedIdsFn := func(ctx context.Context, since time.Time) ([]string, time.Time, error) {
+		// Add the database read timeout to account for any deletions missed due to concurrent
+		// transactions in previous requests.
+		return repo.listDeletedIds(ctx, since.Add(-globals.RefreshReadLookbackDuration))
+	}
+
+	return pagination.ListRefresh(ctx, grantsHash, pageSize, filterItemFn, listItemsFn, repo.estimatedCount, listDeletedIdsFn, tok)
+}

internal/apptoken/service_list_test.go

@@ -6,9 +6,11 @@ package apptoken
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/boundary/globals"
 	"github.com/hashicorp/boundary/internal/db"
+	"github.com/hashicorp/boundary/internal/db/timestamp"
 	"github.com/hashicorp/boundary/internal/iam"
 	"github.com/hashicorp/boundary/internal/listtoken"
 	"github.com/hashicorp/boundary/internal/types/resource"
@@ -32,11 +34,11 @@ func TestList(t *testing.T) {
 
 	var globalAppTokens, org1AppTokens, proj1AppTokens, org2AppTokens, proj2AppTokens, allAppTokens []*AppToken
 	for range 5 {
-		globalAppToken := TestAppToken(t, repo, globals.GlobalPrefix, []string{"ids=*;type=scope;actions=list,read"}, globalUser, true, "individual")
-		org1AppToken := TestAppToken(t, repo, org1.PublicId, []string{"ids=*;type=scope;actions=list,read"}, org1User, true, "individual")
-		proj1AppToken := TestAppToken(t, repo, proj1.PublicId, []string{"ids=*;type=target;actions=list,read"}, org1User, true, "individual")
-		org2AppToken := TestAppToken(t, repo, org2.PublicId, []string{"ids=*;type=scope;actions=list,read"}, org2User, true, "individual")
-		proj2AppToken := TestAppToken(t, repo, proj2.PublicId, []string{"ids=*;type=target;actions=list,read"}, org2User, true, "individual")
+		globalAppToken := TestAppToken(t, repo, globals.GlobalPrefix, globalUser, 0, nil, []string{"ids=*;type=scope;actions=list,read"}, true, "individual")
+		org1AppToken := TestAppToken(t, repo, org1.PublicId, org1User, 0, nil, []string{"ids=*;type=scope;actions=list,read"}, true, "individual")
+		proj1AppToken := TestAppToken(t, repo, proj1.PublicId, org1User, 0, nil, []string{"ids=*;type=target;actions=list,read"}, true, "individual")
+		org2AppToken := TestAppToken(t, repo, org2.PublicId, org2User, 0, nil, []string{"ids=*;type=scope;actions=list,read"}, true, "individual")
+		proj2AppToken := TestAppToken(t, repo, proj2.PublicId, org2User, 0, nil, []string{"ids=*;type=target;actions=list,read"}, true, "individual")
 
 		globalAppTokens = append(globalAppTokens, globalAppToken)
 		org1AppTokens = append(org1AppTokens, org1AppToken)
@@ -55,185 +57,34 @@ func TestList(t *testing.T) {
 	}
 
 	t.Run("List validation", func(t *testing.T) {
-		testCases := []struct {
-			name           string
-			withScopeIds   []string
-			pageSize       int
-			wantTokens     []*AppToken
-			wantErr        bool
-			wantErrMessage string
-		}{
-			{
-				name:         "list global tokens",
-				withScopeIds: []string{globals.GlobalPrefix},
-				pageSize:     10,
-				wantTokens:   globalAppTokens[0:5],
-				wantErr:      false,
-			},
-			{
-				name:         "list org1 tokens",
-				withScopeIds: []string{org1.PublicId},
-				pageSize:     10,
-				wantTokens:   org1AppTokens[0:5],
-				wantErr:      false,
-			},
-			{
-				name:         "list proj1 tokens",
-				withScopeIds: []string{proj1.PublicId},
-				pageSize:     10,
-				wantTokens:   proj1AppTokens[0:5],
-				wantErr:      false,
-			},
-			{
-				name:         "list all org tokens",
-				withScopeIds: []string{org1.PublicId, org2.PublicId},
-				pageSize:     10,
-				wantTokens:   append(org1AppTokens[0:5], org2AppTokens[0:5]...),
-				wantErr:      false,
-			},
-			{
-				name:         "list all proj tokens",
-				withScopeIds: []string{proj1.PublicId, proj2.PublicId},
-				pageSize:     10,
-				wantTokens:   append(proj1AppTokens[0:5], proj2AppTokens[0:5]...),
-				wantErr:      false,
-			},
-			{
-				name: "list all tokens",
-				withScopeIds: []string{
-					globals.GlobalPrefix,
-					org1.PublicId,
-					proj1.PublicId,
-					org2.PublicId,
-					proj2.PublicId,
-				},
-				pageSize:   30,
-				wantTokens: allAppTokens,
-				wantErr:    false,
-			},
-			{
-				name:         "list with no matching scopes",
-				withScopeIds: []string{"nonexistent_scope"},
-				pageSize:     10,
-				wantTokens:   []*AppToken{},
-				wantErr:      false,
-			},
-			{
-				name:           "invalid missing scope ids",
-				withScopeIds:   nil,
-				pageSize:       10,
-				wantTokens:     nil,
-				wantErr:        true,
-				wantErrMessage: "apptoken.List: missing scope ids: parameter violation",
-			},
-			{
-				name:           "invalid empty scope ids",
-				withScopeIds:   []string{},
-				pageSize:       10,
-				wantTokens:     nil,
-				wantErr:        true,
-				wantErrMessage: "apptoken.List: missing scope ids: parameter violation",
-			},
-		}
-
-		for _, tc := range testCases {
-			t.Run(tc.name, func(t *testing.T) {
-				t.Parallel()
-				assert, require := assert.New(t), require.New(t)
-
-				resp, err := List(
-					ctx,
-					[]byte("test_grants_hash"),
-					tc.pageSize,
-					filterNothingFilterFunc,
-					repo,
-					tc.withScopeIds,
-				)
-				if tc.wantErr {
-					require.Error(err)
-					assert.Contains(err.Error(), tc.wantErrMessage)
-					return
-				}
-				require.NoError(err)
-				require.NotNil(resp)
-				assert.Equal(len(tc.wantTokens), len(resp.Items))
-
-				// Verify that all expected tokens are present in the result
-				// The order is not guaranteed, so we check presence rather than position
-				for _, wantToken := range tc.wantTokens {
-					found := false
-					for _, gotToken := range resp.Items {
-						if wantToken.PublicId == gotToken.PublicId && wantToken.ScopeId == gotToken.ScopeId {
-							found = true
-							break
-						}
-					}
-					assert.True(found)
-				}
-			})
-		}
-
-		t.Run("filter out tokens", func(t *testing.T) {
+		t.Run("missing grants hash", func(t *testing.T) {
 			t.Parallel()
-			assert, require := assert.New(t), require.New(t)
-			filterOutOrg2Func := func(_ context.Context, appt *AppToken) (bool, error) {
-				// Filter out tokens associated with org2
-				if appt.ScopeId == org2.PublicId {
-					return false, nil
-				}
-				return true, nil
-			}
 
-			resp, err := List(
-				ctx,
-				[]byte("test_grants_hash"),
-				10,
-				filterOutOrg2Func,
-				repo,
-				[]string{org1.PublicId, org2.PublicId},
-			)
-			require.NoError(err)
-			require.NotNil(resp)
-			assert.Equal(5, len(resp.Items)) // Only respond with org1 tokens
-		})
-
-		t.Run("filter out inactive tokens", func(t *testing.T) {
-			// This test is intentionally not run in parallel because it modifies shared state
-			assert, require := assert.New(t), require.New(t)
-
-			// Create a new global token
-			globalTokenToBeInactive := TestAppToken(t, repo, globals.GlobalPrefix, []string{"ids=*;type=scope;actions=list,read"}, globalUser, true, "individual")
-
-			resp, err := List(
+			_, err := List(
 				ctx,
-				[]byte("test_grants_hash"),
+				[]byte(""),
 				10,
-				filterOutInactiveFunc,
+				filterNothingFilterFunc,
 				repo,
 				[]string{globals.GlobalPrefix},
 			)
-			require.NoError(err)
-			require.NotNil(resp)
-			assert.Equal(6, len(resp.Items)) // globalAppTokens (5) + globalTokenToBeInactive (1)
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), "apptoken.List: missing grants hash: parameter violation")
+		})
 
-			// Revoke
-			tempTestRevokeGlobalAppToken(t, repo, globalTokenToBeInactive.PublicId)
+		t.Run("invalid page size", func(t *testing.T) {
+			t.Parallel()
 
-			// List again and only find the original two active tokens
-			resp, err = List(
+			_, err := List(
 				ctx,
 				[]byte("test_grants_hash"),
-				10,
-				filterOutInactiveFunc,
+				0,
+				filterNothingFilterFunc,
 				repo,
 				[]string{globals.GlobalPrefix},
 			)
-			require.NoError(err)
-			require.NotNil(resp)
-			assert.Equal(5, len(resp.Items)) // globalAppTokens (5)
-
-			// Delete the revoked token to clean up
-			tempTestDeleteAppToken(t, repo, globalTokenToBeInactive.PublicId, globals.GlobalPrefix)
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), "apptoken.List: page size must be at least 1: parameter violation")
 		})
 
 		t.Run("missing filter func", func(t *testing.T) {
@@ -265,34 +116,19 @@ func TestList(t *testing.T) {
 			assert.Contains(t, err.Error(), "apptoken.List: missing repo: parameter violation")
 		})
 
-		t.Run("missing grants hash", func(t *testing.T) {
-			t.Parallel()
-
-			_, err := List(
-				ctx,
-				[]byte(""),
-				10,
-				filterNothingFilterFunc,
-				repo,
-				[]string{globals.GlobalPrefix},
-			)
-			require.Error(t, err)
-			assert.Contains(t, err.Error(), "apptoken.List: missing grants hash: parameter violation")
-		})
-
-		t.Run("invalid page size", func(t *testing.T) {
+		t.Run("missing scope ids", func(t *testing.T) {
 			t.Parallel()
 
 			_, err := List(
 				ctx,
 				[]byte("test_grants_hash"),
-				0,
+				10,
 				filterNothingFilterFunc,
 				repo,
-				[]string{globals.GlobalPrefix},
+				nil,
 			)
 			require.Error(t, err)
-			assert.Contains(t, err.Error(), "apptoken.List: page size must be at least 1: parameter violation")
+			assert.Contains(t, err.Error(), "apptoken.List: missing scope ids: parameter violation")
 		})
 	})
 
@@ -497,4 +333,381 @@ func TestList(t *testing.T) {
 			assert.Contains(err.Error(), "token did not have a pagination token component")
 		})
 	})
+
+	t.Run("ListRefresh validation", func(t *testing.T) {
+		t.Run("missing grants hash", func(t *testing.T) {
+			t.Parallel()
+
+			_, err := ListRefresh(
+				ctx,
+				[]byte(""),
+				10,
+				filterNothingFilterFunc,
+				&listtoken.Token{},
+				repo,
+				[]string{globals.GlobalPrefix},
+			)
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), "apptoken.ListRefresh: missing grants hash: parameter violation")
+		})
+
+		t.Run("invalid page size", func(t *testing.T) {
+			t.Parallel()
+
+			_, err := ListRefresh(
+				ctx,
+				[]byte("test_grants_hash"),
+				0,
+				filterNothingFilterFunc,
+				&listtoken.Token{},
+				repo,
+				[]string{globals.GlobalPrefix},
+			)
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), "apptoken.ListRefresh: page size must be at least 1: parameter violation")
+		})
+
+		t.Run("missing filter func", func(t *testing.T) {
+			t.Parallel()
+
+			_, err := ListRefresh(
+				ctx,
+				[]byte("test_grants_hash"),
+				10,
+				nil,
+				&listtoken.Token{},
+				repo,
+				[]string{globals.GlobalPrefix},
+			)
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), "apptoken.ListRefresh: missing filter item callback: parameter violation")
+		})
+
+		t.Run("missing token", func(t *testing.T) {
+			t.Parallel()
+
+			_, err := ListRefresh(
+				ctx,
+				[]byte("test_grants_hash"),
+				10,
+				filterNothingFilterFunc,
+				nil,
+				repo,
+				[]string{globals.GlobalPrefix},
+			)
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), "apptoken.ListRefresh: missing token: parameter violation")
+		})
+
+		t.Run("missing repo", func(t *testing.T) {
+			t.Parallel()
+
+			_, err := ListRefresh(
+				ctx,
+				[]byte("test_grants_hash"),
+				10,
+				filterNothingFilterFunc,
+				&listtoken.Token{},
+				nil,
+				[]string{globals.GlobalPrefix},
+			)
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), "apptoken.ListRefresh: missing repo: parameter violation")
+		})
+
+		t.Run("missing scope ids", func(t *testing.T) {
+			t.Parallel()
+
+			_, err := ListRefresh(
+				ctx,
+				[]byte("test_grants_hash"),
+				10,
+				filterNothingFilterFunc,
+				&listtoken.Token{},
+				repo,
+				nil,
+			)
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), "apptoken.ListRefresh: missing scope ids: parameter violation")
+		})
+
+		t.Run("invalid resource type in token", func(t *testing.T) {
+			t.Parallel()
+
+			_, err := ListRefresh(
+				ctx,
+				[]byte("test_grants_hash"),
+				10,
+				filterNothingFilterFunc,
+				&listtoken.Token{ResourceType: resource.AuthToken},
+				repo,
+				[]string{globals.GlobalPrefix},
+			)
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), "apptoken.ListRefresh: token did not have an app token resource type: parameter violation")
+		})
+
+		t.Run("invalid subtype in token", func(t *testing.T) {
+			t.Parallel()
+
+			_, err := ListRefresh(
+				ctx,
+				[]byte("test_grants_hash"),
+				10,
+				filterNothingFilterFunc,
+				&listtoken.Token{ResourceType: resource.AppToken, Subtype: nil},
+				repo,
+				[]string{globals.GlobalPrefix},
+			)
+			require.Error(t, err)
+			assert.Contains(t, err.Error(), "apptoken.ListRefresh: token did not have a start-refresh token component: parameter violation")
+		})
+	})
+
+	t.Run("simple listing with pagination", func(t *testing.T) {
+		testCases := []struct {
+			name           string
+			withScopeIds   []string
+			pageSize       int
+			wantTokens     []*AppToken
+			wantErr        bool
+			wantErrMessage string
+		}{
+			{
+				name:         "list global tokens",
+				withScopeIds: []string{globals.GlobalPrefix},
+				pageSize:     10,
+				wantTokens:   globalAppTokens[0:5],
+				wantErr:      false,
+			},
+			{
+				name:         "list org1 tokens",
+				withScopeIds: []string{org1.PublicId},
+				pageSize:     10,
+				wantTokens:   org1AppTokens[0:5],
+				wantErr:      false,
+			},
+			{
+				name:         "list proj1 tokens",
+				withScopeIds: []string{proj1.PublicId},
+				pageSize:     10,
+				wantTokens:   proj1AppTokens[0:5],
+				wantErr:      false,
+			},
+			{
+				name:         "list all org tokens",
+				withScopeIds: []string{org1.PublicId, org2.PublicId},
+				pageSize:     5,
+				wantTokens:   append(org1AppTokens[0:5], org2AppTokens[0:5]...),
+				wantErr:      false,
+			},
+			{
+				name:         "list all proj tokens",
+				withScopeIds: []string{proj1.PublicId, proj2.PublicId},
+				pageSize:     5,
+				wantTokens:   append(proj1AppTokens[0:5], proj2AppTokens[0:5]...),
+				wantErr:      false,
+			},
+			{
+				name: "list all tokens",
+				withScopeIds: []string{
+					globals.GlobalPrefix,
+					org1.PublicId,
+					proj1.PublicId,
+					org2.PublicId,
+					proj2.PublicId,
+				},
+				pageSize:   10,
+				wantTokens: allAppTokens,
+				wantErr:    false,
+			},
+			{
+				name:         "list with no matching scopes",
+				withScopeIds: []string{"nonexistent_scope"},
+				pageSize:     10,
+				wantTokens:   []*AppToken{},
+				wantErr:      false,
+			},
+		}
+
+		for _, tc := range testCases {
+			t.Run(tc.name, func(t *testing.T) {
+				t.Parallel()
+				assert, require := assert.New(t), require.New(t)
+
+				resp, err := List(
+					ctx,
+					[]byte("test_grants_hash"),
+					tc.pageSize,
+					filterNothingFilterFunc,
+					repo,
+					tc.withScopeIds,
+				)
+				if tc.wantErr {
+					require.Error(err)
+					assert.Contains(err.Error(), tc.wantErrMessage)
+					return
+				}
+				require.NoError(err)
+				require.NotNil(resp)
+
+				// Page through the rest of the tokens
+				retrievedTokens := resp.Items
+				nextToken := resp.ListToken
+				for nextToken != nil {
+					if _, ok := nextToken.Subtype.(*listtoken.PaginationToken); !ok {
+						break
+					}
+					resp, err = ListPage(
+						ctx,
+						[]byte("test_grants_hash"),
+						tc.pageSize,
+						filterNothingFilterFunc,
+						nextToken,
+						repo,
+						tc.withScopeIds,
+					)
+					require.NoError(err)
+					require.NotNil(resp)
+					retrievedTokens = append(retrievedTokens, resp.Items...)
+					nextToken = resp.ListToken
+				}
+
+				// Verify we got the expected number of tokens after optional paging
+				assert.Equal(len(tc.wantTokens), len(retrievedTokens))
+
+				// Verify that all expected tokens are present
+				// The order is not guaranteed, so we check presence rather than position
+				for _, wantToken := range tc.wantTokens {
+					found := false
+					for _, gotToken := range retrievedTokens {
+						if wantToken.PublicId == gotToken.PublicId && wantToken.ScopeId == gotToken.ScopeId {
+							found = true
+							break
+						}
+					}
+					assert.True(found)
+				}
+			})
+		}
+	})
+
+	t.Run("listing with aggressive filtering", func(t *testing.T) {
+		t.Run("filter out tokens", func(t *testing.T) {
+			t.Parallel()
+			assert, require := assert.New(t), require.New(t)
+			filterOutOrg2Func := func(_ context.Context, appt *AppToken) (bool, error) {
+				// Filter out tokens associated with org2
+				if appt.ScopeId == org2.PublicId {
+					return false, nil
+				}
+				return true, nil
+			}
+
+			resp, err := List(
+				ctx,
+				[]byte("test_grants_hash"),
+				10,
+				filterOutOrg2Func,
+				repo,
+				[]string{org1.PublicId, org2.PublicId},
+			)
+			require.NoError(err)
+			require.NotNil(resp)
+			assert.Equal(5, len(resp.Items)) // Only respond with org1 tokens
+		})
+
+		t.Run("filter out inactive tokens", func(t *testing.T) {
+			// This test is intentionally not run in parallel because it modifies shared state
+			assert, require := assert.New(t), require.New(t)
+
+			// Create a new global token
+			globalTokenToBeInactive := TestAppToken(t, repo, globals.GlobalPrefix, globalUser, 0, nil, []string{"ids=*;type=scope;actions=list,read"}, true, "individual")
+
+			resp, err := List(
+				ctx,
+				[]byte("test_grants_hash"),
+				10,
+				filterOutInactiveFunc,
+				repo,
+				[]string{globals.GlobalPrefix},
+			)
+			require.NoError(err)
+			require.NotNil(resp)
+			assert.Equal(6, len(resp.Items)) // globalAppTokens (5) + globalTokenToBeInactive (1)
+
+			// Revoke
+			tempTestRevokeGlobalAppToken(t, repo, globalTokenToBeInactive.PublicId)
+
+			// List again and only find the original 5 active tokens
+			resp, err = List(
+				ctx,
+				[]byte("test_grants_hash"),
+				10,
+				filterOutInactiveFunc,
+				repo,
+				[]string{globals.GlobalPrefix},
+			)
+			require.NoError(err)
+			require.NotNil(resp)
+			assert.Equal(5, len(resp.Items)) // globalAppTokens (5)
+
+			// Delete the revoked token to clean up
+			tempTestDeleteAppToken(t, repo, globalTokenToBeInactive.PublicId, globals.GlobalPrefix)
+		})
+	})
+
+	t.Run("listing with refresh", func(t *testing.T) {
+		t.Parallel()
+		assert, require := assert.New(t), require.New(t)
+
+		// Add a couple tokens to ensure there is data to refresh
+		token1ToBeRefreshed := TestAppToken(t, repo, globals.GlobalPrefix, globalUser, 0, nil, []string{"ids=*;type=scope;actions=list,read"}, true, "individual")
+		token2ToBeRefreshed := TestAppToken(t, repo, globals.GlobalPrefix, globalUser, 0, nil, []string{"ids=*;type=scope;actions=list,read"}, true, "individual")
+
+		// Initial list to get a start refresh token
+		firstResp, err := List(
+			ctx,
+			[]byte("test_grants_hash"),
+			10,
+			filterNothingFilterFunc,
+			repo,
+			[]string{globals.GlobalPrefix},
+		)
+		require.NoError(err)
+		require.NotNil(firstResp)
+		require.NotNil(firstResp.ListToken)
+		startRefreshToken, ok := firstResp.ListToken.Subtype.(*listtoken.StartRefreshToken)
+		require.True(ok)
+		require.NotNil(startRefreshToken)
+
+		// Manipulate the token to move it forward in time. This will ensure that
+		// when we do the refresh, we are outside the lookback window and thus
+		// will not pick up the tokens that were just created above.
+		firstResp.ListToken.Subtype.(*listtoken.StartRefreshToken).PreviousPhaseUpperBound = time.Now().Add(globals.RefreshReadLookbackDuration + 1*time.Second)
+		time.Sleep(2 * time.Second)
+
+		// Update tokens
+		testUpdateAppToken(t, repo, token1ToBeRefreshed.PublicId, globals.GlobalPrefix, map[string]any{"name": "updated-token1-name", "update_time": timestamp.New(timestamp.Now().AsTime())})
+		testUpdateAppToken(t, repo, token2ToBeRefreshed.PublicId, globals.GlobalPrefix, map[string]any{"approximate_last_access_time": timestamp.New(timestamp.Now().Timestamp.AsTime()), "update_time": timestamp.New(timestamp.Now().AsTime())})
+
+		// Now do a refresh list
+		refreshResp, err := ListRefresh(
+			ctx,
+			[]byte("test_grants_hash"),
+			10,
+			filterNothingFilterFunc,
+			firstResp.ListToken,
+			repo,
+			[]string{globals.GlobalPrefix},
+		)
+		require.NoError(err)
+		require.NotNil(refreshResp)
+		// Expect to find the two updated tokens
+		assert.Equal(2, len(refreshResp.Items))
+
+		// Clean up
+		tempTestDeleteAppToken(t, repo, token1ToBeRefreshed.PublicId, globals.GlobalPrefix)
+		tempTestDeleteAppToken(t, repo, token2ToBeRefreshed.PublicId, globals.GlobalPrefix)
+	})
 }

internal/apptoken/testing.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/hashicorp/boundary/globals"
 	"github.com/hashicorp/boundary/internal/db"
+	"github.com/hashicorp/boundary/internal/db/timestamp"
 	"github.com/hashicorp/boundary/internal/iam"
 	"github.com/hashicorp/boundary/internal/kms"
 	"github.com/hashicorp/boundary/internal/perms"
@@ -64,91 +65,87 @@ func testPublicId(t testing.TB, prefix string) string {
 
 // TestAppToken creates an app token for testing with the specified grants.
 // TODO: Implement TestAppToken once AppToken functionality is added
-func TestAppToken(t *testing.T, repo *Repository, scopeId string, grants []string, user *iam.User, grantThisScope bool, grantScope string) *AppToken {
+func TestAppToken(t *testing.T, repo *Repository, scopeId string, user *iam.User, timeToStaleSeconds uint32, expirationTime *timestamp.Timestamp, grants []string, grantThisScope bool, grantScope string) *AppToken {
 	t.Helper()
 
-	publicId := testPublicId(t, "appt_")
-	tempTestAddGrants(t, repo, publicId, scopeId, grants, user, grantThisScope, grantScope)
-	return &AppToken{
-		PublicId:    publicId,
-		ScopeId:     scopeId,
-		Description: "test app token",
+	testToken := &AppToken{
+		PublicId:           testPublicId(t, "apt_"),
+		ScopeId:            scopeId,
+		Description:        "test app token",
+		CreatedByUserId:    user.PublicId,
+		TimeToStaleSeconds: timeToStaleSeconds,
+		ExpirationTime:     expirationTime,
 	}
+	tempTestAddGrants(t, repo, testToken, grants, grantThisScope, grantScope)
+	return testToken
 }
 
 // tempTestAddGrants is a temporary test function to add grants to the database for testing
 // TODO: Replace with proper AppToken creation function once AppToken functionality is added
-func tempTestAddGrants(t *testing.T, repo *Repository, tokenId, scopeId string, grants []string, user *iam.User, grantThisScope bool, grantScope string) {
+func tempTestAddGrants(t *testing.T, repo *Repository, token *AppToken, grants []string, grantThisScope bool, grantScope string) {
 	t.Helper()
 	ctx := t.Context()
 	require := require.New(t)
 
 	// Determine which table to insert into based on scope prefix
-	var insertTokenSQL, insertPermissionSQL, insertGrantSQL string
+	var insertTokenSQL, insertPermissionSQL string
+	insertGrantSQL := `
+            insert into app_token_permission_grant (permission_id, raw_grant, canonical_grant)
+            values ($1, $2, $3)
+        `
 
 	switch {
-	case strings.HasPrefix(scopeId, globals.GlobalPrefix):
+	case strings.HasPrefix(token.ScopeId, globals.GlobalPrefix):
 		insertTokenSQL = `
-			insert into app_token_global (public_id, scope_id, name, description, created_by_user_id, expiration_time, create_time, update_time)
-			values ($1, $2, $3, $4, $5, now() + interval '1 day', now(), now())
+			insert into app_token_global (public_id, scope_id, name, description, created_by_user_id, time_to_stale_seconds, expiration_time)
+			values ($1, $2, $3, $4, $5, $6, $7)
 		`
 		insertPermissionSQL = `
             insert into app_token_permission_global (private_id, app_token_id, description, grant_this_scope, grant_scope)
             values ($1, $2, $3, $4, $5)
         `
-		insertGrantSQL = `
-            insert into app_token_permission_grant (permission_id, raw_grant, canonical_grant)
-            values ($1, $2, $3)
-        `
-	case strings.HasPrefix(scopeId, globals.OrgPrefix):
+
+	case strings.HasPrefix(token.ScopeId, globals.OrgPrefix):
 		insertTokenSQL = `
-			insert into app_token_org (public_id, scope_id, name, description, created_by_user_id, expiration_time, create_time, update_time)
-			values ($1, $2, $3, $4, $5, now() + interval '1 day', now(), now())
+			insert into app_token_org (public_id, scope_id, name, description, created_by_user_id, time_to_stale_seconds, expiration_time)
+			values ($1, $2, $3, $4, $5, $6, $7)
 		`
 		insertPermissionSQL = `
             insert into app_token_permission_org (private_id, app_token_id, description, grant_this_scope, grant_scope)
             values ($1, $2, $3, $4, $5)
         `
-		insertGrantSQL = `
-            insert into app_token_permission_grant (permission_id, raw_grant, canonical_grant)
-            values ($1, $2, $3)
-        `
-	case strings.HasPrefix(scopeId, globals.ProjectPrefix):
+	case strings.HasPrefix(token.ScopeId, globals.ProjectPrefix):
 		insertTokenSQL = `
-			insert into app_token_project (public_id, scope_id, name, description, created_by_user_id, expiration_time, create_time, update_time)
-			values ($1, $2, $3, $4, $5, now() + interval '1 day', now(), now())
+			insert into app_token_project (public_id, scope_id, name, description, created_by_user_id, time_to_stale_seconds, expiration_time)
+			values ($1, $2, $3, $4, $5, $6, $7)
 		`
 		insertPermissionSQL = `
             insert into app_token_permission_project (private_id, app_token_id, description, grant_this_scope)
             values ($1, $2, $3, $4)
-        `
-		insertGrantSQL = `
-            insert into app_token_permission_grant (permission_id, raw_grant, canonical_grant)
-            values ($1, $2, $3)
         `
 	default:
-		t.Fatalf("invalid scope id: %s", scopeId)
+		t.Fatalf("invalid scope id: %s", token.ScopeId)
 	}
 
 	// Insert the app token
-	name := fmt.Sprintf("Test App Token %s", tokenId)
-	_, err := repo.writer.Exec(ctx, insertTokenSQL, []any{tokenId, scopeId, name, "test app token", user.PublicId})
+	name := fmt.Sprintf("Test App Token %s", token.PublicId)
+	_, err := repo.writer.Exec(ctx, insertTokenSQL, []any{token.PublicId, token.ScopeId, name, "test app token", token.CreatedByUserId, token.TimeToStaleSeconds, token.ExpirationTime})
 	require.NoError(err)
 
 	// Create a permission for this token
 	permissionId := testPublicId(t, "aptp_")
 
 	// Default to 'descendants' if not specified for global/org scopes
-	if grantScope == "" && (strings.HasPrefix(scopeId, globals.GlobalPrefix) || strings.HasPrefix(scopeId, globals.OrgPrefix)) {
+	if grantScope == "" && (strings.HasPrefix(token.ScopeId, globals.GlobalPrefix) || strings.HasPrefix(token.ScopeId, globals.OrgPrefix)) {
 		grantScope = globals.GrantScopeDescendants
 	}
 
 	var permArgs []any
-	if strings.HasPrefix(scopeId, globals.ProjectPrefix) {
+	if strings.HasPrefix(token.ScopeId, globals.ProjectPrefix) {
 		// Project permissions don't have grant_scope column
-		permArgs = []any{permissionId, tokenId, "test permission", grantThisScope}
+		permArgs = []any{permissionId, token.PublicId, "test permission", grantThisScope}
 	} else {
-		permArgs = []any{permissionId, tokenId, "test permission", grantThisScope, grantScope}
+		permArgs = []any{permissionId, token.PublicId, "test permission", grantThisScope, grantScope}
 	}
 
 	_, err = repo.writer.Exec(ctx, insertPermissionSQL, permArgs)
@@ -158,8 +155,8 @@ func tempTestAddGrants(t *testing.T, repo *Repository, tokenId, scopeId string,
 	for _, grant := range grants {
 		// Parse the grant to get canonical form
 		perm, err := perms.Parse(ctx, perms.GrantTuple{
-			RoleScopeId:  scopeId,
-			GrantScopeId: scopeId,
+			RoleScopeId:  token.ScopeId,
+			GrantScopeId: token.ScopeId,
 			Grant:        grant,
 		}, perms.WithSkipFinalValidation(true))
 		require.NoError(err)
@@ -406,3 +403,43 @@ func tempTestDeleteAppToken(t *testing.T, repo *Repository, tokenId string, scop
 	_, err := repo.writer.Exec(ctx, deleteSQL, []any{tokenId})
 	require.NoError(err)
 }
+
+// testUpdateAppToken is a test helper to update fields on an app token directly in the database
+func testUpdateAppToken(t *testing.T, repo *Repository, tokenId string, scopeId string, fields map[string]any) {
+	t.Helper()
+	ctx := t.Context()
+	require := require.New(t)
+
+	var updateSQL strings.Builder
+	updateSQL.WriteString("update ")
+
+	switch {
+	case strings.HasPrefix(scopeId, globals.GlobalPrefix):
+		updateSQL.WriteString("app_token_global ")
+	case strings.HasPrefix(scopeId, globals.OrgPrefix):
+		updateSQL.WriteString("app_token_org ")
+	case strings.HasPrefix(scopeId, globals.ProjectPrefix):
+		updateSQL.WriteString("app_token_project ")
+	default:
+		t.Fatalf("invalid scope id: %s", scopeId)
+	}
+
+	updateSQL.WriteString("set ")
+
+	args := []any{}
+	i := 1
+	for field, value := range fields {
+		if i > 1 {
+			updateSQL.WriteString(", ")
+		}
+		updateSQL.WriteString(fmt.Sprintf("%s = $%d", field, i))
+		args = append(args, value)
+		i++
+	}
+
+	updateSQL.WriteString(fmt.Sprintf("where public_id = $%d", i))
+	args = append(args, tokenId)
+
+	_, err := repo.writer.Exec(ctx, updateSQL.String(), args)
+	require.NoError(err)
+}