diff --git a/internal/clientcache/internal/cache/refresh.go b/internal/clientcache/internal/cache/refresh.go
index e903bbfb71..517aa32226 100644
--- a/internal/clientcache/internal/cache/refresh.go
+++ b/internal/clientcache/internal/cache/refresh.go
@@ -61,7 +61,7 @@ func (r *RefreshService) cleanAndPickAuthTokens(ctx context.Context, u *user) (m
 				_, err := r.repo.tokenReadFromBoundaryFn(ctx, u.Address, at.Token)
 				var apiErr *api.Error
 				switch {
-				case err != nil && api.ErrUnauthorized.Is(err):
+				case err != nil && (api.ErrUnauthorized.Is(err) || api.ErrNotFound.Is(err)):
 					if err := r.repo.deleteKeyringToken(ctx, *kt); err != nil {
 						return nil, errors.Wrap(ctx, err, op)
 					}
@@ -78,7 +78,7 @@ func (r *RefreshService) cleanAndPickAuthTokens(ctx context.Context, u *user) (m
 				_, err := r.repo.tokenReadFromBoundaryFn(ctx, u.Address, at.Token)
 				var apiErr *api.Error
 				switch {
-				case err != nil && api.ErrUnauthorized.Is(err):
+				case err != nil && (api.ErrUnauthorized.Is(err) || api.ErrNotFound.Is(err)):
 					r.repo.idToKeyringlessAuthToken.Delete(t.Id)
 					continue
 				case err != nil && !errors.Is(err, apiErr):
diff --git a/internal/clientcache/internal/cache/refresh_test.go b/internal/clientcache/internal/cache/refresh_test.go
index 5525aa55a2..4364c06609 100644
--- a/internal/clientcache/internal/cache/refresh_test.go
+++ b/internal/clientcache/internal/cache/refresh_test.go
@@ -106,6 +106,7 @@ func TestCleanAndPickTokens(t *testing.T) {
 
 	boundaryAuthTokens := []*authtokens.AuthToken{at1a, keyringAuthToken1, at1b, keyringAuthToken2}
 	unauthorizedAuthTokens := []*authtokens.AuthToken{}
+	notFoundAuthTokens := []*authtokens.AuthToken{}
 	randomErrorAuthTokens := []*authtokens.AuthToken{}
 	fakeBoundaryLookupFn := func(ctx context.Context, addr, at string) (*authtokens.AuthToken, error) {
 		for _, v := range randomErrorAuthTokens {
@@ -113,6 +114,11 @@ func TestCleanAndPickTokens(t *testing.T) {
 				return nil, errors.New("test error")
 			}
 		}
+		for _, v := range notFoundAuthTokens {
+			if at == v.Token {
+				return nil, api.ErrNotFound
+			}
+		}
 		for _, v := range unauthorizedAuthTokens {
 			if at == v.Token {
 				return nil, api.ErrUnauthorized
@@ -188,6 +194,24 @@ func TestCleanAndPickTokens(t *testing.T) {
 		assert.ElementsMatch(t, maps.Values(got), []string{at1a.Token})
 	})
 
+	t.Run("boundary in memory auth token not found", func(t *testing.T) {
+		require.NoError(t, r.AddRawToken(ctx, boundaryAddr, at1a.Token))
+		require.NoError(t, r.AddRawToken(ctx, boundaryAddr, at1b.Token))
+
+		got, err := rs.cleanAndPickAuthTokens(ctx, u1)
+		assert.NoError(t, err)
+		assert.ElementsMatch(t, maps.Values(got), []string{at1a.Token, at1b.Token})
+
+		t.Cleanup(func() {
+			notFoundAuthTokens = nil
+		})
+
+		notFoundAuthTokens = []*authtokens.AuthToken{at1b}
+		got, err = rs.cleanAndPickAuthTokens(ctx, u1)
+		assert.NoError(t, err)
+		assert.ElementsMatch(t, maps.Values(got), []string{at1a.Token})
+	})
+
 	t.Run("boundary keyring auths token expires", func(t *testing.T) {
 		key1 := ringToken{"k1", "t1"}
 		atMap[key1] = keyringAuthToken1