Fix flaky log rotation test (#4257)

* Fix flaky log rotation test

internal/clientcache/cmd/daemon/logfile_test.go

@@ -12,9 +12,11 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/boundary/internal/event"
 	"github.com/hashicorp/go-hclog"
@@ -131,9 +133,25 @@ func TestRotation(t *testing.T) {
 		_, err := l.Write(toWrite)
 		require.NoError(t, err)
 	}
-	ret, err = os.ReadDir(dir)
-	require.NoError(t, err)
-	require.GreaterOrEqual(t, len(ret), 4)
 
-	require.NoError(t, os.RemoveAll(dir))
+	// Rotation causes the log files to be compressed after they are rotated.
+	// If the test ends before all the files have been compressed and the old
+	// .log files removed then cleanup fails with an error.
+	for {
+		ret, err = os.ReadDir(dir)
+		require.NoError(t, err)
+		require.GreaterOrEqual(t, len(ret), 4)
+
+		filtered := slices.DeleteFunc(ret, func(f fs.DirEntry) bool {
+			return strings.HasSuffix(f.Name(), "cache.log") ||
+				strings.HasSuffix(f.Name(), ".gz")
+		})
+		if len(filtered) == 0 {
+			// Indicates all the rotated log files have finished being
+			// compressed and can now be removed without new files being
+			// written to the directory.
+			break
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
 }