Soak test for kubernetes backend

backend/remote-state/kubernetes/backend_test.go

@@ -1,10 +1,16 @@
 package kubernetes
 
 import (
+	"fmt"
+	"math/rand"
 	"os"
+	"sync"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/terraform/backend"
+	"github.com/hashicorp/terraform/state"
+	"github.com/hashicorp/terraform/states/statemgr"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -30,7 +36,7 @@ func testACC(t *testing.T) {
 		namespace = "default"
 	}
 
-	cleanupK8sSecrets(t)
+	cleanupK8sResources(t)
 }
 
 func TestBackend_impl(t *testing.T) {
@@ -39,7 +45,7 @@ func TestBackend_impl(t *testing.T) {
 
 func TestBackend(t *testing.T) {
 	testACC(t)
-	defer cleanupK8sSecrets(t)
+	defer cleanupK8sResources(t)
 
 	b1 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
 		"secret_suffix": secretSuffix,
@@ -51,7 +57,7 @@ func TestBackend(t *testing.T) {
 
 func TestBackendLocks(t *testing.T) {
 	testACC(t)
-	defer cleanupK8sSecrets(t)
+	defer cleanupK8sResources(t)
 
 	// Get the backend. We need two to test locking.
 	b1 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
@@ -67,7 +73,57 @@ func TestBackendLocks(t *testing.T) {
 	backend.TestBackendStateForceUnlock(t, b1, b2)
 }
 
-func cleanupK8sSecrets(t *testing.T) {
+func TestBackendLocksSoak(t *testing.T) {
+	testACC(t)
+	defer cleanupK8sResources(t)
+
+	clientCount := 1000
+	lockCount := 0
+
+	lockers := []statemgr.Locker{}
+	for i := 0; i < clientCount; i++ {
+		b := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
+			"secret_suffix": secretSuffix,
+		}))
+
+		s, err := b.StateMgr(backend.DefaultStateName)
+		if err != nil {
+			t.Fatalf("Error creating state manager: %v", err)
+		}
+
+		lockers = append(lockers, s.(statemgr.Locker))
+	}
+
+	wg := sync.WaitGroup{}
+	for i, l := range lockers {
+		wg.Add(1)
+		go func(locker statemgr.Locker, i int) {
+			r := rand.Intn(10)
+			time.Sleep(time.Duration(r) * time.Microsecond)
+			li := state.NewLockInfo()
+			li.Operation = "test"
+			li.Who = fmt.Sprintf("client-%v", i)
+			_, err := locker.Lock(li)
+			if err == nil {
+				t.Logf("[INFO] Client %v got the lock\r\n", i)
+				lockCount++
+			}
+			wg.Done()
+		}(l, i)
+	}
+
+	wg.Wait()
+
+	if lockCount > 1 {
+		t.Fatalf("multiple backend clients were able to acquire a lock, count: %v", lockCount)
+	}
+
+	if lockCount == 0 {
+		t.Fatal("no clients were able to acquire a lock")
+	}
+}
+
+func cleanupK8sResources(t *testing.T) {
 	// Get a backend to use the k8s client
 	b1 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
 		"secret_suffix": secretSuffix,
@@ -80,7 +136,7 @@ func cleanupK8sSecrets(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// Get state secrets based off the tfstateKey label
+	// Delete secrets
 	opts := metav1.ListOptions{LabelSelector: tfstateKey + "=true"}
 	secrets, err := sClient.List(opts)
 	if err != nil {
@@ -106,6 +162,32 @@ func cleanupK8sSecrets(t *testing.T) {
 		}
 	}
 
+	leaseClient, err := b.KubernetesLeaseClient()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Delete leases
+	leases, err := leaseClient.List(opts)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, lease := range leases.Items {
+		labels := lease.GetLabels()
+		key, ok := labels[tfstateSecretSuffixKey]
+		if !ok {
+			continue
+		}
+
+		if key == secretSuffix {
+			err = leaseClient.Delete(lease.GetName(), delOps)
+			if err != nil {
+				errs = append(errs, err)
+			}
+		}
+	}
+
 	if len(errs) > 0 {
 		t.Fatal(errs)
 	}

backend/remote-state/kubernetes/client_test.go

@@ -15,7 +15,7 @@ func TestRemoteClient_impl(t *testing.T) {
 
 func TestRemoteClient(t *testing.T) {
 	testACC(t)
-	defer cleanupK8sSecrets(t)
+	defer cleanupK8sResources(t)
 
 	b := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
 		"secret_suffix": secretSuffix,
@@ -31,7 +31,7 @@ func TestRemoteClient(t *testing.T) {
 
 func TestRemoteClientLocks(t *testing.T) {
 	testACC(t)
-	defer cleanupK8sSecrets(t)
+	defer cleanupK8sResources(t)
 
 	b1 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
 		"secret_suffix": secretSuffix,
@@ -56,7 +56,7 @@ func TestRemoteClientLocks(t *testing.T) {
 
 func TestForceUnlock(t *testing.T) {
 	testACC(t)
-	defer cleanupK8sSecrets(t)
+	defer cleanupK8sResources(t)
 
 	b1 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
 		"secret_suffix": secretSuffix,