diff --git a/internal/daemon/controller/worker_tls_config.go b/internal/daemon/controller/worker_tls_config.go
index 703e75f600..d7460e6cf0 100644
--- a/internal/daemon/controller/worker_tls_config.go
+++ b/internal/daemon/controller/worker_tls_config.go
@@ -84,6 +84,9 @@ func (c Controller) v1WorkerAuthConfig(protos []string) (*tls.Config, *base.Work
 	if err := proto.Unmarshal(marshaledEncInfo, encInfo); err != nil {
 		return nil, nil, err
 	}
+	if isNil(c.conf.WorkerAuthKms) {
+		return nil, nil, errors.New("worker auth kms not set")
+	}
 	marshaledInfo, err := c.conf.WorkerAuthKms.Decrypt(context.Background(), encInfo)
 	if err != nil {
 		return nil, nil, err