From b07883e8aa8935b2d0cd8b8840b59b4dbbd3d46f Mon Sep 17 00:00:00 2001
From: Michael Li <michael.li@hashicorp.com>
Date: Mon, 16 Mar 2026 09:53:49 -0400
Subject: [PATCH] ce: chore(e2e): Add support for downstream workers (#6478)

(cherry picked from commit 77a682058ee8c7f0c27f017c61fb444d4dee40dd)
---
 enos/enos-scenario-e2e-aws.hcl                |  3 +-
 .../aws_boundary/boundary-instances.tf        |  1 +
 enos/modules/aws_boundary/outputs.tf          | 17 +++++++++-
 .../modules/aws_boundary/templates/worker.hcl |  8 ++++-
 .../aws_boundary/templates/worker_bsr.hcl     |  8 ++++-
 .../templates/worker_bsr_vault_kms.hcl        | 12 ++++++-
 .../aws_boundary/templates/worker_hcp_bsr.hcl |  8 ++++-
 .../templates/worker_vault_kms.hcl            | 12 ++++++-
 enos/modules/aws_boundary/variables.tf        |  4 +--
 enos/modules/aws_worker/main.tf               | 32 ++++++++++++-------
 enos/modules/aws_worker/outputs.tf            | 11 +++++--
 enos/modules/aws_worker/templates/worker.hcl  |  9 ++++--
 .../aws_worker/templates/worker_bsr.hcl       |  9 ++++--
 .../templates/worker_bsr_vault_kms.hcl        | 13 ++++++--
 enos/modules/aws_worker/variables.tf          | 16 ++++++++--
 15 files changed, 132 insertions(+), 31 deletions(-)

diff --git a/enos/enos-scenario-e2e-aws.hcl b/enos/enos-scenario-e2e-aws.hcl
index 489bd2f568..6cd2115d8c 100644
--- a/enos/enos-scenario-e2e-aws.hcl
+++ b/enos/enos-scenario-e2e-aws.hcl
@@ -209,8 +209,9 @@ scenario "e2e_aws" {
       boundary_install_dir = local.boundary_install_dir
       name_prefix          = step.create_boundary_cluster.name_prefix
       cluster_tag          = step.create_boundary_cluster.cluster_tag
-      controller_addresses = step.create_boundary_cluster.public_controller_addresses
+      upstream_ips         = step.create_boundary_cluster.public_controller_addresses
       controller_sg_id     = step.create_boundary_cluster.controller_aux_sg_id
+      create_subnet        = true
       worker_type_tags     = [local.isolated_tag]
       ip_version           = matrix.ip_version
       config_file_path     = "templates/worker.hcl"
diff --git a/enos/modules/aws_boundary/boundary-instances.tf b/enos/modules/aws_boundary/boundary-instances.tf
index b53879f4d6..8a626c400f 100644
--- a/enos/modules/aws_boundary/boundary-instances.tf
+++ b/enos/modules/aws_boundary/boundary-instances.tf
@@ -242,6 +242,7 @@ resource "enos_file" "worker_config" {
     kms_key_id              = data.aws_kms_key.kms_key.id,
     controller_ips          = var.ip_version == "4" ? jsonencode(aws_instance.controller.*.private_ip) : jsonencode(formatlist("[%s]:9201", flatten(aws_instance.controller.*.ipv6_addresses)))
     listener_address        = var.ip_version == "4" ? "0.0.0.0" : "[::]"
+    listener_proxy_port     = var.listener_proxy_port
     public_address          = var.ip_version == "6" ? format("[%s]", aws_instance.worker[tonumber(each.value)].ipv6_addresses[0]) : aws_instance.worker[tonumber(each.value)].public_ip
     region                  = var.aws_region
     type                    = jsonencode(var.worker_type_tags)
diff --git a/enos/modules/aws_boundary/outputs.tf b/enos/modules/aws_boundary/outputs.tf
index 7a782393f8..15fc128401 100644
--- a/enos/modules/aws_boundary/outputs.tf
+++ b/enos/modules/aws_boundary/outputs.tf
@@ -16,6 +16,11 @@ output "worker_ips" {
   value       = var.ip_version == "6" ? flatten(aws_instance.worker.*.ipv6_addresses) : aws_instance.worker.*.public_ip
 }
 
+output "worker_ips_private" {
+  description = "Private IPs of boundary workers"
+  value       = var.ip_version == "6" || var.ip_version == "dual" ? flatten(aws_instance.worker.*.ipv6_addresses) : aws_instance.worker.*.private_ip
+}
+
 output "alb_hostname" {
   description = "Public hostname of Controller ALB"
   value       = aws_alb.boundary_alb.dns_name
@@ -256,4 +261,14 @@ output "worker_ipv6_cidr" {
 output "alb_cert" {
   description = "Public cert for the alb"
   value       = try(tls_self_signed_cert.certificate[0].cert_pem, null)
-}
\ No newline at end of file
+}
+
+output "controller_upstream_ips" {
+  description = "List of ips that workers can use to reach controllers"
+  value       = var.ip_version == "4" ? [for ip in aws_instance.controller.*.private_ip : "${ip}:${var.listener_cluster_port}"] : [for ip in flatten(aws_instance.controller.*.ipv6_addresses) : "[${ip}]:${var.listener_cluster_port}"]
+}
+
+output "worker_upstream_ips" {
+  description = "List of ips that workers can use to reach upstream workers"
+  value       = var.ip_version == "4" ? [for ip in aws_instance.worker.*.private_ip : "${ip}:${var.listener_proxy_port}"] : [for ip in flatten(aws_instance.worker.*.ipv6_addresses) : "[${ip}]:${var.listener_proxy_port}"]
+}
diff --git a/enos/modules/aws_boundary/templates/worker.hcl b/enos/modules/aws_boundary/templates/worker.hcl
index d71de7aa71..787f0260d6 100644
--- a/enos/modules/aws_boundary/templates/worker.hcl
+++ b/enos/modules/aws_boundary/templates/worker.hcl
@@ -4,7 +4,7 @@
 listener "tcp" {
   purpose     = "proxy"
   tls_disable = true
-  address     = "${listener_address}"
+  address     = "${listener_address}:${listener_proxy_port}"
 }
 
 worker {
@@ -30,6 +30,12 @@ kms "awskms" {
   kms_key_id = "${kms_key_id}"
 }
 
+kms "awskms" {
+  purpose    = "downstream-worker-auth"
+  region     = "${region}"
+  kms_key_id = "${kms_key_id}"
+}
+
 events {
   audit_enabled        = true
   observations_enabled = true
diff --git a/enos/modules/aws_boundary/templates/worker_bsr.hcl b/enos/modules/aws_boundary/templates/worker_bsr.hcl
index 1764e0f39c..3c34377f09 100644
--- a/enos/modules/aws_boundary/templates/worker_bsr.hcl
+++ b/enos/modules/aws_boundary/templates/worker_bsr.hcl
@@ -4,7 +4,7 @@
 listener "tcp" {
   purpose     = "proxy"
   tls_disable = true
-  address     = "${listener_address}"
+  address     = "${listener_address}:${listener_proxy_port}"
 }
 
 worker {
@@ -32,6 +32,12 @@ kms "awskms" {
   kms_key_id = "${kms_key_id}"
 }
 
+kms "awskms" {
+  purpose    = "downstream-worker-auth"
+  region     = "${region}"
+  kms_key_id = "${kms_key_id}"
+}
+
 events {
   audit_enabled        = true
   observations_enabled = true
diff --git a/enos/modules/aws_boundary/templates/worker_bsr_vault_kms.hcl b/enos/modules/aws_boundary/templates/worker_bsr_vault_kms.hcl
index addf195e1d..a53b2eb731 100644
--- a/enos/modules/aws_boundary/templates/worker_bsr_vault_kms.hcl
+++ b/enos/modules/aws_boundary/templates/worker_bsr_vault_kms.hcl
@@ -4,7 +4,7 @@
 listener "tcp" {
   purpose     = "proxy"
   tls_disable = true
-  address     = "${listener_address}:9202"
+  address     = "${listener_address}:${listener_proxy_port}"
 }
 
 worker {
@@ -36,6 +36,16 @@ kms "transit" {
   tls_skip_verify    = "true"
 }
 
+kms "transit" {
+  purpose            = "downstream-worker-auth"
+  address            = "http://${vault_address}:8200"
+  token              = "${vault_transit_token}"
+  disable_renewal    = "false"
+  key_name           = "boundary-worker-auth"
+  mount_path         = "transit/"
+  tls_skip_verify    = "true"
+}
+
 events {
   audit_enabled        = true
   observations_enabled = true
diff --git a/enos/modules/aws_boundary/templates/worker_hcp_bsr.hcl b/enos/modules/aws_boundary/templates/worker_hcp_bsr.hcl
index 30fe45e5bc..18266fc956 100644
--- a/enos/modules/aws_boundary/templates/worker_hcp_bsr.hcl
+++ b/enos/modules/aws_boundary/templates/worker_hcp_bsr.hcl
@@ -4,7 +4,7 @@
 listener "tcp" {
   purpose     = "proxy"
   tls_disable = true
-  address     = "${listener_address}"
+  address     = "${listener_address}:${listener_proxy_port}"
 }
 
 hcp_boundary_cluster_id = "${hcp_boundary_cluster_id}"
@@ -21,6 +21,12 @@ worker {
   recording_storage_path = "${recording_storage_path}"
 }
 
+kms "awskms" {
+  purpose    = "downstream-worker-auth"
+  region     = "${region}"
+  kms_key_id = "${kms_key_id}"
+}
+
 events {
   audit_enabled        = true
   observations_enabled = true
diff --git a/enos/modules/aws_boundary/templates/worker_vault_kms.hcl b/enos/modules/aws_boundary/templates/worker_vault_kms.hcl
index 1f46735d89..cb1301eaac 100644
--- a/enos/modules/aws_boundary/templates/worker_vault_kms.hcl
+++ b/enos/modules/aws_boundary/templates/worker_vault_kms.hcl
@@ -4,7 +4,7 @@
 listener "tcp" {
   purpose     = "proxy"
   tls_disable = true
-  address     = "${listener_address}"
+  address     = "${listener_address}:${listener_proxy_port}"
 }
 
 worker {
@@ -34,6 +34,16 @@ kms "transit" {
   tls_skip_verify    = "true"
 }
 
+kms "transit" {
+  purpose            = "downstream-worker-auth"
+  address            = "http://${vault_address}:8200"
+  token              = "${vault_transit_token}"
+  disable_renewal    = "false"
+  key_name           = "boundary-worker-auth"
+  mount_path         = "transit/"
+  tls_skip_verify    = "true"
+}
+
 events {
   audit_enabled        = true
   observations_enabled = true
diff --git a/enos/modules/aws_boundary/variables.tf b/enos/modules/aws_boundary/variables.tf
index 9293ac2674..22686de0c4 100644
--- a/enos/modules/aws_boundary/variables.tf
+++ b/enos/modules/aws_boundary/variables.tf
@@ -31,7 +31,7 @@ variable "worker_instance_type" {
 variable "worker_type_tags" {
   description = "Tag to set on worker for use in worker filters"
   type        = list(string)
-  default     = ["collocated", "prod", "webservers", "linux"]
+  default     = ["collocated", "ingress", "prod", "webservers", "linux"]
 }
 
 variable "worker_ebs_iops" {
@@ -406,4 +406,4 @@ variable "protocol" {
   description = "http protocol (http/https)"
   type        = string
   default     = "http"
-}
\ No newline at end of file
+}
diff --git a/enos/modules/aws_worker/main.tf b/enos/modules/aws_worker/main.tf
index d208a3d2fa..6535cc1e94 100644
--- a/enos/modules/aws_worker/main.tf
+++ b/enos/modules/aws_worker/main.tf
@@ -44,6 +44,10 @@ data "aws_kms_key" "kms_key" {
   key_id = var.kms_key_arn
 }
 
+data "aws_vpc" "vpc" {
+  id = var.vpc_id
+}
+
 resource "random_integer" "az" {
   min = 0
   max = length(data.aws_availability_zones.available.names) - 1
@@ -56,6 +60,7 @@ resource "random_integer" "az" {
 
 # Create a subnet so that the worker doesn't share one with a controller
 resource "aws_subnet" "default" {
+  count           = var.create_subnet ? 1 : 0
   vpc_id          = var.vpc_id
   cidr_block      = "10.13.9.0/24"
   ipv6_cidr_block = var.ip_version != "4" ? cidrsubnet(var.vpc_cidr_ipv6, 8, 0) : null
@@ -70,6 +75,11 @@ resource "aws_subnet" "default" {
     },
   )
 }
+resource "aws_route_table_association" "worker_rta" {
+  count          = var.create_subnet ? 1 : 0
+  subnet_id      = aws_subnet.default[0].id
+  route_table_id = data.aws_vpc.vpc.main_route_table_id
+}
 
 # The worker instance is a part of this security group, not to be confused with the next rule below
 resource "aws_security_group" "default" {
@@ -98,6 +108,15 @@ resource "aws_security_group" "default" {
     ipv6_cidr_blocks = var.ip_version == "4" ? [] : ["::/0"]
   }
 
+  ingress {
+    description      = "Communication from worker to worker"
+    from_port        = 9202
+    to_port          = 9202
+    protocol         = "tcp"
+    cidr_blocks      = var.ip_version == "6" ? [] : data.aws_vpc.vpc.cidr_block_associations.*.cidr_block
+    ipv6_cidr_blocks = var.ip_version == "4" ? [] : [data.aws_vpc.vpc.ipv6_cidr_block]
+  }
+
   egress {
     from_port        = 0
     to_port          = 0
@@ -126,20 +145,11 @@ resource "aws_vpc_security_group_ingress_rule" "worker_to_controller" {
   ip_protocol       = "tcp"
 }
 
-data "aws_vpc" "vpc" {
-  id = var.vpc_id
-}
-
-resource "aws_route_table_association" "worker_rta" {
-  subnet_id      = aws_subnet.default.id
-  route_table_id = data.aws_vpc.vpc.main_route_table_id
-}
-
 resource "aws_instance" "worker" {
   ami                    = var.ubuntu_ami_id
   instance_type          = var.worker_instance_type
   vpc_security_group_ids = [aws_security_group.default.id]
-  subnet_id              = aws_subnet.default.id
+  subnet_id              = var.create_subnet ? aws_subnet.default[0].id : var.subnet_ids[0]
   key_name               = var.ssh_aws_keypair
   iam_instance_profile   = aws_iam_instance_profile.boundary_profile.name
   monitoring             = var.worker_monitoring
@@ -211,7 +221,7 @@ resource "enos_file" "worker_config" {
   content = templatefile("${path.module}/${var.config_file_path}", {
     id                     = random_pet.worker.id
     kms_key_id             = data.aws_kms_key.kms_key.id
-    controller_ips         = var.ip_version == "4" ? jsonencode(var.controller_addresses) : jsonencode(formatlist("[%s]:9201", var.controller_addresses))
+    upstream_ips           = jsonencode(var.upstream_ips)
     listener_address       = var.ip_version == "4" ? "0.0.0.0" : "[::]"
     public_address         = var.ip_version == "6" ? format("[%s]", aws_instance.worker.ipv6_addresses[0]) : aws_instance.worker.public_ip
     type                   = jsonencode(var.worker_type_tags)
diff --git a/enos/modules/aws_worker/outputs.tf b/enos/modules/aws_worker/outputs.tf
index 081da99354..e7034039dc 100644
--- a/enos/modules/aws_worker/outputs.tf
+++ b/enos/modules/aws_worker/outputs.tf
@@ -6,6 +6,11 @@ output "worker_ip" {
   value       = var.ip_version == "6" ? format("[%s]", aws_instance.worker.ipv6_addresses[0]) : aws_instance.worker.public_ip
 }
 
+output "worker_upstream_ips" {
+  description = "List of ips that workers can use to reach upstream workers"
+  value       = var.ip_version == "4" ? [for ip in aws_instance.worker.*.private_ip : "${ip}:9202"] : [for ip in flatten(aws_instance.worker.*.ipv6_addresses) : "[${ip}]:9201"]
+}
+
 output "worker_tags" {
   description = "The tags used in the worker's configuration"
   value       = var.worker_type_tags
@@ -13,7 +18,7 @@ output "worker_tags" {
 
 output "subnet_ids" {
   description = "The ID of the subnet this worker resides in"
-  value       = [aws_subnet.default.id]
+  value       = length(var.subnet_ids) == 0 ? [aws_subnet.default[0].id] : var.subnet_ids
 }
 
 output "pet_id" {
@@ -28,10 +33,10 @@ output "role_arn" {
 
 output "worker_cidr" {
   description = "The subnet of the isolated worker"
-  value       = var.ip_version == "6" ? [] : [aws_subnet.default.cidr_block]
+  value       = var.ip_version == "6" ? [] : length(var.subnet_ids) == 0 ? [aws_subnet.default[0].cidr_block] : []
 }
 
 output "worker_ipv6_cidr" {
   description = "The ipv6 subnet of the isolated worker"
-  value       = var.ip_version == "4" ? [] : [aws_subnet.default.ipv6_cidr_block]
+  value       = var.ip_version == "4" ? [] : length(var.subnet_ids) == 0 ? [aws_subnet.default[0].ipv6_cidr_block] : []
 }
diff --git a/enos/modules/aws_worker/templates/worker.hcl b/enos/modules/aws_worker/templates/worker.hcl
index 16956137d8..124891f9ae 100644
--- a/enos/modules/aws_worker/templates/worker.hcl
+++ b/enos/modules/aws_worker/templates/worker.hcl
@@ -12,8 +12,7 @@ worker {
   name = "worker-${id}"
   description = "Enos Boundary worker ${id}"
 
-  # Workers must be able to reach controllers on :9201
-  initial_upstreams = ${controller_ips}
+  initial_upstreams = ${upstream_ips}
 
   public_addr = "${public_address}"
 
@@ -30,6 +29,12 @@ kms "awskms" {
   kms_key_id = "${kms_key_id}"
 }
 
+kms "awskms" {
+  purpose    = "downstream-worker-auth"
+  region     = "${region}"
+  kms_key_id = "${kms_key_id}"
+}
+
 events {
   audit_enabled        = true
   observations_enabled = true
diff --git a/enos/modules/aws_worker/templates/worker_bsr.hcl b/enos/modules/aws_worker/templates/worker_bsr.hcl
index f851c9079f..340fbb7fcc 100644
--- a/enos/modules/aws_worker/templates/worker_bsr.hcl
+++ b/enos/modules/aws_worker/templates/worker_bsr.hcl
@@ -12,8 +12,7 @@ worker {
   name = "worker-${id}"
   description = "Enos Boundary worker ${id}"
 
-  # Workers must be able to reach controllers on :9201
-  initial_upstreams = ${controller_ips}
+  initial_upstreams = ${upstream_ips}
 
   public_addr = "${public_address}"
 
@@ -32,6 +31,12 @@ kms "awskms" {
   kms_key_id = "${kms_key_id}"
 }
 
+kms "awskms" {
+  purpose    = "downstream-worker-auth"
+  region     = "${region}"
+  kms_key_id = "${kms_key_id}"
+}
+
 events {
   audit_enabled        = true
   observations_enabled = true
diff --git a/enos/modules/aws_worker/templates/worker_bsr_vault_kms.hcl b/enos/modules/aws_worker/templates/worker_bsr_vault_kms.hcl
index addf195e1d..77e9293040 100644
--- a/enos/modules/aws_worker/templates/worker_bsr_vault_kms.hcl
+++ b/enos/modules/aws_worker/templates/worker_bsr_vault_kms.hcl
@@ -12,8 +12,7 @@ worker {
   name = "demo-worker-${id}"
   description = "Enos Boundary worker ${id}"
 
-  # Workers must be able to reach controllers on :9201
-  initial_upstreams = ${controller_ips}
+  initial_upstreams = ${upstream_ips}
 
   public_addr = "${public_address}"
 
@@ -36,6 +35,16 @@ kms "transit" {
   tls_skip_verify    = "true"
 }
 
+kms "transit" {
+  purpose            = "downstream-worker-auth"
+  address            = "http://${vault_address}:8200"
+  token              = "${vault_transit_token}"
+  disable_renewal    = "false"
+  key_name           = "boundary-worker-auth"
+  mount_path         = "transit/"
+  tls_skip_verify    = "true"
+}
+
 events {
   audit_enabled        = true
   observations_enabled = true
diff --git a/enos/modules/aws_worker/variables.tf b/enos/modules/aws_worker/variables.tf
index 56fde62e4e..047b26778a 100644
--- a/enos/modules/aws_worker/variables.tf
+++ b/enos/modules/aws_worker/variables.tf
@@ -115,8 +115,8 @@ variable "worker_type_tags" {
   default     = ["prod", "webservers"]
 }
 
-variable "controller_addresses" {
-  description = "A list of addresses that will be used as initial_upstreams in the worker's configuration"
+variable "upstream_ips" {
+  description = "The network address of the upstream entity for a downstream worker to connect to"
   type        = list(string)
 }
 
@@ -157,6 +157,12 @@ variable "vpc_cidr_ipv6" {
   default     = null
 }
 
+variable "subnet_ids" {
+  description = "List of subnet ids to use with instances"
+  type        = list(string)
+  default     = []
+}
+
 variable "ip_version" {
   description = "ip version used to setup boundary instance, should be 4, 6, or dual"
   type        = string
@@ -179,3 +185,9 @@ variable "vault_transit_token" {
   type        = string
   default     = ""
 }
+
+variable "create_subnet" {
+  description = "Whether or not to create a separate subnet for the worker in order to isolate it from other resources. If false, subnet_ids must be provided."
+  type        = bool
+  default     = false
+}