From a853c4750f08b3ba7ef17b538d473009e4c8af16 Mon Sep 17 00:00:00 2001 From: Michael Li Date: Wed, 14 Jan 2026 18:14:16 -0500 Subject: [PATCH] chore(e2e): Add network check before setting up SSH service (cherry picked from commit 0dbc96bf334a40f80b1d3963e1d6e5d438fec865) --- .../modules/aws_rdp_domain_controller/main.tf | 26 ++- enos/modules/aws_rdp_member_server/main.tf | 167 ++++++++++-------- .../aws_rdp_member_server_with_worker/main.tf | 19 ++ enos/modules/aws_windows_client/main.tf | 19 ++ 4 files changed, 156 insertions(+), 75 deletions(-) diff --git a/enos/modules/aws_rdp_domain_controller/main.tf b/enos/modules/aws_rdp_domain_controller/main.tf index 785fca079e..2df649b0e0 100644 --- a/enos/modules/aws_rdp_domain_controller/main.tf +++ b/enos/modules/aws_rdp_domain_controller/main.tf @@ -328,6 +328,25 @@ resource "aws_instance" "domain_controller" { # Note: Windows Server 2016 does not support OpenSSH %{if var.server_version != "2016"~} $sshSetupScript = @' + # Wait for network to be available + $networkTimeout = 120 + $networkElapsed = 0 + do { + $network = Test-NetConnection -ComputerName "169.254.169.254" -Port 80 -WarningAction SilentlyContinue + if ($network.TcpTestSucceeded) { + Write-Host "Network is available" + break + } + Write-Host "Waiting for network..." + Start-Sleep -Seconds 10 + $networkElapsed += 10 + } while ($networkElapsed -lt $networkTimeout) + + if ($networkElapsed -ge $networkTimeout) { + Write-Host "Network not available after timeout. Exiting." + exit 1 + } + # set variables for retry loops $timeout = 300 $interval = 30 @@ -401,6 +420,7 @@ resource "aws_instance" "domain_controller" { # Register a scheduled task to run the SSH setup script at next boot $Action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument "-NoProfile -ExecutionPolicy Bypass -File C:\ssh-setup.ps1" $Trigger = New-ScheduledTaskTrigger -AtStartup + $Trigger.Delay = 'PT2M' # Wait 2 minutes after startup to allow networking services to load $Principal = New-ScheduledTaskPrincipal -UserId "SYSTEM" -LogonType ServiceAccount -RunLevel Highest Register-ScheduledTask -TaskName "SetupOpenSSH" -Action $Action -Trigger $Trigger -Principal $Principal; %{endif~} @@ -447,16 +467,16 @@ resource "local_sensitive_file" "private_key" { file_permission = "0400" } -resource "time_sleep" "wait_10_minutes" { +resource "time_sleep" "wait_for_reboot" { depends_on = [aws_instance.domain_controller] - create_duration = "10m" + create_duration = "20m" } # wait for the SSH service to be available on the instance. We specifically use # BatchMode=Yes to prevent SSH from prompting for a password to ensure that we # can just SSH using the private key resource "enos_local_exec" "wait_for_ssh" { - depends_on = [time_sleep.wait_10_minutes] + depends_on = [time_sleep.wait_for_reboot] count = var.server_version != "2016" ? 1 : 0 inline = ["timeout 600s bash -c 'until ssh -i ${abspath(local_sensitive_file.private_key.filename)} -o BatchMode=Yes -o IdentitiesOnly=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no Administrator@${aws_instance.domain_controller.public_ip} \"echo ready\"; do sleep 10; done'"] } diff --git a/enos/modules/aws_rdp_member_server/main.tf b/enos/modules/aws_rdp_member_server/main.tf index 8411fefc51..03230410f1 100644 --- a/enos/modules/aws_rdp_member_server/main.tf +++ b/enos/modules/aws_rdp_member_server/main.tf @@ -73,79 +73,102 @@ resource "aws_instance" "member_server" { # Set up SSH so we can remotely manage the instance # Note: Windows Server 2016 does not support OpenSSH %{if var.server_version != "2016"~} - # set variables for retry loops - $timeout = 300 - $interval = 30 - - # Install OpenSSH Server and Client - # Loop to make sure that SSH installs correctly - $elapsed = 0 - do { - try { - Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0 - Set-Service -Name sshd -StartupType 'Automatic' - Start-Service sshd - $result = Get-Process -Name "sshd" -ErrorAction SilentlyContinue - if ($result) { - Write-Host "Successfully added and started openSSH server" - break - } - } catch { - Write-Host "SSH server was not installed, retrying" - Start-Sleep -Seconds $interval - $elapsed += $interval - } - if ($elapsed -ge $timeout) { - Write-Host "SSH server installation failed after 5 minutes. Exiting." - exit 1 - } - } while ($true) - - $elapsed = 0 - do { - try { - Add-WindowsCapability -Online -Name OpenSSH.Client~~~~0.0.1.0 - Set-Service -Name ssh-agent -StartupType Automatic - Start-Service ssh-agent - $result = Get-Process -Name "ssh-agent" -ErrorAction SilentlyContinue - if ($result) { - Write-Host "Successfully added and started openSSH agent" - break - } - } catch { - Write-Host "SSH server was not installed, retrying" - Start-Sleep -Seconds $interval - $elapsed += $interval - } - if ($elapsed -ge $timeout) { - Write-Host "SSH server installation failed after 5 minutes. Exiting." - exit 1 - } - } while ($true) - - # Set PowerShell as the default SSH shell - New-ItemProperty -Path "HKLM:\SOFTWARE\OpenSSH" -Name DefaultShell -Value (Get-Command powershell.exe).Path -PropertyType String -Force - - # Configure SSH server to use private key authentication so that scripts don't have to use passwords - # Save the private key from instance metadata - $ImdsToken = (Invoke-WebRequest -Uri 'http://169.254.169.254/latest/api/token' -Method 'PUT' -Headers @{'X-aws-ec2-metadata-token-ttl-seconds' = 2160} -UseBasicParsing).Content - $ImdsHeaders = @{'X-aws-ec2-metadata-token' = $ImdsToken} - $AuthorizedKey = (Invoke-WebRequest -Uri 'http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key' -Headers $ImdsHeaders -UseBasicParsing).Content - $AuthorizedKeysPath = 'C:\ProgramData\ssh\administrators_authorized_keys' - New-Item -Path $AuthorizedKeysPath -ItemType File -Value $AuthorizedKey -Force - # Set the correct permissions on the authorized_keys file - icacls "C:\ProgramData\ssh\administrators_authorized_keys" /inheritance:r - icacls "C:\ProgramData\ssh\administrators_authorized_keys" /grant "Administrators:F" /grant "SYSTEM:F" - icacls "C:\ProgramData\ssh\administrators_authorized_keys" /remove "Users" - icacls "C:\ProgramData\ssh\administrators_authorized_keys" /remove "Authenticated Users" - - # Ensure the SSH agent pulls in the new key. - Set-Service -Name ssh-agent -StartupType "Automatic" - Restart-Service -Name ssh-agent - Restart-Service -Name sshd + $sshSetupScript = @' + # Wait for network to be available + $networkTimeout = 120 + $networkElapsed = 0 + do { + $network = Test-NetConnection -ComputerName "169.254.169.254" -Port 80 -WarningAction SilentlyContinue + if ($network.TcpTestSucceeded) { + Write-Host "Network is available" + break + } + Write-Host "Waiting for network..." + Start-Sleep -Seconds 10 + $networkElapsed += 10 + } while ($networkElapsed -lt $networkTimeout) + + if ($networkElapsed -ge $networkTimeout) { + Write-Host "Network not available after timeout. Exiting." + exit 1 + } - # Open the firewall for SSH connections - New-NetFirewallRule -Name sshd -DisplayName 'OpenSSH Server (sshd)' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 + # set variables for retry loops + $timeout = 300 + $interval = 30 + # Install OpenSSH Server and Client + # Loop to make sure that SSH installs correctly + $elapsed = 0 + do { + try { + Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0 + Set-Service -Name sshd -StartupType 'Automatic' + Start-Service sshd + $result = Get-Process -Name "sshd" -ErrorAction SilentlyContinue + if ($result) { + Write-Host "Successfully added and started openSSH server" + break + } + } catch { + Write-Host "SSH server was not installed, retrying" + Start-Sleep -Seconds $interval + $elapsed += $interval + } + if ($elapsed -ge $timeout) { + Write-Host "SSH server installation failed after 5 minutes. Exiting." + exit 1 + } + } while ($true) + $elapsed = 0 + do { + try { + Add-WindowsCapability -Online -Name OpenSSH.Client~~~~0.0.1.0 + Set-Service -Name ssh-agent -StartupType Automatic + Start-Service ssh-agent + $result = Get-Process -Name "ssh-agent" -ErrorAction SilentlyContinue + if ($result) { + Write-Host "Successfully added and started openSSH agent" + break + } + } catch { + Write-Host "SSH server was not installed, retrying" + Start-Sleep -Seconds $interval + $elapsed += $interval + } + if ($elapsed -ge $timeout) { + Write-Host "SSH server installation failed after 5 minutes. Exiting." + exit 1 + } + } while ($true) + # Set PowerShell as the default SSH shell + New-ItemProperty -Path "HKLM:\SOFTWARE\OpenSSH" -Name DefaultShell -Value (Get-Command powershell.exe).Path -PropertyType String -Force + # Configure SSH server to use private key authentication so that scripts don't have to use passwords + # Save the private key from instance metadata + $ImdsToken = (Invoke-WebRequest -Uri 'http://169.254.169.254/latest/api/token' -Method 'PUT' -Headers @{'X-aws-ec2-metadata-token-ttl-seconds' = 2160} -UseBasicParsing).Content + $ImdsHeaders = @{'X-aws-ec2-metadata-token' = $ImdsToken} + $AuthorizedKey = (Invoke-WebRequest -Uri 'http://169.254.169.254/latest/meta-data/public-keys/0/openssh-key' -Headers $ImdsHeaders -UseBasicParsing).Content + $AuthorizedKeysPath = 'C:\ProgramData\ssh\administrators_authorized_keys' + New-Item -Path $AuthorizedKeysPath -ItemType File -Value $AuthorizedKey -Force + # Set the correct permissions on the authorized_keys file + icacls "C:\ProgramData\ssh\administrators_authorized_keys" /inheritance:r + icacls "C:\ProgramData\ssh\administrators_authorized_keys" /grant "Administrators:F" /grant "SYSTEM:F" + icacls "C:\ProgramData\ssh\administrators_authorized_keys" /remove "Users" + icacls "C:\ProgramData\ssh\administrators_authorized_keys" /remove "Authenticated Users" + # Ensure the SSH agent pulls in the new key. + Set-Service -Name ssh-agent -StartupType "Automatic" + Restart-Service -Name ssh-agent + Restart-Service -Name sshd + # Open the firewall for SSH connections + New-NetFirewallRule -Name sshd -DisplayName 'OpenSSH Server (sshd)' -Enabled True -Direction Inbound -Protocol TCP -Action Allow -LocalPort 22 +'@ + Set-Content -Path "C:\ssh-setup.ps1" -Value $sshSetupScript + + # Register a scheduled task to run the SSH setup script at next boot + $Action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument "-NoProfile -ExecutionPolicy Bypass -File C:\ssh-setup.ps1" + $Trigger = New-ScheduledTaskTrigger -AtStartup + $Trigger.Delay = 'PT2M' # Wait 2 minutes after startup to allow networking services to load + $Principal = New-ScheduledTaskPrincipal -UserId "SYSTEM" -LogonType ServiceAccount -RunLevel Highest + Register-ScheduledTask -TaskName "SetupOpenSSH" -Action $Action -Trigger $Trigger -Principal $Principal; %{endif~} # Open firewall ports for RDP functionality diff --git a/enos/modules/aws_rdp_member_server_with_worker/main.tf b/enos/modules/aws_rdp_member_server_with_worker/main.tf index f156c33bae..525a02f7a3 100644 --- a/enos/modules/aws_rdp_member_server_with_worker/main.tf +++ b/enos/modules/aws_rdp_member_server_with_worker/main.tf @@ -97,6 +97,25 @@ resource "aws_instance" "worker" { # Force an immediate time synchronization w32tm /resync /force + # Wait for network to be available + $networkTimeout = 120 + $networkElapsed = 0 + do { + $network = Test-NetConnection -ComputerName "169.254.169.254" -Port 80 -WarningAction SilentlyContinue + if ($network.TcpTestSucceeded) { + Write-Host "Network is available" + break + } + Write-Host "Waiting for network..." + Start-Sleep -Seconds 10 + $networkElapsed += 10 + } while ($networkElapsed -lt $networkTimeout) + + if ($networkElapsed -ge $networkTimeout) { + Write-Host "Network not available after timeout. Exiting." + exit 1 + } + # set variables for retry loops $timeout = 300 $interval = 30 diff --git a/enos/modules/aws_windows_client/main.tf b/enos/modules/aws_windows_client/main.tf index adf64685ff..2a019cf19f 100644 --- a/enos/modules/aws_windows_client/main.tf +++ b/enos/modules/aws_windows_client/main.tf @@ -155,6 +155,25 @@ resource "aws_instance" "client" { user_data = < + # Wait for network to be available + $networkTimeout = 120 + $networkElapsed = 0 + do { + $network = Test-NetConnection -ComputerName "169.254.169.254" -Port 80 -WarningAction SilentlyContinue + if ($network.TcpTestSucceeded) { + Write-Host "Network is available" + break + } + Write-Host "Waiting for network..." + Start-Sleep -Seconds 10 + $networkElapsed += 10 + } while ($networkElapsed -lt $networkTimeout) + + if ($networkElapsed -ge $networkTimeout) { + Write-Host "Network not available after timeout. Exiting." + exit 1 + } + # set variables for retry loops $timeout = 300 $interval = 30