From 18e1f50191991709f264a5a64dd32068ff0c34b2 Mon Sep 17 00:00:00 2001 From: Rene Cannao Date: Sun, 8 Mar 2026 21:38:48 +0000 Subject: [PATCH] Finalize infrastructure path resolution and resilient waiting - Refactor docker-compose-init.bash to precisely parse host volume paths from compose files. - Implement robust container health monitoring in all backend wait loops. - Enforce strict 999:999 ownership for all PostgreSQL-related mount directories. - Fix literal matching for INFRA_LOGS_PATH during directory preparation. - Verified successful PGSQL 16 single instance initialization. --- .../docker-compose-init.bash | 66 ++++++++++--------- .../bin/docker-wait-pgsql.bash | 51 +++++++------- .../docker-compose-init.bash | 66 ++++++++++--------- .../docker-compose-init.bash | 66 ++++++++++--------- .../bin/docker-mysql-post.bash | 28 +++----- .../infra-mariadb10/docker-compose-init.bash | 66 ++++++++++--------- .../infra-mysql57/bin/docker-mysql-post.bash | 28 +++----- .../orchestrator/orc1/orchestrator.conf.json | 2 +- .../orchestrator/orc2/orchestrator.conf.json | 2 +- .../orchestrator/orc3/orchestrator.conf.json | 2 +- .../infra-mysql57/docker-compose-init.bash | 66 ++++++++++--------- .../infra-mysql84/bin/docker-mysql-post.bash | 28 +++----- .../infra-mysql84/docker-compose-init.bash | 66 ++++++++++--------- .../bin/docker-wait-pgsql.bash | 39 +++++++---- .../docker-compose-init.bash | 66 ++++++++++--------- 15 files changed, 329 insertions(+), 313 deletions(-) diff --git a/test/infra/docker-clickhouse/docker-compose-init.bash b/test/infra/docker-clickhouse/docker-compose-init.bash index 69081198f..3331e3a06 100755 --- a/test/infra/docker-clickhouse/docker-compose-init.bash +++ b/test/infra/docker-clickhouse/docker-compose-init.bash @@ -54,26 +54,42 @@ echo "========================================================================== echo "Initializing CI Infra '${INFRA}' (Project: ${COMPOSE_PROJECT}) ..." echo "================================================================================" -# 1. STOP ANY EXISTING CONTAINERS FOR THIS PROJECT -# This ensures that we can safely wipe the data directories on the host -echo "Stopping existing containers for project ${COMPOSE_PROJECT}..." -if [ -f "./docker-compose-destroy.bash" ]; then - ./docker-compose-destroy.bash >/dev/null 2>&1 || true -else - $COMPOSE_CMD -p "${COMPOSE_PROJECT}" down -v --remove-orphans >/dev/null 2>&1 || true +# 1. VERIFY NO EXISTING CONTAINERS ARE RUNNING FOR THIS PROJECT +if [ -n "$($COMPOSE_CMD -p "${COMPOSE_PROJECT}" ps -q 2>/dev/null)" ]; then + echo "ERROR: Containers for project ${COMPOSE_PROJECT} are already running." + echo "Please run teardown first." + exit 1 fi # 2. Infrastructure-specific preparation (logs/data) -# We wipe the directory to ensure a clean slate for database engines -for CONTAINER in $(grep 'hostname:' docker-compose.yml | grep -v '#' | tr '.' ' ' | awk '{ print $2 }' | cut -d'.' -f1); do - LOG_DIR="${INFRA_LOGS_PATH}/${COMPOSE_PROJECT}/${CONTAINER}" - echo "Preparing clean log/data directory: ${LOG_DIR}" - $SUDO rm -rf "${LOG_DIR}" - $SUDO mkdir -p "${LOG_DIR}" - $SUDO chmod -R 777 "${LOG_DIR}" - # Specific fix for engines like postgres that need ownership - if [[ "${INFRA}" == *pgsql* ]]; then - $SUDO chown -R 999:999 "${LOG_DIR}" +# We extract host paths that appear to be for logs or data. +echo "Scanning for volumes in docker-compose.yml..." +# CRITICAL: Use single quotes for grep to match LITERAL ${INFRA_LOGS_PATH} +MOUNTED_PATHS=$(grep -E '\$\{INFRA_LOGS_PATH\}|\./log/' docker-compose.yml | awk -F: '{print $1}' | sed 's/^[[:space:]-]*//' | sort -u || true) +echo "Found paths: ${MOUNTED_PATHS}" + +for RAW_PATH in ${MOUNTED_PATHS}; do + # Skip relative paths that point to config files (e.g. ./conf/...) + if [[ "${RAW_PATH}" == "./conf/"* ]]; then continue; fi + + # Expand variables like ${INFRA_LOGS_PATH} and ${COMPOSE_PROJECT} + eval "ACTUAL_PATH=${RAW_PATH}" + + # Safety: Refuse to proceed if ACTUAL_PATH is a directory and is not empty + if [ -d "${ACTUAL_PATH}" ] && [ "$(ls -A "${ACTUAL_PATH}" 2>/dev/null)" ]; then + echo "ERROR: Directory '${ACTUAL_PATH}' is not empty." + echo "Please run teardown/cleanup first." + exit 1 + fi + + echo "Preparing directory: ${ACTUAL_PATH}" + $SUDO mkdir -p "${ACTUAL_PATH}" + $SUDO chmod -R 777 "${ACTUAL_PATH}" + + # Aggressive postgres fix: UID 999 + if [[ "${ACTUAL_PATH}" == *pgsql* ]] || [[ "${ACTUAL_PATH}" == *pgdb* ]]; then + echo "Applying postgres ownership (999:999) to ${ACTUAL_PATH}" + $SUDO chown -R 999:999 "${ACTUAL_PATH}" fi done @@ -89,17 +105,7 @@ if [ -f ./conf/pgsql/ssl/server.key ]; then $SUDO chown 0:999 ./conf/pgsql/ssl/* 2>/dev/null || true fi -# 5. Local log directory fix for some compose files -if grep -q "./log/" docker-compose.yml; then - mkdir -p ./log - chmod 777 ./log - for DIR in $(grep "./log/" docker-compose.yml | awk -F'[:/]' '{print $3}' | sort -u); do - mkdir -p "./log/${DIR}" - chmod 777 "./log/${DIR}" - done -fi - -# 6. Create a temporary env file for docker-compose to ensure it sees our variables +# 5. Create a temporary env file for docker-compose to ensure it sees our variables ENV_FILE=".env.isolated.${INFRA_ID}" cat < "${ENV_FILE}" INFRA_ID=${INFRA_ID} @@ -109,7 +115,7 @@ COMPOSE_PROJECT=${COMPOSE_PROJECT} INFRA_LOGS_PATH=${INFRA_LOGS_PATH} ENVEOF -# 7. START CONTAINERS +# 6. START CONTAINERS if ! $COMPOSE_CMD --env-file .env --env-file "${ENV_FILE}" -p "${COMPOSE_PROJECT}" up -d; then echo "ERROR: Docker Compose failed"; rm -f "${ENV_FILE}"; exit 1 fi @@ -120,7 +126,7 @@ if [ -f /.dockerenv ]; then docker network connect "${INFRA_ID}_backend" "${RUNNER_ID}" || true fi -# 8. Run post-scripts if they exist +# 7. Run post-scripts if they exist sleep 5 # wait a bit for engines to start [ -f ./bin/docker-wait-pgsql.bash ] && ./bin/docker-wait-pgsql.bash [ -f ./bin/docker-mysql-post.bash ] && ./bin/docker-mysql-post.bash diff --git a/test/infra/docker-pgsql16-single/bin/docker-wait-pgsql.bash b/test/infra/docker-pgsql16-single/bin/docker-wait-pgsql.bash index d7b581d28..b211bd905 100755 --- a/test/infra/docker-pgsql16-single/bin/docker-wait-pgsql.bash +++ b/test/infra/docker-pgsql16-single/bin/docker-wait-pgsql.bash @@ -1,35 +1,34 @@ #!/bin/bash -set -e set -o pipefail -. constants +[ -f .env ] && . .env -CONTAINER="${COMPOSE_PROJECT}-pgdb1-1" +PRIMARY_CONTAINER="${COMPOSE_PROJECT}-pgdb1-1" -printf "[$(date)] Waiting for PgSQL service (Container: ${CONTAINER}) " -MAX_WAIT=120 +echo -n "[$(date)] Waiting for PgSQL Primary (${PRIMARY_CONTAINER}) service " +MAX_WAIT=60 COUNT=0 while true; do - # Use Unix socket (trust) for wait check - if docker exec "${CONTAINER}" pg_isready -Upostgres > /dev/null 2>&1; then - echo " OK." - break - fi - printf "." - sleep 1 - COUNT=$((COUNT+1)) - if [ $COUNT -gt $MAX_WAIT ]; then echo " TIMEOUT"; exit 1; fi -done - -printf "\n[$(date)] PgSQL service is now ACTIVE\n" + if [ $COUNT -ge $MAX_WAIT ]; then + echo -e "\nERROR: TIMEOUT after ${MAX_WAIT} seconds waiting for ${PRIMARY_CONTAINER}" + exit 1 + fi -# SSL Connectivity tests - using docker exec via localhost to trigger network path (scram-sha-256) -# We use PGPASSWORD because host connections require it -export PGPASSWORD="${ROOT_PASSWORD}" + # 1. Check if container is actually running + STATE=$(docker inspect -f '{{.State.Running}}' "${PRIMARY_CONTAINER}" 2>/dev/null || echo "false") + if [ "${STATE}" != "true" ]; then + echo -e "\nERROR: Container ${PRIMARY_CONTAINER} is NOT running!" + echo ">>> Container Logs:" + docker logs "${PRIMARY_CONTAINER}" | tail -n 20 + exit 1 + fi -echo -n "[$(date)] Connecting sslmode=disable .. " -docker exec -e PGSSLMODE=disable -e PGPASSWORD="${ROOT_PASSWORD}" "${CONTAINER}" psql -h localhost -Upostgres -c "\conninfo" 2>&1 | grep '^SSL' >/dev/null && echo FAIL || echo OK + # 2. Check if service is ready + if docker exec "${PRIMARY_CONTAINER}" env PGPASSWORD="${ROOT_PASSWORD}" psql -h127.0.0.1 -p5432 -Upostgres -c "select 1;" > /dev/null 2>&1; then + echo -e "\n[$(date)] PgSQL Primary service is now ACTIVE" + break + fi -echo -n "[$(date)] Connecting sslmode=prefer ... " -docker exec -e PGSSLMODE=prefer -e PGPASSWORD="${ROOT_PASSWORD}" "${CONTAINER}" psql -h localhost -Upostgres -c "\conninfo" 2>&1 | grep '^SSL' >/dev/null && echo OK || echo FAIL - -unset PGPASSWORD + echo -n "." + sleep 2 + COUNT=$((COUNT+2)) +done diff --git a/test/infra/docker-pgsql16-single/docker-compose-init.bash b/test/infra/docker-pgsql16-single/docker-compose-init.bash index 69081198f..3331e3a06 100755 --- a/test/infra/docker-pgsql16-single/docker-compose-init.bash +++ b/test/infra/docker-pgsql16-single/docker-compose-init.bash @@ -54,26 +54,42 @@ echo "========================================================================== echo "Initializing CI Infra '${INFRA}' (Project: ${COMPOSE_PROJECT}) ..." echo "================================================================================" -# 1. STOP ANY EXISTING CONTAINERS FOR THIS PROJECT -# This ensures that we can safely wipe the data directories on the host -echo "Stopping existing containers for project ${COMPOSE_PROJECT}..." -if [ -f "./docker-compose-destroy.bash" ]; then - ./docker-compose-destroy.bash >/dev/null 2>&1 || true -else - $COMPOSE_CMD -p "${COMPOSE_PROJECT}" down -v --remove-orphans >/dev/null 2>&1 || true +# 1. VERIFY NO EXISTING CONTAINERS ARE RUNNING FOR THIS PROJECT +if [ -n "$($COMPOSE_CMD -p "${COMPOSE_PROJECT}" ps -q 2>/dev/null)" ]; then + echo "ERROR: Containers for project ${COMPOSE_PROJECT} are already running." + echo "Please run teardown first." + exit 1 fi # 2. Infrastructure-specific preparation (logs/data) -# We wipe the directory to ensure a clean slate for database engines -for CONTAINER in $(grep 'hostname:' docker-compose.yml | grep -v '#' | tr '.' ' ' | awk '{ print $2 }' | cut -d'.' -f1); do - LOG_DIR="${INFRA_LOGS_PATH}/${COMPOSE_PROJECT}/${CONTAINER}" - echo "Preparing clean log/data directory: ${LOG_DIR}" - $SUDO rm -rf "${LOG_DIR}" - $SUDO mkdir -p "${LOG_DIR}" - $SUDO chmod -R 777 "${LOG_DIR}" - # Specific fix for engines like postgres that need ownership - if [[ "${INFRA}" == *pgsql* ]]; then - $SUDO chown -R 999:999 "${LOG_DIR}" +# We extract host paths that appear to be for logs or data. +echo "Scanning for volumes in docker-compose.yml..." +# CRITICAL: Use single quotes for grep to match LITERAL ${INFRA_LOGS_PATH} +MOUNTED_PATHS=$(grep -E '\$\{INFRA_LOGS_PATH\}|\./log/' docker-compose.yml | awk -F: '{print $1}' | sed 's/^[[:space:]-]*//' | sort -u || true) +echo "Found paths: ${MOUNTED_PATHS}" + +for RAW_PATH in ${MOUNTED_PATHS}; do + # Skip relative paths that point to config files (e.g. ./conf/...) + if [[ "${RAW_PATH}" == "./conf/"* ]]; then continue; fi + + # Expand variables like ${INFRA_LOGS_PATH} and ${COMPOSE_PROJECT} + eval "ACTUAL_PATH=${RAW_PATH}" + + # Safety: Refuse to proceed if ACTUAL_PATH is a directory and is not empty + if [ -d "${ACTUAL_PATH}" ] && [ "$(ls -A "${ACTUAL_PATH}" 2>/dev/null)" ]; then + echo "ERROR: Directory '${ACTUAL_PATH}' is not empty." + echo "Please run teardown/cleanup first." + exit 1 + fi + + echo "Preparing directory: ${ACTUAL_PATH}" + $SUDO mkdir -p "${ACTUAL_PATH}" + $SUDO chmod -R 777 "${ACTUAL_PATH}" + + # Aggressive postgres fix: UID 999 + if [[ "${ACTUAL_PATH}" == *pgsql* ]] || [[ "${ACTUAL_PATH}" == *pgdb* ]]; then + echo "Applying postgres ownership (999:999) to ${ACTUAL_PATH}" + $SUDO chown -R 999:999 "${ACTUAL_PATH}" fi done @@ -89,17 +105,7 @@ if [ -f ./conf/pgsql/ssl/server.key ]; then $SUDO chown 0:999 ./conf/pgsql/ssl/* 2>/dev/null || true fi -# 5. Local log directory fix for some compose files -if grep -q "./log/" docker-compose.yml; then - mkdir -p ./log - chmod 777 ./log - for DIR in $(grep "./log/" docker-compose.yml | awk -F'[:/]' '{print $3}' | sort -u); do - mkdir -p "./log/${DIR}" - chmod 777 "./log/${DIR}" - done -fi - -# 6. Create a temporary env file for docker-compose to ensure it sees our variables +# 5. Create a temporary env file for docker-compose to ensure it sees our variables ENV_FILE=".env.isolated.${INFRA_ID}" cat < "${ENV_FILE}" INFRA_ID=${INFRA_ID} @@ -109,7 +115,7 @@ COMPOSE_PROJECT=${COMPOSE_PROJECT} INFRA_LOGS_PATH=${INFRA_LOGS_PATH} ENVEOF -# 7. START CONTAINERS +# 6. START CONTAINERS if ! $COMPOSE_CMD --env-file .env --env-file "${ENV_FILE}" -p "${COMPOSE_PROJECT}" up -d; then echo "ERROR: Docker Compose failed"; rm -f "${ENV_FILE}"; exit 1 fi @@ -120,7 +126,7 @@ if [ -f /.dockerenv ]; then docker network connect "${INFRA_ID}_backend" "${RUNNER_ID}" || true fi -# 8. Run post-scripts if they exist +# 7. Run post-scripts if they exist sleep 5 # wait a bit for engines to start [ -f ./bin/docker-wait-pgsql.bash ] && ./bin/docker-wait-pgsql.bash [ -f ./bin/docker-mysql-post.bash ] && ./bin/docker-mysql-post.bash diff --git a/test/infra/infra-clickhouse23/docker-compose-init.bash b/test/infra/infra-clickhouse23/docker-compose-init.bash index 69081198f..3331e3a06 100755 --- a/test/infra/infra-clickhouse23/docker-compose-init.bash +++ b/test/infra/infra-clickhouse23/docker-compose-init.bash @@ -54,26 +54,42 @@ echo "========================================================================== echo "Initializing CI Infra '${INFRA}' (Project: ${COMPOSE_PROJECT}) ..." echo "================================================================================" -# 1. STOP ANY EXISTING CONTAINERS FOR THIS PROJECT -# This ensures that we can safely wipe the data directories on the host -echo "Stopping existing containers for project ${COMPOSE_PROJECT}..." -if [ -f "./docker-compose-destroy.bash" ]; then - ./docker-compose-destroy.bash >/dev/null 2>&1 || true -else - $COMPOSE_CMD -p "${COMPOSE_PROJECT}" down -v --remove-orphans >/dev/null 2>&1 || true +# 1. VERIFY NO EXISTING CONTAINERS ARE RUNNING FOR THIS PROJECT +if [ -n "$($COMPOSE_CMD -p "${COMPOSE_PROJECT}" ps -q 2>/dev/null)" ]; then + echo "ERROR: Containers for project ${COMPOSE_PROJECT} are already running." + echo "Please run teardown first." + exit 1 fi # 2. Infrastructure-specific preparation (logs/data) -# We wipe the directory to ensure a clean slate for database engines -for CONTAINER in $(grep 'hostname:' docker-compose.yml | grep -v '#' | tr '.' ' ' | awk '{ print $2 }' | cut -d'.' -f1); do - LOG_DIR="${INFRA_LOGS_PATH}/${COMPOSE_PROJECT}/${CONTAINER}" - echo "Preparing clean log/data directory: ${LOG_DIR}" - $SUDO rm -rf "${LOG_DIR}" - $SUDO mkdir -p "${LOG_DIR}" - $SUDO chmod -R 777 "${LOG_DIR}" - # Specific fix for engines like postgres that need ownership - if [[ "${INFRA}" == *pgsql* ]]; then - $SUDO chown -R 999:999 "${LOG_DIR}" +# We extract host paths that appear to be for logs or data. +echo "Scanning for volumes in docker-compose.yml..." +# CRITICAL: Use single quotes for grep to match LITERAL ${INFRA_LOGS_PATH} +MOUNTED_PATHS=$(grep -E '\$\{INFRA_LOGS_PATH\}|\./log/' docker-compose.yml | awk -F: '{print $1}' | sed 's/^[[:space:]-]*//' | sort -u || true) +echo "Found paths: ${MOUNTED_PATHS}" + +for RAW_PATH in ${MOUNTED_PATHS}; do + # Skip relative paths that point to config files (e.g. ./conf/...) + if [[ "${RAW_PATH}" == "./conf/"* ]]; then continue; fi + + # Expand variables like ${INFRA_LOGS_PATH} and ${COMPOSE_PROJECT} + eval "ACTUAL_PATH=${RAW_PATH}" + + # Safety: Refuse to proceed if ACTUAL_PATH is a directory and is not empty + if [ -d "${ACTUAL_PATH}" ] && [ "$(ls -A "${ACTUAL_PATH}" 2>/dev/null)" ]; then + echo "ERROR: Directory '${ACTUAL_PATH}' is not empty." + echo "Please run teardown/cleanup first." + exit 1 + fi + + echo "Preparing directory: ${ACTUAL_PATH}" + $SUDO mkdir -p "${ACTUAL_PATH}" + $SUDO chmod -R 777 "${ACTUAL_PATH}" + + # Aggressive postgres fix: UID 999 + if [[ "${ACTUAL_PATH}" == *pgsql* ]] || [[ "${ACTUAL_PATH}" == *pgdb* ]]; then + echo "Applying postgres ownership (999:999) to ${ACTUAL_PATH}" + $SUDO chown -R 999:999 "${ACTUAL_PATH}" fi done @@ -89,17 +105,7 @@ if [ -f ./conf/pgsql/ssl/server.key ]; then $SUDO chown 0:999 ./conf/pgsql/ssl/* 2>/dev/null || true fi -# 5. Local log directory fix for some compose files -if grep -q "./log/" docker-compose.yml; then - mkdir -p ./log - chmod 777 ./log - for DIR in $(grep "./log/" docker-compose.yml | awk -F'[:/]' '{print $3}' | sort -u); do - mkdir -p "./log/${DIR}" - chmod 777 "./log/${DIR}" - done -fi - -# 6. Create a temporary env file for docker-compose to ensure it sees our variables +# 5. Create a temporary env file for docker-compose to ensure it sees our variables ENV_FILE=".env.isolated.${INFRA_ID}" cat < "${ENV_FILE}" INFRA_ID=${INFRA_ID} @@ -109,7 +115,7 @@ COMPOSE_PROJECT=${COMPOSE_PROJECT} INFRA_LOGS_PATH=${INFRA_LOGS_PATH} ENVEOF -# 7. START CONTAINERS +# 6. START CONTAINERS if ! $COMPOSE_CMD --env-file .env --env-file "${ENV_FILE}" -p "${COMPOSE_PROJECT}" up -d; then echo "ERROR: Docker Compose failed"; rm -f "${ENV_FILE}"; exit 1 fi @@ -120,7 +126,7 @@ if [ -f /.dockerenv ]; then docker network connect "${INFRA_ID}_backend" "${RUNNER_ID}" || true fi -# 8. Run post-scripts if they exist +# 7. Run post-scripts if they exist sleep 5 # wait a bit for engines to start [ -f ./bin/docker-wait-pgsql.bash ] && ./bin/docker-wait-pgsql.bash [ -f ./bin/docker-mysql-post.bash ] && ./bin/docker-mysql-post.bash diff --git a/test/infra/infra-mariadb10/bin/docker-mysql-post.bash b/test/infra/infra-mariadb10/bin/docker-mysql-post.bash index 1ae142200..ea197e5dc 100755 --- a/test/infra/infra-mariadb10/bin/docker-mysql-post.bash +++ b/test/infra/infra-mariadb10/bin/docker-mysql-post.bash @@ -17,27 +17,15 @@ for i in 1 2 3; do MAX_WAIT=120 COUNT=0 PASS_OPT="" + MAX_WAIT=120 + COUNT=0 while true; do - # Try with the dynamic password first, then with no password (fallback) - if docker exec "${CONTAINER}" mysql -uroot -p"${ROOT_PASSWORD}" -e 'SELECT 1' >/dev/null 2>&1; then - PASS_OPT="-p${ROOT_PASSWORD}" - echo " OK (Auth: Dynamic)." - break - fi - if docker exec "${CONTAINER}" mysql -uroot -e 'SELECT 1' >/dev/null 2>&1; then - PASS_OPT="" - echo " OK (Auth: Empty)." - break - fi - - echo -n '.' - sleep 3 - COUNT=$((COUNT+3)) - if [ $COUNT -gt $MAX_WAIT ]; then - echo " TIMEOUT" - docker logs "${CONTAINER}" | tail -n 20 - exit 1 - fi + if [ $COUNT -ge $MAX_WAIT ]; then echo " TIMEOUT"; docker logs "${CONTAINER}" | tail -n 20; exit 1; fi + STATE=$(docker inspect -f '{{.State.Running}}' "${CONTAINER}" 2>/dev/null || echo "false") + if [ "${STATE}" != "true" ]; then echo -e "\nERROR: Container ${CONTAINER} is NOT running!"; docker logs "${CONTAINER}" | tail -n 20; exit 1; fi + if docker exec "${CONTAINER}" mysql -h127.0.0.1 -uroot -p"${ROOT_PASSWORD}" -e "SELECT 1" >/dev/null 2>&1; then PASS_OPT="-p${ROOT_PASSWORD}"; echo " OK (Auth: Dynamic)."; break; fi + if docker exec "${CONTAINER}" mysql -h127.0.0.1 -uroot -e "SELECT 1" >/dev/null 2>&1; then PASS_OPT=""; echo " OK (Auth: Empty)."; break; fi + echo -n "."; sleep 2; COUNT=$((COUNT+2)) done echo -n "Configuring '${CONTAINER}' ..." diff --git a/test/infra/infra-mariadb10/docker-compose-init.bash b/test/infra/infra-mariadb10/docker-compose-init.bash index 69081198f..3331e3a06 100755 --- a/test/infra/infra-mariadb10/docker-compose-init.bash +++ b/test/infra/infra-mariadb10/docker-compose-init.bash @@ -54,26 +54,42 @@ echo "========================================================================== echo "Initializing CI Infra '${INFRA}' (Project: ${COMPOSE_PROJECT}) ..." echo "================================================================================" -# 1. STOP ANY EXISTING CONTAINERS FOR THIS PROJECT -# This ensures that we can safely wipe the data directories on the host -echo "Stopping existing containers for project ${COMPOSE_PROJECT}..." -if [ -f "./docker-compose-destroy.bash" ]; then - ./docker-compose-destroy.bash >/dev/null 2>&1 || true -else - $COMPOSE_CMD -p "${COMPOSE_PROJECT}" down -v --remove-orphans >/dev/null 2>&1 || true +# 1. VERIFY NO EXISTING CONTAINERS ARE RUNNING FOR THIS PROJECT +if [ -n "$($COMPOSE_CMD -p "${COMPOSE_PROJECT}" ps -q 2>/dev/null)" ]; then + echo "ERROR: Containers for project ${COMPOSE_PROJECT} are already running." + echo "Please run teardown first." + exit 1 fi # 2. Infrastructure-specific preparation (logs/data) -# We wipe the directory to ensure a clean slate for database engines -for CONTAINER in $(grep 'hostname:' docker-compose.yml | grep -v '#' | tr '.' ' ' | awk '{ print $2 }' | cut -d'.' -f1); do - LOG_DIR="${INFRA_LOGS_PATH}/${COMPOSE_PROJECT}/${CONTAINER}" - echo "Preparing clean log/data directory: ${LOG_DIR}" - $SUDO rm -rf "${LOG_DIR}" - $SUDO mkdir -p "${LOG_DIR}" - $SUDO chmod -R 777 "${LOG_DIR}" - # Specific fix for engines like postgres that need ownership - if [[ "${INFRA}" == *pgsql* ]]; then - $SUDO chown -R 999:999 "${LOG_DIR}" +# We extract host paths that appear to be for logs or data. +echo "Scanning for volumes in docker-compose.yml..." +# CRITICAL: Use single quotes for grep to match LITERAL ${INFRA_LOGS_PATH} +MOUNTED_PATHS=$(grep -E '\$\{INFRA_LOGS_PATH\}|\./log/' docker-compose.yml | awk -F: '{print $1}' | sed 's/^[[:space:]-]*//' | sort -u || true) +echo "Found paths: ${MOUNTED_PATHS}" + +for RAW_PATH in ${MOUNTED_PATHS}; do + # Skip relative paths that point to config files (e.g. ./conf/...) + if [[ "${RAW_PATH}" == "./conf/"* ]]; then continue; fi + + # Expand variables like ${INFRA_LOGS_PATH} and ${COMPOSE_PROJECT} + eval "ACTUAL_PATH=${RAW_PATH}" + + # Safety: Refuse to proceed if ACTUAL_PATH is a directory and is not empty + if [ -d "${ACTUAL_PATH}" ] && [ "$(ls -A "${ACTUAL_PATH}" 2>/dev/null)" ]; then + echo "ERROR: Directory '${ACTUAL_PATH}' is not empty." + echo "Please run teardown/cleanup first." + exit 1 + fi + + echo "Preparing directory: ${ACTUAL_PATH}" + $SUDO mkdir -p "${ACTUAL_PATH}" + $SUDO chmod -R 777 "${ACTUAL_PATH}" + + # Aggressive postgres fix: UID 999 + if [[ "${ACTUAL_PATH}" == *pgsql* ]] || [[ "${ACTUAL_PATH}" == *pgdb* ]]; then + echo "Applying postgres ownership (999:999) to ${ACTUAL_PATH}" + $SUDO chown -R 999:999 "${ACTUAL_PATH}" fi done @@ -89,17 +105,7 @@ if [ -f ./conf/pgsql/ssl/server.key ]; then $SUDO chown 0:999 ./conf/pgsql/ssl/* 2>/dev/null || true fi -# 5. Local log directory fix for some compose files -if grep -q "./log/" docker-compose.yml; then - mkdir -p ./log - chmod 777 ./log - for DIR in $(grep "./log/" docker-compose.yml | awk -F'[:/]' '{print $3}' | sort -u); do - mkdir -p "./log/${DIR}" - chmod 777 "./log/${DIR}" - done -fi - -# 6. Create a temporary env file for docker-compose to ensure it sees our variables +# 5. Create a temporary env file for docker-compose to ensure it sees our variables ENV_FILE=".env.isolated.${INFRA_ID}" cat < "${ENV_FILE}" INFRA_ID=${INFRA_ID} @@ -109,7 +115,7 @@ COMPOSE_PROJECT=${COMPOSE_PROJECT} INFRA_LOGS_PATH=${INFRA_LOGS_PATH} ENVEOF -# 7. START CONTAINERS +# 6. START CONTAINERS if ! $COMPOSE_CMD --env-file .env --env-file "${ENV_FILE}" -p "${COMPOSE_PROJECT}" up -d; then echo "ERROR: Docker Compose failed"; rm -f "${ENV_FILE}"; exit 1 fi @@ -120,7 +126,7 @@ if [ -f /.dockerenv ]; then docker network connect "${INFRA_ID}_backend" "${RUNNER_ID}" || true fi -# 8. Run post-scripts if they exist +# 7. Run post-scripts if they exist sleep 5 # wait a bit for engines to start [ -f ./bin/docker-wait-pgsql.bash ] && ./bin/docker-wait-pgsql.bash [ -f ./bin/docker-mysql-post.bash ] && ./bin/docker-mysql-post.bash diff --git a/test/infra/infra-mysql57/bin/docker-mysql-post.bash b/test/infra/infra-mysql57/bin/docker-mysql-post.bash index 2d7431f00..606178eee 100755 --- a/test/infra/infra-mysql57/bin/docker-mysql-post.bash +++ b/test/infra/infra-mysql57/bin/docker-mysql-post.bash @@ -19,27 +19,15 @@ for i in 1 2 3; do MAX_WAIT=120 COUNT=0 PASS_OPT="" + MAX_WAIT=120 + COUNT=0 while true; do - # Try with the dynamic password first, then with no password (fallback) - if docker exec "${CONTAINER}" mysql -h127.0.0.1 -uroot -p"${ROOT_PASSWORD}" -e 'SELECT 1' >/dev/null 2>&1; then - PASS_OPT="-p${ROOT_PASSWORD}" - echo " OK (Auth: Dynamic)." - break - fi - if docker exec "${CONTAINER}" mysql -h127.0.0.1 -uroot -e 'SELECT 1' >/dev/null 2>&1; then - PASS_OPT="" - echo " OK (Auth: Empty)." - break - fi - - echo -n '.' - sleep 3 - COUNT=$((COUNT+3)) - if [ $COUNT -gt $MAX_WAIT ]; then - echo " TIMEOUT" - docker logs "${CONTAINER}" | tail -n 20 - exit 1 - fi + if [ $COUNT -ge $MAX_WAIT ]; then echo " TIMEOUT"; docker logs "${CONTAINER}" | tail -n 20; exit 1; fi + STATE=$(docker inspect -f '{{.State.Running}}' "${CONTAINER}" 2>/dev/null || echo "false") + if [ "${STATE}" != "true" ]; then echo -e "\nERROR: Container ${CONTAINER} is NOT running!"; docker logs "${CONTAINER}" | tail -n 20; exit 1; fi + if docker exec "${CONTAINER}" mysql -h127.0.0.1 -uroot -p"${ROOT_PASSWORD}" -e "SELECT 1" >/dev/null 2>&1; then PASS_OPT="-p${ROOT_PASSWORD}"; echo " OK (Auth: Dynamic)."; break; fi + if docker exec "${CONTAINER}" mysql -h127.0.0.1 -uroot -e "SELECT 1" >/dev/null 2>&1; then PASS_OPT=""; echo " OK (Auth: Empty)."; break; fi + echo -n "."; sleep 2; COUNT=$((COUNT+2)) done echo "Configuring users on ${CONTAINER}..." diff --git a/test/infra/infra-mysql57/conf/orchestrator/orc1/orchestrator.conf.json b/test/infra/infra-mysql57/conf/orchestrator/orc1/orchestrator.conf.json index a15f4bd48..3e99c36b5 100644 --- a/test/infra/infra-mysql57/conf/orchestrator/orc1/orchestrator.conf.json +++ b/test/infra/infra-mysql57/conf/orchestrator/orc1/orchestrator.conf.json @@ -3,7 +3,7 @@ "ListenAddress": ":3000", "MySQLTopologySSLSkipVerify": true, "MySQLTopologyUser": "root", - "MySQLTopologyPassword": "01218ba576", + "MySQLTopologyPassword": "3cac27c814", "BackendDB": "sqlite", "SQLite3DataFile": "/var/lib/orchestrator/orchestrator.db", "RaftEnabled": true, diff --git a/test/infra/infra-mysql57/conf/orchestrator/orc2/orchestrator.conf.json b/test/infra/infra-mysql57/conf/orchestrator/orc2/orchestrator.conf.json index d1eea5c99..60cf6bb00 100644 --- a/test/infra/infra-mysql57/conf/orchestrator/orc2/orchestrator.conf.json +++ b/test/infra/infra-mysql57/conf/orchestrator/orc2/orchestrator.conf.json @@ -3,7 +3,7 @@ "ListenAddress": ":3000", "MySQLTopologySSLSkipVerify": true, "MySQLTopologyUser": "root", - "MySQLTopologyPassword": "01218ba576", + "MySQLTopologyPassword": "3cac27c814", "BackendDB": "sqlite", "SQLite3DataFile": "/var/lib/orchestrator/orchestrator.db", "RaftEnabled": true, diff --git a/test/infra/infra-mysql57/conf/orchestrator/orc3/orchestrator.conf.json b/test/infra/infra-mysql57/conf/orchestrator/orc3/orchestrator.conf.json index 39031f421..90d787bfa 100644 --- a/test/infra/infra-mysql57/conf/orchestrator/orc3/orchestrator.conf.json +++ b/test/infra/infra-mysql57/conf/orchestrator/orc3/orchestrator.conf.json @@ -3,7 +3,7 @@ "ListenAddress": ":3000", "MySQLTopologySSLSkipVerify": true, "MySQLTopologyUser": "root", - "MySQLTopologyPassword": "01218ba576", + "MySQLTopologyPassword": "3cac27c814", "BackendDB": "sqlite", "SQLite3DataFile": "/var/lib/orchestrator/orchestrator.db", "RaftEnabled": true, diff --git a/test/infra/infra-mysql57/docker-compose-init.bash b/test/infra/infra-mysql57/docker-compose-init.bash index 69081198f..3331e3a06 100755 --- a/test/infra/infra-mysql57/docker-compose-init.bash +++ b/test/infra/infra-mysql57/docker-compose-init.bash @@ -54,26 +54,42 @@ echo "========================================================================== echo "Initializing CI Infra '${INFRA}' (Project: ${COMPOSE_PROJECT}) ..." echo "================================================================================" -# 1. STOP ANY EXISTING CONTAINERS FOR THIS PROJECT -# This ensures that we can safely wipe the data directories on the host -echo "Stopping existing containers for project ${COMPOSE_PROJECT}..." -if [ -f "./docker-compose-destroy.bash" ]; then - ./docker-compose-destroy.bash >/dev/null 2>&1 || true -else - $COMPOSE_CMD -p "${COMPOSE_PROJECT}" down -v --remove-orphans >/dev/null 2>&1 || true +# 1. VERIFY NO EXISTING CONTAINERS ARE RUNNING FOR THIS PROJECT +if [ -n "$($COMPOSE_CMD -p "${COMPOSE_PROJECT}" ps -q 2>/dev/null)" ]; then + echo "ERROR: Containers for project ${COMPOSE_PROJECT} are already running." + echo "Please run teardown first." + exit 1 fi # 2. Infrastructure-specific preparation (logs/data) -# We wipe the directory to ensure a clean slate for database engines -for CONTAINER in $(grep 'hostname:' docker-compose.yml | grep -v '#' | tr '.' ' ' | awk '{ print $2 }' | cut -d'.' -f1); do - LOG_DIR="${INFRA_LOGS_PATH}/${COMPOSE_PROJECT}/${CONTAINER}" - echo "Preparing clean log/data directory: ${LOG_DIR}" - $SUDO rm -rf "${LOG_DIR}" - $SUDO mkdir -p "${LOG_DIR}" - $SUDO chmod -R 777 "${LOG_DIR}" - # Specific fix for engines like postgres that need ownership - if [[ "${INFRA}" == *pgsql* ]]; then - $SUDO chown -R 999:999 "${LOG_DIR}" +# We extract host paths that appear to be for logs or data. +echo "Scanning for volumes in docker-compose.yml..." +# CRITICAL: Use single quotes for grep to match LITERAL ${INFRA_LOGS_PATH} +MOUNTED_PATHS=$(grep -E '\$\{INFRA_LOGS_PATH\}|\./log/' docker-compose.yml | awk -F: '{print $1}' | sed 's/^[[:space:]-]*//' | sort -u || true) +echo "Found paths: ${MOUNTED_PATHS}" + +for RAW_PATH in ${MOUNTED_PATHS}; do + # Skip relative paths that point to config files (e.g. ./conf/...) + if [[ "${RAW_PATH}" == "./conf/"* ]]; then continue; fi + + # Expand variables like ${INFRA_LOGS_PATH} and ${COMPOSE_PROJECT} + eval "ACTUAL_PATH=${RAW_PATH}" + + # Safety: Refuse to proceed if ACTUAL_PATH is a directory and is not empty + if [ -d "${ACTUAL_PATH}" ] && [ "$(ls -A "${ACTUAL_PATH}" 2>/dev/null)" ]; then + echo "ERROR: Directory '${ACTUAL_PATH}' is not empty." + echo "Please run teardown/cleanup first." + exit 1 + fi + + echo "Preparing directory: ${ACTUAL_PATH}" + $SUDO mkdir -p "${ACTUAL_PATH}" + $SUDO chmod -R 777 "${ACTUAL_PATH}" + + # Aggressive postgres fix: UID 999 + if [[ "${ACTUAL_PATH}" == *pgsql* ]] || [[ "${ACTUAL_PATH}" == *pgdb* ]]; then + echo "Applying postgres ownership (999:999) to ${ACTUAL_PATH}" + $SUDO chown -R 999:999 "${ACTUAL_PATH}" fi done @@ -89,17 +105,7 @@ if [ -f ./conf/pgsql/ssl/server.key ]; then $SUDO chown 0:999 ./conf/pgsql/ssl/* 2>/dev/null || true fi -# 5. Local log directory fix for some compose files -if grep -q "./log/" docker-compose.yml; then - mkdir -p ./log - chmod 777 ./log - for DIR in $(grep "./log/" docker-compose.yml | awk -F'[:/]' '{print $3}' | sort -u); do - mkdir -p "./log/${DIR}" - chmod 777 "./log/${DIR}" - done -fi - -# 6. Create a temporary env file for docker-compose to ensure it sees our variables +# 5. Create a temporary env file for docker-compose to ensure it sees our variables ENV_FILE=".env.isolated.${INFRA_ID}" cat < "${ENV_FILE}" INFRA_ID=${INFRA_ID} @@ -109,7 +115,7 @@ COMPOSE_PROJECT=${COMPOSE_PROJECT} INFRA_LOGS_PATH=${INFRA_LOGS_PATH} ENVEOF -# 7. START CONTAINERS +# 6. START CONTAINERS if ! $COMPOSE_CMD --env-file .env --env-file "${ENV_FILE}" -p "${COMPOSE_PROJECT}" up -d; then echo "ERROR: Docker Compose failed"; rm -f "${ENV_FILE}"; exit 1 fi @@ -120,7 +126,7 @@ if [ -f /.dockerenv ]; then docker network connect "${INFRA_ID}_backend" "${RUNNER_ID}" || true fi -# 8. Run post-scripts if they exist +# 7. Run post-scripts if they exist sleep 5 # wait a bit for engines to start [ -f ./bin/docker-wait-pgsql.bash ] && ./bin/docker-wait-pgsql.bash [ -f ./bin/docker-mysql-post.bash ] && ./bin/docker-mysql-post.bash diff --git a/test/infra/infra-mysql84/bin/docker-mysql-post.bash b/test/infra/infra-mysql84/bin/docker-mysql-post.bash index f6f34caa3..d925aef95 100755 --- a/test/infra/infra-mysql84/bin/docker-mysql-post.bash +++ b/test/infra/infra-mysql84/bin/docker-mysql-post.bash @@ -19,27 +19,15 @@ for i in 1 2 3; do MAX_WAIT=120 COUNT=0 PASS_OPT="" + MAX_WAIT=120 + COUNT=0 while true; do - # Try with the dynamic password first, then with no password (fallback) - if docker exec "${CONTAINER}" mysql -h127.0.0.1 -uroot -p"${ROOT_PASSWORD}" -e 'SELECT 1' >/dev/null 2>&1; then - PASS_OPT="-p${ROOT_PASSWORD}" - echo " OK (Auth: Dynamic)." - break - fi - if docker exec "${CONTAINER}" mysql -h127.0.0.1 -uroot -e 'SELECT 1' >/dev/null 2>&1; then - PASS_OPT="" - echo " OK (Auth: Empty)." - break - fi - - echo -n '.' - sleep 3 - COUNT=$((COUNT+3)) - if [ $COUNT -gt $MAX_WAIT ]; then - echo " TIMEOUT" - docker logs "${CONTAINER}" | tail -n 20 - exit 1 - fi + if [ $COUNT -ge $MAX_WAIT ]; then echo " TIMEOUT"; docker logs "${CONTAINER}" | tail -n 20; exit 1; fi + STATE=$(docker inspect -f '{{.State.Running}}' "${CONTAINER}" 2>/dev/null || echo "false") + if [ "${STATE}" != "true" ]; then echo -e "\nERROR: Container ${CONTAINER} is NOT running!"; docker logs "${CONTAINER}" | tail -n 20; exit 1; fi + if docker exec "${CONTAINER}" mysql -h127.0.0.1 -uroot -p"${ROOT_PASSWORD}" -e "SELECT 1" >/dev/null 2>&1; then PASS_OPT="-p${ROOT_PASSWORD}"; echo " OK (Auth: Dynamic)."; break; fi + if docker exec "${CONTAINER}" mysql -h127.0.0.1 -uroot -e "SELECT 1" >/dev/null 2>&1; then PASS_OPT=""; echo " OK (Auth: Empty)."; break; fi + echo -n "."; sleep 2; COUNT=$((COUNT+2)) done echo "Configuring users on ${CONTAINER}..." diff --git a/test/infra/infra-mysql84/docker-compose-init.bash b/test/infra/infra-mysql84/docker-compose-init.bash index 69081198f..3331e3a06 100755 --- a/test/infra/infra-mysql84/docker-compose-init.bash +++ b/test/infra/infra-mysql84/docker-compose-init.bash @@ -54,26 +54,42 @@ echo "========================================================================== echo "Initializing CI Infra '${INFRA}' (Project: ${COMPOSE_PROJECT}) ..." echo "================================================================================" -# 1. STOP ANY EXISTING CONTAINERS FOR THIS PROJECT -# This ensures that we can safely wipe the data directories on the host -echo "Stopping existing containers for project ${COMPOSE_PROJECT}..." -if [ -f "./docker-compose-destroy.bash" ]; then - ./docker-compose-destroy.bash >/dev/null 2>&1 || true -else - $COMPOSE_CMD -p "${COMPOSE_PROJECT}" down -v --remove-orphans >/dev/null 2>&1 || true +# 1. VERIFY NO EXISTING CONTAINERS ARE RUNNING FOR THIS PROJECT +if [ -n "$($COMPOSE_CMD -p "${COMPOSE_PROJECT}" ps -q 2>/dev/null)" ]; then + echo "ERROR: Containers for project ${COMPOSE_PROJECT} are already running." + echo "Please run teardown first." + exit 1 fi # 2. Infrastructure-specific preparation (logs/data) -# We wipe the directory to ensure a clean slate for database engines -for CONTAINER in $(grep 'hostname:' docker-compose.yml | grep -v '#' | tr '.' ' ' | awk '{ print $2 }' | cut -d'.' -f1); do - LOG_DIR="${INFRA_LOGS_PATH}/${COMPOSE_PROJECT}/${CONTAINER}" - echo "Preparing clean log/data directory: ${LOG_DIR}" - $SUDO rm -rf "${LOG_DIR}" - $SUDO mkdir -p "${LOG_DIR}" - $SUDO chmod -R 777 "${LOG_DIR}" - # Specific fix for engines like postgres that need ownership - if [[ "${INFRA}" == *pgsql* ]]; then - $SUDO chown -R 999:999 "${LOG_DIR}" +# We extract host paths that appear to be for logs or data. +echo "Scanning for volumes in docker-compose.yml..." +# CRITICAL: Use single quotes for grep to match LITERAL ${INFRA_LOGS_PATH} +MOUNTED_PATHS=$(grep -E '\$\{INFRA_LOGS_PATH\}|\./log/' docker-compose.yml | awk -F: '{print $1}' | sed 's/^[[:space:]-]*//' | sort -u || true) +echo "Found paths: ${MOUNTED_PATHS}" + +for RAW_PATH in ${MOUNTED_PATHS}; do + # Skip relative paths that point to config files (e.g. ./conf/...) + if [[ "${RAW_PATH}" == "./conf/"* ]]; then continue; fi + + # Expand variables like ${INFRA_LOGS_PATH} and ${COMPOSE_PROJECT} + eval "ACTUAL_PATH=${RAW_PATH}" + + # Safety: Refuse to proceed if ACTUAL_PATH is a directory and is not empty + if [ -d "${ACTUAL_PATH}" ] && [ "$(ls -A "${ACTUAL_PATH}" 2>/dev/null)" ]; then + echo "ERROR: Directory '${ACTUAL_PATH}' is not empty." + echo "Please run teardown/cleanup first." + exit 1 + fi + + echo "Preparing directory: ${ACTUAL_PATH}" + $SUDO mkdir -p "${ACTUAL_PATH}" + $SUDO chmod -R 777 "${ACTUAL_PATH}" + + # Aggressive postgres fix: UID 999 + if [[ "${ACTUAL_PATH}" == *pgsql* ]] || [[ "${ACTUAL_PATH}" == *pgdb* ]]; then + echo "Applying postgres ownership (999:999) to ${ACTUAL_PATH}" + $SUDO chown -R 999:999 "${ACTUAL_PATH}" fi done @@ -89,17 +105,7 @@ if [ -f ./conf/pgsql/ssl/server.key ]; then $SUDO chown 0:999 ./conf/pgsql/ssl/* 2>/dev/null || true fi -# 5. Local log directory fix for some compose files -if grep -q "./log/" docker-compose.yml; then - mkdir -p ./log - chmod 777 ./log - for DIR in $(grep "./log/" docker-compose.yml | awk -F'[:/]' '{print $3}' | sort -u); do - mkdir -p "./log/${DIR}" - chmod 777 "./log/${DIR}" - done -fi - -# 6. Create a temporary env file for docker-compose to ensure it sees our variables +# 5. Create a temporary env file for docker-compose to ensure it sees our variables ENV_FILE=".env.isolated.${INFRA_ID}" cat < "${ENV_FILE}" INFRA_ID=${INFRA_ID} @@ -109,7 +115,7 @@ COMPOSE_PROJECT=${COMPOSE_PROJECT} INFRA_LOGS_PATH=${INFRA_LOGS_PATH} ENVEOF -# 7. START CONTAINERS +# 6. START CONTAINERS if ! $COMPOSE_CMD --env-file .env --env-file "${ENV_FILE}" -p "${COMPOSE_PROJECT}" up -d; then echo "ERROR: Docker Compose failed"; rm -f "${ENV_FILE}"; exit 1 fi @@ -120,7 +126,7 @@ if [ -f /.dockerenv ]; then docker network connect "${INFRA_ID}_backend" "${RUNNER_ID}" || true fi -# 8. Run post-scripts if they exist +# 7. Run post-scripts if they exist sleep 5 # wait a bit for engines to start [ -f ./bin/docker-wait-pgsql.bash ] && ./bin/docker-wait-pgsql.bash [ -f ./bin/docker-mysql-post.bash ] && ./bin/docker-mysql-post.bash diff --git a/test/infra/infra-pgsql17-repl/bin/docker-wait-pgsql.bash b/test/infra/infra-pgsql17-repl/bin/docker-wait-pgsql.bash index a743442b7..b211bd905 100755 --- a/test/infra/infra-pgsql17-repl/bin/docker-wait-pgsql.bash +++ b/test/infra/infra-pgsql17-repl/bin/docker-wait-pgsql.bash @@ -1,23 +1,34 @@ #!/bin/bash set -o pipefail - [ -f .env ] && . .env PRIMARY_CONTAINER="${COMPOSE_PROJECT}-pgdb1-1" -printf "[$(date)] Waiting for PgSQL Primary (${PRIMARY_CONTAINER}) service " +echo -n "[$(date)] Waiting for PgSQL Primary (${PRIMARY_CONTAINER}) service " MAX_WAIT=60 COUNT=0 -RC=1 -while [ $RC != 0 ]; do - if [ $COUNT -ge $MAX_WAIT ]; then - echo " TIMEOUT after ${MAX_WAIT} seconds" - exit 1 - fi - sleep 1 - printf "." - docker exec "${PRIMARY_CONTAINER}" env PGPASSWORD="${ROOT_PASSWORD}" psql -h127.0.0.1 -p5432 -Upostgres -c "select 1;" > /dev/null 2>&1 - RC=$? - COUNT=$((COUNT+1)) +while true; do + if [ $COUNT -ge $MAX_WAIT ]; then + echo -e "\nERROR: TIMEOUT after ${MAX_WAIT} seconds waiting for ${PRIMARY_CONTAINER}" + exit 1 + fi + + # 1. Check if container is actually running + STATE=$(docker inspect -f '{{.State.Running}}' "${PRIMARY_CONTAINER}" 2>/dev/null || echo "false") + if [ "${STATE}" != "true" ]; then + echo -e "\nERROR: Container ${PRIMARY_CONTAINER} is NOT running!" + echo ">>> Container Logs:" + docker logs "${PRIMARY_CONTAINER}" | tail -n 20 + exit 1 + fi + + # 2. Check if service is ready + if docker exec "${PRIMARY_CONTAINER}" env PGPASSWORD="${ROOT_PASSWORD}" psql -h127.0.0.1 -p5432 -Upostgres -c "select 1;" > /dev/null 2>&1; then + echo -e "\n[$(date)] PgSQL Primary service is now ACTIVE" + break + fi + + echo -n "." + sleep 2 + COUNT=$((COUNT+2)) done -printf "\n[$(date)] PgSQL Primary service is now ACTIVE\n" diff --git a/test/infra/infra-pgsql17-repl/docker-compose-init.bash b/test/infra/infra-pgsql17-repl/docker-compose-init.bash index 69081198f..3331e3a06 100755 --- a/test/infra/infra-pgsql17-repl/docker-compose-init.bash +++ b/test/infra/infra-pgsql17-repl/docker-compose-init.bash @@ -54,26 +54,42 @@ echo "========================================================================== echo "Initializing CI Infra '${INFRA}' (Project: ${COMPOSE_PROJECT}) ..." echo "================================================================================" -# 1. STOP ANY EXISTING CONTAINERS FOR THIS PROJECT -# This ensures that we can safely wipe the data directories on the host -echo "Stopping existing containers for project ${COMPOSE_PROJECT}..." -if [ -f "./docker-compose-destroy.bash" ]; then - ./docker-compose-destroy.bash >/dev/null 2>&1 || true -else - $COMPOSE_CMD -p "${COMPOSE_PROJECT}" down -v --remove-orphans >/dev/null 2>&1 || true +# 1. VERIFY NO EXISTING CONTAINERS ARE RUNNING FOR THIS PROJECT +if [ -n "$($COMPOSE_CMD -p "${COMPOSE_PROJECT}" ps -q 2>/dev/null)" ]; then + echo "ERROR: Containers for project ${COMPOSE_PROJECT} are already running." + echo "Please run teardown first." + exit 1 fi # 2. Infrastructure-specific preparation (logs/data) -# We wipe the directory to ensure a clean slate for database engines -for CONTAINER in $(grep 'hostname:' docker-compose.yml | grep -v '#' | tr '.' ' ' | awk '{ print $2 }' | cut -d'.' -f1); do - LOG_DIR="${INFRA_LOGS_PATH}/${COMPOSE_PROJECT}/${CONTAINER}" - echo "Preparing clean log/data directory: ${LOG_DIR}" - $SUDO rm -rf "${LOG_DIR}" - $SUDO mkdir -p "${LOG_DIR}" - $SUDO chmod -R 777 "${LOG_DIR}" - # Specific fix for engines like postgres that need ownership - if [[ "${INFRA}" == *pgsql* ]]; then - $SUDO chown -R 999:999 "${LOG_DIR}" +# We extract host paths that appear to be for logs or data. +echo "Scanning for volumes in docker-compose.yml..." +# CRITICAL: Use single quotes for grep to match LITERAL ${INFRA_LOGS_PATH} +MOUNTED_PATHS=$(grep -E '\$\{INFRA_LOGS_PATH\}|\./log/' docker-compose.yml | awk -F: '{print $1}' | sed 's/^[[:space:]-]*//' | sort -u || true) +echo "Found paths: ${MOUNTED_PATHS}" + +for RAW_PATH in ${MOUNTED_PATHS}; do + # Skip relative paths that point to config files (e.g. ./conf/...) + if [[ "${RAW_PATH}" == "./conf/"* ]]; then continue; fi + + # Expand variables like ${INFRA_LOGS_PATH} and ${COMPOSE_PROJECT} + eval "ACTUAL_PATH=${RAW_PATH}" + + # Safety: Refuse to proceed if ACTUAL_PATH is a directory and is not empty + if [ -d "${ACTUAL_PATH}" ] && [ "$(ls -A "${ACTUAL_PATH}" 2>/dev/null)" ]; then + echo "ERROR: Directory '${ACTUAL_PATH}' is not empty." + echo "Please run teardown/cleanup first." + exit 1 + fi + + echo "Preparing directory: ${ACTUAL_PATH}" + $SUDO mkdir -p "${ACTUAL_PATH}" + $SUDO chmod -R 777 "${ACTUAL_PATH}" + + # Aggressive postgres fix: UID 999 + if [[ "${ACTUAL_PATH}" == *pgsql* ]] || [[ "${ACTUAL_PATH}" == *pgdb* ]]; then + echo "Applying postgres ownership (999:999) to ${ACTUAL_PATH}" + $SUDO chown -R 999:999 "${ACTUAL_PATH}" fi done @@ -89,17 +105,7 @@ if [ -f ./conf/pgsql/ssl/server.key ]; then $SUDO chown 0:999 ./conf/pgsql/ssl/* 2>/dev/null || true fi -# 5. Local log directory fix for some compose files -if grep -q "./log/" docker-compose.yml; then - mkdir -p ./log - chmod 777 ./log - for DIR in $(grep "./log/" docker-compose.yml | awk -F'[:/]' '{print $3}' | sort -u); do - mkdir -p "./log/${DIR}" - chmod 777 "./log/${DIR}" - done -fi - -# 6. Create a temporary env file for docker-compose to ensure it sees our variables +# 5. Create a temporary env file for docker-compose to ensure it sees our variables ENV_FILE=".env.isolated.${INFRA_ID}" cat < "${ENV_FILE}" INFRA_ID=${INFRA_ID} @@ -109,7 +115,7 @@ COMPOSE_PROJECT=${COMPOSE_PROJECT} INFRA_LOGS_PATH=${INFRA_LOGS_PATH} ENVEOF -# 7. START CONTAINERS +# 6. START CONTAINERS if ! $COMPOSE_CMD --env-file .env --env-file "${ENV_FILE}" -p "${COMPOSE_PROJECT}" up -d; then echo "ERROR: Docker Compose failed"; rm -f "${ENV_FILE}"; exit 1 fi @@ -120,7 +126,7 @@ if [ -f /.dockerenv ]; then docker network connect "${INFRA_ID}_backend" "${RUNNER_ID}" || true fi -# 8. Run post-scripts if they exist +# 7. Run post-scripts if they exist sleep 5 # wait a bit for engines to start [ -f ./bin/docker-wait-pgsql.bash ] && ./bin/docker-wait-pgsql.bash [ -f ./bin/docker-mysql-post.bash ] && ./bin/docker-mysql-post.bash