Fix code coverage collection in multi-group test runner

1. Add set -e to run-multi-group.bash heredoc for proper error handling 2. Add EXIT trap in run-tests-isolated.bash to collect coverage on timeout/failure 3. Pass COVERAGE_MODE, COVERAGE_REPORT_DIR, and SCRIPT_DIR env vars to container 4. Fix fastcov invocation to use -e deps/ instead of restrictive -i filters 5. Add per-group HTML report generation 6. Run coverage combination in container for host without tools Coverage is now collected for: - include/ (189 header files) - lib/ (109 library files) - src/ (4 source files) The EXIT trap ensures coverage is collected even when tests timeout.
2 months ago · 20fc53e45c
parent 30620f37c4
commit 20fc53e45c
2 changed files with 98 additions and 32 deletions
--- a/test/infra/control/run-multi-group.bash
+++ b/test/infra/control/run-multi-group.bash
@ -132,7 +132,7 @@ run_single_group() {
    export COVERAGE="${COVERAGE}"

    timeout "${TIMEOUT_MINUTES}m" bash <<INNERSHELL || cmd_exit_code=$?
-        set -uo pipefail
+        set -euo pipefail

        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Setting up infrastructure..." | tee -a "${log_file}"
        if ! "${SCRIPT_DIR}/ensure-infras.bash" >> "${log_file}" 2>&1; then
@ -335,15 +335,34 @@ if [ "${COVERAGE}" -eq 1 ]; then
    done

    if [ -n "${COVERAGE_FILES}" ]; then
-        # Check if fastcov is available
-        if command -v fastcov >/dev/null 2>&1; then
-            COMBINED_INFO="${COMBINED_COVERAGE_DIR}/combined-coverage.info"
-            echo ">>> Combining coverage reports into: ${COMBINED_INFO}"
-            # shellcheck disable=SC2086
-            fastcov -b -l -C ${COVERAGE_FILES} -o "${COMBINED_INFO}" 2>/dev/null || {
-                echo ">>> WARNING: fastcov combine failed, trying lcov..."
-                # Fallback to lcov if fastcov fails
-                if command -v lcov >/dev/null 2>&1; then
+        COMBINED_INFO="${COMBINED_COVERAGE_DIR}/combined-coverage.info"
+        echo ">>> Combining coverage reports into: ${COMBINED_INFO}"
+
+        # Run coverage combination in container (tools may not be on host)
+        docker run --rm \
+            -v "${WORKSPACE}:${WORKSPACE}" \
+            -e COVERAGE_FILES="${COVERAGE_FILES}" \
+            -e COMBINED_INFO="${COMBINED_INFO}" \
+            proxysql-ci-base:latest \
+            bash -c '
+                set -e
+                if command -v fastcov >/dev/null 2>&1; then
+                    fastcov -b -l -C ${COVERAGE_FILES} -o "${COMBINED_INFO}" 2>&1 || {
+                        echo ">>> WARNING: fastcov combine failed, trying lcov..."
+                        if command -v lcov >/dev/null 2>&1; then
+                            FIRST_FILE=true
+                            for info_file in ${COVERAGE_FILES}; do
+                                if [ "${FIRST_FILE}" = true ]; then
+                                    cp "${info_file}" "${COMBINED_INFO}"
+                                    FIRST_FILE=false
+                                else
+                                    lcov -a "${COMBINED_INFO}" -a "${info_file}" -o "${COMBINED_INFO}".tmp 2>/dev/null && \
+                                        mv "${COMBINED_INFO}".tmp "${COMBINED_INFO}"
+                                fi
+                            done
+                        fi
+                    }
+                elif command -v lcov >/dev/null 2>&1; then
                    FIRST_FILE=true
                    for info_file in ${COVERAGE_FILES}; do
                        if [ "${FIRST_FILE}" = true ]; then
@ -354,30 +373,39 @@ if [ "${COVERAGE}" -eq 1 ]; then
                                mv "${COMBINED_INFO}".tmp "${COMBINED_INFO}"
                        fi
                    done
+                else
+                    echo ">>> ERROR: Neither fastcov nor lcov available"
+                    exit 1
                fi
-            }
-
-            if [ -f "${COMBINED_INFO}" ]; then
-                echo ">>> Combined coverage report: ${COMBINED_INFO}"
-
-                # Generate HTML report
-                if command -v genhtml >/dev/null 2>&1; then
-                    COMBINED_HTML="${COMBINED_COVERAGE_DIR}/html"
-                    mkdir -p "${COMBINED_HTML}"
-                    genhtml --branch-coverage "${COMBINED_INFO}" --output-directory "${COMBINED_HTML}" 2>/dev/null || \
-                        echo ">>> WARNING: HTML generation failed"
-                    [ -f "${COMBINED_HTML}/index.html" ] && echo ">>> Combined HTML report: ${COMBINED_HTML}/index.html"
-                fi
-            else
-                echo ">>> WARNING: Failed to generate combined coverage report"
+            ' || echo ">>> WARNING: Coverage combination failed"
+
+        if [ -f "${COMBINED_INFO}" ]; then
+            echo ">>> Combined coverage report: ${COMBINED_INFO}"
+
+            # Generate HTML report
+            echo ">>> Generating HTML coverage report..."
+            COMBINED_HTML="${COMBINED_COVERAGE_DIR}/html"
+            mkdir -p "${COMBINED_HTML}"
+
+            docker run --rm \
+                -v "${WORKSPACE}:${WORKSPACE}" \
+                -e COMBINED_INFO="${COMBINED_INFO}" \
+                -e COMBINED_HTML="${COMBINED_HTML}" \
+                proxysql-ci-base:latest \
+                bash -c '
+                    if command -v genhtml >/dev/null 2>&1; then
+                        genhtml --branch-coverage "${COMBINED_INFO}" --output-directory "${COMBINED_HTML}" 2>&1 || \
+                            echo ">>> WARNING: HTML generation failed"
+                    else
+                        echo ">>> WARNING: genhtml not available"
+                    fi
+                '
+
+            if [ -f "${COMBINED_HTML}/index.html" ]; then
+                echo ">>> Combined HTML report: ${COMBINED_HTML}/index.html"
            fi
        else
-            echo ">>> WARNING: fastcov/lcov not found, skipping coverage combination"
-            echo ">>> Individual coverage files are available at:"
-            for group in ${TAP_GROUPS}; do
-                infra_id="${group}-${RUN_ID}"
-                echo "  ${group}: ${WORKSPACE}/ci_infra_logs/${infra_id}/coverage-report/"
-            done
+            echo ">>> WARNING: Failed to generate combined coverage report"
        fi
    else
        echo ">>> No coverage files found to combine"
--- a/test/infra/control/run-tests-isolated.bash
+++ b/test/infra/control/run-tests-isolated.bash
@ -138,11 +138,49 @@ docker run \
    -e TAP_GROUP="${TAP_GROUP}" \
    -e SKIP_CLUSTER_START="${SKIP_CLUSTER_START}" \
    -e PROXYSQL_CLUSTER_NODES="${PROXYSQL_CLUSTER_NODES}" \
+    -e COVERAGE_MODE="${COVERAGE_MODE}" \
+    -e COVERAGE_REPORT_DIR="${COVERAGE_REPORT_DIR}" \
+    -e SCRIPT_DIR="${SCRIPT_DIR}" \
+    -e MYSQL_BINLOG_BIN="${MYSQL_BINLOG_BIN}" \
+    -e BINLOG_READER_BIN="${BINLOG_READER_BIN}" \
    proxysql-ci-base:latest \
    /bin/bash -c "
        set -e
+
+        # Coverage collection trap - runs on exit regardless of success/failure/timeout
+        collect_coverage() {
+            local exit_code=\$?
+            if [ \"\${COVERAGE_MODE}\" = \"1\" ]; then
+                echo \">>> Collecting code coverage data (exit code was: \${exit_code})...\"
+                if command -v fastcov >/dev/null 2>&1; then
+                    mkdir -p \"\${COVERAGE_REPORT_DIR}\"
+                    local coverage_file=\"\${COVERAGE_REPORT_DIR}/\${INFRA_ID}.info\"
+                    echo \">>> Generating coverage report: \${coverage_file}\"
+                    local nproc_val=\$(nproc)
+                    cd \"\${WORKSPACE}\"
+                    fastcov -b -j\"\${nproc_val}\" --process-gcno -l -e /usr/include/ -e test/tap/tests -e deps/ -d . -o \"\${coverage_file}\" 2>&1 || echo \">>> WARNING: Coverage generation failed\"
+                    if [ -f \"\${coverage_file}\" ]; then
+                        echo \">>> Coverage report generated: \${coverage_file}\"
+                        # Generate HTML report
+                        if command -v genhtml >/dev/null 2>&1; then
+                            local html_dir=\"\${COVERAGE_REPORT_DIR}/html\"
+                            mkdir -p \"\${html_dir}\"
+                            genhtml --branch-coverage \"\${coverage_file}\" --output-directory \"\${html_dir}\" 2>&1 || echo \">>> WARNING: HTML generation failed\"
+                            [ -f \"\${html_dir}/index.html\" ] && echo \">>> HTML coverage report: \${html_dir}/index.html\"
+                        fi
+                    else
+                        echo \">>> WARNING: Coverage info file not generated\"
+                    fi
+                else
+                    echo \">>> WARNING: fastcov not found in container, skipping coverage collection\"
+                fi
+            fi
+            exit \${exit_code}
+        }
+        trap collect_coverage EXIT
+
        git config --global --add safe.directory \"${WORKSPACE}\"
-        
+
        # Redirection: Replace reference to legacy scripts with local infra control scripts
        find \"${WORKSPACE}/test/tap/groups\" -name \"*.bash\" | xargs -r sed -i \"s|\\\$JENKINS_SCRIPTS_PATH|${WORKSPACE}/test/infra/control|g\"