proxysql/.github/workflows/ci-builds.yml

name: CI-builds

on:
  workflow_dispatch:
  workflow_call:
    inputs:
      trigger:
        type: string

permissions: write-all

env:
  LOCK: CI_BUILDS_${{ inputs.trigger && fromJson(inputs.trigger).event.workflow_run.head_sha || github.sha }}
  SHA: ${{ inputs.trigger && fromJson(inputs.trigger).event.workflow_run.head_sha || github.sha }}

jobs:

#  lock:
#    runs-on: ubuntu-22.04
#    env:
#      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
#
#    steps:
#
#    - name: Concurency locking
#      run: |
#        # lock fast
#        echo "Setting lock '${LOCK^^}' to '${{ github.workflow }}'"
#        gh variable -R ${{ github.repository }} list | grep -i ${LOCK} >/dev/null || gh variable -R ${{ github.repository }} set ${LOCK} -b '${{ github.workflow }}' >/dev/null
#
#        # wait for unlock
#        sleep 5
#        PROBE="$(gh variable -R ${{ github.repository }} list | grep -i ${LOCK} || echo '')"
#        if [[ ! ${PROBE} =~ ${{ github.workflow }} ]]; then
#          while [[ ! -z ${PROBE} ]]; do
#            echo "Wating for unlock '${LOCK^^}' by '$(echo ${PROBE} | awk '{ print $2 }')' ..."
#            sleep 15
#            PROBE="$(gh variable -R ${{ github.repository }} list | grep -i ${LOCK} || echo '')"
#          done
#        else
#          echo "Aquired lock '${LOCK^^}' by '${{ github.workflow }}'"
#        fi

  builds:
    runs-on: ubuntu-24.04
#    needs: [ lock ]
#    outputs:
#      matrix: ${{ steps.set-matrix.outputs.matrix }}
    strategy:
      fail-fast: false
      matrix:
        include:

          # debian12 is needed for 3p testing
          - dist: 'debian12'
            type: '-dbg'
#          - dist: 'opensuse15'
#            type: '-clang'

          # ubuntu22 is needed for TAP testing
          - dist: 'ubuntu22'
            type: '-tap'

          # ubuntu24 GenAI build with code coverage
          - dist: 'ubuntu24'
            type: '-tap-genai-gcov'
          # ubuntu22 is needed for ASAN TAP testing
          # disabled to save cache space
          #- dist: 'ubuntu22'
          #  type: '-tap-asan'
#          - dist: 'ubuntu22'
#            type: '-clang'
#          - dist: 'ubuntu22'
#            type: '-clang-tap'
#          - dist: 'ubuntu22'
#            type: '-clang-tap-asan'
    env:
      BLDCACHE: ${{ inputs.trigger && fromJson(inputs.trigger).event.workflow_run.head_sha || github.sha }}_${{ matrix.dist }}${{ matrix.type }}
      MATRIX: '(${{ matrix.dist }},${{ matrix.type }})'

    steps:

    - uses: LouisBrunner/checks-action@v2.0.0
      id: checks
      if: always()
      with:
        token: ${{ secrets.GITHUB_TOKEN }}
        name: '${{ github.workflow }} / ${{ github.job }} ${{ env.MATRIX }}'
        repo: ${{ github.repository }}
        sha: ${{ env.SHA }}
        status: 'in_progress'
#        action_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}'
        details_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}'
    
    - name: Cache check
      id: cache-check
      uses: actions/cache/restore@v4
      with:
        key: ${{ env.BLDCACHE }}_bin
        enableCrossOsArchive: true
        lookup-only: true
        path: |
                    proxysql/binaries/

    - name: Checkout repository
      if: ${{ steps.cache-check.outputs.cache-hit != 'true' }}
      uses: actions/checkout@v4
      with:
        repository: ${{ github.repository }}
        ref: ${{ env.SHA }}
        fetch-depth: 0
        path: 'proxysql'

#    - name: Patch TAP-tests
#      if: ${{ steps.cache-check.outputs.cache-hit != 'true' }}
#      run: |
#         apply patches
#        for PATCH in $(cd jenkins-build-scripts/test-scripts/patches && find . -type f); do
#          if [[ $PATCH =~ \.patch ]]; then
#            patch --verbose proxysql/${PATCH%.patch} jenkins-build-scripts/test-scripts/patches/${PATCH} || true
#          elif [[ ! -f jenkins-build-scripts/test-scripts/patches/${PATCH#./}.patch ]]; then
#            cp -v jenkins-build-scripts/test-scripts/patches/${PATCH#./} proxysql/${PATCH#./} || true
#          fi
#        done
#        ls -l proxysql/test/tap

    - name: Build
      id: build
      if: ${{ steps.cache-check.outputs.cache-hit != 'true' }}
      run: |
        cd proxysql/
        #sed -i 's/docker-compose/docker compose/g' Makefile
        git fetch --tags --force
        mkdir ci_build_log
        if [[ "${{ matrix.type }}" =~ "-asan" ]]; then
          sed -i "/command/i \      - WITHASAN=1" docker-compose.yml
          sed -i "/command/i \      - TEST_WITHASAN=1" docker-compose.yml
        fi
        if [[ "${{ matrix.type }}" =~ "-genai" ]]; then
          sed -i "/command/i \      - PROXYSQLGENAI=1" docker-compose.yml
        fi
        if [[ "${{ matrix.type }}" =~ "-gcov" ]]; then
          sed -i "/command/i \      - WITHGCOV=1" docker-compose.yml
        fi
        if [[ "${{ matrix.type }}" =~ "-tap" ]]; then
          # prebuild dependencies without debug
          #sed -i "/command/c \    command: bash -l -c '${ASAN} cd /opt/proxysql && make -j$(nproc) build_deps_clickhouse'" docker-compose.yml
          #make ${{ matrix.dist }} | tee ci_build_log/build-deps.log
          # build tap tests
          #sed -i "s/^build_tap_test_debug: build_src_debug$/build_tap_test_debug: build_src_debug_clickhouse/" Makefile
          #sed -i "/command/c \    command: bash -l -c 'cd /opt/proxysql && make -j$(nproc) build_tap_test_debug'" docker-compose.yml
          # NOTE: target must be -tap (not -dbg): entrypoint.bash routes on BLD_NAME
          # matching \-test|\-tap and selects build_tap_test_debug; -dbg takes the else
          # branch which only builds src/proxysql, leaving test/tap/tests/**/*-t unbuilt
          # (silent false-green — see v3.0 doc/GH-Actions/README.md debugging section).
          make ${{ matrix.dist }}-tap | tee ci_build_log/build-tap.log
          
          # create matrix of all test folders containing *-t
          echo "[ "$(find test/tap/ -name '*-t' | xargs -n1 dirname | uniq | sed 's|test/tap/||' | xargs -I{} -n1 echo -n "'{}', ")"]" > tap-matrix.json
          find test/tap/ -name *-t | xargs -n1 dirname | uniq  > tap-matrix.txt
          echo "===================="
          echo ">>>tap-matrix.txt<<<"
          cat tap-matrix.txt
          echo "===================="
          # create matrixes of all tests
          for TESTS in test/tap/tests*; do
            echo "[ "$(find -L $TESTS -type f -name '*-t' -executable -printf "'%f', ")" ]" > tap-matrix-${TESTS##*/}.json
            find -L $TESTS -type f -name '*-t' -executable -printf "%f\n" > tap-matrix-${TESTS##*/}.txt
            echo "===================="
            echo ">>>${TESTS##*/}<<<"
            cat tap-matrix-${TESTS##*/}.txt
            echo "===================="
          done

          # ------------------------------------------------------------
          # Cache size control: prune artifacts the _test cache doesn't
          # need before it gets saved. Without this pruning, the _test
          # cache grows to ~4 GB compressed (~16 GB uncompressed), and
          # two concurrent cascades (each saving one ubuntu22-tap cache
          # and one ubuntu24-tap-genai-gcov cache) exceed GitHub's 10 GB
          # per-repo cache quota. LRU eviction then deletes caches
          # between when CI-builds saves them and when downstream test
          # workflows try to restore them, causing "Cache restore test"
          # to fail and the entire TAP pipeline to break.
          # ------------------------------------------------------------

          # Compile-time safety check: make sure unit tests actually got
          # built. If they didn't, the build step above silently ignored
          # a compile error and the delete below would "succeed" on an
          # empty set. Fail loudly instead.
          UNIT_COUNT=$(find test/tap/tests/unit -type f -name '*-t' -executable 2>/dev/null | wc -l)
          echo ">>> Compiled ${UNIT_COUNT} unit test binaries"
          if [ "${UNIT_COUNT}" -lt 1 ]; then
            echo "ERROR: no unit test binaries found at test/tap/tests/unit/*-t"
            echo "       This typically means a unit test failed to compile."
            exit 1
          fi

          # Delete unit test binaries before the cache save. Each is ~170 MB
          # because they statically link libproxysql.a in debug mode
          # (52 binaries × 170 MB = 8.8 GB). CI-unittests on v3.0 is
          # disabled for now; when it's re-enabled it will need its own
          # rebuild strategy (probably a full docker-compose rebuild).
          #
          # sudo is required: files under test/tap/tests/unit/ and test/deps/
          # are created inside the docker build container running as root,
          # and the host-side runner user can't always remove root-owned
          # files under root-owned subdirectories (unlink needs write access
          # to the parent directory, which the runner doesn't always have).
          sudo find test/tap/tests/unit -type f -name '*-t' -delete
          echo ">>> Deleted ${UNIT_COUNT} unit test binaries from the cache payload"

          # Delete test/deps (~2.8 GB on disk). These are the test-framework
          # dependencies (mariadb-connector-c, mysql-connector-c 5.7,
          # mysql-connector-c 8.4) that TAP binaries statically link against
          # at build time. Nothing at runtime looks at test/deps — the code
          # it contains already lives inside the compiled *-t binaries.
          # Dropping it from the cache saves ~700 MB per variant after
          # zstd compression.
          #
          # Safety check: verify no TAP binary dynamically links against
          # libraries that would be under test/deps, or has a RUNPATH /
          # RPATH pointing at test/deps. If this check fails, someone has
          # added a test that dynamically links to a test/deps library
          # and deleting test/deps will break that test at runtime (with
          # a cryptic .so-not-found error). Fail loudly in CI-builds now
          # instead of silently corrupting the cache.
          BAD_BINS=""
          while IFS= read -r b; do
            deps_needed=$(readelf -d "$b" 2>/dev/null | grep -E "NEEDED.*\b(libmysqlclient|libjson_binlog|libstrings_shared|libmariadbclient)\.so" || true)
            deps_rpath=$(readelf -d "$b" 2>/dev/null | grep -E "R(UN)?PATH.*test/deps" || true)
            if [ -n "${deps_needed}" ] || [ -n "${deps_rpath}" ]; then
              BAD_BINS="${BAD_BINS}${b}\n"
              echo ">>> UNSAFE: $(basename ${b})"
              [ -n "${deps_needed}" ] && echo "    ${deps_needed}"
              [ -n "${deps_rpath}" ] && echo "    ${deps_rpath}"
            fi
          done < <(find test/tap/tests test/tap/tests_with_deps -type f -name '*-t' 2>/dev/null)
          if [ -n "${BAD_BINS}" ]; then
            echo "ERROR: the following TAP binaries depend on test/deps at runtime:"
            printf "${BAD_BINS}"
            echo "       Cannot drop test/deps from the _test cache without"
            echo "       breaking these tests. Fix the Makefile to link"
            echo "       them statically, or update ci-builds.yml to keep"
            echo "       test/deps in the cache."
            exit 1
          fi
          echo ">>> test/deps runtime-dependency check passed"

          # sudo required: test/deps contents are owned by root (created
          # inside the docker build container). See comment above on the
          # unit test delete.
          if [ -d test/deps ]; then
            sudo rm -rf test/deps
            echo ">>> Deleted test/deps from the cache payload"
          fi

        elif [[ "${{ matrix.type }}" =~ "-test" ]]; then
          TYPE=${{ matrix.type }}
          # build TYPE
          #sed -i "/command/c \    command: bash -l -c 'cd /opt/proxysql && make -j$(nproc) ${TYPE#-}'" docker-compose.yml
          make ${{ matrix.dist }}-test | tee ci_build_log/build.log
        else
          make ${{ matrix.dist }}${{ matrix.type }} | tee ci_build_log/build.log
        fi        

    - name: Check build
      if: ${{ steps.cache-check.outputs.cache-hit != 'true' }}
      run: |
        for LOG in proxysql/ci_build_log/build*.log ; do
          grep 'exited with code 0' ${LOG} || exit 1
        done        

    - name: Pack bin cache with zstd-15
      id: cache-pack-bin
      if: ${{ success() && steps.cache-check.outputs.cache-hit != 'true' }}
      # `actions/cache@v4` compresses with zstd level ~3; we pre-compress
      # with zstd -15 to fit more under the 10 GB repo quota. Level 15
      # captures most of the size win (~80% of what --ultra -22 gives
      # over default -3) at roughly 1/10 the time on this codebase --
      # `--ultra -22` measured 14-16 min just on the test cache step per
      # ubuntu build job (vs ~17 min for the build itself), pushing
      # CI-builds wall clock to ~38 min and gating every downstream TAP
      # workflow on it. Re-compression by actions/cache on the already-
      # compressed .tar.zst is a near-no-op. Symbols are preserved
      # bit-for-bit.
      run: |
        command -v zstd >/dev/null || sudo apt-get install -y zstd
        cd proxysql/
        tar -cf - .git/ binaries/ | zstd -15 -T0 -o ../cache_bin.tar.zst        

    - name: Cache save bin
      id: cache-save-bin
      if: ${{ success() && steps.cache-check.outputs.cache-hit != 'true' }}
      uses: actions/cache/save@v4
      with:
        key: ${{ env.BLDCACHE }}_bin
        enableCrossOsArchive: true
        path: cache_bin.tar.zst

    - name: Cache save src
      id: cache-save-src
      if: ${{ success() && steps.cache-check.outputs.cache-hit != 'true' && contains(matrix.type,'-tap') }}
      uses: actions/cache/save@v4
      with:
        key: ${{ env.BLDCACHE }}_src
        enableCrossOsArchive: true
        # For -gcov variants, include lib/obj/*.gcno so the test
        # workflow's coverage collector can match .gcda (runtime data
        # written by the proxysql daemon under /gcov) to .gcno
        # (compile-time data). Without this, fastcov sees .gcda files
        # with no associated source-file mapping
        # (`Missing 'current_working_directory'`, `files: []`) and
        # produces a 0-byte .info report -- the bug that hid coverage
        # for as long as COVERAGE=1 has existed in run-tests-isolated.bash.
        # src/obj/*.gcno is already captured by `proxysql/src/`. The
        # extra entry is a no-op for non-gcov builds (lib/obj/*.gcno
        # won't exist) and adds only a few MB to the gcov cache.
        path: |
          proxysql/src/
          proxysql/lib/obj/*.gcno          

    - name: Pack test cache with zstd-15
      id: cache-pack-test
      if: ${{ success() && steps.cache-check.outputs.cache-hit != 'true' && contains(matrix.type,'-tap') }}
      # `actions/cache@v4` compresses with zstd level ~3; we pre-compress
      # with zstd -15 to fit more under the 10 GB repo quota. See the
      # bin-cache pack step above for the rationale on dropping from
      # `--ultra -22` to -15: this step alone was taking 14-16 min per
      # ubuntu build job at --ultra -22, doubling CI-builds wall clock
      # and blocking every downstream TAP workflow on it. Re-compression
      # by actions/cache on the already-compressed .tar.zst is a near-
      # no-op. Symbols are preserved bit-for-bit.
      run: |
        command -v zstd >/dev/null || sudo apt-get install -y zstd
        cd proxysql/
        tar -cf - test/ | zstd -15 -T0 -o ../cache_test.tar.zst        

    - name: Cache save test
      id: cache-save-test
      if: ${{ success() && steps.cache-check.outputs.cache-hit != 'true' && contains(matrix.type,'-tap') }}
      uses: actions/cache/save@v4
      with:
        key: ${{ env.BLDCACHE }}_test
        enableCrossOsArchive: true
        path: cache_test.tar.zst

    - name: Cache save matrix
      id: cache-save-matrix
      if: ${{ success() && steps.cache-check.outputs.cache-hit != 'true' && contains(matrix.type,'-tap') }}
      uses: actions/cache/save@v4
      with:
        key: ${{ env.BLDCACHE }}_matrix
        enableCrossOsArchive: true
        path: |
                    proxysql/tap-matrix*

    - name: Archive artifacts
      if: ${{ failure() && !cancelled() }}
      uses: actions/upload-artifact@v4
      with:
        name: ci-builds-${{ env.SHA }}-run#${{ github.run_number }}-${{ matrix.dist }}${{ matrix.type }}
        path: |
                    proxysql/

    - uses: LouisBrunner/checks-action@v2.0.0
      if: always()
      with:
        token: ${{ secrets.GITHUB_TOKEN }}
        check_id: ${{ steps.checks.outputs.check_id }}
        repo: ${{ github.repository }}
        sha: ${{ env.SHA }}
        conclusion: ${{ job.status }}
#        action_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}'
        details_url: 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}'

#  unlock:
#    runs-on: ubuntu-22.04
#    if: ${{ always() }}
#    needs: [ builds ]
#    env:
#      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
#
#    steps:
#    
#    - name: Concurency unlock
#      run: |
#        # delete lock
#        PROBE="$(gh variable -R ${{ github.repository }} list | grep -i ${LOCK} || echo '')"
#        if [[ ${PROBE} =~ ${{ github.workflow }} ]]; then
#          echo "Releasing lock '${LOCK^^}' by '$(echo ${PROBE} | awk '{ print $2 }')'"
#          gh variable -R ${{ github.repository }} delete ${LOCK} || true
#        elif [[ ! -z ${PROBE} ]]; then
#          echo "Lock raced '${LOCK^^}' by '$(echo ${PROBE} | awk '{ print $2 }')'"
#        #  exit -1
#        else
#          echo "Not locked '${LOCK^^}'"
#        fi