proxysql/.github/workflows/ci-taptests.yml

name: CI-taptests

on:
  workflow_dispatch:
  workflow_call:
    inputs:
      trigger:
        type: string
      testgroup:
        type: string

env:
  TAPGROUP: tests
  SHA: ${{ inputs.trigger && fromJson(inputs.trigger).event.workflow_run.head_sha || github.sha }}

jobs:
  select:
    runs-on: ubuntu-22.04
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    strategy:
      fail-fast: false
      matrix:
        testdist: [ 'ubuntu22-tap' ]
#        infradb: [ 'mysql57', 'mysql80', 'mysql81', 'mysql82', 'mariadb10', 'mariadb11' ]
        infradb: [ 'mysql57' ]
    env:
      TESTDIST: ${{ matrix.testdist }}
      INFRADB:  ${{ matrix.infradb }}
      BLDCACHE: ${{ inputs.trigger && fromJson(inputs.trigger).event.workflow_run.head_sha || github.sha }}_${{ matrix.testdist }}_matrix

    steps:

#    - name: Wait for cache
#      if: ${{ env.BRANCH != 'none' }}
#      env:
#        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
#      run: |
#        echo "Wating for cache '${BLDCACHE}' ..."
#        sleep $(( $RANDOM / 1024 ))
#        PROBE="$(gh cache list --limit 300 --repo ${{ github.repository }} | grep -i ${BLDCACHE} || echo '')"
#        while [[ -z ${PROBE} ]]; do
#          echo "Wating for cache '${BLDCACHE}' ..."
#          sleep 30
#          PROBE="$(gh cache list --limit 300 --repo ${{ github.repository }} | grep -i ${BLDCACHE} || echo '')"
#        done
#        echo "Cache available '${BLDCACHE}'"

    - name: Cache restore
      id: cache-matrix
      uses: actions/cache/restore@v4
      with:
        key: ${{ env.BLDCACHE }}
        fail-on-cache-miss: true
        path: |
                    proxysql/tap-matrix*

    - name: Set matrix
      id: set-matrix
      run: |
        sed -i "s/'clickhouse_php_conn-t', //" proxysql/tap-matrix-$TAPGROUP.json
        sed -i "s/'reg_test_3603-stmt_metadata-t', //" proxysql/tap-matrix-$TAPGROUP.json
        sed -i "s/'reg_test_3992_fast_forward_malformed_packet-mysqlsh-t', //" proxysql/tap-matrix-$TAPGROUP.json
        sed -i "s/'reg_test_3992_fast_forward_malformed_packet-pymysql-t', //" proxysql/tap-matrix-$TAPGROUP.json
        sed -i "s/'reg_test_3992_fast_forward_malformed_packet-t', //" proxysql/tap-matrix-$TAPGROUP.json
        sed -i "s/'reg_test_fast_forward_split_packet-t', //" proxysql/tap-matrix-$TAPGROUP.json
        #sed -i "s/'set_testing-240-t', //" proxysql/tap-matrix-$TAPGROUP.json
        sed -i "s/'test_binlog_reader-t', //" proxysql/tap-matrix-$TAPGROUP.json
        sed -i "s/'test_clickhouse_server-t', //" proxysql/tap-matrix-$TAPGROUP.json
        sed -i "s/'test_clickhouse_server_libmysql-t', //" proxysql/tap-matrix-$TAPGROUP.json
        sed -i "s/'test_cluster1-t', //" proxysql/tap-matrix-$TAPGROUP.json
        sed -i "s/'test_cluster_sync-t', //" proxysql/tap-matrix-$TAPGROUP.json
        sed -i "s/'test_com_reset_connection_com_change_user-t', //" proxysql/tap-matrix-$TAPGROUP.json
        sed -i "s/'test_log_last_insert_id-t', //" proxysql/tap-matrix-$TAPGROUP.json
        sed -i "s/'test_read_only_actions_offline_hard_servers-t', //" proxysql/tap-matrix-$TAPGROUP.json
        sed -i "s/'test_rw_binary_data-t', //" proxysql/tap-matrix-$TAPGROUP.json
        sed -i "s/'test_server_sess_status-t', //" proxysql/tap-matrix-$TAPGROUP.json
        sed -i "s/'test_session_status_flags-t', //" proxysql/tap-matrix-$TAPGROUP.json
        sed -i "s/'test_auth_methods-t', //" proxysql/tap-matrix-$TAPGROUP.json
        sed -i "s/'test_sqlite3_pass_exts-t', //" proxysql/tap-matrix-$TAPGROUP.json
        
        echo "matrix=$(cat proxysql/tap-matrix-$TAPGROUP.json)" >> $GITHUB_OUTPUT        

  tests:
    runs-on: ubuntu-22.04
    needs: [ "select" ]
    strategy:
      fail-fast: false
      matrix:
        testdist: [ 'ubuntu22-tap' ]
#        infradb: [ 'mysql57', 'mysql80', 'mysql81', 'mysql82', 'mariadb10', 'mariadb11' ]
        infradb: [ 'mysql57' ]
#        taptests: [ '.*admin.*-t|.*basic.*-t|.*charset.*-t|.*firewall.*-t|.*kill.*-t|.*max.*-t', 'multiple.*-t', '.*mysql.*-t', '.*prepare.*-t', 'reg_test.*-t', 'save.*-t', '.*set.*-t', '.*sqlite.*-t', '.*ssl.*-t', 'test_.*-t' ]
#        taptests: [ '' ]
#        taptests: [ 'kill_connection3-t', 'reg_test_3847_admin_lock-t', 'set_testing-240-t', 'max_connections_ff-t', 'test_server_sess_status-t' ]
        taptests: ${{ fromJson(needs.select.outputs.matrix) }}
    env:
      TESTDIST: ${{ matrix.testdist }}
      TAPTESTS: ${{ matrix.taptests }}
      INFRADB:  ${{ matrix.infradb }}
      BLDCACHE: ${{ inputs.trigger && fromJson(inputs.trigger).event.workflow_run.head_sha || github.sha }}_${{ matrix.testdist }}

    steps:

    - name: Install dependencies
      run: |
        sudo apt-get update -y
        sudo apt-get full-upgrade -y
        sudo apt-get install -y python3-pymysql python3-structlog sysbench mycli
        sudo apt-get install -y --no-install-recommends python3-pip
        sudo pip3 install fastcov
        
        wget https://github.com/openark/orchestrator/releases/download/v3.2.6/orchestrator-client_3.2.6_amd64.deb
        sudo dpkg -i orchestrator-client_3.2.6_amd64.deb
        
        sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29
        sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C
        wget https://repo.mysql.com/mysql-apt-config_0.8.29-1_all.deb
        sudo dpkg -i mysql-apt-config_0.8.29-1_all.deb
        sudo apt-get update -y
        sudo apt-cache policy mysql-shell
        sudo apt-get install -y mysql-shell

        sudo sed -i 's/8.0/5.7/' /etc/apt/sources.list.d/mysql.list
        sudo sed -i 's/jammy/bionic/' /etc/apt/sources.list.d/mysql.list
        sudo apt-get update -y
        sudo apt-cache policy libmysqlclient-dev
        sudo apt-get install -y --allow-downgrades libmysqlclient-dev=5.7*        

    - name: Cache restore src
      id: cache-src
      uses: actions/cache/restore@v4
      with:
        key: ${{ env.BLDCACHE }}_src
        fail-on-cache-miss: true
        path: |
                    proxysql/src/

    - name: Cache restore test
      id: cache-test
      uses: actions/cache/restore@v4
      with:
        key: ${{ env.BLDCACHE }}_test
        fail-on-cache-miss: true
        path: |
                    proxysql/test/

    - name: Checkout jenkins_build_scripts
      uses: actions/checkout@v4
      with:
        repository: 'proxysql/jenkins-build-scripts'
        ref: 'kubernetes'
#        fetch-depth: 0
        path: 'jenkins-build-scripts'
        token: ${{ secrets.GH_TOKEN_PROXYSQL }}
        submodules: 'false'

    - name: Configure env.sh
      run: |
        cd jenkins-build-scripts
        
        # configure paths
        sed -i "s|JENKINS_SCRIPTS_PATH=.*|JENKINS_SCRIPTS_PATH=${{ github.workspace }}/jenkins-build-scripts|" env.sh
        sed -i "s|WORKSPACE=.*|WORKSPACE=${{ github.workspace }}/proxysql|" env.sh        

    - name: Configure test/env.sh
      run: |
        cd proxysql
        #mkdir test
        
        # select tests
        echo "export TAP_WORKDIR=\$WORKSPACE/test/tap/$TAPGROUP" >> test/env.sh
        echo "export TEST_PY_INTERNAL=0" >> test/env.sh
        echo "export TEST_PY_BENCHMARK=0" >> test/env.sh
        echo "export TEST_PY_CHUSER=0" >> test/env.sh
        echo "export TEST_PY_STATS=0" >> test/env.sh
        echo "export TEST_PY_TAP=1" >> test/env.sh
        echo "export TEST_PY_TAPINT=0" >> test/env.sh
        echo "export TEST_PY_FAILOVER=0" >> test/env.sh
        echo 'export TEST_PY_TAP_INCL="${{ matrix.taptests }}"' >> test/env.sh
        #echo 'export TEST_PY_TAP_EXCL=""' >> test/env.sh
        echo 'export MALLOC_CONF="retain:false"' >> test/env.sh
        echo 'export WITHGCOV=0' >> test/env.sh
        #echo 'export' >> test/env.sh
        #cat test/env.sh        

    - name: Run proxysql
      run: |
        #set -x
        set +e
        cd ${{ github.workspace }}/jenkins-build-scripts
        source ./env.sh
        
        ./cluster_start.bash
        sleep 10
        
        cd ${{ github.workspace }}/proxysql
        mkdir -p ci_infra_logs/regular_infra/proxysql
        cd src
        mkdir coverage_reports

        (./proxysql --clickhouse-server --sqlite3-server --idle-threads -f -c "$DOCKER_SCRIPT_PATH/conf/proxysql/proxysql.cnf" -D $REGULAR_INFRA_DATADIR &> $REGULAR_INFRA_DATADIR/proxysql.log) &
        sleep 10
        mysql -uadmin -padmin -h127.0.0.1 -P6032 -e "SELECT version();" 2>&1 | grep -v "Using a password"
        
        cd ${{ github.workspace }}/jenkins-build-scripts
        ./cluster_init.bash
        sleep 10                

    - name: Infra
      run: |
        cd ${{ github.workspace }}/jenkins-build-scripts
        source ./env.sh
        
        cd ${{ github.workspace }}/jenkins-build-scripts/infra-docker-hoster
        sed -i 's|image: .*|image: ghcr.io/mrmohebi/docker-hoster|' docker-compose.yml
        ./docker-compose-init.bash

        cd ${{ github.workspace }}/jenkins-build-scripts/infra-mysql57
        sed -i "s/\${INFRA}-\${CONTAINER}-1/\${INFRA}_\${CONTAINER}_1/" docker-compose-init.bash
        ./docker-compose-init.bash
        
        docker ps -a
        cat /etc/hosts        

    - name: TAP-tests
      run: |
        set +e        
        cd ${{ github.workspace }}/jenkins-build-scripts
        source ./env.sh
        
        sudo -E ./test-scripts/bin/proxysql-tester.py
        RC=$?
        
        exit $RC        

    - name: Cleanup
      if: ${{ failure() && !cancelled() }}
      run: |
        set +e
        cd ${{ github.workspace }}/jenkins-build-scripts
        source ./env.sh
        
        mysql -h127.0.0.1 -P6032 -uadmin -padmin -e "PROXYSQL SHUTDOWN SLOW" &> /dev/null
        sleep 10
        killall proxysql
        
        cd ${{ github.workspace }}/jenkins-build-scripts/infra-docker-hoster
        sed -i 's|image: .*|image: ghcr.io/mrmohebi/docker-hoster|' docker-compose.yml
        ./docker-compose-destroy.bash
        
        cd ${{ github.workspace }}/jenkins-build-scripts/infra-mysql57
        sed -i "s/\${INFRA}-\${CONTAINER}-1/\${INFRA}_\${CONTAINER}_1/" docker-compose-init.bash
        ./docker-compose-destroy.bash
        
        sudo chmod -R 777 ${{ github.workspace }}/*
        echo "${{ matrix.taptests }}" | tr -d '*|' | xargs -0 printf 'TAPTESTS=%s' >> $GITHUB_ENV        

    - name: Archive artifacts logs
      if: ${{ failure() && !cancelled() }}
      uses: actions/upload-artifact@v4
      with:
        name: ${{ github.workflow }}-${{ env.SHA }}-logs-run#${{ github.run_number }}-${{ env.TAPTESTS }}
        path: |
                    proxysql/ci_*_logs/

    - name: Archive artifacts bins
      if: ${{ failure() && !cancelled() }}
      uses: actions/upload-artifact@v4
      with:
        name: ${{ github.workflow }}-${{ env.SHA }}-bins-run#${{ github.run_number }}-${{ env.TAPTESTS }}
        path: |
          proxysql/binaries/
          proxysql/src/
          proxysql/libs/          
#          proxysql/deps/

    - name: Archive artifacts tests
      if: ${{ failure() && !cancelled() }}
      uses: actions/upload-artifact@v4
      with:
        name: ${{ github.workflow }}-${{ env.SHA }}-tests-run#${{ github.run_number }}-${{ env.TAPTESTS }}
        path: |
          proxysql/test/tap/tests*