Create ci-taptests.yml

3 years ago · 5b7d57ac1c
parent 48240b232c
commit 5b7d57ac1c
1 changed files with 321 additions and 0 deletions
--- a/.github/workflows/ci-taptests.yml
+++ b/.github/workflows/ci-taptests.yml
@ -0,0 +1,321 @@
+
+name: CI-taptests
+
+on:
+  push:
+    branches: [ "v2.x" ]
+    paths-ignore:
+    - '.github/**'
+    - '**.md'
+  pull_request:
+    branches: [ "v2.x" ]
+    paths-ignore:
+    - '.github/**'
+    - '**.md'
+#  schedule:
+#    - cron: '15 13 * * 3'
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+    
+    - name: Install dependencies
+      run: |
+        sudo apt-get update -y
+        sudo apt-get install -y python3-pymysql python3-structlog sysbench mycli
+        sudo pip3 install fastcov
+        
+        wget https://github.com/openark/orchestrator/releases/download/v3.2.6/orchestrator-client_3.2.6_amd64.deb
+        sudo dpkg -i orchestrator-client_3.2.6_amd64.deb
+        
+        sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29
+        wget https://repo.mysql.com/mysql-apt-config_0.8.24-1_all.deb
+        sudo dpkg -i mysql-apt-config_0.8.24-1_all.deb
+        sudo apt-get update -y
+        sudo apt-cache policy mysql-shell
+        sudo apt-get install -y mysql-shell
+        
+        sudo sed -i 's/8.0/5.7/' /etc/apt/sources.list.d/mysql.list
+        sudo sed -i 's/jammy/bionic/' /etc/apt/sources.list.d/mysql.list
+        sudo apt-get update -y
+        sudo apt-cache policy libmysqlclient-dev
+        sudo apt-get install -y --allow-downgrades libmysqlclient-dev=5.7*
+        
+    - name: Checkout proxysql
+      uses: actions/checkout@v3
+      with:
+        repository: 'sysown/proxysql'
+#        ref: 'v2.x'
+        fetch-depth: 0
+        path: 'proxysql'
+        
+    - name: Set GIT_VERSION
+      run: |
+        cd proxysql/
+        git fetch --tags --force
+        echo "GIT_VERSION=$(git describe --long --abbrev=7)" >> $GITHUB_ENV
+        
+    - name: Checkout jenkins_build_scripts
+      uses: actions/checkout@v3
+      with:
+        repository: 'proxysql/jenkins-build-scripts'
+        ref: 'kubernetes'
+        fetch-depth: 0
+        path: 'jenkins-build-scripts'
+        token: ${{ secrets.GH_TOKEN }}
+        submodules: 'false'
+        
+    - name: Configure env.sh
+      run: |
+        cd jenkins-build-scripts
+        git rm --cached proxysql_qa_test_suite
+        # configure paths
+        sed -i "s|JENKINS_SCRIPTS_PATH=.*|JENKINS_SCRIPTS_PATH=${{ github.workspace }}/jenkins-build-scripts|" env.sh
+        sed -i "s|WORKSPACE=.*|WORKSPACE=${{ github.workspace }}/proxysql|" env.sh
+        # select tests
+        sed -i "s|TEST_PY_INTERNAL=.*|TEST_PY_INTERNAL=0|" env.sh
+        sed -i "s|TEST_PY_BENCHMARK=.*|TEST_PY_BENCHMARK=0|" env.sh
+        sed -i "s|TEST_PY_CHUSER=.*|TEST_PY_CHUSER=0|" env.sh
+        sed -i "s|TEST_PY_STATS=.*|TEST_PY_STATS=0|" env.sh
+        sed -i "s|TEST_PY_TAP=.*|TEST_PY_TAP=1|" env.sh
+        sed -i "s|TEST_PY_TAPINT=.*|TEST_PY_TAPINT=0|" env.sh
+        sed -i "s|TEST_PY_FAILOVER=.*|TEST_PY_FAILOVER=0|" env.sh
+        sed -i "s/TEST_PY_TAP_INCL=.*/TEST_PY_TAP_INCL=\"${{ matrix.testgroup }}\"/" env.sh
+        #sed -i "s|TEST_PY_TAP_EXCL=.*|TEST_PY_TAP_EXCL=\"\"|" env.sh
+        
+    - name: Patch TAP-tests
+      run: |
+        # apply patches
+        for PATCH in $(cd jenkins-build-scripts/test-scripts/patches; find . -type f); do
+          if [[ $PATCH =~ \.patch ]]; then
+            patch proxysql/test/tap/${PATCH%.patch} jenkins-build-scripts/test-scripts/patches/${PATCH}
+          elif [[ ! -f jenkins-build-scripts/test-scripts/patches/${PATCH#./}.patch ]]; then
+            cp jenkins-build-scripts/test-scripts/patches/${PATCH#./} proxysql/test/tap/${PATCH#./}
+          fi
+        done
+        
+    - name: Build
+      run: |
+        cd proxysql/
+        # fix compile issues
+        grep -rl "<curl/curl.h>" test/ | xargs sed -i 's|<curl/curl.h>|"curl/curl.h"|'
+        sed -i 's|-I/usr/include/mysql |-I/usr/include/mysql -I$(CURL_IDIR) |' test/tap/tests/Makefile
+        
+        # build proxysql
+        sed -i "/command/c \    command: bash -l -c 'cd /opt/proxysql && make debug_clickhouse WITHGCOV=1'" docker-compose.yml
+        #cat docker-compose.yml
+        make ubuntu22-dbg
+        
+        # build tap tests
+        sed -i "/command/c \    command: bash -l -c 'cd /opt/proxysql && make build_tap_test_debug WITHGCOV=1'" docker-compose.yml
+        #cat docker-compose.yml
+        make ubuntu22-dbg
+        
+        # remove irrelevant tests (failing)
+        cd test/tap/tests
+        rm -f *clickhouse*-t
+        #rm -f *cluster*-t
+        rm -f *binlog*-t
+        rm -f *mariadb*-t
+        rm -f reg_test_3992_fast_forward_malformed_packet*-t
+        #rm -f reg_test_3847_admin_lock-t
+        
+        #rm -f set_testing-240-t
+        #rm -f test_com_reset_connection_com_change_user*-t
+        #rm -f max_connections_ff-t
+        #rm -f test_server_sess_status-t
+        
+        # remove long running tests (passing)
+        #rm -f multiple_prepared_statements-t
+        #rm -f mysql-mirror1-t
+        
+    - name: Prepare tests
+      id: set-matrix
+      if: always()
+      run: |
+        find proxysql/src -type f -executable -print > proxysql/executable.txt
+        find proxysql/test -type f -executable -print >> proxysql/executable.txt
+        MATRIX="[ "$(find proxysql/test/tap/tests/ -type f -name '*-t' -executable -printf "'%f', ")" ]"
+        #echo "matrix=${MATRIX}"
+        echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT
+        
+    - name: Archive artifacts
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: ci-taptests-${{ env.GIT_VERSION }}-run#${{ github.run_number }}
+        path: |
+          proxysql/src/
+          proxysql/test/
+          proxysql/deps/libssl/openssl-*/include/openssl
+          proxysql/executable.txt
+          
+
+###############################################################################################################
+
+
+  test:
+    runs-on: ubuntu-22.04
+    needs: [ build ]
+    strategy:
+      fail-fast: false
+      matrix:
+#        testgroup: [ '.*admin.*-t|.*basic.*-t|.*charset.*-t|.*firewall.*-t|.*kill.*-t|.*max.*-t', 'multiple.*-t', '.*mysql.*-t', '.*prepare.*-t', 'reg_test.*-t', 'save.*-t', '.*set.*-t', '.*sqlite.*-t', '.*ssl.*-t', 'test_.*-t' ]
+#        testgroup: [ '' ]
+        testgroup: [ 'kill_connection3-t', 'reg_test_3847_admin_lock-t', 'set_testing-240-t', 'max_connections_ff-t', 'test_server_sess_status-t' ]
+#        testgroup: ${{ fromJson(needs.build.outputs.matrix) }}
+    steps:
+    
+    - name: Install dependencies
+      run: |
+        sudo apt-get update -y
+        sudo apt-get install -y python3-pymysql python3-structlog sysbench mycli
+        sudo pip3 install fastcov
+        
+        wget https://github.com/openark/orchestrator/releases/download/v3.2.6/orchestrator-client_3.2.6_amd64.deb
+        sudo dpkg -i orchestrator-client_3.2.6_amd64.deb
+        
+        sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29
+        wget https://repo.mysql.com/mysql-apt-config_0.8.24-1_all.deb
+        sudo dpkg -i mysql-apt-config_0.8.24-1_all.deb
+        sudo apt-get update -y
+        sudo apt-cache policy mysql-shell
+        sudo apt-get install -y mysql-shell
+
+        sudo sed -i 's/8.0/5.7/' /etc/apt/sources.list.d/mysql.list
+        sudo sed -i 's/jammy/bionic/' /etc/apt/sources.list.d/mysql.list
+        sudo apt-get update -y
+        sudo apt-cache policy libmysqlclient-dev
+        sudo apt-get install -y --allow-downgrades libmysqlclient-dev=5.7*
+        
+    - name: Checkout proxysql
+      uses: actions/checkout@v3
+      with:
+        repository: 'sysown/proxysql'
+#        ref: 'v2.x'
+        fetch-depth: 0
+        path: 'proxysql'
+        
+    - name: Set GIT_VERSION
+      run: |
+        cd proxysql/
+        git fetch --tags --force
+        echo "GIT_VERSION=$(git describe --long --abbrev=7)" >> $GITHUB_ENV
+        
+    - name: Download build
+      uses: actions/download-artifact@v3
+      with:
+        name: ci-taptests-${{ env.GIT_VERSION }}-run#${{ github.run_number }}
+        path: proxysql
+        
+    - name: Fix build
+      run: |
+        cat proxysql/executable.txt | xargs -n1 chmod +x
+        sudo ln -s ${{ github.workspace }}/proxysql /opt/proxysql
+        
+    - name: Checkout jenkins_build_scripts
+      uses: actions/checkout@v3
+      with:
+        repository: 'proxysql/jenkins-build-scripts'
+        ref: 'kubernetes'
+        fetch-depth: 0
+        path: 'jenkins-build-scripts'
+        token: ${{ secrets.GH_TOKEN }}
+        submodules: 'false'
+        
+    - name: Configure env.sh
+      run: |
+        cd jenkins-build-scripts
+        git rm --cached proxysql_qa_test_suite
+        # configure paths
+        sed -i "s|JENKINS_SCRIPTS_PATH=.*|JENKINS_SCRIPTS_PATH=${{ github.workspace }}/jenkins-build-scripts|" env.sh
+        sed -i "s|WORKSPACE=.*|WORKSPACE=${{ github.workspace }}/proxysql|" env.sh
+        # select tests
+        sed -i "s|TEST_PY_INTERNAL=.*|TEST_PY_INTERNAL=0|" env.sh
+        sed -i "s|TEST_PY_BENCHMARK=.*|TEST_PY_BENCHMARK=0|" env.sh
+        sed -i "s|TEST_PY_CHUSER=.*|TEST_PY_CHUSER=0|" env.sh
+        sed -i "s|TEST_PY_STATS=.*|TEST_PY_STATS=0|" env.sh
+        sed -i "s|TEST_PY_TAP=.*|TEST_PY_TAP=1|" env.sh
+        sed -i "s|TEST_PY_TAPINT=.*|TEST_PY_TAPINT=0|" env.sh
+        sed -i "s|TEST_PY_FAILOVER=.*|TEST_PY_FAILOVER=1|" env.sh
+        sed -i "s/TEST_PY_TAP_INCL=.*/TEST_PY_TAP_INCL=\"${{ matrix.testgroup }}\"/" env.sh
+        #sed -i "s|TEST_PY_TAP_EXCL=.*|TEST_PY_TAP_EXCL=\"\"|" env.sh
+       
+    - name: Docker-hoster
+      run: |
+        cd ${{ github.workspace }}/jenkins-build-scripts/infra-docker-hoster
+        docker-compose up -d
+        
+    - name: Run proxysql
+      run: |
+        set -x
+        set +e
+        cd ${{ github.workspace }}/jenkins-build-scripts
+        source ./env.sh
+        ./cluster_start.bash
+        sleep 10
+        
+        cd ${{ github.workspace }}/proxysql
+        mkdir -p ci_infra_logs/regular_infra/proxysql
+        cd src
+        mkdir coverage_reports
+
+        (./proxysql --clickhouse-server --sqlite3-server --idle-threads -f -c "$DOCKER_SCRIPT_PATH/conf/proxysql/proxysql.cnf" -D $REGULAR_INFRA_DATADIR &> $REGULAR_INFRA_DATADIR/proxysql.log) &
+        sleep 10
+        mysql -uadmin -padmin -h127.0.0.1 -P6032 -e "SELECT version();"
+        
+        cd ${{ github.workspace }}/jenkins-build-scripts
+        ./cluster_init.bash
+        sleep 10        
+        
+    - name: Infra
+      run: |
+        cd ${{ github.workspace }}/jenkins-build-scripts
+        source ./env.sh
+        cd ${{ github.workspace }}/jenkins-build-scripts/infra-mysql57
+        
+        #sed -i "s|#!/bin/bash.*|#!/bin/bash -x|" docker-compose-init.bash
+        #sed -i "s|#!/bin/bash.*|#!/bin/bash -x|" bin/docker-mysql-post.bash
+        #sed -i "s|#!/bin/bash.*|#!/bin/bash -x|" bin/docker-orchestrator-post.bash
+        #sed -i "s|#!/bin/bash.*|#!/bin/bash -x|" bin/docker-proxy-post.bash
+        #sed -i "/orchestrator-post/i docker ps; cat /etc/hosts;" docker-compose-init.bash
+        #sed -i "s|&>/dev/null||" bin/docker-orchestrator-post.bash
+        #sed -i "/docker-orchestrator-post.bash/d" docker-compose-init.bash
+        
+        sed -i "s/\${INFRA}-\${CONTAINER}-1/\${INFRA}_\${CONTAINER}_1/" docker-compose-init.bash
+        ./docker-compose-init.bash
+        docker ps -a
+        cat /etc/hosts
+        
+    - name: TAP-tests
+      run: |
+        set +e
+        cd ${{ github.workspace }}/jenkins-build-scripts
+        
+        source ./env.sh
+        env | sort
+        sudo -E ./test-scripts/bin/proxysql-tester.py
+        RC=$?
+        
+        exit $RC
+        
+    - name: Fix premissions
+      if: always()
+      run: |
+        sudo chmod -R 777 ${{ github.workspace }}/*
+        echo "${{ matrix.testgroup }}" | tr -d '*|' | xargs -0 printf 'TESTGROUP=%s' >> $GITHUB_ENV
+        
+    - name: Archive artifacts
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: ci-selftests-${{ env.GIT_VERSION }}-run#${{ github.run_number }}-${{ env.TESTGROUP }}
+        path: |
+          proxysql/ci_gcov_logs/
+          proxysql/ci_infra_logs/
+          proxysql/ci_tests_logs/
+