From 5b7d57ac1c357cca845603941ccf8e64351c60ba Mon Sep 17 00:00:00 2001 From: Miro Stauder Date: Thu, 4 May 2023 08:48:03 +0200 Subject: [PATCH] Create ci-taptests.yml --- .github/workflows/ci-taptests.yml | 321 ++++++++++++++++++++++++++++++ 1 file changed, 321 insertions(+) create mode 100644 .github/workflows/ci-taptests.yml diff --git a/.github/workflows/ci-taptests.yml b/.github/workflows/ci-taptests.yml new file mode 100644 index 000000000..fd5cec39e --- /dev/null +++ b/.github/workflows/ci-taptests.yml @@ -0,0 +1,321 @@ + +name: CI-taptests + +on: + push: + branches: [ "v2.x" ] + paths-ignore: + - '.github/**' + - '**.md' + pull_request: + branches: [ "v2.x" ] + paths-ignore: + - '.github/**' + - '**.md' +# schedule: +# - cron: '15 13 * * 3' + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-22.04 + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y python3-pymysql python3-structlog sysbench mycli + sudo pip3 install fastcov + + wget https://github.com/openark/orchestrator/releases/download/v3.2.6/orchestrator-client_3.2.6_amd64.deb + sudo dpkg -i orchestrator-client_3.2.6_amd64.deb + + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + wget https://repo.mysql.com/mysql-apt-config_0.8.24-1_all.deb + sudo dpkg -i mysql-apt-config_0.8.24-1_all.deb + sudo apt-get update -y + sudo apt-cache policy mysql-shell + sudo apt-get install -y mysql-shell + + sudo sed -i 's/8.0/5.7/' /etc/apt/sources.list.d/mysql.list + sudo sed -i 's/jammy/bionic/' /etc/apt/sources.list.d/mysql.list + sudo apt-get update -y + sudo apt-cache policy libmysqlclient-dev + sudo apt-get install -y --allow-downgrades libmysqlclient-dev=5.7* + + - name: Checkout proxysql + uses: actions/checkout@v3 + with: + repository: 'sysown/proxysql' +# ref: 'v2.x' + fetch-depth: 0 + path: 'proxysql' + + - name: Set GIT_VERSION + run: | + cd proxysql/ + git fetch --tags --force + echo "GIT_VERSION=$(git describe --long --abbrev=7)" >> $GITHUB_ENV + + - name: Checkout jenkins_build_scripts + uses: actions/checkout@v3 + with: + repository: 'proxysql/jenkins-build-scripts' + ref: 'kubernetes' + fetch-depth: 0 + path: 'jenkins-build-scripts' + token: ${{ secrets.GH_TOKEN }} + submodules: 'false' + + - name: Configure env.sh + run: | + cd jenkins-build-scripts + git rm --cached proxysql_qa_test_suite + # configure paths + sed -i "s|JENKINS_SCRIPTS_PATH=.*|JENKINS_SCRIPTS_PATH=${{ github.workspace }}/jenkins-build-scripts|" env.sh + sed -i "s|WORKSPACE=.*|WORKSPACE=${{ github.workspace }}/proxysql|" env.sh + # select tests + sed -i "s|TEST_PY_INTERNAL=.*|TEST_PY_INTERNAL=0|" env.sh + sed -i "s|TEST_PY_BENCHMARK=.*|TEST_PY_BENCHMARK=0|" env.sh + sed -i "s|TEST_PY_CHUSER=.*|TEST_PY_CHUSER=0|" env.sh + sed -i "s|TEST_PY_STATS=.*|TEST_PY_STATS=0|" env.sh + sed -i "s|TEST_PY_TAP=.*|TEST_PY_TAP=1|" env.sh + sed -i "s|TEST_PY_TAPINT=.*|TEST_PY_TAPINT=0|" env.sh + sed -i "s|TEST_PY_FAILOVER=.*|TEST_PY_FAILOVER=0|" env.sh + sed -i "s/TEST_PY_TAP_INCL=.*/TEST_PY_TAP_INCL=\"${{ matrix.testgroup }}\"/" env.sh + #sed -i "s|TEST_PY_TAP_EXCL=.*|TEST_PY_TAP_EXCL=\"\"|" env.sh + + - name: Patch TAP-tests + run: | + # apply patches + for PATCH in $(cd jenkins-build-scripts/test-scripts/patches; find . -type f); do + if [[ $PATCH =~ \.patch ]]; then + patch proxysql/test/tap/${PATCH%.patch} jenkins-build-scripts/test-scripts/patches/${PATCH} + elif [[ ! -f jenkins-build-scripts/test-scripts/patches/${PATCH#./}.patch ]]; then + cp jenkins-build-scripts/test-scripts/patches/${PATCH#./} proxysql/test/tap/${PATCH#./} + fi + done + + - name: Build + run: | + cd proxysql/ + # fix compile issues + grep -rl "" test/ | xargs sed -i 's||"curl/curl.h"|' + sed -i 's|-I/usr/include/mysql |-I/usr/include/mysql -I$(CURL_IDIR) |' test/tap/tests/Makefile + + # build proxysql + sed -i "/command/c \ command: bash -l -c 'cd /opt/proxysql && make debug_clickhouse WITHGCOV=1'" docker-compose.yml + #cat docker-compose.yml + make ubuntu22-dbg + + # build tap tests + sed -i "/command/c \ command: bash -l -c 'cd /opt/proxysql && make build_tap_test_debug WITHGCOV=1'" docker-compose.yml + #cat docker-compose.yml + make ubuntu22-dbg + + # remove irrelevant tests (failing) + cd test/tap/tests + rm -f *clickhouse*-t + #rm -f *cluster*-t + rm -f *binlog*-t + rm -f *mariadb*-t + rm -f reg_test_3992_fast_forward_malformed_packet*-t + #rm -f reg_test_3847_admin_lock-t + + #rm -f set_testing-240-t + #rm -f test_com_reset_connection_com_change_user*-t + #rm -f max_connections_ff-t + #rm -f test_server_sess_status-t + + # remove long running tests (passing) + #rm -f multiple_prepared_statements-t + #rm -f mysql-mirror1-t + + - name: Prepare tests + id: set-matrix + if: always() + run: | + find proxysql/src -type f -executable -print > proxysql/executable.txt + find proxysql/test -type f -executable -print >> proxysql/executable.txt + MATRIX="[ "$(find proxysql/test/tap/tests/ -type f -name '*-t' -executable -printf "'%f', ")" ]" + #echo "matrix=${MATRIX}" + echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT + + - name: Archive artifacts + if: always() + uses: actions/upload-artifact@v3 + with: + name: ci-taptests-${{ env.GIT_VERSION }}-run#${{ github.run_number }} + path: | + proxysql/src/ + proxysql/test/ + proxysql/deps/libssl/openssl-*/include/openssl + proxysql/executable.txt + + +############################################################################################################### + + + test: + runs-on: ubuntu-22.04 + needs: [ build ] + strategy: + fail-fast: false + matrix: +# testgroup: [ '.*admin.*-t|.*basic.*-t|.*charset.*-t|.*firewall.*-t|.*kill.*-t|.*max.*-t', 'multiple.*-t', '.*mysql.*-t', '.*prepare.*-t', 'reg_test.*-t', 'save.*-t', '.*set.*-t', '.*sqlite.*-t', '.*ssl.*-t', 'test_.*-t' ] +# testgroup: [ '' ] + testgroup: [ 'kill_connection3-t', 'reg_test_3847_admin_lock-t', 'set_testing-240-t', 'max_connections_ff-t', 'test_server_sess_status-t' ] +# testgroup: ${{ fromJson(needs.build.outputs.matrix) }} + steps: + + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y python3-pymysql python3-structlog sysbench mycli + sudo pip3 install fastcov + + wget https://github.com/openark/orchestrator/releases/download/v3.2.6/orchestrator-client_3.2.6_amd64.deb + sudo dpkg -i orchestrator-client_3.2.6_amd64.deb + + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + wget https://repo.mysql.com/mysql-apt-config_0.8.24-1_all.deb + sudo dpkg -i mysql-apt-config_0.8.24-1_all.deb + sudo apt-get update -y + sudo apt-cache policy mysql-shell + sudo apt-get install -y mysql-shell + + sudo sed -i 's/8.0/5.7/' /etc/apt/sources.list.d/mysql.list + sudo sed -i 's/jammy/bionic/' /etc/apt/sources.list.d/mysql.list + sudo apt-get update -y + sudo apt-cache policy libmysqlclient-dev + sudo apt-get install -y --allow-downgrades libmysqlclient-dev=5.7* + + - name: Checkout proxysql + uses: actions/checkout@v3 + with: + repository: 'sysown/proxysql' +# ref: 'v2.x' + fetch-depth: 0 + path: 'proxysql' + + - name: Set GIT_VERSION + run: | + cd proxysql/ + git fetch --tags --force + echo "GIT_VERSION=$(git describe --long --abbrev=7)" >> $GITHUB_ENV + + - name: Download build + uses: actions/download-artifact@v3 + with: + name: ci-taptests-${{ env.GIT_VERSION }}-run#${{ github.run_number }} + path: proxysql + + - name: Fix build + run: | + cat proxysql/executable.txt | xargs -n1 chmod +x + sudo ln -s ${{ github.workspace }}/proxysql /opt/proxysql + + - name: Checkout jenkins_build_scripts + uses: actions/checkout@v3 + with: + repository: 'proxysql/jenkins-build-scripts' + ref: 'kubernetes' + fetch-depth: 0 + path: 'jenkins-build-scripts' + token: ${{ secrets.GH_TOKEN }} + submodules: 'false' + + - name: Configure env.sh + run: | + cd jenkins-build-scripts + git rm --cached proxysql_qa_test_suite + # configure paths + sed -i "s|JENKINS_SCRIPTS_PATH=.*|JENKINS_SCRIPTS_PATH=${{ github.workspace }}/jenkins-build-scripts|" env.sh + sed -i "s|WORKSPACE=.*|WORKSPACE=${{ github.workspace }}/proxysql|" env.sh + # select tests + sed -i "s|TEST_PY_INTERNAL=.*|TEST_PY_INTERNAL=0|" env.sh + sed -i "s|TEST_PY_BENCHMARK=.*|TEST_PY_BENCHMARK=0|" env.sh + sed -i "s|TEST_PY_CHUSER=.*|TEST_PY_CHUSER=0|" env.sh + sed -i "s|TEST_PY_STATS=.*|TEST_PY_STATS=0|" env.sh + sed -i "s|TEST_PY_TAP=.*|TEST_PY_TAP=1|" env.sh + sed -i "s|TEST_PY_TAPINT=.*|TEST_PY_TAPINT=0|" env.sh + sed -i "s|TEST_PY_FAILOVER=.*|TEST_PY_FAILOVER=1|" env.sh + sed -i "s/TEST_PY_TAP_INCL=.*/TEST_PY_TAP_INCL=\"${{ matrix.testgroup }}\"/" env.sh + #sed -i "s|TEST_PY_TAP_EXCL=.*|TEST_PY_TAP_EXCL=\"\"|" env.sh + + - name: Docker-hoster + run: | + cd ${{ github.workspace }}/jenkins-build-scripts/infra-docker-hoster + docker-compose up -d + + - name: Run proxysql + run: | + set -x + set +e + cd ${{ github.workspace }}/jenkins-build-scripts + source ./env.sh + ./cluster_start.bash + sleep 10 + + cd ${{ github.workspace }}/proxysql + mkdir -p ci_infra_logs/regular_infra/proxysql + cd src + mkdir coverage_reports + + (./proxysql --clickhouse-server --sqlite3-server --idle-threads -f -c "$DOCKER_SCRIPT_PATH/conf/proxysql/proxysql.cnf" -D $REGULAR_INFRA_DATADIR &> $REGULAR_INFRA_DATADIR/proxysql.log) & + sleep 10 + mysql -uadmin -padmin -h127.0.0.1 -P6032 -e "SELECT version();" + + cd ${{ github.workspace }}/jenkins-build-scripts + ./cluster_init.bash + sleep 10 + + - name: Infra + run: | + cd ${{ github.workspace }}/jenkins-build-scripts + source ./env.sh + cd ${{ github.workspace }}/jenkins-build-scripts/infra-mysql57 + + #sed -i "s|#!/bin/bash.*|#!/bin/bash -x|" docker-compose-init.bash + #sed -i "s|#!/bin/bash.*|#!/bin/bash -x|" bin/docker-mysql-post.bash + #sed -i "s|#!/bin/bash.*|#!/bin/bash -x|" bin/docker-orchestrator-post.bash + #sed -i "s|#!/bin/bash.*|#!/bin/bash -x|" bin/docker-proxy-post.bash + #sed -i "/orchestrator-post/i docker ps; cat /etc/hosts;" docker-compose-init.bash + #sed -i "s|&>/dev/null||" bin/docker-orchestrator-post.bash + #sed -i "/docker-orchestrator-post.bash/d" docker-compose-init.bash + + sed -i "s/\${INFRA}-\${CONTAINER}-1/\${INFRA}_\${CONTAINER}_1/" docker-compose-init.bash + ./docker-compose-init.bash + docker ps -a + cat /etc/hosts + + - name: TAP-tests + run: | + set +e + cd ${{ github.workspace }}/jenkins-build-scripts + + source ./env.sh + env | sort + sudo -E ./test-scripts/bin/proxysql-tester.py + RC=$? + + exit $RC + + - name: Fix premissions + if: always() + run: | + sudo chmod -R 777 ${{ github.workspace }}/* + echo "${{ matrix.testgroup }}" | tr -d '*|' | xargs -0 printf 'TESTGROUP=%s' >> $GITHUB_ENV + + - name: Archive artifacts + if: always() + uses: actions/upload-artifact@v3 + with: + name: ci-selftests-${{ env.GIT_VERSION }}-run#${{ github.run_number }}-${{ env.TESTGROUP }} + path: | + proxysql/ci_gcov_logs/ + proxysql/ci_infra_logs/ + proxysql/ci_tests_logs/ +