From 22e0713cc48468c1a010fb07238a8a74f5814aa8 Mon Sep 17 00:00:00 2001 From: Miro Stauder Date: Tue, 6 Jun 2023 15:44:57 +0200 Subject: [PATCH] Create ci-cb_taptests.yml --- .github/workflows/ci-cb_taptests.yml | 311 +++++++++++++++++++++++++++ 1 file changed, 311 insertions(+) create mode 100644 .github/workflows/ci-cb_taptests.yml diff --git a/.github/workflows/ci-cb_taptests.yml b/.github/workflows/ci-cb_taptests.yml new file mode 100644 index 000000000..7e3c7ce7a --- /dev/null +++ b/.github/workflows/ci-cb_taptests.yml @@ -0,0 +1,311 @@ + +name: CI-cb_taptests + +on: + push: + branches: [ "v2.x" ] + paths-ignore: + - '.github/**' + - '**.md' + pull_request: + branches: [ "v2.x" ] + paths-ignore: + - '.github/**' + - '**.md' +# schedule: +# - cron: '15 13 * * 3' + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-22.04 + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y python3-pymysql python3-structlog sysbench mycli + sudo pip3 install fastcov + + wget https://github.com/openark/orchestrator/releases/download/v3.2.6/orchestrator-client_3.2.6_amd64.deb + sudo dpkg -i orchestrator-client_3.2.6_amd64.deb + + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + wget https://repo.mysql.com/mysql-apt-config_0.8.24-1_all.deb + sudo dpkg -i mysql-apt-config_0.8.24-1_all.deb + sudo apt-get update -y + sudo apt-cache policy mysql-shell + sudo apt-get install -y mysql-shell + + sudo sed -i 's/8.0/5.7/' /etc/apt/sources.list.d/mysql.list + sudo sed -i 's/jammy/bionic/' /etc/apt/sources.list.d/mysql.list + sudo apt-get update -y + sudo apt-cache policy libmysqlclient-dev + sudo apt-get install -y --allow-downgrades libmysqlclient-dev=5.7* + + - name: Checkout proxysql + uses: actions/checkout@v3 + with: + repository: 'sysown/proxysql' +# ref: 'v2.x' + fetch-depth: 0 + path: 'proxysql' + + - name: Set GIT_VERSION + run: | + cd proxysql/ + git fetch --tags --force + echo "GIT_VERSION=$(git describe --long --abbrev=7)" >> $GITHUB_ENV + + - name: Checkout jenkins_build_scripts + uses: actions/checkout@v3 + with: + repository: 'proxysql/jenkins-build-scripts' + ref: 'kubernetes' +# fetch-depth: 0 + path: 'jenkins-build-scripts' + token: ${{ secrets.GH_TOKEN }} + submodules: 'false' + + - name: Patch TAP-tests + run: | + # apply patches + for PATCH in $(cd jenkins-build-scripts/test-scripts/patches; find . -type f); do + if [[ $PATCH =~ \.patch ]]; then + patch proxysql/test/tap/${PATCH%.patch} jenkins-build-scripts/test-scripts/patches/${PATCH} + elif [[ ! -f jenkins-build-scripts/test-scripts/patches/${PATCH#./}.patch ]]; then + cp jenkins-build-scripts/test-scripts/patches/${PATCH#./} proxysql/test/tap/${PATCH#./} + fi + done + # patch multi port listening + cd proxysql/ + sed -i '61 s/"0.0.0.0:6032"/ current.c_str()/' test/tap/tests/admin-listen_on_unix-t.cpp + sed -i 's/:6035/:6036/' test/tap/tests/test_sqlite3_server-t.cpp + + - name: Build + run: | + cd proxysql/ + # fix compile issues + #grep -rl "" test/ | xargs sed -i 's||"curl/curl.h"|' + #sed -i 's|-I/usr/include/mysql |-I/usr/include/mysql -I$(CURL_IDIR) |' test/tap/tests/Makefile + + # build proxysql + sed -i "/command/c \ command: bash -l -c 'cd /opt/proxysql && make debug_clickhouse WITHGCOV=1'" docker-compose.yml + #cat docker-compose.yml + make ubuntu22-dbg + + # build tap tests + sed -i "/command/c \ command: bash -l -c 'cd /opt/proxysql && make build_tap_test_debug WITHGCOV=1'" docker-compose.yml + #cat docker-compose.yml + make ubuntu22-dbg + + # remove irrelevant tests (missing infra) + cd test/tap/tests + rm -f *clickhouse*-t + #rm -f *cluster*-t + rm -f *binlog*-t + rm -f *mariadb*-t + rm -f reg_test_3992_fast_forward_malformed_packet*-t + + # remove always failing tests + rm -f test_rw_binary_data-t + rm -f reg_test_3603-stmt_metadata-t + rm -f test_log_last_insert_id-t + rm -f test_session_status_flags-t + rm -f set_testing-240-t + rm -f reg_test_fast_forward_split_packet-t + rm -f test_com_reset_connection_com_change_user-t + rm -f test_cluster1-t + rm -f test_server_sess_status-t + rm -f test_cluster_sync-t + + # remove occasionaly failing tests + rm -f max_connections_ff-t + rm -f test_backend_conn_ping-t + + - name: Prepare tests + id: set-matrix + if: ${{ always() }} + run: | + find proxysql/src -type f -executable -print > proxysql/executable.txt + find proxysql/test -type f -executable -print >> proxysql/executable.txt + MATRIX="[ "$(find proxysql/test/tap/tests/ -type f -name '*-t' -executable -printf "'%f', ")" ]" + #echo "matrix=${MATRIX}" + echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT + + - name: Archive artifacts + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: ci-taptests-${{ env.GIT_VERSION }}-run#${{ github.run_number }} + path: | + proxysql/src/ + proxysql/test/ + proxysql/deps/libssl/openssl-*/include/openssl + proxysql/executable.txt + + +############################################################################################################### + + + test: + runs-on: ubuntu-22.04 + needs: [ build ] + strategy: + fail-fast: false + matrix: +# testgroup: [ '.*admin.*-t|.*basic.*-t|.*charset.*-t|.*firewall.*-t|.*kill.*-t|.*max.*-t', 'multiple.*-t', '.*mysql.*-t', '.*prepare.*-t', 'reg_test.*-t', 'save.*-t', '.*set.*-t', '.*sqlite.*-t', '.*ssl.*-t', 'test_.*-t' ] +# testgroup: [ '' ] +# testgroup: [ 'kill_connection3-t', 'reg_test_3847_admin_lock-t', 'set_testing-240-t', 'max_connections_ff-t', 'test_server_sess_status-t' ] + testgroup: ${{ fromJson(needs.build.outputs.matrix) }} + steps: + + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y python3-pymysql python3-structlog sysbench mycli + sudo pip3 install fastcov + + wget https://github.com/openark/orchestrator/releases/download/v3.2.6/orchestrator-client_3.2.6_amd64.deb + sudo dpkg -i orchestrator-client_3.2.6_amd64.deb + + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + wget https://repo.mysql.com/mysql-apt-config_0.8.24-1_all.deb + sudo dpkg -i mysql-apt-config_0.8.24-1_all.deb + sudo apt-get update -y + sudo apt-cache policy mysql-shell + sudo apt-get install -y mysql-shell + + sudo sed -i 's/8.0/5.7/' /etc/apt/sources.list.d/mysql.list + sudo sed -i 's/jammy/bionic/' /etc/apt/sources.list.d/mysql.list + sudo apt-get update -y + sudo apt-cache policy libmysqlclient-dev + sudo apt-get install -y --allow-downgrades libmysqlclient-dev=5.7* + + - name: Checkout proxysql + uses: actions/checkout@v3 + with: + repository: 'sysown/proxysql' +# ref: 'v2.x' + fetch-depth: 0 + path: 'proxysql' + + - name: Set GIT_VERSION + run: | + cd proxysql/ + git fetch --tags --force + echo "GIT_VERSION=$(git describe --long --abbrev=7)" >> $GITHUB_ENV + + - name: Download build + uses: actions/download-artifact@v3 + with: + name: ci-taptests-${{ env.GIT_VERSION }}-run#${{ github.run_number }} + path: proxysql + + - name: Fix build + run: | + cat proxysql/executable.txt | xargs -n1 chmod +x + sudo ln -s ${{ github.workspace }}/proxysql /opt/proxysql + + - name: Checkout jenkins_build_scripts + uses: actions/checkout@v3 + with: + repository: 'proxysql/jenkins-build-scripts' + ref: 'kubernetes' +# fetch-depth: 0 + path: 'jenkins-build-scripts' + token: ${{ secrets.GH_TOKEN }} + submodules: 'false' + + - name: Configure env.sh + run: | + cd jenkins-build-scripts + # configure paths + sed -i "s|JENKINS_SCRIPTS_PATH=.*|JENKINS_SCRIPTS_PATH=${{ github.workspace }}/jenkins-build-scripts|" env.sh + sed -i "s|WORKSPACE=.*|WORKSPACE=${{ github.workspace }}/proxysql|" env.sh + + - name: Configure test/env.sh + run: | + cd proxysql + # select tests + echo "export TEST_PY_INTERNAL=0" >> test/env.sh + echo "export TEST_PY_BENCHMARK=0" >> test/env.sh + echo "export TEST_PY_CHUSER=0" >> test/env.sh + echo "export TEST_PY_STATS=0" >> test/env.sh + echo "export TEST_PY_TAP=1" >> test/env.sh + echo "export TEST_PY_TAPINT=0" >> test/env.sh + echo "export TEST_PY_FAILOVER=0" >> test/env.sh + echo 'export TEST_PY_TAP_INCL="${{ matrix.testgroup }}"' >> test/env.sh + #echo 'export TEST_PY_TAP_EXCL=""' >> test/env.sh + echo 'export MALLOC_CONF="retain:false"' >> test/env.sh + echo 'export' >> test/env.sh + cat test/env.sh + + - name: Docker-hoster + run: | + cd ${{ github.workspace }}/jenkins-build-scripts/infra-docker-hoster + docker-compose up -d + + - name: Run proxysql + run: | + set -x + set +e + cd ${{ github.workspace }}/jenkins-build-scripts + source ./env.sh + ./cluster_start.bash + sleep 10 + + cd ${{ github.workspace }}/proxysql + mkdir -p ci_infra_logs/regular_infra/proxysql + cd src + mkdir coverage_reports + + (./proxysql --clickhouse-server --sqlite3-server --idle-threads -f -c "$DOCKER_SCRIPT_PATH/conf/proxysql/proxysql.cnf" -D $REGULAR_INFRA_DATADIR &> $REGULAR_INFRA_DATADIR/proxysql.log) & + sleep 10 + mysql -uadmin -padmin -h127.0.0.1 -P6032 -e "SELECT version();" + + cd ${{ github.workspace }}/jenkins-build-scripts + ./cluster_init.bash + sleep 10 + + - name: Infra + run: | + cd ${{ github.workspace }}/jenkins-build-scripts + source ./env.sh + cd ${{ github.workspace }}/jenkins-build-scripts/infra-mysql57 + + sed -i "s/\${INFRA}-\${CONTAINER}-1/\${INFRA}_\${CONTAINER}_1/" docker-compose-init.bash + ./docker-compose-init.bash + docker ps -a + cat /etc/hosts + + - name: TAP-tests + run: | + set +e + cd ${{ github.workspace }}/jenkins-build-scripts + + source ./env.sh + env | sort + sudo -E ./test-scripts/bin/proxysql-tester.py + RC=$? + + exit $RC + + - name: Fix premissions + if: ${{ failure() }} + run: | + sudo chmod -R 777 ${{ github.workspace }}/* + echo "${{ matrix.testgroup }}" | tr -d '*|' | xargs -0 printf 'TESTGROUP=%s' >> $GITHUB_ENV + + - name: Archive artifacts + if: ${{ failure() }} + uses: actions/upload-artifact@v3 + with: + name: ci-selftests-${{ env.GIT_VERSION }}-run#${{ github.run_number }}-${{ env.TESTGROUP }} + path: | + proxysql/ci_gcov_logs/ + proxysql/ci_infra_logs/ + proxysql/ci_tests_logs/ +