diff --git a/.gitignore b/.gitignore index 67983ce52..45cf9fb40 100644 --- a/.gitignore +++ b/.gitignore @@ -121,7 +121,7 @@ deps/libhttpserver/libhttpserver-master_20191121/ deps/libhttpserver/libhttpserver-0.18.1/ #clickhouse -deps/clickhouse-cpp/clickhouse-cpp/ +deps/clickhouse-cpp/clickhouse-cpp-* deps/cityhash/cityhash/ deps/lz4/lz4-1.7.5/ deps/lz4/lz4/ diff --git a/deps/Makefile b/deps/Makefile index 9c45e073e..e1c0e0dfe 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -1,3 +1,5 @@ +#!/bin/make -f + UNAME_S := $(shell uname -s) UNAME_MS := $(word 2, $(shell uname -ms)) @@ -35,6 +37,24 @@ endif endif + +# determine good compiler version for stdc++17 +IS_CXX17 := 0 +ifeq ($(CXX),clang++) + CLANG_VERSION := $(shell clang -dumpversion | tr '.' ' ' | awk '{ printf("%04d.%04d.%04d", $$1, $$2, $$3) }') + CLANG_MIN_VER := $(shell echo 14.0 | tr '.' ' ' | awk '{ printf("%04d.%04d.%04d", $$1, $$2, $$3) }') +ifeq ($(CLANG_MIN_VER),$(firstword $(sort $(CLANG_VERSION) $(CLANG_MIN_VER)))) + IS_CXX17 := 1 +endif +else + GCC_VERSION := $(shell gcc -dumpversion | tr '.' ' ' | awk '{ printf("%04d.%04d.%04d", $$1, $$2, $$3) }') + GCC_MIN_VER := $(shell echo 8.2.0 | tr '.' ' ' | awk '{ printf("%04d.%04d.%04d", $$1, $$2, $$3) }') +ifeq ($(GCC_MIN_VER),$(firstword $(sort $(GCC_VERSION) $(GCC_MIN_VER)))) + IS_CXX17 := 1 +endif +endif + + PROXYSQLCLICKHOUSE := $(shell echo $(PROXYSQLCLICKHOUSE)) ifeq ($(PROXYSQLCLICKHOUSE),1) default: libconfig libdaemon jemalloc mariadb_client re2 sqlite3 pcre clickhouse-cpp lz4 cityhash microhttpd curl ev libssl libhttpserver libinjection prometheus-cpp @@ -140,17 +160,24 @@ lz4/lz4/liblz4.a: cd lz4/lz4 && CC=${CC} CXX=${CXX} ${MAKE} lz4: lz4/lz4/liblz4.a -clickhouse-cpp/clickhouse-cpp/clickhouse/libclickhouse-cpp-lib.a: - cd clickhouse-cpp && rm -rf clickhouse-cpp/ || true - cd clickhouse-cpp && tar -zxf clickhouse-cpp.tar.gz -ifeq ($(CXX),clang++) - cd clickhouse-cpp/clickhouse-cpp && sed -i 's/"-O2 -pthread -Wall -Wextra -Werror"/"-O2 -pthread -Wall -Wextra -Werror -Wno-deprecated-copy"/' CMakeLists.txt + +clickhouse-cpp/clickhouse-cpp/clickhouse/libclickhouse-cpp-lib-static.a: +ifeq ($(IS_CXX17), 1) + echo ">>> Clickhouse CXX17" + cd clickhouse-cpp && ln -fs clickhouse-cpp-2.1.0 clickhouse-cpp + cd clickhouse-cpp && rm -rf clickhouse-cpp-*/ || true + cd clickhouse-cpp && tar -zxf v2.1.0.tar.gz +else + echo ">>> Clickhouse CXX11" + cd clickhouse-cpp && ln -fs clickhouse-cpp-1.0.0 clickhouse-cpp + cd clickhouse-cpp && rm -rf clickhouse-cpp-*/ || true + cd clickhouse-cpp && tar -zxf v1.0.0.tar.gz + cd clickhouse-cpp && sed -i 's/SET (CMAKE_CXX_STANDARD_REQUIRED ON)//' clickhouse-cpp/cmake/cpp17.cmake endif cd clickhouse-cpp/clickhouse-cpp && cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo . - cd clickhouse-cpp/clickhouse-cpp && patch clickhouse/columns/column.h < ../column.h.diff cd clickhouse-cpp/clickhouse-cpp && CC=${CC} CXX=${CXX} ${MAKE} -clickhouse-cpp: clickhouse-cpp/clickhouse-cpp/clickhouse/libclickhouse-cpp-lib.a +clickhouse-cpp: clickhouse-cpp/clickhouse-cpp/clickhouse/libclickhouse-cpp-lib-static.a libdaemon/libdaemon/libdaemon/.libs/libdaemon.a: @@ -281,7 +308,7 @@ cleanall: cd re2 && rm -rf re2-*/ || true cd pcre && rm -rf pcre-*/ || true cd sqlite3 && rm -rf sqlite-amalgamation-*/ || true - cd clickhouse-cpp/ && rm -rf clickhouse-cpp/ || true + cd clickhouse-cpp/ && rm -rf clickhouse-cpp-* || true cd lz4 && rm -rf lz4-*/ || true cd libmicrohttpd && rm -rf libmicrohttpd-*/ || true cd libmicrohttpd && rm -f libmicrohttpd || true diff --git a/deps/clickhouse-cpp/clickhouse-cpp b/deps/clickhouse-cpp/clickhouse-cpp new file mode 120000 index 000000000..64c429a75 --- /dev/null +++ b/deps/clickhouse-cpp/clickhouse-cpp @@ -0,0 +1 @@ +clickhouse-cpp-2.1.0 \ No newline at end of file diff --git a/deps/clickhouse-cpp/clickhouse-cpp.tar.gz b/deps/clickhouse-cpp/clickhouse-cpp.tar.gz deleted file mode 100644 index 6df8f3176..000000000 Binary files a/deps/clickhouse-cpp/clickhouse-cpp.tar.gz and /dev/null differ diff --git a/deps/clickhouse-cpp/column.h.diff b/deps/clickhouse-cpp/column.h.diff deleted file mode 100644 index 7205089f2..000000000 --- a/deps/clickhouse-cpp/column.h.diff +++ /dev/null @@ -1,9 +0,0 @@ -@@ -4,6 +4,8 @@ - #include "../base/coded.h" - #include "../types/types.h" - -+#include -+ - namespace clickhouse { - - using ColumnRef = std::shared_ptr; diff --git a/deps/clickhouse-cpp/v1.0.0.tar.gz b/deps/clickhouse-cpp/v1.0.0.tar.gz new file mode 100644 index 000000000..d50894db6 Binary files /dev/null and b/deps/clickhouse-cpp/v1.0.0.tar.gz differ diff --git a/deps/clickhouse-cpp/v2.1.0.tar.gz b/deps/clickhouse-cpp/v2.1.0.tar.gz new file mode 100644 index 000000000..36954c4bb Binary files /dev/null and b/deps/clickhouse-cpp/v2.1.0.tar.gz differ diff --git a/docker-compose.yml b/docker-compose.yml index c86e2cf45..3b9cfd5c2 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,8 @@ services: centos6_build: image: proxysql/packaging:build-centos6 volumes: - - ./docker/images/proxysql/rhel-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/rhel6/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -13,13 +14,13 @@ services: - MAKEOPT - CURVER - PKG_RELEASE=centos6 - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash centos6_dbg_build: image: proxysql/packaging:build-centos6 volumes: - - ./docker/images/proxysql/rhel-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/rhel6/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -28,13 +29,13 @@ services: - CURVER - PKG_RELEASE=dbg-centos6 - PROXYSQL_BUILD_TYPE=debug - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash centos7_build: image: proxysql/packaging:build-centos7 volumes: - - ./docker/images/proxysql/rhel-compliant/rhel7/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rhel7/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/rhel7/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -43,13 +44,13 @@ services: - CURVER - PKG_RELEASE=centos7 - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash centos7_dbg_build: image: proxysql/packaging:build-centos7 volumes: - - ./docker/images/proxysql/rhel-compliant/rhel7/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rhel7/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/rhel7/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -58,13 +59,13 @@ services: - CURVER - PKG_RELEASE=dbg-centos7 - PROXYSQL_BUILD_TYPE=debug - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash centos8_build: image: proxysql/packaging:build-centos8 volumes: - - ./docker/images/proxysql/rhel-compliant/rhel7/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rhel7/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/rhel7/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -73,13 +74,13 @@ services: - CURVER - PKG_RELEASE=centos8 - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash centos8_clang_build: image: proxysql/packaging:build-clang-centos8 volumes: - - ./docker/images/proxysql/rhel-compliant/rhel7/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rhel7/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/rhel7/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -88,13 +89,13 @@ services: - CURVER - PKG_RELEASE=centos8-clang - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash centos8_dbg_build: image: proxysql/packaging:build-centos8 volumes: - - ./docker/images/proxysql/rhel-compliant/rhel7/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rhel7/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/rhel7/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -103,14 +104,14 @@ services: - CURVER - PKG_RELEASE=dbg-centos8 - PROXYSQL_BUILD_TYPE=debug - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash fedora27_build: image: proxysql/packaging:build-fedora27 volumes: - - ./docker/images/proxysql/rhel-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -119,13 +120,13 @@ services: - CURVER - PKG_RELEASE=fedora27 - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash fedora27_dbg_build: image: proxysql/packaging:build-fedora27 volumes: - - ./docker/images/proxysql/rhel-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -134,13 +135,13 @@ services: - CURVER - PKG_RELEASE=dbg-fedora27 - PROXYSQL_BUILD_TYPE=debug - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash fedora28_build: image: proxysql/packaging:build-fedora28 volumes: - - ./docker/images/proxysql/rhel-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -149,13 +150,13 @@ services: - CURVER - PKG_RELEASE=fedora28 - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash fedora28_dbg_build: image: proxysql/packaging:build-fedora28 volumes: - - ./docker/images/proxysql/rhel-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -164,13 +165,13 @@ services: - CURVER - PKG_RELEASE=dbg-fedora28 - PROXYSQL_BUILD_TYPE=debug - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash fedora33_build: image: proxysql/packaging:build-fedora33 volumes: - - ./docker/images/proxysql/rhel-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -179,13 +180,13 @@ services: - CURVER - PKG_RELEASE=fedora33 - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash fedora33_dbg_build: image: proxysql/packaging:build-fedora33 volumes: - - ./docker/images/proxysql/rhel-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -194,13 +195,13 @@ services: - CURVER - PKG_RELEASE=dbg-fedora33 - PROXYSQL_BUILD_TYPE=debug - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash fedora34_build: image: proxysql/packaging:build-fedora34 volumes: - - ./docker/images/proxysql/rhel-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -209,13 +210,13 @@ services: - CURVER - PKG_RELEASE=fedora34 - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash fedora34_clang_build: image: proxysql/packaging:build-clang-fedora34 volumes: - - ./docker/images/proxysql/rhel-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -224,13 +225,13 @@ services: - CURVER - PKG_RELEASE=fedora34-clang - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash fedora34_dbg_build: image: proxysql/packaging:build-fedora34 volumes: - - ./docker/images/proxysql/rhel-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -239,8 +240,7 @@ services: - CURVER - PKG_RELEASE=dbg-fedora34 - PROXYSQL_BUILD_TYPE=debug - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash debian8_build: @@ -254,8 +254,7 @@ services: - MAKEOPT - CURVER - PKG_RELEASE=debian8 - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash debian8_dbg_build: image: proxysql/packaging:build-debian8 @@ -269,8 +268,7 @@ services: - CURVER - PKG_RELEASE=dbg-debian8 - PROXYSQL_BUILD_TYPE=debug - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash debian9_build: image: proxysql/packaging:build-debian9 @@ -284,8 +282,7 @@ services: - CURVER - PKG_RELEASE=debian9 - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash debian9_dbg_build: image: proxysql/packaging:build-debian9 @@ -299,8 +296,7 @@ services: - CURVER - PKG_RELEASE=dbg-debian9 - PROXYSQL_BUILD_TYPE=debug - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash debian10_build: image: proxysql/packaging:build-debian10 @@ -314,8 +310,7 @@ services: - CURVER - PKG_RELEASE=debian10 - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash debian10_dbg_build: image: proxysql/packaging:build-debian10 @@ -329,8 +324,7 @@ services: - CURVER - PKG_RELEASE=dbg-debian10 - PROXYSQL_BUILD_TYPE=debug - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash debian11_build: image: proxysql/packaging:build-debian11 @@ -344,8 +338,7 @@ services: - CURVER - PKG_RELEASE=debian11 - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash debian11_clang_build: image: proxysql/packaging:build-clang-debian11 @@ -359,8 +352,7 @@ services: - CURVER - PKG_RELEASE=debian11-clang - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash debian11_dbg_build: image: proxysql/packaging:build-debian11 @@ -374,8 +366,7 @@ services: - CURVER - PKG_RELEASE=dbg-debian11 - PROXYSQL_BUILD_TYPE=debug - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash ubuntu14_build: @@ -389,8 +380,7 @@ services: - MAKEOPT - CURVER - PKG_RELEASE=ubuntu14 - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash ubuntu14_dbg_build: image: proxysql/packaging:build-ubuntu14 @@ -404,8 +394,7 @@ services: - CURVER - PKG_RELEASE=dbg-ubuntu14 - PROXYSQL_BUILD_TYPE=debug - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash ubuntu16_build: image: proxysql/packaging:build-ubuntu16 @@ -419,8 +408,7 @@ services: - CURVER - PKG_RELEASE=ubuntu16 - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash ubuntu16_dbg_build: image: proxysql/packaging:build-ubuntu16 @@ -434,8 +422,7 @@ services: - CURVER - PKG_RELEASE=dbg-ubuntu16 - PROXYSQL_BUILD_TYPE=debug - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash ubuntu18_build: image: proxysql/packaging:build-ubuntu18 @@ -449,8 +436,7 @@ services: - CURVER - PKG_RELEASE=ubuntu18 - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash ubuntu18_dbg_build: image: proxysql/packaging:build-ubuntu18 @@ -464,8 +450,7 @@ services: - CURVER - PKG_RELEASE=dbg-ubuntu18 - PROXYSQL_BUILD_TYPE=debug - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash ubuntu20_build: image: proxysql/packaging:build-ubuntu20 @@ -479,8 +464,7 @@ services: - CURVER - PKG_RELEASE=ubuntu20 - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash ubuntu20_clang_build: image: proxysql/packaging:build-clang-ubuntu20 @@ -494,8 +478,7 @@ services: - CURVER - PKG_RELEASE=ubuntu20-clang - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash ubuntu20_dbg_build: image: proxysql/packaging:build-ubuntu20 @@ -509,8 +492,7 @@ services: - CURVER - PKG_RELEASE=dbg-ubuntu20 - PROXYSQL_BUILD_TYPE=debug - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash ubuntu22_build: image: proxysql/packaging:build-ubuntu22 @@ -524,8 +506,7 @@ services: - CURVER - PKG_RELEASE=ubuntu22 - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash ubuntu22_clang_build: image: proxysql/packaging:build-clang-ubuntu22 @@ -539,8 +520,7 @@ services: - CURVER - PKG_RELEASE=ubuntu22-clang - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash ubuntu22_dbg_build: image: proxysql/packaging:build-ubuntu22 @@ -554,14 +534,14 @@ services: - CURVER - PKG_RELEASE=dbg-ubuntu22 - PROXYSQL_BUILD_TYPE=debug - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash opensuse15_build: image: proxysql/packaging:build-opensuse15 volumes: - - ./docker/images/proxysql/suse-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/suse-compliant/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -570,13 +550,13 @@ services: - CURVER - PKG_RELEASE=opensuse15 - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash opensuse15_clang_build: image: proxysql/packaging:build-clang-opensuse15 volumes: - - ./docker/images/proxysql/suse-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/suse-compliant/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -585,13 +565,13 @@ services: - CURVER - PKG_RELEASE=opensuse15-clang - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash opensuse15_dbg_build: image: proxysql/packaging:build-opensuse15 volumes: - - ./docker/images/proxysql/suse-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/suse-compliant/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -600,14 +580,14 @@ services: - CURVER - PKG_RELEASE=dbg-opensuse15 - PROXYSQL_BUILD_TYPE=debug - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash almalinux8_build: image: proxysql/packaging:build-almalinux8 volumes: - - ./docker/images/proxysql/rhel-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -616,13 +596,13 @@ services: - CURVER - PKG_RELEASE=almalinux8 - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash almalinux8_clang_build: image: proxysql/packaging:build-clang-almalinux8 volumes: - - ./docker/images/proxysql/rhel-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -631,13 +611,13 @@ services: - CURVER - PKG_RELEASE=almalinux8-clang - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash almalinux8_dbg_build: image: proxysql/packaging:build-almalinux8 volumes: - - ./docker/images/proxysql/rhel-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -646,13 +626,13 @@ services: - CURVER - PKG_RELEASE=dbg-almalinux8 - PROXYSQL_BUILD_TYPE=debug - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash almalinux9_build: image: proxysql/packaging:build-almalinux9 volumes: - - ./docker/images/proxysql/rhel-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -661,13 +641,13 @@ services: - CURVER - PKG_RELEASE=almalinux9 - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash almalinux9_clang_build: image: proxysql/packaging:build-clang-almalinux9 volumes: - - ./docker/images/proxysql/rhel-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -676,13 +656,13 @@ services: - CURVER - PKG_RELEASE=almalinux9-clang - PROXYSQL_BUILD_TYPE=clickhouse - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash almalinux9_dbg_build: image: proxysql/packaging:build-almalinux9 volumes: - - ./docker/images/proxysql/rhel-compliant/rpmmacros/:/root/ + - ./docker/images/proxysql/rhel-compliant/rpmmacros/rpmbuild/:/root/rpmbuild/ + - ./docker/images/proxysql/suse-compliant/rpmmacros/.rpmmacros:/root/.rpmmacros - ./docker/images/proxysql/rhel-compliant/entrypoint/:/opt/entrypoint/ - ./:/opt/proxysql/ environment: @@ -691,6 +671,5 @@ services: - CURVER - PKG_RELEASE=dbg-almalinux9 - PROXYSQL_BUILD_TYPE=debug - command: - - /opt/entrypoint/entrypoint.bash + command: bash -l -c /opt/entrypoint/entrypoint.bash diff --git a/docker/images/proxysql/rhel-compliant/entrypoint/entrypoint.bash b/docker/images/proxysql/rhel-compliant/entrypoint/entrypoint.bash index 787b70f3b..d97c385c9 100755 --- a/docker/images/proxysql/rhel-compliant/entrypoint/entrypoint.bash +++ b/docker/images/proxysql/rhel-compliant/entrypoint/entrypoint.bash @@ -10,8 +10,8 @@ echo "==> '${ARCH}' architecture detected for package" DIST=$(source /etc/os-release; echo ${ID%%[-._ ]*}${VERSION%%[-._ ]*}) echo "==> '${DIST}' distro detected for package" -#echo -e "==> C compiler: ${CC} -> $(readlink -e $(which ${CC}))\n$(${CC} --version)" -#echo -e "==> C++ compiler: ${CXX} -> $(readlink -e $(which ${CXX}))\n$(${CXX} --version)" +echo -e "==> C compiler: ${CC} -> $(readlink -e $(which ${CC}))\n$(${CC} --version)" +echo -e "==> C++ compiler: ${CXX} -> $(readlink -e $(which ${CXX}))\n$(${CXX} --version)" #echo -e "==> linker version:\n$ ${LD} -> $(readlink -e $(which ${LD}))\n$(${LD} --version)" echo "==> Cleaning" diff --git a/docker/images/proxysql/rhel-compliant/rhel6/entrypoint/entrypoint.bash b/docker/images/proxysql/rhel-compliant/rhel6/entrypoint/entrypoint.bash index a9e475a59..eff2d101f 100755 --- a/docker/images/proxysql/rhel-compliant/rhel6/entrypoint/entrypoint.bash +++ b/docker/images/proxysql/rhel-compliant/rhel6/entrypoint/entrypoint.bash @@ -10,8 +10,8 @@ echo "==> '${ARCH}' architecture detected for package" DIST=$(cat /etc/redhat-release| sed 's/ .*//') echo "==> '${DIST}' distro detected for package" -#echo -e "==> C compiler: ${CC} -> $(readlink -e $(which ${CC}))\n$(${CC} --version)" -#echo -e "==> C++ compiler: ${CXX} -> $(readlink -e $(which ${CXX}))\n$(${CXX} --version)" +echo -e "==> C compiler: ${CC} -> $(readlink -e $(which ${CC}))\n$(${CC} --version)" +echo -e "==> C++ compiler: ${CXX} -> $(readlink -e $(which ${CXX}))\n$(${CXX} --version)" #echo -e "==> linker version:\n$ ${LD} -> $(readlink -e $(which ${LD}))\n$(${LD} --version)" echo "==> Cleaning" diff --git a/docker/images/proxysql/rhel-compliant/rhel7/entrypoint/entrypoint.bash b/docker/images/proxysql/rhel-compliant/rhel7/entrypoint/entrypoint.bash index ec0e95f11..3133ffdea 100755 --- a/docker/images/proxysql/rhel-compliant/rhel7/entrypoint/entrypoint.bash +++ b/docker/images/proxysql/rhel-compliant/rhel7/entrypoint/entrypoint.bash @@ -10,8 +10,8 @@ echo "==> '${ARCH}' architecture detected for package" DIST=$(source /etc/os-release; echo ${ID%%[-._ ]*}${VERSION%%[-._ ]*}) echo "==> '${DIST}' distro detected for package" -#echo -e "==> C compiler: ${CC} -> $(readlink -e $(which ${CC}))\n$(${CC} --version)" -#echo -e "==> C++ compiler: ${CXX} -> $(readlink -e $(which ${CXX}))\n$(${CXX} --version)" +echo -e "==> C compiler: ${CC} -> $(readlink -e $(which ${CC}))\n$(${CC} --version)" +echo -e "==> C++ compiler: ${CXX} -> $(readlink -e $(which ${CXX}))\n$(${CXX} --version)" #echo -e "==> linker version:\n$ ${LD} -> $(readlink -e $(which ${LD}))\n$(${LD} --version)" echo "==> Cleaning" diff --git a/include/ClickHouse_Authentication.hpp b/include/ClickHouse_Authentication.hpp index 62b9a31c3..1faf18078 100644 --- a/include/ClickHouse_Authentication.hpp +++ b/include/ClickHouse_Authentication.hpp @@ -67,7 +67,7 @@ class ClickHouse_Authentication { bool set_lock = true); bool reset(); void print_version(); - bool exists(char *username); +// bool exists(char *username); char *lookup(char *username, enum cred_username_type usertype, bool *use_ssl, int *default_hostgroup, char **default_schema, bool *schema_locked, bool *transaction_persistent, @@ -77,8 +77,8 @@ class ClickHouse_Authentication { void decrease_frontend_user_connections(char *username); void set_all_inactive(enum cred_username_type usertype); void remove_inactives(enum cred_username_type usertype); - bool set_SHA1(char *username, enum cred_username_type usertype, - void *sha_pass); +// bool set_SHA1(char *username, enum cred_username_type usertype, +// void *sha_pass); }; #endif // CLASS_PROXYSQL_CLICKHOUSE_AUTHENTICATION_H diff --git a/include/proxysql_utils.h b/include/proxysql_utils.h index 555f27564..84f53860a 100644 --- a/include/proxysql_utils.h +++ b/include/proxysql_utils.h @@ -6,12 +6,19 @@ #include #include +#ifdef CXX17 +template struct conjunction : std::true_type { }; +template struct std::conjunction : B1 { }; +template +struct std::conjunction + : std::conditional, B1>::type {}; +#else template struct conjunction : std::true_type { }; template struct conjunction : B1 { }; template struct conjunction : std::conditional, B1>::type {}; - +#endif // CXX17 /** * @brief Stores the result of formatting the first parameter with the provided * arguments, into the std::string reference provided in the second parameter. @@ -24,7 +31,11 @@ struct conjunction */ template< typename... Args, +#ifdef CXX17 + typename std::enable_if...>::value,int>::type = 0 +#else typename std::enable_if...>::value,int>::type = 0 +#endif // CXX17 > int string_format(const std::string& str, std::string& result, Args... args) { int err = 0; diff --git a/lib/ClickHouse_Authentication.cpp b/lib/ClickHouse_Authentication.cpp index a7ecfeb5b..7bf86dcbb 100644 --- a/lib/ClickHouse_Authentication.cpp +++ b/lib/ClickHouse_Authentication.cpp @@ -318,6 +318,7 @@ bool ClickHouse_Authentication::del(char * username, enum cred_username_type use return ret; }; +/* unused? bool ClickHouse_Authentication::set_SHA1(char * username, enum cred_username_type usertype, void *sha_pass) { bool ret=false; uint64_t hash1, hash2; @@ -352,7 +353,12 @@ bool ClickHouse_Authentication::set_SHA1(char * username, enum cred_username_typ #endif return ret; }; +*/ +/* +// REMOVING THIS FUNCTION BECAUSE IT IS USED ONLY FOR LDAP +// AND CLICKHOUSE MODULE DOESN'T SUPPORT LDAP. +// See also MySQL_Protocol::process_pkt_handshake_response() bool ClickHouse_Authentication::exists(char * username) { bool ret = false; uint64_t hash1, hash2; @@ -371,6 +377,7 @@ bool ClickHouse_Authentication::exists(char * username) { pthread_rwlock_unlock(&cg.lock); return ret; } +*/ char * ClickHouse_Authentication::lookup(char * username, enum cred_username_type usertype, bool *use_ssl, int *default_hostgroup, char **default_schema, bool *schema_locked, bool *transaction_persistent, bool *fast_forward, int *max_connections, void **sha1_pass) { char *ret=NULL; diff --git a/lib/ClickHouse_Server.cpp b/lib/ClickHouse_Server.cpp index 73e6485f3..4c6a9aaa9 100644 --- a/lib/ClickHouse_Server.cpp +++ b/lib/ClickHouse_Server.cpp @@ -70,7 +70,6 @@ using namespace clickhouse; __thread MySQL_Session * clickhouse_thread___mysql_sess; -//static void ClickHouse_to_MySQL(SQLite3_result *result, char *error, int affected_rows, MySQL_Protocol *myprot) { inline void ClickHouse_to_MySQL(const Block& block) { MySQL_Session *sess = clickhouse_thread___mysql_sess; MySQL_Protocol *myprot=NULL; @@ -88,8 +87,103 @@ inline void ClickHouse_to_MySQL(const Block& block) { columns=block.GetColumnCount(); //int rows=block.GetRowCount(); myprot->generate_pkt_column_count(true,NULL,NULL,sid,block.GetColumnCount()); sid++; + // Return proper types for: + // - Int8/Int16/Int32/Int64/Float/Double/NULL/DATE/Datetime for (Block::Iterator bi(block); bi.IsValid(); bi.Next()) { - myprot->generate_pkt_field(true,NULL,NULL,sid,(char *)"",(char *)"",(char *)"",(char *)bi.Name().c_str(),(char *)"",33,15,MYSQL_TYPE_VAR_STRING,1,0x1f,false,0,NULL); + clickhouse::Type::Code cc = bi.Type()->GetCode(); + + uint8_t is_null = 1; + + if (cc != clickhouse::Type::Code::Nullable) { + is_null = 0; + } else { + auto s_t = bi.Column()->As(); + cc = s_t->Nested()->GetType().GetCode(); + } + + if (cc >= clickhouse::Type::Code::Int8 && cc <= clickhouse::Type::Code::Float64) { + bool _unsigned = false; + uint16_t flags = is_null | 128; + + // NOTE: Both 'size' and 'decimals' are just used for representation purposes. + // For this reason, the values we specify here are always the 'MAX' length of these + // fields without any computation specific to the current value. See note: + // - https://dev.mysql.com/doc/internals/en/com-query-response.html#column-definition + uint32_t size = 0; + uint8_t decimals = 0; + + enum_field_types type = MYSQL_TYPE_LONG; + + switch(cc) { + case clickhouse::Type::Code::UInt8: + _unsigned = true; + case clickhouse::Type::Code::Int8: + type = MYSQL_TYPE_TINY; + flags |= (_unsigned ? 32 : 0); + size = 4; + break; + case clickhouse::Type::Code::UInt16: + _unsigned = true; + case clickhouse::Type::Code::Int16: + type = MYSQL_TYPE_SHORT; + flags |= (_unsigned ? 32 : 0); + size = 6; + break; + case clickhouse::Type::Code::UInt32: + _unsigned = true; + case clickhouse::Type::Code::Int32: + type = MYSQL_TYPE_LONG; + flags |= (_unsigned ? 32 : 0); + size = 11; + break; + case clickhouse::Type::Code::UInt64: + _unsigned = true; + case clickhouse::Type::Code::Int64: + type = MYSQL_TYPE_LONGLONG; + flags |= (_unsigned ? 32 : 0); + size = 20; + break; + case clickhouse::Type::Code::Float32: + type = MYSQL_TYPE_FLOAT; + size = 12; + decimals = 31; + break; + case clickhouse::Type::Code::Float64: + type = MYSQL_TYPE_DOUBLE; + size = 22; + decimals = 31; + break; + default: + _unsigned = false; + flags = 128; + size = 22; + } + + myprot->generate_pkt_field( + true, NULL, NULL, sid, (char*)"", (char*)"", (char*)"", (char*)bi.Name().c_str(), + (char*)"", 63, size, type, flags, decimals, false, 0, NULL + ); + } else if (cc == clickhouse::Type::Code::Date || cc == clickhouse::Type::Code::DateTime) { + if (cc == clickhouse::Type::Code::Date) { + const uint32_t size = strlen("YYYY-MM-DD") + 1; + myprot->generate_pkt_field( + true, NULL, NULL, sid, (char*)"", (char*)"", (char*)"", (char*)bi.Name().c_str(), + (char*)"", 33, size, MYSQL_TYPE_DATE, 0, 0x0, false, 0, NULL + ); + } else { + const uint32_t size = strlen("YYYY-MM-DD hh:mm:ss") + 1; + myprot->generate_pkt_field( + true, NULL, NULL, sid, (char*)"", (char*)"", (char*)"", (char*)bi.Name().c_str(), + (char*)"", 33, size, MYSQL_TYPE_DATETIME, 0, 0x0, false, 0, NULL + ); + } + } else { + myprot->generate_pkt_field( + true, NULL, NULL, sid, (char *)"", (char *)"", (char *)"", (char *)bi.Name().c_str(), + (char *)"", 33, 15, MYSQL_TYPE_VAR_STRING, 0, 0x1f, false, 0, NULL + ); + } + sid++; } /* @@ -101,7 +195,10 @@ inline void ClickHouse_to_MySQL(const Block& block) { unsigned int nTrx=0; uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0 ); //if (autocommit) setStatus += SERVER_STATUS_AUTOCOMMIT; - myprot->generate_pkt_EOF(true,NULL,NULL,sid,0, setStatus ); sid++; + bool deprecate_eof_active = sess->client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; + if (!deprecate_eof_active) { + myprot->generate_pkt_EOF(true,NULL,NULL,sid,0, setStatus); sid++; + } } char **p=(char **)malloc(sizeof(char*)*columns); unsigned long *l=(unsigned long *)malloc(sizeof(unsigned long *)*columns); @@ -109,6 +206,7 @@ inline void ClickHouse_to_MySQL(const Block& block) { for (int r=0; rType()->GetCode(); + bool is_null = false; string s; switch (cc) { case clickhouse::Type::Code::Int8: @@ -177,9 +275,9 @@ inline void ClickHouse_to_MySQL(const Block& block) { { auto s_t = block[i]->As(); if (s_t->IsNull(r)) { - s = "\\N"; + is_null = true; } else { - clickhouse::Type::Code cnc = block[i]->Type()->GetNestedType()->GetCode(); + clickhouse::Type::Code cnc = s_t->Nested()->Type()->GetCode(); switch (cnc) { case clickhouse::Type::Code::Int8: s=std::to_string(s_t->Nested()->As()->At(r)); @@ -246,8 +344,12 @@ inline void ClickHouse_to_MySQL(const Block& block) { default: break; } - l[i]=s.length(); - p[i]=strdup((char *)s.c_str()); + if (is_null == false) { + l[i]=s.length(); + p[i]=strdup((char *)s.c_str()); + } else { + p[i]=NULL; + } } myprot->generate_pkt_row(true,NULL,NULL,sid,columns,l,p); sid++; for (int i=0; isize==(strlen("SELECT @@storage_engine;")+5) && strncasecmp((char *)"SELECT @@storage_engine;",(char *)pkt->ptr+5,pkt->size-5)==0) - ) { - l_free(query_length,query); - query=l_strdup("SELECT 'MergeTree' AS '@@storage_engine'"); - query_length=strlen(query)+1; - run_query_sqlite = true; - goto __run_query_sqlite; - } - if ( - (pkt->size==(strlen("SELECT @@storage_engine;")+5) && strncasecmp((char *)"SELECT @@storage_engine;",(char *)pkt->ptr+5,pkt->size-5)==0) + (pkt->size==(strlen("SELECT @@storage_engine")+5) && strncasecmp((char *)"SELECT @@storage_engine",(char *)pkt->ptr+5,pkt->size-5)==0) ) { l_free(query_length,query); query=l_strdup("SELECT 'MergeTree' AS '@@storage_engine'"); @@ -792,18 +886,28 @@ void ClickHouse_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t myprot->generate_pkt_column_count(true,NULL,NULL,sid,1); sid++; myprot->generate_pkt_field(true,NULL,NULL,sid,(char *)"",(char *)"",(char *)"",(char *)"CONNECTION_ID()",(char *)"",63,31,MYSQL_TYPE_LONGLONG,161,0,false,0,NULL); sid++; myds->DSS=STATE_COLUMN_DEFINITION; - myprot->generate_pkt_EOF(true,NULL,NULL,sid,0, setStatus); sid++; + bool deprecate_eof_active = sess->client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; + if (!deprecate_eof_active) { + myprot->generate_pkt_EOF(true,NULL,NULL,sid,0, setStatus); sid++; + } char **p=(char **)malloc(sizeof(char*)*1); unsigned long *l=(unsigned long *)malloc(sizeof(unsigned long *)*1); l[0]=strlen(buf);; p[0]=buf; myprot->generate_pkt_row(true,NULL,NULL,sid,1,l,p); sid++; myds->DSS=STATE_ROW; - myprot->generate_pkt_EOF(true,NULL,NULL,sid,0, setStatus); sid++; + if (!deprecate_eof_active) { + myprot->generate_pkt_EOF(true, NULL, NULL, sid, 0, setStatus); + sid++; + } else { + myprot->generate_pkt_OK(true, NULL, NULL, sid, 0, 0, setStatus, 0, NULL, true); + sid++; + } myds->DSS=STATE_SLEEP; run_query=false; goto __run_query; } +/* if ( (pkt->size==(strlen("SELECT current_user()")+5) && strncasecmp((char *)"SELECT current_user()",(char *)pkt->ptr+5,pkt->size-5)==0) ) { @@ -833,6 +937,7 @@ void ClickHouse_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t run_query=false; goto __run_query; } +*/ } if (query_no_space_length==SELECT_VERSION_COMMENT_LEN) { @@ -847,7 +952,7 @@ void ClickHouse_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t if (query_no_space_length==SELECT_DB_USER_LEN) { if (!strncasecmp(SELECT_DB_USER, query_no_space, query_no_space_length)) { l_free(query_length,query); - char *query1=(char *)"SELECT \"admin\" AS 'DATABASE()', \"%s\" AS 'USER()'"; + char *query1=(char *)"SELECT 'admin' AS \"DATABASE()\", '%s' AS \"USER()\""; char *query2=(char *)malloc(strlen(query1)+strlen(sess->client_myds->myconn->userinfo->username)+10); sprintf(query2,query1,sess->client_myds->myconn->userinfo->username); query=l_strdup(query2); @@ -860,25 +965,16 @@ void ClickHouse_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t if (query_no_space_length==SELECT_CHARSET_VARIOUS_LEN) { if (!strncasecmp(SELECT_CHARSET_VARIOUS, query_no_space, query_no_space_length)) { l_free(query_length,query); - char *query1=(char *)"select 'utf8' as '@@character_set_client', 'utf8' as '@@character_set_connection', 'utf8' as '@@character_set_server', 'utf8' as '@@character_set_database' limit 1"; + char *query1=(char *)"select 'utf8' as \"@@character_set_client\", 'utf8' as \"@@character_set_connection\", 'utf8' as \"@@character_set_server\", 'utf8' as \"@@character_set_database\" limit 1"; query=l_strdup(query1); query_length=strlen(query1)+1; goto __run_query; } } - if (!strncasecmp("SELECT @@version", query_no_space, strlen("SELECT @@version"))) { - l_free(query_length,query); - char *q=(char *)"SELECT '%s' AS '@@version'"; - query_length=strlen(q)+20; - query=(char *)l_alloc(query_length); - sprintf(query,q,PROXYSQL_VERSION); - goto __run_query; - } - if (!strncasecmp("SELECT version()", query_no_space, strlen("SELECT version()"))) { l_free(query_length,query); - char *q=(char *)"SELECT '%s' AS 'version()'"; + char *q=(char *)"SELECT '%s' AS \"version()\""; query_length=strlen(q)+20; query=(char *)l_alloc(query_length); sprintf(query,q,PROXYSQL_VERSION); @@ -944,11 +1040,12 @@ __end_show_commands: // see issue #1022 if (query_no_space_length==strlen("SELECT DATABASE() AS name") && !strncasecmp("SELECT DATABASE() AS name",query_no_space, query_no_space_length)) { l_free(query_length,query); - query=l_strdup("SELECT \"main\" AS 'DATABASE()'"); + query=l_strdup("SELECT 'main' AS \"DATABASE()\""); query_length=strlen(query)+1; goto __run_query; } +/* if (sess->session_type == PROXYSQL_SESSION_SQLITE) { // no admin if ( (strncasecmp("PRAGMA",query_no_space,6)==0) @@ -961,8 +1058,9 @@ __end_show_commands: goto __run_query; } } - +*/ if (sess->session_type == PROXYSQL_SESSION_CLICKHOUSE) { // no admin +/* if ( (strncasecmp("SHOW SESSION VARIABLES",query_no_space,22)==0) || @@ -978,6 +1076,7 @@ __end_show_commands: run_query_sqlite = true; goto __run_query_sqlite; } +*/ if ( (strncasecmp("SET NAMES",query_no_space,9)==0) || @@ -1099,7 +1198,15 @@ __run_query: MySQL_Data_Stream *myds=myprot->get_myds(); if (clickhouse_sess->transfer_started) { - myprot->generate_pkt_EOF(true,NULL,NULL,clickhouse_sess->sid,0, 2); clickhouse_sess->sid++; + myds->DSS=STATE_ROW; + bool deprecate_eof_active = sess->client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; + if (deprecate_eof_active) { + myprot->generate_pkt_OK(true, NULL, NULL, clickhouse_sess->sid, 0, 0, 2, 0, NULL, true); + clickhouse_sess->sid++; + } else { + myprot->generate_pkt_EOF(true, NULL, NULL, clickhouse_sess->sid, 0, 2); + clickhouse_sess->sid++; + } } else { myprot->generate_pkt_OK(true,NULL,NULL,1,0,0,2,0,(char *)""); } @@ -1148,7 +1255,8 @@ __run_query_sqlite: // we are introducing this new section to send some query to if (run_query_sqlite) { ClickHouse_Session *sqlite_sess = (ClickHouse_Session *)sess->thread->gen_args; sqlite_sess->sessdb->execute_statement(query, &error , &cols , &affected_rows , &resultset); - sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); + bool deprecate_eof_active = sess->client_myds->myconn->options.client_flag & CLIENT_DEPRECATE_EOF; + sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot, false, deprecate_eof_active); delete resultset; l_free(pkt->size-sizeof(mysql_hdr),query_no_space); // it is always freed here l_free(query_length,query); @@ -1173,6 +1281,7 @@ bool ClickHouse_Session::init() { co.SetHost(hostname); co.SetPort(atoi(port)); co.SetCompressionMethod(CompressionMethod::None); + client = NULL; client = new clickhouse::Client(co); ret=true; } catch (const std::exception& e) { @@ -1234,7 +1343,7 @@ static void *child_mysql(void *arg) { fds[0].revents=0; fds[0].events=POLLIN|POLLOUT; free(arg); - sess->client_myds->myprot.generate_pkt_initial_handshake(true,NULL,NULL, &sess->thread_session_id, false); + sess->client_myds->myprot.generate_pkt_initial_handshake(true,NULL,NULL, &sess->thread_session_id, true); while (__sync_fetch_and_add(&glovars.shutdown,0)==0) { if (myds->available_data_out()) { @@ -1441,7 +1550,7 @@ void ClickHouse_Server::print_version() { }; bool ClickHouse_Server::init() { - cpu_timer cpt; +// cpu_timer cpt; child_func[0]=child_mysql; main_shutdown=0; @@ -1467,9 +1576,11 @@ bool ClickHouse_Server::init() { perror("Thread creation"); exit(EXIT_FAILURE); } +/* #ifdef DEBUG std::cerr << "SQLite3 Server initialized in "; #endif +*/ return true; }; diff --git a/lib/Makefile b/lib/Makefile index b583ab668..ba611c745 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -1,3 +1,6 @@ +#!/bin/make -f + + ifndef GIT_VERSION GIT_VERSION := $(shell git describe --long --abbrev=7) ifndef GIT_VERSION @@ -54,7 +57,7 @@ PROMETHEUS_LDIR=$(PROMETHEUS_PATH)/lib IDIR=../include -IDIRS=-I$(IDIR) -I$(JEMALLOC_IDIR) -I$(MARIADB_IDIR) $(LIBCONFIG_IDIR) -I$(RE2_IDIR) -I$(SQLITE3_DIR) -I$(PCRE_PATH) -I/usr/local/include -I$(CLICKHOUSE_CPP_DIR) $(MICROHTTPD_IDIR) $(LIBHTTPSERVER_IDIR) $(LIBINJECTION_IDIR) $(CURL_IDIR) -I$(EV_DIR) -I$(SSL_IDIR) -I$(PROMETHEUS_IDIR) +IDIRS=-I$(IDIR) -I$(JEMALLOC_IDIR) -I$(MARIADB_IDIR) $(LIBCONFIG_IDIR) -I$(RE2_IDIR) -I$(SQLITE3_DIR) -I$(PCRE_PATH) -I/usr/local/include -I$(CLICKHOUSE_CPP_DIR) -I$(CLICKHOUSE_CPP_DIR)/contrib/ $(MICROHTTPD_IDIR) $(LIBHTTPSERVER_IDIR) $(LIBINJECTION_IDIR) $(CURL_IDIR) -I$(EV_DIR) -I$(SSL_IDIR) -I$(PROMETHEUS_IDIR) LDIRS=-L$(JEMALLOC_PATH)/lib -L$(RE2_PATH)/obj -L$(INJECTION_PATH) -L$(PROMETHEUS_LDIR) @@ -97,17 +100,39 @@ ifeq ($(UNAME_S),Darwin) NOJEM=-DNOJEM endif -PROXYSQLCLICKHOUSE := $(shell echo $(PROXYSQLCLICKHOUSE)) -ifeq ($(PROXYSQLCLICKHOUSE),1) -PSQLCH=-DPROXYSQLCLICKHOUSE + +# determine good compiler version for stdc++17 +IS_CXX17 := 0 +ifeq ($(CXX),clang++) + CLANG_VERSION := $(shell clang -dumpversion | tr '.' ' ' | awk '{ printf("%04d.%04d.%04d", $$1, $$2, $$3) }') + CLANG_MIN_VER := $(shell echo 14.0 | tr '.' ' ' | awk '{ printf("%04d.%04d.%04d", $$1, $$2, $$3) }') +ifeq ($(CLANG_MIN_VER),$(firstword $(sort $(CLANG_VERSION) $(CLANG_MIN_VER)))) + IS_CXX17 := 1 +endif else + GCC_VERSION := $(shell gcc -dumpversion | tr '.' ' ' | awk '{ printf("%04d.%04d.%04d", $$1, $$2, $$3) }') + GCC_MIN_VER := $(shell echo 8.2.0 | tr '.' ' ' | awk '{ printf("%04d.%04d.%04d", $$1, $$2, $$3) }') +ifeq ($(GCC_MIN_VER),$(firstword $(sort $(GCC_VERSION) $(GCC_MIN_VER)))) + IS_CXX17 := 1 +endif +endif + + +PROXYSQLCLICKHOUSE := $(shell echo $(PROXYSQLCLICKHOUSE)) PSQLCH= +ifeq ($(PROXYSQLCLICKHOUSE),1) + PSQLCH=-DPROXYSQLCLICKHOUSE +ifeq ($(IS_CXX17),1) + PSQLCH=-DPROXYSQLCLICKHOUSE -std=c++17 -DCXX17 endif +endif + ifeq ($(UNAME_S),Darwin) IDIRS+= -I/usr/local/opt/openssl/include endif + # 'libhttpserver': Add 'ENABLE_EPOLL' by default for all platforms except # for 'Darwin'. This is required when compiling 'libhttpserver' for avoiding # internal use of 'SELECT' in favor of 'EPOLL'. See #3591. diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 3cac9df5e..911a38f3d 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3561,7 +3561,7 @@ void MySQL_HostGroups_Manager::group_replication_lag_action( bool writer_is_also_reader = false; // Get the reader_hostgroup for the supplied writter hostgroup - std::string t_reader_hostgroup_query { + const std::string t_reader_hostgroup_query { "SELECT reader_hostgroup,writer_is_also_reader FROM mysql_group_replication_hostgroups WHERE writer_hostgroup=%d" }; std::string reader_hostgroup_query {}; diff --git a/lib/MySQL_Protocol.cpp b/lib/MySQL_Protocol.cpp index 26e66cb0d..c33bafac0 100644 --- a/lib/MySQL_Protocol.cpp +++ b/lib/MySQL_Protocol.cpp @@ -1829,7 +1829,7 @@ bool MySQL_Protocol::process_pkt_handshake_response(unsigned char *pkt, unsigned if (GloMyLdapAuth) { // we check if user exists only if GloMyLdapAuth is enabled #ifdef PROXYSQLCLICKHOUSE if (session_type == PROXYSQL_SESSION_CLICKHOUSE) { - user_exists = GloClickHouseAuth->exists((char *)user); + //user_exists = GloClickHouseAuth->exists((char *)user); // for clickhouse, we currently do not support clear text or LDAP user_exists = true; } else { @@ -1859,7 +1859,7 @@ bool MySQL_Protocol::process_pkt_handshake_response(unsigned char *pkt, unsigned bool user_exists = true; #ifdef PROXYSQLCLICKHOUSE if (session_type == PROXYSQL_SESSION_CLICKHOUSE) { - user_exists = GloClickHouseAuth->exists((char *)user); + //user_exists = GloClickHouseAuth->exists((char *)user); // for clickhouse, we currently do not support clear text or LDAP user_exists = true; } else { diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 49e3402d7..42be36324 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -5949,6 +5949,7 @@ bool ProxySQL_Admin::init() { flush_debug_levels_runtime_to_database(admindb, true); #endif /* DEBUG */ + // Set default values for the module variables in the target 'dbs' flush_mysql_variables___runtime_to_database(configdb, false, false, false); flush_mysql_variables___runtime_to_database(admindb, false, true, false); @@ -5957,6 +5958,7 @@ bool ProxySQL_Admin::init() { load_or_update_global_settings(configdb); + // Insert or update the configuration from 'disk' __insert_or_replace_maintable_select_disktable(); // removing this line of code. It seems redundant @@ -6323,6 +6325,10 @@ void ProxySQL_Admin::load_or_update_global_settings(SQLite3DB *db) { uuid=NULL; } } + + if (resultset) { + delete resultset; + } } } diff --git a/src/Makefile b/src/Makefile index 4a9cb8a22..563357f5c 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,3 +1,6 @@ +#!/bin/make -f + + ifndef GIT_VERSION GIT_VERSION := $(shell git describe --long --abbrev=7) ifndef GIT_VERSION @@ -76,7 +79,7 @@ EV_LDIR=$(EV_DIR)/.libs IDIR=../include LDIR=../lib -IDIRS=-I$(IDIR) -I$(JEMALLOC_IDIR) -I$(MARIADB_IDIR) $(LIBCONFIG_IDIR) -I$(DAEMONPATH_IDIR) -I$(SQLITE3_DIR) -I$(CLICKHOUSE_CPP_DIR) -I$(MICROHTTPD_IDIR) -I$(LIBHTTPSERVER_IDIR) -I$(LIBINJECTION_IDIR) -I$(CURL_IDIR) -I$(EV_IDIR) -I$(SSL_IDIR) -I$(RE2_IDIR) -I$(PROMETHEUS_IDIR) +IDIRS=-I$(IDIR) -I$(JEMALLOC_IDIR) -I$(MARIADB_IDIR) $(LIBCONFIG_IDIR) -I$(DAEMONPATH_IDIR) -I$(SQLITE3_DIR) -I$(CLICKHOUSE_CPP_DIR) -I$(CLICKHOUSE_CPP_DIR)/contrib/ -I$(MICROHTTPD_IDIR) -I$(LIBHTTPSERVER_IDIR) -I$(LIBINJECTION_IDIR) -I$(CURL_IDIR) -I$(EV_IDIR) -I$(SSL_IDIR) -I$(RE2_IDIR) -I$(PROMETHEUS_IDIR) LDIRS=-L$(LDIR) -L$(JEMALLOC_LDIR) $(LIBCONFIG_LDIR) -L$(RE2_PATH)/obj -L$(MARIADB_LDIR) -L$(DAEMONPATH_LDIR) -L$(PCRE_LDIR) -L$(MICROHTTPD_LDIR) -L$(LIBHTTPSERVER_LDIR) -L$(LIBINJECTION_LDIR) -L$(CURL_LDIR) -L$(EV_LDIR) -L$(SSL_LDIR) -L$(PROMETHEUS_LDIR) UNAME_S := $(shell uname -s) @@ -86,13 +89,34 @@ ifeq ($(UNAME_S),Darwin) LDIRS+= -L/usr/local/lib endif -PROXYSQLCLICKHOUSE := $(shell echo $(PROXYSQLCLICKHOUSE)) -ifeq ($(PROXYSQLCLICKHOUSE),1) -PSQLCH=-DPROXYSQLCLICKHOUSE + +# determine good compiler version for stdc++17 +IS_CXX17 := 0 +ifeq ($(CXX),clang++) + CLANG_VERSION := $(shell clang -dumpversion | tr '.' ' ' | awk '{ printf("%04d.%04d.%04d", $$1, $$2, $$3) }') + CLANG_MIN_VER := $(shell echo 14.0 | tr '.' ' ' | awk '{ printf("%04d.%04d.%04d", $$1, $$2, $$3) }') +ifeq ($(CLANG_MIN_VER),$(firstword $(sort $(CLANG_VERSION) $(CLANG_MIN_VER)))) + IS_CXX17 := 1 +endif else + GCC_VERSION := $(shell gcc -dumpversion | tr '.' ' ' | awk '{ printf("%04d.%04d.%04d", $$1, $$2, $$3) }') + GCC_MIN_VER := $(shell echo 8.2.0 | tr '.' ' ' | awk '{ printf("%04d.%04d.%04d", $$1, $$2, $$3) }') +ifeq ($(GCC_MIN_VER),$(firstword $(sort $(GCC_VERSION) $(GCC_MIN_VER)))) + IS_CXX17 := 1 +endif +endif + + +PROXYSQLCLICKHOUSE := $(shell echo $(PROXYSQLCLICKHOUSE)) PSQLCH= +ifeq ($(PROXYSQLCLICKHOUSE),1) + PSQLCH=-DPROXYSQLCLICKHOUSE +ifeq ($(IS_CXX17),1) + PSQLCH=-DPROXYSQLCLICKHOUSE -std=c++17 -DCXX17 +endif endif + WITHGCOVVAR := $(shell echo $(WITHGCOV)) ifeq ($(WITHGCOVVAR),1) WGCOV=-DWITHGCOV --coverage @@ -170,7 +194,7 @@ $(ODIR)/%.o: %.cpp $(EXECUTABLE): $(ODIR) $(OBJ) $(LIBPROXYSQLAR) ifeq ($(PROXYSQLCLICKHOUSE),1) - $(CXX) -o $@ $(OBJ) $(LIBPROXYSQLAR) $(CLICKHOUSE_CPP_DIR)/clickhouse/libclickhouse-cpp-lib.a $(CITYHASH_DIR)/libcityhash.a $(LZ4_DIR)/liblz4.a $(MYCXXFLAGS) $(CXXFLAGS) $(LDIRS) $(LIBS) $(LDFLAGS) $(MYLIBS) + $(CXX) -o $@ $(OBJ) $(LIBPROXYSQLAR) $(CLICKHOUSE_CPP_DIR)/clickhouse/libclickhouse-cpp-lib-static.a $(CITYHASH_DIR)/libcityhash.a $(LZ4_DIR)/liblz4.a $(MYCXXFLAGS) $(CXXFLAGS) $(LDIRS) $(LIBS) $(LDFLAGS) $(MYLIBS) else $(CXX) -o $@ $(OBJ) $(LIBPROXYSQLAR) $(MYCXXFLAGS) $(CXXFLAGS) $(LDIRS) $(LIBS) $(LDFLAGS) $(MYLIBS) endif diff --git a/src/main.cpp b/src/main.cpp index 2b5c9e212..14e476081 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -748,6 +748,9 @@ void ProxySQL_Main_init_SQLite3Server() { // start SQLite3Server GloSQLite3Server = new SQLite3_Server(); GloSQLite3Server->init(); + // NOTE: Always perform the 'load_*_to_runtime' after module start, otherwise values won't be properly + // loaded from disk at ProxySQL startup. + GloAdmin->load_sqliteserver_variables_to_runtime(); GloAdmin->init_sqliteserver_variables(); GloSQLite3Server->print_version(); } @@ -756,6 +759,9 @@ void ProxySQL_Main_init_ClickHouseServer() { // start SQServer GloClickHouseServer = new ClickHouse_Server(); GloClickHouseServer->init(); + // NOTE: Always perform the 'load_*_to_runtime' after module start, otherwise values won't be properly + // loaded from disk at ProxySQL startup. + GloAdmin->load_clickhouse_variables_to_runtime(); GloAdmin->init_clickhouse_variables(); GloClickHouseServer->print_version(); GloClickHouseAuth = new ClickHouse_Authentication(); diff --git a/test/tap/tests/Makefile b/test/tap/tests/Makefile index 1e86c60f3..22b2f2765 100644 --- a/test/tap/tests/Makefile +++ b/test/tap/tests/Makefile @@ -101,7 +101,7 @@ OPT=-O2 $(WGCOV) -Wl,--no-as-needed debug: OPT=-O0 -DDEBUG -ggdb -Wl,--no-as-needed $(WGCOV) $(WASAN) debug: tests -tests: $(patsubst %.cpp,%,$(wildcard *-t.cpp)) setparser_test reg_test_3504-change_user_libmariadb_helper reg_test_3504-change_user_libmysql_helper set_testing-240.csv +tests: $(patsubst %.cpp,%,$(wildcard *-t.cpp)) setparser_test reg_test_3504-change_user_libmariadb_helper reg_test_3504-change_user_libmysql_helper set_testing-240.csv test_clickhouse_server_libmysql testgalera: galera_1_timeout_count galera_2_timeout_no_count testaurora: aurora @@ -149,3 +149,6 @@ reg_test_3504-change_user_libmariadb_helper: reg_test_3504-change_user_helper.cp reg_test_3504-change_user_libmysql_helper: reg_test_3504-change_user_helper.cpp $(CXX) -DLIBMYSQL_HELPER -DDEBUG reg_test_3504-change_user_helper.cpp -I/usr/include/mysql -I$(IDIR) -I$(JSON_IDIR) -I../tap -L$(TAP_LIBDIR) -lpthread -ldl -std=c++11 -ltap -lmysqlclient -o reg_test_3504-change_user_libmysql_helper -DGITVERSION=\"$(GIT_VERSION)\" + +test_clickhouse_server_libmysql: test_clickhouse_server-t.cpp + $(CXX) -DLIBMYSQL_HELPER -DDEBUG test_clickhouse_server-t.cpp -I/usr/include/mysql -I$(IDIR) -I$(JSON_IDIR) -I../tap -L$(TAP_LIBDIR) -lpthread -ldl -std=c++11 -ltap -lmysqlclient -o test_clickhouse_server_libmysql-t -DGITVERSION=\"$(GIT_VERSION)\" diff --git a/test/tap/tests/clickhouse_php_conn-t b/test/tap/tests/clickhouse_php_conn-t new file mode 100755 index 000000000..259e919b1 --- /dev/null +++ b/test/tap/tests/clickhouse_php_conn-t @@ -0,0 +1,274 @@ +#!/usr/bin/env php + +connect_errno) { + die("PorxySQL connect failed: " . $proxy->connect_error); +} +echo ":: ProxySQL ProxySQL Admin connection completed".PHP_EOL; + +echo ":: Creating required users for test...".PHP_EOL; +$proxy_admin->query("INSERT OR REPLACE INTO clickhouse_users (username,password,active,max_connections) VALUES ('{$username}','{$password}',1,100)"); +$proxy_admin->query("LOAD CLICKHOUSE USERS TO RUNTIME"); +echo ":: Finished creating users".PHP_EOL; + +echo ":: Creating ProxySQL connection...".PHP_EOL; +$proxy = new mysqli("127.0.0.1", $username, $password, "", $port); +if ($proxy->connect_errno) { + die("PorxySQL connect failed: " . $proxy->connect_error); +} +echo ":: ProxySQL connection completed".PHP_EOL; + + +echo ":: Starting schema and table creation...".PHP_EOL; +if ($port !== 6090) { + $proxy->query("CREATE DATABASE IF NOT EXISTS test_clickhouse_types_php"); + $proxy->query("USE test_clickhouse_types_php"); + $proxy->query("DROP TABLE IF EXISTS types_table"); + + $proxy->query("CREATE TABLE IF NOT EXISTS types_table (EventDate DATE, DateTime DATETIME, col1 TINYINT, col2 SMALLINT, col3 INT, col4 BIGINT, col5 FLOAT, col6 DOUBLE)"); + + echo ":: Inserting data directly to MySQL".PHP_EOL; + $proxy->query("INSERT INTO types_table SELECT NOW(),NOW(),127,-32768,2147483647,9223372036854775807,340282346638528859811704183484516925440.0,340282346638528859811704183484516925440.0"); + + echo ":: Fetching inserted data".PHP_EOL; + $result = $proxy->query("SELECT EventDate,DateTime,col1,col2,col3,col4,col5,round(col6,0) FROM types_table"); + while ($row = mysqli_fetch_row($result)) { + echo " * ROW: ["; + + foreach ($row as $val) { + echo $val.","; + } + + echo "]".PHP_EOL; + } + + echo ":: Finished operations on MySQL conn".PHP_EOL; + exit(0); +} else { + $proxy->query("CREATE DATABASE IF NOT EXISTS test_clickhouse_types_php"); + $proxy->query("USE test_clickhouse_types_php"); + $proxy->query("DROP TABLE IF EXISTS types_table"); + $proxy->query("CREATE TABLE IF NOT EXISTS types_table (EventDate DATE, DateTime DATETIME, col1 UInt8, col2 Int16, col3 Int32, col4 Int64, col5 Nullable(Float32), col6 Float64) ENGINE=MergeTree(EventDate, (EventDate), 8192)"); +} + +$shortName = exec('date +%Z'); +$longName = timezone_name_from_abbr($shortName); +$timezone = timezone_open($longName); +$datetime_db = date_create("now", timezone_open("UTC")); +$timezone_off = timezone_offset_get($timezone, $datetime_db); + +$cur_date = date("Y-m-d"); +$cur_datetime = date("Y-m-d H:i:s"); + +echo ":: Schema and table creation completed".PHP_EOL; + +$exp_rows = [ + [ + "insert" => "INSERT INTO types_table SELECT '{$cur_date}','{$cur_datetime}',127,-32768,2147483647,9223372036854775807,340282346638528859811704183484516925440,340282346638528859811704183484516925440.0", + "select" => "SELECT EventDate,DateTime,col1,col2,col3,col4,col5,round(col6,0) FROM types_table", + "types" => [10, 12, 1, 2, 3, 8, 4, 5], + "vals" => [$cur_date, $cur_datetime, 127, -32768, 2147483647, 9223372036854775807, 340282346638528859811704183484516925440, 340282346638528859811704183484516925440.0] + ], + [ + "insert" => "INSERT INTO types_table SELECT '{$cur_date}','{$cur_datetime}',127,-32768,2147483647,9223372036854775807,1.2,340282346638528859811704183484516925440.0", + "select" => "SELECT EventDate,DateTime,col1,col2,col3,col4,ROUND(col5,20),round(col6,0) FROM types_table", + "types" => [10, 12, 1, 2, 3, 8, 4, 5], + "vals" => [$cur_date, $cur_datetime, 127, -32768, 2147483647, 9223372036854775807, 1.2, 340282346638528859811704183484516925440.0] + ], + [ + "insert" => "INSERT INTO types_table SELECT '{$cur_date}','{$cur_datetime}',127,-32768,2147483647,9223372036854775807,NULL,340282346638528859811704183484516925440.0", + "select" => "SELECT EventDate,DateTime,col1,col2,col3,col4,col5,round(col6,0) FROM types_table", + "types" => [10, 12, 1, 2, 3, 8, 4, 5], + "vals" => [$cur_date, $cur_datetime, 127, -32768, 2147483647, 9223372036854775807, NULL, 340282346638528859811704183484516925440.0] + ] +]; + +echo ":: Checking expected data definition...".PHP_EOL; + +foreach ($exp_rows as $row) { + $c_row_types = count($row["types"]); + $c_row_vals = count($row["vals"]); + + if ($c_row_types !== $c_row_vals) { + echo " * Invalid exp row definition for query '{$row["select"]}'. Expected type count '{$c_row_types}' != '{$c_row_vals}'".PHP_EOL; + exit(1); + } +} +echo ":: Checking expected data completed".PHP_EOL; + +$exit_code = 0; + +$count = 0; +foreach ($exp_rows as $exp_row) { + echo ":: Performing operation for payload num '$count'".PHP_EOL; + echo " * Issuing INSERT query '{$exp_row["insert"]}'".PHP_EOL; + $proxy->query($exp_row["insert"]); + + echo " * Issuing SELECT query '{$exp_row["select"]}'".PHP_EOL; + $result = $proxy->query($exp_row["select"]); + + /* Get field information for all columns */ + $finfo = $result->fetch_fields(); + + echo " * START: Received columns info".PHP_EOL; + + $act_col_defs = array(); + + foreach ($finfo as $val) { + printf(" - Name: %s\n", $val->name); + printf(" - Table: %s\n", $val->table); + printf(" - Max. Len: %d\n", $val->max_length); + printf(" - Length: %d\n", $val->length); + printf(" - charsetnr: %d\n", $val->charsetnr); + printf(" - Flags: %d\n", $val->flags); + printf(" - Type: %d\n\n", $val->type); + + array_push($act_col_defs, $val); + } + + echo " * END: Received columns info".PHP_EOL; + + echo ":: Checking fetched data...".PHP_EOL; + + $fetch_rows = array(); + while ($row = mysqli_fetch_row($result)) { + array_push($fetch_rows, $row); + } + + $c_exp_rows = 1; + $c_fetch_rows = count($fetch_rows); + + if ($c_exp_rows !== $c_fetch_rows) { + echo "Expected received row number doesn't match actual received rows - exp: {$c_exp_rows}, act: {$c_fetch_rows}".PHP_EOL; + exit(1); + } + + $types_match = true; + $type_count = 0; + + foreach ($act_col_defs as $act_col_def) { + $act_type = $act_col_def->type; + $exp_type = $exp_row["types"][$type_count]; + + $type_match = $act_type === $exp_type; + + if ($type_match) { + $ok_msg = "ok"; + } else { + $ok_msg = "not ok"; + } + + echo " {$ok_msg} - Type match result for column '{$act_col_def->name}' was '{$type_match}'."; + echo " ExpType: '{$exp_type}'. ActType '{$act_type}'".PHP_EOL; + + $type_count += 1; + $types_match &= $type_match; + } + echo PHP_EOL; + + $vals_match = true; + $val_count = 0; + foreach ($fetch_rows as $row) { + foreach ($row as $val) { + $col_name = $act_col_defs[$val_count]->name; + + $exp_val = $exp_row["vals"][$val_count]; + $exp_type = $exp_row["types"][$val_count]; + + if (is_null($exp_val) == false) { + if ($exp_type == 1 || $exp_type == 2 || $exp_type == 3 || $exp_type == 8) { + $cast_val = (int)$val; + } else if ($exp_type == 4 | $exp_type == 5) { + $cast_val = (float)$val; + } else if ($exp_type == 12) { + $timestamp = strtotime($val) - $timezone_off; + $cast_val = date("Y-m-d H:i:s", $timestamp); + } else { + $cast_val = $val; + } + } else { + $cast_val = $val; + } + + $val_match = $exp_val === $cast_val; + + if ($val_match) { + $ok_msg = "ok"; + } else { + $ok_msg = "not ok"; + } + + echo " {$ok_msg} - Value result match for column '{$col_name}' was '{$val_match}'."; + echo " ExpVal: '{$exp_val}'. ActVal '{$cast_val}'".PHP_EOL; + + $vals_match &= $val_match; + $val_count += 1; + } + $val_count = 0; + break; + } + + $count += 1; + + echo ":: Cleaning up table before next queries...".PHP_EOL; + $proxy->query("ALTER TABLE types_table DELETE WHERE 1=1"); + + $result = $proxy->query("SELECT COUNT(*) FROM types_table"); + $row = mysqli_fetch_row($result); + echo ":: Waiting for table cleaning"; + + while ($row[0] != 0) { + echo "."; + sleep(1); + + $result = $proxy->query("SELECT COUNT(*) FROM types_table"); + $row = mysqli_fetch_row($result); + } + + echo PHP_EOL; + echo ":: Table cleaning completed".PHP_EOL; + + $exit_code |= !($types_match & $vals_match); +} + +$proxy->query("DROP DATABASE IF EXISTS test_clickhouse_types_php"); +$result->free(); + +exit($exit_code); + +?> diff --git a/test/tap/tests/modules_server_test.h b/test/tap/tests/modules_server_test.h new file mode 100644 index 000000000..dfd4e11ef --- /dev/null +++ b/test/tap/tests/modules_server_test.h @@ -0,0 +1,94 @@ +/** + * @brief Extract the current 'MODULE-mysql_ifaces' from ProxySQL config. + * @param proxysql_admin An already opened connection to ProxySQL Admin. + * @return EXIT_SUCCESS, or one of the following error codes: + * - EINVAL if supplied 'proxysql_admin' is NULL. + * - EXIT_FAILURE in case other operation failed. + */ +int get_module_ifaces(MYSQL* proxysql_admin, const std::string varname, std::string& module_ifaces) { + if (proxysql_admin == NULL) { + return EINVAL; + } + + int res = EXIT_FAILURE; + + std::string query = "SELECT * FROM global_variables WHERE Variable_name='" + varname + "'"; + diag("Running query: %s", query.c_str()); + MYSQL_QUERY(proxysql_admin, query.c_str()); + + MYSQL_RES* admin_res = mysql_store_result(proxysql_admin); + if (!admin_res) { + diag("'mysql_store_result' at line %d failed: %s", __LINE__, mysql_error(proxysql_admin)); + goto cleanup; + } + + { + MYSQL_ROW row = mysql_fetch_row(admin_res); + if (!row || row[0] == nullptr || row[1] == nullptr) { + diag("'mysql_fetch_row' at line %d returned 'NULL'", __LINE__); + res = -1; + goto cleanup; + } + + std::string _module_ifaces { row[1] }; + module_ifaces = _module_ifaces; + res = EXIT_SUCCESS; + } + +cleanup: + + return res; +} + +int extract_module_host_port(MYSQL* proxysql_admin, const std::string varname, std::pair& host_port) { + if (proxysql_admin == nullptr) { return EINVAL; } + int res = EXIT_SUCCESS; + + std::string module_ifaces {}; + int ifaces_err = get_module_ifaces(proxysql_admin, varname, module_ifaces); + + // ProxySQL is likely to have been launched without "--MODULE-server" flag + if (ifaces_err == -1) { + if (varname=="sqliteserver-mysql_ifaces") { + diag("ProxySQL was launched without '--sqlite3-server' flag"); + } else if (varname=="clickhouse-mysql_ifaces") { + diag("ProxySQL was launched without '--clickhouse-server' flag"); + } else { + diag("Unknown variable %s", varname.c_str()); + } + res = EXIT_FAILURE; + return res; + } + + // Extract the correct port to connect to MODULE server + std::string::size_type colon_pos = module_ifaces.find(":"); + if (colon_pos == std::string::npos) { + diag("ProxySQL returned a malformed '%s': %s", varname.c_str(), module_ifaces.c_str()); + res = EXIT_FAILURE; + return res; + } + + std::string module_host { module_ifaces.substr(0, colon_pos) }; + std::string module_port { module_ifaces.substr(colon_pos + 1) }; + + // Check that port has valid conversion + char* end_pos = nullptr; + int i_module_port = std::strtol(module_port.c_str(), &end_pos, 10); + + if (errno == ERANGE || (end_pos != &module_port.back() + 1)) { + diag( + "ProxySQL returned a invalid port number within '%s': %s", + varname.c_str(), module_ifaces.c_str() + ); + res = EXIT_FAILURE; + return res; + } + + if (res == EXIT_SUCCESS) { + host_port = { module_host, i_module_port }; + } + + return res; +} + + diff --git a/test/tap/tests/sqlite_3_server_test.h b/test/tap/tests/sqlite_3_server_test.h deleted file mode 100644 index a31510466..000000000 --- a/test/tap/tests/sqlite_3_server_test.h +++ /dev/null @@ -1,91 +0,0 @@ -/** - * @brief Extract the current 'sqliteserver-mysql_ifaces' from ProxySQL config. - * @param proxysql_admin An already opened connection to ProxySQL Admin. - * @return EXIT_SUCCESS, or one of the following error codes: - * - EINVAL if supplied 'proxysql_admin' is NULL. - * - '-1' in case of ProxySQL returns an 'NULL' row for the query selecting - * the variable 'sqliteserver-read_only'. - * - EXIT_FAILURE in case other operation failed. - */ -int get_sqlite3_ifaces(MYSQL* proxysql_admin, std::string& sqlite3_ifaces) { - if (proxysql_admin == NULL) { - return EINVAL; - } - - int res = EXIT_FAILURE; - - MYSQL_QUERY( - proxysql_admin, - "SELECT * FROM global_variables WHERE Variable_name='sqliteserver-mysql_ifaces'" - ); - - MYSQL_RES* admin_res = mysql_store_result(proxysql_admin); - if (!admin_res) { - diag("'mysql_store_result' at line %d failed: %s", __LINE__, mysql_error(proxysql_admin)); - goto cleanup; - } - - { - MYSQL_ROW row = mysql_fetch_row(admin_res); - if (!row || row[0] == nullptr || row[1] == nullptr) { - diag("'mysql_fetch_row' at line %d returned 'NULL'", __LINE__); - res = -1; - goto cleanup; - } - - std::string _sqlite3_ifaces { row[1] }; - sqlite3_ifaces = _sqlite3_ifaces; - res = EXIT_SUCCESS; - } - -cleanup: - - return res; -} - -int extract_sqlite3_host_port(MYSQL* proxysql_admin, std::pair& host_port) { - if (proxysql_admin == nullptr) { return EINVAL; } - int res = EXIT_SUCCESS; - - std::string sqlite3_ifaces {}; - int ifaces_err = get_sqlite3_ifaces(proxysql_admin, sqlite3_ifaces); - - // ProxySQL is likely to have been launched without "--sqlite3-server" flag - if (ifaces_err == -1) { - diag("ProxySQL was launched without '--sqlite3-server' flag"); - res = EXIT_FAILURE; - return res; - } - - // Extract the correct port to connect to SQLite server - std::string::size_type colon_pos = sqlite3_ifaces.find(":"); - if (colon_pos == std::string::npos) { - diag("ProxySQL returned a malformed 'sqliteserver-mysql_ifaces': %s", sqlite3_ifaces.c_str()); - res = EXIT_FAILURE; - return res; - } - - std::string sqlite3_host { sqlite3_ifaces.substr(0, colon_pos) }; - std::string sqlite3_port { sqlite3_ifaces.substr(colon_pos + 1) }; - - // Check that port has valid conversion - char* end_pos = nullptr; - int i_sqlite3_port = std::strtol(sqlite3_port.c_str(), &end_pos, 10); - - if (errno == ERANGE || (end_pos != &sqlite3_port.back() + 1)) { - diag( - "ProxySQL returned a invalid port number within 'sqliteserver-mysql_ifaces': %s", - sqlite3_ifaces.c_str() - ); - res = EXIT_FAILURE; - return res; - } - - if (res == EXIT_SUCCESS) { - host_port = { sqlite3_host, i_sqlite3_port }; - } - - return res; -} - - diff --git a/test/tap/tests/test_clickhouse_server-t.cpp b/test/tap/tests/test_clickhouse_server-t.cpp new file mode 100644 index 000000000..9280c7ce3 --- /dev/null +++ b/test/tap/tests/test_clickhouse_server-t.cpp @@ -0,0 +1,546 @@ +/** + * @file test_clickhouse_server-t.cpp + * @brief Test to perform multiple operations over ProxySQL Clickhouse server. + * @details It performs the following operations: + * - Connects to clickhouse with a wrong username. + * - Connects to clickhouse with a right username but wrong password. + * - Successfully connects to clickhouse and runs several queries. + * + SHOW SCHEMAS + * + SHOW DATABASES + * + SELECT DATABASE() + * - Successfully connects to clickhouse and runs a variety of queries: + * + CREATE TABLE, SHOW CREATE TABLE, INSERT, SELECT, DROP TABLE... + * + Queries that induce errors: syntax error, duplicate keys, etc... + * - Changes 'clickhouse-mysql_ifaces' and tries to connect to the new interface. + * - Connects to ProxySQL Admin and performs the following operations: + * + LOAD|SAVE SQLITESERVER TO|FROM RUNTIME|MEMORY|DISK + * - This test is also compiled against 'libmysqlclient' resulting in the binary + * 'test_clickhouse_server_libmysql-t'. This duplicate test exists for testing 'deprecate_eof' support + * against ProxySQL ClickHouse server. + * + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "tap.h" +#include "command_line.h" +#include "utils.h" + +using query_spec = std::tuple; + +const int proxysql_clickhouse_port = 6090; +const int crash_loops = 4; + +#include "modules_server_test.h" + +int fetch_and_discard_results(MYSQL_RES* result, bool verbose=false); + +std::vector> credentials = { + {"cliuser1", "clipass1"}, + {"cliuser2", "clipass2"}, + {"cliuser3", "clipass3"}, + {"cliuser4", "clipass4"} +}; + +int set_clickhouse_port(MYSQL *pa, int p) { + std::string query = "SET clickhouse-port=" + std::to_string(p); + diag("Line: %d . Setting clickhouse-port to %d", __LINE__ , p); + MYSQL_QUERY(pa, query.c_str()); + MYSQL_QUERY(pa, "LOAD CLICKHOUSE VARIABLES TO RUNTIME"); + return 0; +} + +int test_crash(const char *host, int port) { + // try to connect and run queries while there is no backend + for (int i=0; i>::iterator it = credentials.begin(); it!=credentials.end(); it++) { + std::string query = "INSERT INTO clickhouse_users VALUES ('" + it->first + "', '" + it->second + "', 1, 100)"; + diag("Adding user %s : %s", it->first.c_str(), query.c_str()); + MYSQL_QUERY(pa, query.c_str()); + } + q = (char *)"SELECT * FROM clickhouse_users"; + query_err = mysql_query(pa, q); + result = mysql_store_result(pa); + if (result) { + int j = fetch_and_discard_results(result, true); + mysql_free_result(result); + ok(j==4, "Line %d : Rows in clickhouse_users should be 4. Actual: %d" , __LINE__, j); + } else { + ok(false,"Line %d : Query failed: %s . Error: %s", __LINE__, q, mysql_error(pa)); + return exit_status(); + } + diag("Loading clickhouse_users to runtime"); + MYSQL_QUERY(pa, "LOAD CLICKHOUSE USERS TO RUNTIME"); + q = (char *)"SELECT * FROM runtime_clickhouse_users"; + query_err = mysql_query(pa, q); + result = mysql_store_result(pa); + if (result) { + int j = fetch_and_discard_results(result, true); + mysql_free_result(result); + ok(j==4, "Line %d : Rows in clickhouse_users should be 4. Actual: %d" , __LINE__, j); + } else { + ok(false,"Line %d : Query failed: %s . Error: %s", __LINE__, q, mysql_error(pa)); + return exit_status(); + } + return 0; +} + +int fetch_and_discard_results(MYSQL_RES* result, bool verbose) { + MYSQL_ROW row = nullptr; + unsigned int num_fields = 0; + unsigned int i = 0; + unsigned int j = 0; + + num_fields = mysql_num_fields(result); + while ((row = mysql_fetch_row(result))) { + unsigned long *lengths = mysql_fetch_lengths(result); + + if (verbose) { + printf("# RowNum_%d: ", j); + } + + for(i = 0; i < num_fields; i++) { + if (verbose) { + printf("[%.*s] ", (int) lengths[i], row[i] ? row[i] : "NULL"); + } + } + + if (verbose) { + printf("\n"); + } + + j++; + } + return j; +} + +/** + * @brief Execute the supplied queries and check that the return codes are the + * ones specified. + * + * @param proxysql_clickhouse An already opened MYSQL connection to ProxySQL + * Clickhouse server. + * @param queries The queries to be performed and check. + */ +int execute_and_check_queries(MYSQL* proxysql_clickhouse, const std::vector& queries) { + for (const auto& supp_query : queries) { + const std::string query = std::get<0>(supp_query); + const int exp_err_code = std::get<1>(supp_query); + const int exp_rows = std::get<2>(supp_query); // if >= 0 , it is a select and expects data + + diag("Line: %d . Running query: %s" , __LINE__ , query.c_str()); + int query_err = mysql_query(proxysql_clickhouse, query.c_str()); + MYSQL_RES* result = nullptr; + // NOTE: For test compatibility with 'libmysqlclient', 'mysql_store_result' should only be called + // in case no error is present. Otherwise would modify the error itself, thus making test fail. + if (!query_err) { + result = mysql_store_result(proxysql_clickhouse); + } + if (exp_rows >= 0 && result == NULL) { + diag ("We were expecting %d rows, but we didn't receive a resultset", exp_rows); + return exit_status(); + } + if (exp_rows < 0 && result != NULL) { + diag ("We were expecting no result, but we received a resultset"); + return exit_status(); + } + if (result) { + int j = fetch_and_discard_results(result, true); + mysql_free_result(result); + if (j != exp_rows) { + diag ("We were expecting a result of %d rows, but we received a resultset of %d rows", exp_rows, j); + return exit_status(); + } + } + + int m_errno = mysql_errno(proxysql_clickhouse); + const char* m_error = mysql_error(proxysql_clickhouse); + + if (exp_err_code == 0) { + ok( + exp_err_code == m_errno, + "Line: %d . Query '%s' should succeed. Error code: (Expected:'%d' == Actual:'%d'), Err: '%s'", + __LINE__, query.c_str(), exp_err_code, m_errno, m_error + ); + } else { + ok( + exp_err_code == m_errno, + "Line: %d . Query '%s' should fail. Error code: (Expected:'%d' == Actual:'%d'), Err: '%s'", + __LINE__, query.c_str(), exp_err_code, m_errno, m_error + ); + } + } + return 0; +} + +std::vector queries_set1 { + std::make_tuple("SHOW SCHEMAS", 0, 4), + std::make_tuple("SHOW DATABASES", 0, 4), + std::make_tuple("SELECT DATABASE()", 0, 1), + std::make_tuple("SELECT USER()", 0, 1), + std::make_tuple("SELECT CURRENT_USER()", 0, 1), + std::make_tuple("SELECT VERSION()", 0, 1), + std::make_tuple("SELECT CONCAT(version(),'')", 0, 1), + std::make_tuple("SELECT 1", 0, 1), + std::make_tuple("SELECT 1+1", 0, 1), + std::make_tuple("SELECT CONCAT('AAA','BBB')", 0, 1), + std::make_tuple("SELECT NULL", 0, 1), + std::make_tuple("SELECT NULL AS a", 0, 1), + std::make_tuple("SELECT NULL+2 AS a, 'hello', NULL+1, 'world', NULL AS b", 0, 1), + std::make_tuple("SELECT CONCAT('AAA',NULL)", 0, 1), + std::make_tuple("DROP TABLE IF EXISTS table1", 0, -1), + std::make_tuple("CREATE TABLE table1 (CounterID INT, EventDate DATE, col1 INT) ENGINE=MergeTree(EventDate, (CounterID, EventDate), 8192)", 0, -1), + std::make_tuple("CREATE TABLE table1 (CounterID INT, EventDate DATE, col1 INT) ENGINE=MergeTree(EventDate, (CounterID, EventDate), 8192)", 1148, -1), // the second time it must fails + std::make_tuple("INSERT INTO table1 VALUES (1,NOW(),1)", 1148, -1), + std::make_tuple("INSERT INTO table1 SELECT 1,NOW(),1", 0, -1), + std::make_tuple("SELECT * FROM table1", 0, 1), + std::make_tuple("INSERT INTO table1 SELECT * FROM table1", 0, -1), + std::make_tuple("SELECT * FROM table1", 0, 2), + std::make_tuple("TRUNCATE TABLE table1", 1148, -1), + std::make_tuple("DROP TABLE IF EXISTS table1", 0, -1), + std::make_tuple("CREATE TABLE table1 (CounterID INT, EventDate DATE, col1 INT) ENGINE=MergeTree(EventDate, (CounterID, EventDate), 8192)", 0, -1), + std::make_tuple("SELECT * FROM table1", 0, 0), + std::make_tuple("INSERT INTO table1 SELECT 1,'2022-06-23',1", 0, -1), + std::make_tuple("INSERT INTO table1 SELECT 2,'2022-06-23',1", 0, -1), + std::make_tuple("INSERT INTO table1 SELECT CounterID+2, '2022-06-23', 1 FROM table1", 0, -1), + std::make_tuple("SELECT * FROM table1 ORDER BY CounterID", 0, 4), + std::make_tuple("INSERT INTO table1 SELECT * FROM table1", 0, -1), + std::make_tuple("INSERT INTO table1 SELECT * FROM table1", 0, -1), + std::make_tuple("SELECT CounterID, EventDate, SUM(col1) s FROM table1 GROUP BY CounterID,EventDate ORDER BY CounterID", 0, 4), + std::make_tuple("SELECT * FROM table1 t1 JOIN table1 t2 ON t1.CounterID==t2.CounterID ORDER BY t1.CounterID", 0, 64), + std::make_tuple("DESC table1", 0, 3), + std::make_tuple("SHOW COLUMNS FROM table1", 0, 3), + std::make_tuple("LOCK TABLE table1", 0, -1), + std::make_tuple("UNLOCK TABLE table1", 0, -1), +}; +std::vector queries_set2 { + std::make_tuple("DROP TABLE IF EXISTS table2", 0, -1), + std::make_tuple("CREATE TABLE table2 (CounterID INT, EventDate DATE, col0 INT, col1 Nullable(INT), col2 Nullable(UInt8), col3 Nullable(UInt16), col4 Nullable(UInt32), col5 Nullable(UInt64), col6 Nullable(Float32), col7 Nullable(Float64), col8 Nullable(Enum8('hello' = 1, 'world' = 2)) , col9 Nullable(Enum16('hello' = 1, 'world' = 2))) ENGINE=MergeTree(EventDate, (CounterID, EventDate), 8192)", 0, -1), + std::make_tuple("INSERT INTO table2 SELECT 1,'2022-06-23', 0, 1, 2, 3, 4, 5, 6, 7, 'hello', 'world'", 0, -1), + std::make_tuple("INSERT INTO table2 SELECT 1,'2022-06-23', 1, 2, 3, 4, 5, 6, 7, 'hello', 'world'", 1148, -1), // incorrect number of values + std::make_tuple("INSERT INTO table2 SELECT 1,'2022-06-23', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL", 1148, -1), // col0 can't be null + std::make_tuple("INSERT INTO table2 SELECT 1,'2022-06-23', 1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL", 0, -1), + std::make_tuple("SELECT * FROM table2 ORDER BY CounterID", 0, 2), + std::make_tuple("DESC table2", 0, 12), + std::make_tuple("SHOW COLUMNS FROM table2", 0, 12), +}; + +std::vector queries_set3 { + std::make_tuple("SHOW FULL TABLES FROM `default`", 0, 2), // table1 and table2 + std::make_tuple("SHOW CHARSET", 0, 42), + std::make_tuple("SET AUTOCOMMIT=0", 0, -1), + std::make_tuple("SET foreign_key_checks=0", 0, -1), + std::make_tuple("/*!40101 SET whatever", 0, -1), + std::make_tuple("SET NAMES utf8", 0, -1), + std::make_tuple("SET WAIT_TIMEOUT=10", 0, -1), + std::make_tuple("SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED", 0, -1), + std::make_tuple("SET SQL_SAFE_UPDATES = OFF", 0, -1), + std::make_tuple("SET SQL_AUTO_IS_NULL = OFF", 0, -1), + std::make_tuple("SHOW GLOBAL VARIABLES", 0, 6), + std::make_tuple("SHOW ALL VARIABLES", 0, 6), + std::make_tuple("SHOW GLOBAL STATUS", 0, 6), + std::make_tuple("SHOW ENGINES", 0, 1), + std::make_tuple("SHOW VARIABLES LIKE 'lower_case_table_names'", 0, 1), + std::make_tuple("SELECT * FROM INFORMATION_SCHEMA.COLLATIONS", 0, 375), + std::make_tuple("SHOW COLLATION", 0, 375), + std::make_tuple("SHOW CHARSET", 0, 42), + std::make_tuple("SELECT * FROM INFORMATION_SCHEMA.CHARACTER_SETS", 0, 42), + std::make_tuple("SELECT @@collation_server", 0, 1), + std::make_tuple("SELECT @@character_set_results", 0, 1), + std::make_tuple("SELECT @@have_profiling", 0, 1), + std::make_tuple("SELECT @@lower_case_table_names", 0, 1), + std::make_tuple("SELECT @@version, @@version_comment", 0, 1), + std::make_tuple("SELECT @@storage_engine", 0, 1), + std::make_tuple("SELECT `SCHEMA_NAME` FROM `INFORMATION_SCHEMA`.`SCHEMATA`", 0, 4), + std::make_tuple("select name, type FROM mysql.proc where db='default'", 0, 0), + std::make_tuple("SELECT logfile_group_name FROM information_schema.FILES", 0, 0), + std::make_tuple("SELECT tablespace_name FROM information_schema.FILES", 0, 0), + std::make_tuple("SELECT CONNECTION_ID()", 0, 1), + std::make_tuple("select @@version_comment limit 1", 0, 1), + std::make_tuple("select DATABASE(), USER() limit 1", 0, 1), + std::make_tuple("select @@character_set_client, @@character_set_connection, @@character_set_server, @@character_set_database limit 1", 0, 1), + std::make_tuple("SELECT @@version", 0, 1), + std::make_tuple("SHOW TABLES FROM default", 0, 2), + std::make_tuple("SELECT DATABASE() AS name", 0, 1), + std::make_tuple("SHOW MASTER STATUS", 1045, -1), + std::make_tuple("SHOW SLAVE STATUS", 1045, -1), + std::make_tuple("SHOW MASTER LOGS", 1045, -1), + std::make_tuple("LOCK TABLE table1", 0, -1), + std::make_tuple("UNLOCK TABLE table1", 0, -1), +}; +/** + * @brief Perform several admin queries to exercise more paths. + */ +std::vector admin_queries { + "LOAD CLICKHOUSE VARIABLES FROM DISK", + "LOAD CLICKHOUSE VARIABLES TO RUNTIME", + "SAVE CLICKHOUSE VARIABLES FROM RUNTIME", + "SAVE CLICKHOUSE VARIABLES TO DISK" +}; + +/** + * @brief Perform several admin queries to exercise more paths. + */ +std::vector ch_intf_queries { + "SET clickhouse-mysql_ifaces='127.0.0.1:6091'", + "LOAD CLICKHOUSE VARIABLES TO RUNTIME" +}; + +int main(int argc, char** argv) { + CommandLine cl; + + // plan as many tests as queries + plan( + crash_loops + + 2 /* Fail to connect with wrong username and password */ + + 4 // during LOAD USERS TO RUNTIME + + 4 // during LOAD USERS TO RUNTIME , second time + + queries_set1.size() + + queries_set2.size() + + queries_set3.size() + + admin_queries.size() + ch_intf_queries.size() + + 1 /* Connect to new setup interface */ + ); + + if (cl.getEnv()) { + diag("Failed to get the required environmental variables."); + return EXIT_FAILURE; + } + + MYSQL* proxysql_admin = mysql_init(NULL); + + // Connect to ProxySQL Admin and check current clickhouse configuration + if ( + !mysql_real_connect( + proxysql_admin, cl.host, cl.admin_username, cl.admin_password, + NULL, cl.admin_port, NULL, 0 + ) + ) { + fprintf( + stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, + mysql_error(proxysql_admin) + ); + return EXIT_FAILURE; + } + + + create_users(proxysql_admin); + create_users(proxysql_admin); // to trigger more code coverage + + + { + std::pair host_port {}; + int host_port_err = extract_module_host_port(proxysql_admin, "clickhouse-mysql_ifaces", host_port); + if (host_port_err) { + diag("Failed to get and parse 'clickhouse-mysql_ifaces' at line '%d'", __LINE__); + goto cleanup; + } + + set_clickhouse_port(proxysql_admin,8000); + test_crash(host_port.first.c_str(), host_port.second); + set_clickhouse_port(proxysql_admin,19000); + + MYSQL* proxysql_clickhouse = mysql_init(NULL); + + // Connect with invalid username + std::string inv_user_err {}; + bool failed_to_connect = false; + if ( + !mysql_real_connect( + proxysql_clickhouse, host_port.first.c_str(), "foobar_user", cl.password, + NULL, host_port.second, NULL, 0 + ) + ) { + inv_user_err = mysql_error(proxysql_clickhouse); + failed_to_connect = true; + } + + ok( + failed_to_connect, + "An invalid user should fail to connect to Clickhouse server, error was: %s", + inv_user_err.c_str() + ); + + // Reinitialize MYSQL handle + mysql_close(proxysql_clickhouse); + proxysql_clickhouse = mysql_init(NULL); + + // Connect with invalid password + std::string inv_pass_err {}; + failed_to_connect = false; + if ( + !mysql_real_connect( + proxysql_clickhouse, host_port.first.c_str(), credentials[0].first.c_str(), "foobar_pass", + NULL, host_port.second, NULL, 0 + ) + ) { + inv_pass_err = mysql_error(proxysql_clickhouse); + failed_to_connect = true; + } + + ok( + failed_to_connect, + "An invalid pass should fail to connect to Clickhouse server, error was: %s", + inv_pass_err.c_str() + ); + + // Reinitialize MYSQL handle + mysql_close(proxysql_clickhouse); + proxysql_clickhouse = mysql_init(NULL); + + // Correctly connect to Clickhouse server + if ( + !mysql_real_connect( + proxysql_clickhouse, host_port.first.c_str(), credentials[0].first.c_str(), credentials[0].second.c_str(), + NULL, host_port.second, NULL, 0 + ) + ) { + fprintf( + stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, + mysql_error(proxysql_clickhouse) + ); + goto cleanup; + } + + diag("Started performing queries set 1"); + if (execute_and_check_queries(proxysql_clickhouse, queries_set1)) { + return exit_status(); + } + + diag("Started performing queries set 2"); + if (execute_and_check_queries(proxysql_clickhouse, queries_set2)) { + return exit_status(); + } + + diag("Started performing queries set 3"); + if (execute_and_check_queries(proxysql_clickhouse, queries_set3)) { + return exit_status(); + } + + // Reinitialize MYSQL handle + mysql_close(proxysql_clickhouse); + proxysql_clickhouse = mysql_init(NULL); + + // Change Clickhouse interface and connect to new port + for (const auto& admin_query : ch_intf_queries) { + int query_err = mysql_query(proxysql_admin, admin_query.c_str()); + ok( + query_err == 0, "Admin query '%s' should succeed. Line: %d, Err: '%s'", + admin_query.c_str(), __LINE__, mysql_error(proxysql_admin) + ); + } + // NOTE: Wait for ProxySQL to reconfigure, changing Clickhous interface. + // Trying to perform a connection immediately after changing the + // interface could lead to 'EADDRINUSE' in ProxySQL side. + sleep(1); + + // Connect to the new interface + std::pair new_host_port {}; + int ext_intf_err = extract_module_host_port(proxysql_admin, "clickhouse-mysql_ifaces", new_host_port); + if (ext_intf_err) { + diag("Failed to get and parse 'clickhouse-mysql_ifaces' at line '%d'", __LINE__); + goto cleanup; + } + + // Connect with invalid username + bool success_to_connect = true; + std::string new_intf_conn_err {}; + if ( + !mysql_real_connect( + proxysql_clickhouse, new_host_port.first.c_str(), credentials[1].first.c_str(), credentials[1].second.c_str(), + NULL, new_host_port.second, NULL, 0 + ) + ) { + new_intf_conn_err = mysql_error(proxysql_clickhouse); + success_to_connect = false; + } + + ok( + success_to_connect, + "A connection to the new selected interface should success, error was: '%s'", + new_intf_conn_err.c_str() + ); + + mysql_close(proxysql_clickhouse); + + // Perform the final Admin queries + for (const auto& admin_query : admin_queries) { + int query_err = mysql_query(proxysql_admin, admin_query.c_str()); + ok( + query_err == 0, "Admin query '%s' should succeed. Line: %d, Err: '%s'", + admin_query.c_str(), __LINE__, mysql_error(proxysql_admin) + ); + } + } + +cleanup: + + mysql_close(proxysql_admin); + + return exit_status(); +} diff --git a/test/tap/tests/test_sqlite3_server-t.cpp b/test/tap/tests/test_sqlite3_server-t.cpp index 49832c3e7..1012362c1 100644 --- a/test/tap/tests/test_sqlite3_server-t.cpp +++ b/test/tap/tests/test_sqlite3_server-t.cpp @@ -41,7 +41,7 @@ using query_spec = std::tuple; const int sqlite3_port = 0; -#include "sqlite_3_server_test.h" +#include "modules_server_test.h" void fetch_and_discard_results(MYSQL_RES* result, bool verbose=false) { MYSQL_ROW row = nullptr; @@ -246,7 +246,7 @@ int main(int argc, char** argv) { { std::pair host_port {}; - int host_port_err = extract_sqlite3_host_port(proxysql_admin, host_port); + int host_port_err = extract_module_host_port(proxysql_admin, "sqliteserver-mysql_ifaces", host_port); if (host_port_err) { diag("Failed to get and parse 'sqliteserver-mysql_ifaces' at line '%d'", __LINE__); goto cleanup; @@ -292,7 +292,7 @@ int main(int argc, char** argv) { ok( failed_to_connect, - "An invalid user should fail to connect to SQLite3 server, error was: %s", + "An invalid password should fail to connect to SQLite3 server, error was: %s", inv_pass_err.c_str() ); @@ -340,7 +340,7 @@ int main(int argc, char** argv) { // Connect to the new interface std::pair new_host_port {}; - int ext_intf_err = extract_sqlite3_host_port(proxysql_admin, new_host_port); + int ext_intf_err = extract_module_host_port(proxysql_admin, "sqliteserver-mysql_ifaces", new_host_port); if (ext_intf_err) { diag("Failed to get and parse 'sqliteserver-mysql_ifaces' at line '%d'", __LINE__); goto cleanup; diff --git a/test/tap/tests/test_sqlite3_server_and_fast_routing-t.cpp b/test/tap/tests/test_sqlite3_server_and_fast_routing-t.cpp index 6181fc27a..6c7f7f472 100644 --- a/test/tap/tests/test_sqlite3_server_and_fast_routing-t.cpp +++ b/test/tap/tests/test_sqlite3_server_and_fast_routing-t.cpp @@ -18,7 +18,7 @@ using query_spec = std::tuple; const int sqlite3_port = 0; -#include "sqlite_3_server_test.h" +#include "modules_server_test.h" inline unsigned long long monotonic_time() { struct timespec ts; @@ -61,7 +61,7 @@ int main(int argc, char** argv) { } // { - host_port_err = extract_sqlite3_host_port(proxysql_admin, host_port); + host_port_err = extract_module_host_port(proxysql_admin, "sqliteserver-mysql_ifaces", host_port); if (host_port_err) { diag("Failed to get and parse 'sqliteserver-mysql_ifaces' at line '%d'", __LINE__); goto cleanup;