From c5040ab8546b8291903e6840d9b33ead9da8f7aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Mon, 2 Apr 2018 05:01:48 +0200 Subject: [PATCH 1/8] Monitor: do not consider unhealthy ProxySQL's as backends if monitor is unable to login --- lib/MySQL_Monitor.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index e021ec6ab..95da46dc4 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -400,7 +400,11 @@ void * monitor_connect_thread(void *arg) { rc=sqlite3_reset(statement); assert(rc==SQLITE_OK); sqlite3_finalize(statement); if (mmsd->mysql_error_msg) { - if (strncmp(mmsd->mysql_error_msg,"Access denied for user",strlen("Access denied for user"))==0) { + if ( + (strncmp(mmsd->mysql_error_msg,"Access denied for user",strlen("Access denied for user"))==0) + || + (strncmp(mmsd->mysql_error_msg,"ProxySQL Error: Access denied for user",strlen("ProxySQL Error: Access denied for user"))==0) + ) { proxy_error("Server %s:%d is returning \"Access denied\" for monitoring user\n", mmsd->hostname, mmsd->port); } } @@ -1403,7 +1407,7 @@ __end_monitor_ping_loop: resultset=NULL; } char *new_query=NULL; - new_query=(char *)"SELECT 1 FROM (SELECT hostname,port,ping_error FROM mysql_server_ping_log WHERE hostname='%s' AND port='%s' ORDER BY time_start_us DESC LIMIT %d) a WHERE ping_error IS NOT NULL AND ping_error NOT LIKE 'Access denied for user%%' GROUP BY hostname,port HAVING COUNT(*)=%d"; + new_query=(char *)"SELECT 1 FROM (SELECT hostname,port,ping_error FROM mysql_server_ping_log WHERE hostname='%s' AND port='%s' ORDER BY time_start_us DESC LIMIT %d) a WHERE ping_error IS NOT NULL AND ping_error NOT LIKE 'Access denied for user%%' AND ping_error NOT LIKE 'ProxySQL Error: Access denied for user%%' GROUP BY hostname,port HAVING COUNT(*)=%d"; for (j=0;j Date: Mon, 2 Apr 2018 05:49:41 +0200 Subject: [PATCH 2/8] Missing flag SERVER_STATUS_NO_BACKSLASH_ESCAPES for SQLite3 backends #1446 --- lib/MySQL_Protocol.cpp | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/MySQL_Protocol.cpp b/lib/MySQL_Protocol.cpp index 25bb6592c..d7f99d12b 100644 --- a/lib/MySQL_Protocol.cpp +++ b/lib/MySQL_Protocol.cpp @@ -478,8 +478,20 @@ bool MySQL_Protocol::generate_pkt_EOF(bool send, void **ptr, unsigned int *len, memcpy(_ptr, &myhdr, sizeof(mysql_hdr)); int l=sizeof(mysql_hdr); _ptr[l]=0xfe; l++; + int16_t internal_status = status; + if (sess) { + switch (sess->session_type) { + case PROXYSQL_SESSION_SQLITE: + case PROXYSQL_SESSION_ADMIN: + case PROXYSQL_SESSION_STATS: + internal_status += SERVER_STATUS_NO_BACKSLASH_ESCAPES; + break; + default: + break; + } + } memcpy(_ptr+l, &warnings, sizeof(uint16_t)); l+=sizeof(uint16_t); - memcpy(_ptr+l, &status, sizeof(uint16_t)); + memcpy(_ptr+l, &internal_status, sizeof(uint16_t)); if (send==true) { (*myds)->PSarrayOUT->add((void *)_ptr,size); @@ -570,7 +582,19 @@ bool MySQL_Protocol::generate_pkt_OK(bool send, void **ptr, unsigned int *len, u _ptr[l]=0x00; l++; l+=write_encoded_length(_ptr+l, affected_rows, affected_rows_len, affected_rows_prefix); l+=write_encoded_length(_ptr+l, last_insert_id, last_insert_id_len, last_insert_id_prefix); - memcpy(_ptr+l, &status, sizeof(uint16_t)); l+=sizeof(uint16_t); + int16_t internal_status = status; + if (sess) { + switch (sess->session_type) { + case PROXYSQL_SESSION_SQLITE: + case PROXYSQL_SESSION_ADMIN: + case PROXYSQL_SESSION_STATS: + internal_status += SERVER_STATUS_NO_BACKSLASH_ESCAPES; + break; + default: + break; + } + } + memcpy(_ptr+l, &internal_status, sizeof(uint16_t)); l+=sizeof(uint16_t); memcpy(_ptr+l, &warnings, sizeof(uint16_t)); l+=sizeof(uint16_t); if (msg) { l+=write_encoded_length(_ptr+l, msg_len, msg_len_len, msg_prefix); From 7b3757a273f82228a88bcdac65fb09720aea4159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Mon, 2 Apr 2018 16:15:07 +0200 Subject: [PATCH 3/8] Remember if the current connection has an unknown transaction status The fix for bug #1038 was to not return a connection to the connection pool if it has an error. Although this is correct, it also has the side effect that connections coming from connection pool and failing during the first query because the connection was already broken, would be considered as possibly to run a transaction. That is incorrect. Now at connection level it is tracked if the transaction status is known or not. --- include/mysql_connection.h | 2 ++ lib/MySQL_Session.cpp | 5 ++++ lib/mysql_connection.cpp | 55 +++++++++++++++++++++++++++++++++++++- 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/include/mysql_connection.h b/include/mysql_connection.h index 4f7573cd3..a0bb1f5e3 100644 --- a/include/mysql_connection.h +++ b/include/mysql_connection.h @@ -92,6 +92,8 @@ class MySQL_Connection { bool processing_prepared_statement_execute; bool processing_multi_statement; bool multiplex_delayed; + bool unknown_transaction_status; + void compute_unknown_transaction_status(); MySQL_Connection(); ~MySQL_Connection(); bool set_autocommit(bool); diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index c5862fb95..14d8b3467 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -942,6 +942,7 @@ int MySQL_Session::handler_again___status_PINGING_SERVER() { int rc=myconn->async_ping(myds->revents); if (rc==0) { myconn->async_state_machine=ASYNC_IDLE; + myconn->unknown_transaction_status = false; if (mysql_thread___multiplexing && (myconn->reusable==true) && myds->myconn->IsActiveTransaction()==false && myds->myconn->MultiplexDisabled()==false) { myds->return_MySQL_Connection_To_Pool(); } else { @@ -2730,6 +2731,7 @@ handler_again: enum session_status st=status; size_t sts=previous_status.size(); if (sts) { + myconn->unknown_transaction_status = false; myconn->async_state_machine=ASYNC_IDLE; myds->DSS=STATE_MARIADB_GENERIC; st=previous_status.top(); @@ -2769,6 +2771,7 @@ handler_again: if (mysql_thread___multiplexing && (myds->myconn->reusable==true) && myds->myconn->IsActiveTransaction()==false && myds->myconn->MultiplexDisabled()==false) { if (mysql_thread___connection_delay_multiplex_ms && mirror==false) { myds->wait_until=thread->curtime+mysql_thread___connection_delay_multiplex_ms*1000; + myconn->unknown_transaction_status = false; myconn->async_state_machine=ASYNC_IDLE; myconn->multiplex_delayed=true; myds->DSS=STATE_MARIADB_GENERIC; @@ -2796,6 +2799,7 @@ handler_again: } } else { myconn->multiplex_delayed=false; + myconn->unknown_transaction_status = false; myconn->async_state_machine=ASYNC_IDLE; myds->DSS=STATE_MARIADB_GENERIC; if (transaction_persistent==true) { @@ -2813,6 +2817,7 @@ handler_again: } else { if (rc==-1) { int myerr=mysql_errno(myconn->mysql); + myconn->compute_unknown_transaction_status(); char *errmsg = NULL; if (myerr == 0) { if (CurrentQuery.mysql_stmt) { diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index e566f10aa..d88b517cc 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -49,6 +49,29 @@ MySQL_Connection_userinfo::~MySQL_Connection_userinfo() { if (schemaname) free(schemaname); } +void MySQL_Connection::compute_unknown_transaction_status() { + if (mysql) { + int _myerrno=mysql_errno(mysql); + if (_myerrno == 0) { + unknown_transaction_status = false; // no error + return; + } + if (_myerrno >= 2000 && _myerrno < 3000) { // client error + // do not change it + return; + } + if (_myerrno >= 1000 && _myerrno < 2000) { // server error + unknown_transaction_status = true; + return; + } + if (_myerrno >= 3000 && _myerrno < 4000) { // server error + unknown_transaction_status = true; + return; + } + // all other cases, server error + } +} + uint64_t MySQL_Connection_userinfo::compute_hash() { int l=0; if (username) @@ -189,6 +212,7 @@ MySQL_Connection::MySQL_Connection() { largest_query_length=0; multiplex_delayed=false; MyRS=NULL; + unknown_transaction_status = false; creation_time=0; processing_multi_statement=false; proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "Creating new MySQL_Connection %p\n", this); @@ -981,6 +1005,14 @@ handler_again: } break; case ASYNC_QUERY_END: + if (mysql) { + int _myerrno=mysql_errno(mysql); + if (_myerrno == 0) { + unknown_transaction_status = false; + } else { + compute_unknown_transaction_status(); + } + } if (mysql_result) { mysql_free_result(mysql_result); mysql_result=NULL; @@ -1137,6 +1169,7 @@ int MySQL_Connection::async_connect(short event) { return 0; } if (async_state_machine==ASYNC_CONNECT_SUCCESSFUL) { + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; myds->wait_until=0; creation_time = monotonic_time(); @@ -1145,6 +1178,7 @@ int MySQL_Connection::async_connect(short event) { handler(event); switch (async_state_machine) { case ASYNC_CONNECT_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; myds->wait_until=0; return 0; @@ -1224,8 +1258,10 @@ int MySQL_Connection::async_query(short event, char *stmt, unsigned long length, if (async_state_machine==ASYNC_QUERY_END) { if (mysql_errno(mysql)) { + compute_unknown_transaction_status(); return -1; } else { + unknown_transaction_status = false; return 0; } } @@ -1233,17 +1269,21 @@ int MySQL_Connection::async_query(short event, char *stmt, unsigned long length, query.stmt_meta=NULL; async_state_machine=ASYNC_QUERY_END; if (mysql_stmt_errno(query.stmt)) { + compute_unknown_transaction_status(); return -1; } else { + unknown_transaction_status = false; return 0; } } if (async_state_machine==ASYNC_STMT_PREPARE_SUCCESSFUL || async_state_machine==ASYNC_STMT_PREPARE_FAILED) { query.stmt_meta=NULL; if (async_state_machine==ASYNC_STMT_PREPARE_FAILED) { + compute_unknown_transaction_status(); return -1; } else { *_stmt=query.stmt; + unknown_transaction_status = false; return 0; } } @@ -1273,6 +1313,7 @@ int MySQL_Connection::async_ping(short event) { assert(ret_mysql); switch (async_state_machine) { case ASYNC_PING_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1292,6 +1333,7 @@ int MySQL_Connection::async_ping(short event) { // check again switch (async_state_machine) { case ASYNC_PING_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1314,6 +1356,7 @@ int MySQL_Connection::async_change_user(short event) { assert(ret_mysql); switch (async_state_machine) { case ASYNC_CHANGE_USER_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1333,6 +1376,7 @@ int MySQL_Connection::async_change_user(short event) { // check again switch (async_state_machine) { case ASYNC_CHANGE_USER_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1355,6 +1399,7 @@ int MySQL_Connection::async_select_db(short event) { assert(ret_mysql); switch (async_state_machine) { case ASYNC_INITDB_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1371,6 +1416,7 @@ int MySQL_Connection::async_select_db(short event) { // check again switch (async_state_machine) { case ASYNC_INITDB_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1390,6 +1436,7 @@ int MySQL_Connection::async_set_autocommit(short event, bool ac) { assert(ret_mysql); switch (async_state_machine) { case ASYNC_SET_AUTOCOMMIT_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1407,6 +1454,7 @@ int MySQL_Connection::async_set_autocommit(short event, bool ac) { // check again switch (async_state_machine) { case ASYNC_SET_AUTOCOMMIT_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1426,6 +1474,7 @@ int MySQL_Connection::async_set_names(short event, uint8_t c) { assert(ret_mysql); switch (async_state_machine) { case ASYNC_SET_NAMES_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1443,6 +1492,7 @@ int MySQL_Connection::async_set_names(short event, uint8_t c) { // check again switch (async_state_machine) { case ASYNC_SET_NAMES_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1487,6 +1537,7 @@ void MySQL_Connection::async_free_result() { mysql_result=NULL; } } + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; if (MyRS) { delete MyRS; @@ -1499,7 +1550,7 @@ bool MySQL_Connection::IsActiveTransaction() { bool ret=false; if (mysql) { ret = (mysql->server_status & SERVER_STATUS_IN_TRANS); - if (ret == false && (mysql)->net.last_errno) { + if (ret == false && (mysql)->net.last_errno && unknown_transaction_status == true) { ret = true; } if (ret == false) { @@ -1685,8 +1736,10 @@ int MySQL_Connection::async_send_simple_command(short event, char *stmt, unsigne } if (async_state_machine==ASYNC_QUERY_END) { if (mysql_errno(mysql)) { + compute_unknown_transaction_status(); return -1; } else { + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; } From e3603de5d82f841a5fe9ef0075302ef99ffdf842 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Mon, 2 Apr 2018 17:03:22 +0200 Subject: [PATCH 4/8] Remember if the current connection has an unknown transaction status Further improvement on previous commit --- lib/MySQL_Session.cpp | 8 +++----- lib/mysql_connection.cpp | 18 +++++++----------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index 14d8b3467..fae5bb79d 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -942,7 +942,7 @@ int MySQL_Session::handler_again___status_PINGING_SERVER() { int rc=myconn->async_ping(myds->revents); if (rc==0) { myconn->async_state_machine=ASYNC_IDLE; - myconn->unknown_transaction_status = false; + myconn->compute_unknown_transaction_status(); if (mysql_thread___multiplexing && (myconn->reusable==true) && myds->myconn->IsActiveTransaction()==false && myds->myconn->MultiplexDisabled()==false) { myds->return_MySQL_Connection_To_Pool(); } else { @@ -2731,7 +2731,6 @@ handler_again: enum session_status st=status; size_t sts=previous_status.size(); if (sts) { - myconn->unknown_transaction_status = false; myconn->async_state_machine=ASYNC_IDLE; myds->DSS=STATE_MARIADB_GENERIC; st=previous_status.top(); @@ -2771,7 +2770,6 @@ handler_again: if (mysql_thread___multiplexing && (myds->myconn->reusable==true) && myds->myconn->IsActiveTransaction()==false && myds->myconn->MultiplexDisabled()==false) { if (mysql_thread___connection_delay_multiplex_ms && mirror==false) { myds->wait_until=thread->curtime+mysql_thread___connection_delay_multiplex_ms*1000; - myconn->unknown_transaction_status = false; myconn->async_state_machine=ASYNC_IDLE; myconn->multiplex_delayed=true; myds->DSS=STATE_MARIADB_GENERIC; @@ -2799,7 +2797,7 @@ handler_again: } } else { myconn->multiplex_delayed=false; - myconn->unknown_transaction_status = false; + myconn->compute_unknown_transaction_status(); myconn->async_state_machine=ASYNC_IDLE; myds->DSS=STATE_MARIADB_GENERIC; if (transaction_persistent==true) { @@ -2817,7 +2815,6 @@ handler_again: } else { if (rc==-1) { int myerr=mysql_errno(myconn->mysql); - myconn->compute_unknown_transaction_status(); char *errmsg = NULL; if (myerr == 0) { if (CurrentQuery.mysql_stmt) { @@ -4402,6 +4399,7 @@ void MySQL_Session::RequestEnd(MySQL_Data_Stream *myds) { // if there is a mysql connection, clean its status if (myds->myconn) { myds->myconn->async_free_result(); + myds->myconn->compute_unknown_transaction_status(); } myds->free_mysql_real_query(); } diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index d88b517cc..7d7c09b57 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -1169,7 +1169,7 @@ int MySQL_Connection::async_connect(short event) { return 0; } if (async_state_machine==ASYNC_CONNECT_SUCCESSFUL) { - unknown_transaction_status = false; + compute_unknown_transaction_status(); async_state_machine=ASYNC_IDLE; myds->wait_until=0; creation_time = monotonic_time(); @@ -1178,7 +1178,7 @@ int MySQL_Connection::async_connect(short event) { handler(event); switch (async_state_machine) { case ASYNC_CONNECT_SUCCESSFUL: - unknown_transaction_status = false; + compute_unknown_transaction_status(); async_state_machine=ASYNC_IDLE; myds->wait_until=0; return 0; @@ -1257,33 +1257,30 @@ int MySQL_Connection::async_query(short event, char *stmt, unsigned long length, } if (async_state_machine==ASYNC_QUERY_END) { + compute_unknown_transaction_status(); if (mysql_errno(mysql)) { - compute_unknown_transaction_status(); return -1; } else { - unknown_transaction_status = false; return 0; } } if (async_state_machine==ASYNC_STMT_EXECUTE_END) { query.stmt_meta=NULL; async_state_machine=ASYNC_QUERY_END; + compute_unknown_transaction_status(); if (mysql_stmt_errno(query.stmt)) { - compute_unknown_transaction_status(); return -1; } else { - unknown_transaction_status = false; return 0; } } if (async_state_machine==ASYNC_STMT_PREPARE_SUCCESSFUL || async_state_machine==ASYNC_STMT_PREPARE_FAILED) { query.stmt_meta=NULL; + compute_unknown_transaction_status(); if (async_state_machine==ASYNC_STMT_PREPARE_FAILED) { - compute_unknown_transaction_status(); return -1; } else { *_stmt=query.stmt; - unknown_transaction_status = false; return 0; } } @@ -1537,7 +1534,7 @@ void MySQL_Connection::async_free_result() { mysql_result=NULL; } } - unknown_transaction_status = false; + compute_unknown_transaction_status(); async_state_machine=ASYNC_IDLE; if (MyRS) { delete MyRS; @@ -1735,11 +1732,10 @@ int MySQL_Connection::async_send_simple_command(short event, char *stmt, unsigne proxy_error("Retrieved a resultset while running a simple command. This is an error!! Simple command: %s\n", stmt); } if (async_state_machine==ASYNC_QUERY_END) { + compute_unknown_transaction_status(); if (mysql_errno(mysql)) { - compute_unknown_transaction_status(); return -1; } else { - unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; } From 993ef216276e9b51e72f097f6b6233cd07bdd783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Eeden?= Date: Mon, 2 Apr 2018 18:46:12 +0200 Subject: [PATCH 5/8] Disable libssh2 support in curl to fix building on Fedora 27 Without this building failed with undefined references to libssh2: ``` g++ -o proxysql obj/main.o obj/proxysql_global.o ../lib/libproxysql.a -std=c++11 -I../include -I../deps/jemalloc/jemalloc/include/jemalloc -I../deps/mariadb-client-library/mariadb_client/include -I../deps/libconfig/libconfig-1.4.9/lib -I../deps/libdaemon/libdaemon -I../deps/sqlite3/sqlite3 -I../deps/clickhouse-cpp/clickhouse-cpp -I../deps/libmicrohttpd/libmicrohttpd/src/include -I../deps/curl/curl//include -O2 -ggdb -L../lib -L../deps/jemalloc/jemalloc/lib -L../deps/libconfig/libconfig-1.4.9/lib/.libs -L../deps/re2/re2/obj -L../deps/mariadb-client-library/mariadb_client/libmariadb -L../deps/libdaemon/libdaemon/libdaemon/.libs -L../deps/pcre/pcre/.libs -L../deps/libmicrohttpd/libmicrohttpd/src/microhttpd/.libs -L../deps/curl/curl//lib/.libs -Wl,--export-dynamic -Wl,-Bstatic -lconfig -lproxysql -ldaemon -ljemalloc -lconfig++ -lre2 -lpcrecpp -lpcre -lmariadbclient -lmicrohttpd -lcurl -Wl,-Bdynamic -lpthread -lm -lz -lrt -lcrypto -lssl -ldl ../deps/curl/curl//lib/.libs/libcurl.a(libcurl_la-easy.o): In function `global_init': easy.c:(.text+0x7d): undefined reference to `libssh2_init' ../deps/curl/curl//lib/.libs/libcurl.a(libcurl_la-easy.o): In function `curl_global_cleanup': easy.c:(.text+0x2c8): undefined reference to `libssh2_exit' ``` I don't think libssh2 support is needed and this fixes the build. The other option would be to add `-lssh2`. --- deps/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/Makefile b/deps/Makefile index 87d19d87a..2329423ac 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -34,7 +34,7 @@ coredumper: google-coredumper/google-coredumper/.libs/libcoredumper.a curl/curl/lib/.libs/libcurl.a: cd curl && rm -rf curl-7.57.0 || true cd curl && tar -zxf curl-7.57.0.tar.gz - cd curl/curl && ./configure --disable-debug --disable-ftp --disable-ldap --disable-ldaps --disable-rtsp --disable-proxy --disable-dict --disable-telnet --disable-tftp --disable-pop3 --disable-imap --disable-smb --disable-smtp --disable-gopher --disable-manual --disable-ipv6 --disable-sspi --disable-crypto-auth --disable-ntlm-wb --disable-tls-srp --without-nghttp2 --without-libidn2 && CC=${CC} CXX=${CXX} ${MAKE} + cd curl/curl && ./configure --disable-debug --disable-ftp --disable-ldap --disable-ldaps --disable-rtsp --disable-proxy --disable-dict --disable-telnet --disable-tftp --disable-pop3 --disable-imap --disable-smb --disable-smtp --disable-gopher --disable-manual --disable-ipv6 --disable-sspi --disable-crypto-auth --disable-ntlm-wb --disable-tls-srp --without-nghttp2 --without-libidn2 --without-libssh2 && CC=${CC} CXX=${CXX} ${MAKE} curl: curl/curl/lib/.libs/libcurl.a libmicrohttpd/libmicrohttpd/src/microhttpd/.libs/libmicrohttpd.a: From 405f8363b0b1fe1c991bc73a473906f8045bf936 Mon Sep 17 00:00:00 2001 From: Nikolaos Vyzas Date: Wed, 4 Apr 2018 18:43:30 +0300 Subject: [PATCH 6/8] Fix to return OK packet for SET WAIT_TIMEOUT --- lib/ClickHouse_Server.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/ClickHouse_Server.cpp b/lib/ClickHouse_Server.cpp index b9504d1a4..9dddcdf8d 100644 --- a/lib/ClickHouse_Server.cpp +++ b/lib/ClickHouse_Server.cpp @@ -495,7 +495,7 @@ void ClickHouse_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t if (sess->session_type == PROXYSQL_SESSION_CLICKHOUSE) { - if (!strncmp("SET ", query_no_space, 4)) { + if (!strncasecmp("SET ", query_no_space, 4)) { if ( !strncasecmp("SET AUTOCOMMIT", query_no_space, 14) || !strncasecmp("SET NAMES ", query_no_space, 10) || @@ -503,7 +503,8 @@ void ClickHouse_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t !strncasecmp("SET COLLATION", query_no_space, 13) || !strncasecmp("SET SQL_AUTO_", query_no_space, 13) || !strncasecmp("SET SQL_SAFE_", query_no_space, 13) || - !strncasecmp("SET SESSION TRANSACTION", query_no_space, 23) + !strncasecmp("SET SESSION TRANSACTION", query_no_space, 23) || + !strncasecmp("SET WAIT_TIMEOUT", query_no_space, 16) ) { GloClickHouseServer->send_MySQL_OK(&sess->client_myds->myprot, NULL); run_query=false; From c15ee4979722acf3f8ba8eb8b054bd89371a90a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Fri, 6 Apr 2018 08:00:33 +0200 Subject: [PATCH 7/8] Add support for wildcard in mysql_query_rules.client_addr #1450 Input validation: - client_addr not longer than INET6_ADDRSTRLEN - % allowed only at the end of client_addr Query rule itself remembers if there is a wildcard or not. If there is a wildcard compares string till the wildcard. If client_addr=='%' , it is a match all. --- include/query_processor.h | 1 + lib/ProxySQL_Admin.cpp | 16 ++++++++++++++++ lib/Query_Processor.cpp | 32 +++++++++++++++++++++++++++++--- 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/include/query_processor.h b/include/query_processor.h index cad2de7a2..1e489371c 100644 --- a/include/query_processor.h +++ b/include/query_processor.h @@ -14,6 +14,7 @@ struct _Query_Processor_rule_t { char *schemaname; int flagIN; char *client_addr; + int client_addr_wildcard_position; char *proxy_addr; int proxy_port; uint64_t digest; diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 51db65dd7..0fb692562 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -7070,6 +7070,22 @@ char * ProxySQL_Admin::load_mysql_query_rules_to_runtime() { QP_rule_t * nqpr; for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; + if (r->fields[4]) { + char *pct = NULL; + if (strlen(r->fields[4]) >= INET6_ADDRSTRLEN) { + proxy_error("Query rule with rule_id=%s has an invalid client_addr: %s\n", r->fields[0], r->fields[4]); + continue; + } + pct = strchr(r->fields[4],'%'); + if (pct) { // there is a wildcard + if (strlen(pct) == 1) { + // % is at the end of the string, good + } else { + proxy_error("Query rule with rule_id=%s has a wildcard that is not at the end of client_addr: %s\n", r->fields[0], r->fields[4]); + continue; + } + } + } nqpr=GloQPro->new_query_rule( atoi(r->fields[0]), // rule_id true, diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index f0407076a..9c6865a74 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -495,7 +495,23 @@ QP_rule_t * Query_Processor::new_query_rule(int rule_id, bool active, char *user newQR->regex_engine2=NULL; newQR->hits=0; + newQR->client_addr_wildcard_position = -1; // not existing by default newQR->client_addr=(client_addr ? strdup(client_addr) : NULL); + if (newQR->client_addr) { + char *pct = strchr(newQR->client_addr,'%'); + if (pct) { // there is a wildcard . We assume Admin did already all the input validation + if (pct == newQR->client_addr) { + // client_addr == '%' + // % is at the end of the string, but also at the beginning + // becoming a catch all + newQR->client_addr_wildcard_position = 0; + } else { + // this math is valid also if (pct == newQR->client_addr) + // but we separate it to clarify that client_addr_wildcard_position is a match all + newQR->client_addr_wildcard_position = strlen(newQR->client_addr) - strlen(pct); + } + } + } newQR->proxy_addr=(proxy_addr ? strdup(proxy_addr) : NULL); newQR->proxy_port=proxy_port; newQR->log=log; @@ -957,9 +973,19 @@ __internal_loop: // match on client address if (qr->client_addr && strlen(qr->client_addr)) { if (sess->client_myds->addr.addr) { - if (strcmp(qr->client_addr,sess->client_myds->addr.addr)!=0) { - proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has no matching client_addr\n", qr->rule_id); - continue; + if (qr->client_addr_wildcard_position == -1) { // no wildcard , old algorithm + if (strcmp(qr->client_addr,sess->client_myds->addr.addr)!=0) { + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has no matching client_addr\n", qr->rule_id); + continue; + } + } else if (qr->client_addr_wildcard_position==0) { + // catch all! + // therefore we have a match + } else { // client_addr_wildcard_position > 0 + if (strncmp(qr->client_addr,sess->client_myds->addr.addr,qr->client_addr_wildcard_position)!=0) { + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has no matching client_addr\n", qr->rule_id); + continue; + } } } } From 0e9d055000427381a5159060fc363ed849a0a6a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Fri, 6 Apr 2018 09:25:35 +0200 Subject: [PATCH 8/8] Disable brotli from curl #1458 --- deps/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/Makefile b/deps/Makefile index 2329423ac..1797e8e79 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -34,7 +34,7 @@ coredumper: google-coredumper/google-coredumper/.libs/libcoredumper.a curl/curl/lib/.libs/libcurl.a: cd curl && rm -rf curl-7.57.0 || true cd curl && tar -zxf curl-7.57.0.tar.gz - cd curl/curl && ./configure --disable-debug --disable-ftp --disable-ldap --disable-ldaps --disable-rtsp --disable-proxy --disable-dict --disable-telnet --disable-tftp --disable-pop3 --disable-imap --disable-smb --disable-smtp --disable-gopher --disable-manual --disable-ipv6 --disable-sspi --disable-crypto-auth --disable-ntlm-wb --disable-tls-srp --without-nghttp2 --without-libidn2 --without-libssh2 && CC=${CC} CXX=${CXX} ${MAKE} + cd curl/curl && ./configure --disable-debug --disable-ftp --disable-ldap --disable-ldaps --disable-rtsp --disable-proxy --disable-dict --disable-telnet --disable-tftp --disable-pop3 --disable-imap --disable-smb --disable-smtp --disable-gopher --disable-manual --disable-ipv6 --disable-sspi --disable-crypto-auth --disable-ntlm-wb --disable-tls-srp --without-nghttp2 --without-libidn2 --without-libssh2 --without-brotli && CC=${CC} CXX=${CXX} ${MAKE} curl: curl/curl/lib/.libs/libcurl.a libmicrohttpd/libmicrohttpd/src/microhttpd/.libs/libmicrohttpd.a: