diff --git a/deps/Makefile b/deps/Makefile index 87d19d87a..1797e8e79 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -34,7 +34,7 @@ coredumper: google-coredumper/google-coredumper/.libs/libcoredumper.a curl/curl/lib/.libs/libcurl.a: cd curl && rm -rf curl-7.57.0 || true cd curl && tar -zxf curl-7.57.0.tar.gz - cd curl/curl && ./configure --disable-debug --disable-ftp --disable-ldap --disable-ldaps --disable-rtsp --disable-proxy --disable-dict --disable-telnet --disable-tftp --disable-pop3 --disable-imap --disable-smb --disable-smtp --disable-gopher --disable-manual --disable-ipv6 --disable-sspi --disable-crypto-auth --disable-ntlm-wb --disable-tls-srp --without-nghttp2 --without-libidn2 && CC=${CC} CXX=${CXX} ${MAKE} + cd curl/curl && ./configure --disable-debug --disable-ftp --disable-ldap --disable-ldaps --disable-rtsp --disable-proxy --disable-dict --disable-telnet --disable-tftp --disable-pop3 --disable-imap --disable-smb --disable-smtp --disable-gopher --disable-manual --disable-ipv6 --disable-sspi --disable-crypto-auth --disable-ntlm-wb --disable-tls-srp --without-nghttp2 --without-libidn2 --without-libssh2 --without-brotli && CC=${CC} CXX=${CXX} ${MAKE} curl: curl/curl/lib/.libs/libcurl.a libmicrohttpd/libmicrohttpd/src/microhttpd/.libs/libmicrohttpd.a: diff --git a/include/mysql_connection.h b/include/mysql_connection.h index a407a0944..f8397fb53 100644 --- a/include/mysql_connection.h +++ b/include/mysql_connection.h @@ -93,6 +93,8 @@ class MySQL_Connection { bool processing_prepared_statement_execute; bool processing_multi_statement; bool multiplex_delayed; + bool unknown_transaction_status; + void compute_unknown_transaction_status(); MySQL_Connection(); ~MySQL_Connection(); bool set_autocommit(bool); diff --git a/include/query_processor.h b/include/query_processor.h index cad2de7a2..1e489371c 100644 --- a/include/query_processor.h +++ b/include/query_processor.h @@ -14,6 +14,7 @@ struct _Query_Processor_rule_t { char *schemaname; int flagIN; char *client_addr; + int client_addr_wildcard_position; char *proxy_addr; int proxy_port; uint64_t digest; diff --git a/lib/ClickHouse_Server.cpp b/lib/ClickHouse_Server.cpp index b9504d1a4..9dddcdf8d 100644 --- a/lib/ClickHouse_Server.cpp +++ b/lib/ClickHouse_Server.cpp @@ -495,7 +495,7 @@ void ClickHouse_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t if (sess->session_type == PROXYSQL_SESSION_CLICKHOUSE) { - if (!strncmp("SET ", query_no_space, 4)) { + if (!strncasecmp("SET ", query_no_space, 4)) { if ( !strncasecmp("SET AUTOCOMMIT", query_no_space, 14) || !strncasecmp("SET NAMES ", query_no_space, 10) || @@ -503,7 +503,8 @@ void ClickHouse_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t !strncasecmp("SET COLLATION", query_no_space, 13) || !strncasecmp("SET SQL_AUTO_", query_no_space, 13) || !strncasecmp("SET SQL_SAFE_", query_no_space, 13) || - !strncasecmp("SET SESSION TRANSACTION", query_no_space, 23) + !strncasecmp("SET SESSION TRANSACTION", query_no_space, 23) || + !strncasecmp("SET WAIT_TIMEOUT", query_no_space, 16) ) { GloClickHouseServer->send_MySQL_OK(&sess->client_myds->myprot, NULL); run_query=false; diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index e021ec6ab..95da46dc4 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -400,7 +400,11 @@ void * monitor_connect_thread(void *arg) { rc=sqlite3_reset(statement); assert(rc==SQLITE_OK); sqlite3_finalize(statement); if (mmsd->mysql_error_msg) { - if (strncmp(mmsd->mysql_error_msg,"Access denied for user",strlen("Access denied for user"))==0) { + if ( + (strncmp(mmsd->mysql_error_msg,"Access denied for user",strlen("Access denied for user"))==0) + || + (strncmp(mmsd->mysql_error_msg,"ProxySQL Error: Access denied for user",strlen("ProxySQL Error: Access denied for user"))==0) + ) { proxy_error("Server %s:%d is returning \"Access denied\" for monitoring user\n", mmsd->hostname, mmsd->port); } } @@ -1403,7 +1407,7 @@ __end_monitor_ping_loop: resultset=NULL; } char *new_query=NULL; - new_query=(char *)"SELECT 1 FROM (SELECT hostname,port,ping_error FROM mysql_server_ping_log WHERE hostname='%s' AND port='%s' ORDER BY time_start_us DESC LIMIT %d) a WHERE ping_error IS NOT NULL AND ping_error NOT LIKE 'Access denied for user%%' GROUP BY hostname,port HAVING COUNT(*)=%d"; + new_query=(char *)"SELECT 1 FROM (SELECT hostname,port,ping_error FROM mysql_server_ping_log WHERE hostname='%s' AND port='%s' ORDER BY time_start_us DESC LIMIT %d) a WHERE ping_error IS NOT NULL AND ping_error NOT LIKE 'Access denied for user%%' AND ping_error NOT LIKE 'ProxySQL Error: Access denied for user%%' GROUP BY hostname,port HAVING COUNT(*)=%d"; for (j=0;jsession_type) { + case PROXYSQL_SESSION_SQLITE: + case PROXYSQL_SESSION_ADMIN: + case PROXYSQL_SESSION_STATS: + internal_status += SERVER_STATUS_NO_BACKSLASH_ESCAPES; + break; + default: + break; + } + } memcpy(_ptr+l, &warnings, sizeof(uint16_t)); l+=sizeof(uint16_t); - memcpy(_ptr+l, &status, sizeof(uint16_t)); + memcpy(_ptr+l, &internal_status, sizeof(uint16_t)); if (send==true) { (*myds)->PSarrayOUT->add((void *)_ptr,size); @@ -570,7 +582,19 @@ bool MySQL_Protocol::generate_pkt_OK(bool send, void **ptr, unsigned int *len, u _ptr[l]=0x00; l++; l+=write_encoded_length(_ptr+l, affected_rows, affected_rows_len, affected_rows_prefix); l+=write_encoded_length(_ptr+l, last_insert_id, last_insert_id_len, last_insert_id_prefix); - memcpy(_ptr+l, &status, sizeof(uint16_t)); l+=sizeof(uint16_t); + int16_t internal_status = status; + if (sess) { + switch (sess->session_type) { + case PROXYSQL_SESSION_SQLITE: + case PROXYSQL_SESSION_ADMIN: + case PROXYSQL_SESSION_STATS: + internal_status += SERVER_STATUS_NO_BACKSLASH_ESCAPES; + break; + default: + break; + } + } + memcpy(_ptr+l, &internal_status, sizeof(uint16_t)); l+=sizeof(uint16_t); memcpy(_ptr+l, &warnings, sizeof(uint16_t)); l+=sizeof(uint16_t); if (msg) { l+=write_encoded_length(_ptr+l, msg_len, msg_len_len, msg_prefix); diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index 6a779397c..a95e017f0 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -942,6 +942,7 @@ int MySQL_Session::handler_again___status_PINGING_SERVER() { int rc=myconn->async_ping(myds->revents); if (rc==0) { myconn->async_state_machine=ASYNC_IDLE; + myconn->compute_unknown_transaction_status(); if (mysql_thread___multiplexing && (myconn->reusable==true) && myds->myconn->IsActiveTransaction()==false && myds->myconn->MultiplexDisabled()==false) { myds->return_MySQL_Connection_To_Pool(); } else { @@ -2796,6 +2797,7 @@ handler_again: } } else { myconn->multiplex_delayed=false; + myconn->compute_unknown_transaction_status(); myconn->async_state_machine=ASYNC_IDLE; myds->DSS=STATE_MARIADB_GENERIC; if (transaction_persistent==true) { @@ -4401,6 +4403,7 @@ void MySQL_Session::RequestEnd(MySQL_Data_Stream *myds) { // if there is a mysql connection, clean its status if (myds->myconn) { myds->myconn->async_free_result(); + myds->myconn->compute_unknown_transaction_status(); } myds->free_mysql_real_query(); } diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 51db65dd7..0fb692562 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -7070,6 +7070,22 @@ char * ProxySQL_Admin::load_mysql_query_rules_to_runtime() { QP_rule_t * nqpr; for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; + if (r->fields[4]) { + char *pct = NULL; + if (strlen(r->fields[4]) >= INET6_ADDRSTRLEN) { + proxy_error("Query rule with rule_id=%s has an invalid client_addr: %s\n", r->fields[0], r->fields[4]); + continue; + } + pct = strchr(r->fields[4],'%'); + if (pct) { // there is a wildcard + if (strlen(pct) == 1) { + // % is at the end of the string, good + } else { + proxy_error("Query rule with rule_id=%s has a wildcard that is not at the end of client_addr: %s\n", r->fields[0], r->fields[4]); + continue; + } + } + } nqpr=GloQPro->new_query_rule( atoi(r->fields[0]), // rule_id true, diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index f0407076a..9c6865a74 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -495,7 +495,23 @@ QP_rule_t * Query_Processor::new_query_rule(int rule_id, bool active, char *user newQR->regex_engine2=NULL; newQR->hits=0; + newQR->client_addr_wildcard_position = -1; // not existing by default newQR->client_addr=(client_addr ? strdup(client_addr) : NULL); + if (newQR->client_addr) { + char *pct = strchr(newQR->client_addr,'%'); + if (pct) { // there is a wildcard . We assume Admin did already all the input validation + if (pct == newQR->client_addr) { + // client_addr == '%' + // % is at the end of the string, but also at the beginning + // becoming a catch all + newQR->client_addr_wildcard_position = 0; + } else { + // this math is valid also if (pct == newQR->client_addr) + // but we separate it to clarify that client_addr_wildcard_position is a match all + newQR->client_addr_wildcard_position = strlen(newQR->client_addr) - strlen(pct); + } + } + } newQR->proxy_addr=(proxy_addr ? strdup(proxy_addr) : NULL); newQR->proxy_port=proxy_port; newQR->log=log; @@ -957,9 +973,19 @@ __internal_loop: // match on client address if (qr->client_addr && strlen(qr->client_addr)) { if (sess->client_myds->addr.addr) { - if (strcmp(qr->client_addr,sess->client_myds->addr.addr)!=0) { - proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has no matching client_addr\n", qr->rule_id); - continue; + if (qr->client_addr_wildcard_position == -1) { // no wildcard , old algorithm + if (strcmp(qr->client_addr,sess->client_myds->addr.addr)!=0) { + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has no matching client_addr\n", qr->rule_id); + continue; + } + } else if (qr->client_addr_wildcard_position==0) { + // catch all! + // therefore we have a match + } else { // client_addr_wildcard_position > 0 + if (strncmp(qr->client_addr,sess->client_myds->addr.addr,qr->client_addr_wildcard_position)!=0) { + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has no matching client_addr\n", qr->rule_id); + continue; + } } } } diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index c9e67b6fd..f02a51776 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -49,6 +49,29 @@ MySQL_Connection_userinfo::~MySQL_Connection_userinfo() { if (schemaname) free(schemaname); } +void MySQL_Connection::compute_unknown_transaction_status() { + if (mysql) { + int _myerrno=mysql_errno(mysql); + if (_myerrno == 0) { + unknown_transaction_status = false; // no error + return; + } + if (_myerrno >= 2000 && _myerrno < 3000) { // client error + // do not change it + return; + } + if (_myerrno >= 1000 && _myerrno < 2000) { // server error + unknown_transaction_status = true; + return; + } + if (_myerrno >= 3000 && _myerrno < 4000) { // server error + unknown_transaction_status = true; + return; + } + // all other cases, server error + } +} + uint64_t MySQL_Connection_userinfo::compute_hash() { int l=0; if (username) @@ -190,6 +213,7 @@ MySQL_Connection::MySQL_Connection() { multiplex_delayed=false; MyRS=NULL; MyRS_reuse=NULL; + unknown_transaction_status = false; creation_time=0; processing_multi_statement=false; proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "Creating new MySQL_Connection %p\n", this); @@ -1004,6 +1028,14 @@ handler_again: } break; case ASYNC_QUERY_END: + if (mysql) { + int _myerrno=mysql_errno(mysql); + if (_myerrno == 0) { + unknown_transaction_status = false; + } else { + compute_unknown_transaction_status(); + } + } if (mysql_result) { mysql_free_result(mysql_result); mysql_result=NULL; @@ -1160,6 +1192,7 @@ int MySQL_Connection::async_connect(short event) { return 0; } if (async_state_machine==ASYNC_CONNECT_SUCCESSFUL) { + compute_unknown_transaction_status(); async_state_machine=ASYNC_IDLE; myds->wait_until=0; creation_time = monotonic_time(); @@ -1168,6 +1201,7 @@ int MySQL_Connection::async_connect(short event) { handler(event); switch (async_state_machine) { case ASYNC_CONNECT_SUCCESSFUL: + compute_unknown_transaction_status(); async_state_machine=ASYNC_IDLE; myds->wait_until=0; return 0; @@ -1246,6 +1280,7 @@ int MySQL_Connection::async_query(short event, char *stmt, unsigned long length, } if (async_state_machine==ASYNC_QUERY_END) { + compute_unknown_transaction_status(); if (mysql_errno(mysql)) { return -1; } else { @@ -1255,6 +1290,7 @@ int MySQL_Connection::async_query(short event, char *stmt, unsigned long length, if (async_state_machine==ASYNC_STMT_EXECUTE_END) { query.stmt_meta=NULL; async_state_machine=ASYNC_QUERY_END; + compute_unknown_transaction_status(); if (mysql_stmt_errno(query.stmt)) { return -1; } else { @@ -1263,6 +1299,7 @@ int MySQL_Connection::async_query(short event, char *stmt, unsigned long length, } if (async_state_machine==ASYNC_STMT_PREPARE_SUCCESSFUL || async_state_machine==ASYNC_STMT_PREPARE_FAILED) { query.stmt_meta=NULL; + compute_unknown_transaction_status(); if (async_state_machine==ASYNC_STMT_PREPARE_FAILED) { return -1; } else { @@ -1296,6 +1333,7 @@ int MySQL_Connection::async_ping(short event) { assert(ret_mysql); switch (async_state_machine) { case ASYNC_PING_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1315,6 +1353,7 @@ int MySQL_Connection::async_ping(short event) { // check again switch (async_state_machine) { case ASYNC_PING_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1337,6 +1376,7 @@ int MySQL_Connection::async_change_user(short event) { assert(ret_mysql); switch (async_state_machine) { case ASYNC_CHANGE_USER_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1356,6 +1396,7 @@ int MySQL_Connection::async_change_user(short event) { // check again switch (async_state_machine) { case ASYNC_CHANGE_USER_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1378,6 +1419,7 @@ int MySQL_Connection::async_select_db(short event) { assert(ret_mysql); switch (async_state_machine) { case ASYNC_INITDB_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1394,6 +1436,7 @@ int MySQL_Connection::async_select_db(short event) { // check again switch (async_state_machine) { case ASYNC_INITDB_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1413,6 +1456,7 @@ int MySQL_Connection::async_set_autocommit(short event, bool ac) { assert(ret_mysql); switch (async_state_machine) { case ASYNC_SET_AUTOCOMMIT_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1430,6 +1474,7 @@ int MySQL_Connection::async_set_autocommit(short event, bool ac) { // check again switch (async_state_machine) { case ASYNC_SET_AUTOCOMMIT_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1449,6 +1494,7 @@ int MySQL_Connection::async_set_names(short event, uint8_t c) { assert(ret_mysql); switch (async_state_machine) { case ASYNC_SET_NAMES_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1466,6 +1512,7 @@ int MySQL_Connection::async_set_names(short event, uint8_t c) { // check again switch (async_state_machine) { case ASYNC_SET_NAMES_SUCCESSFUL: + unknown_transaction_status = false; async_state_machine=ASYNC_IDLE; return 0; break; @@ -1510,6 +1557,7 @@ void MySQL_Connection::async_free_result() { mysql_result=NULL; } } + compute_unknown_transaction_status(); async_state_machine=ASYNC_IDLE; if (MyRS) { if (MyRS_reuse) { @@ -1525,7 +1573,7 @@ bool MySQL_Connection::IsActiveTransaction() { bool ret=false; if (mysql) { ret = (mysql->server_status & SERVER_STATUS_IN_TRANS); - if (ret == false && (mysql)->net.last_errno) { + if (ret == false && (mysql)->net.last_errno && unknown_transaction_status == true) { ret = true; } if (ret == false) { @@ -1710,6 +1758,7 @@ int MySQL_Connection::async_send_simple_command(short event, char *stmt, unsigne proxy_error("Retrieved a resultset while running a simple command. This is an error!! Simple command: %s\n", stmt); } if (async_state_machine==ASYNC_QUERY_END) { + compute_unknown_transaction_status(); if (mysql_errno(mysql)) { return -1; } else {