From b0f31c97841ba2f449c91bde4fa9402cf99ea7a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Mon, 31 Oct 2016 07:57:11 +0000 Subject: [PATCH 01/21] Allow rules on proxy_port without proxy_addr #712 --- lib/Query_Processor.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index c27f9f7f7..b523c4c63 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -718,19 +718,21 @@ __internal_loop: } } - // match on proxy_addr & proxy_port + // match on proxy_addr if (qr->proxy_addr && strlen(qr->proxy_addr)) { if (sess->client_myds->proxy_addr.addr) { if (strcmp(qr->proxy_addr,sess->client_myds->proxy_addr.addr)!=0) { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has no matching proxy_addr\n", qr->rule_id); continue; } - if (qr->proxy_port>=0) { - if (qr->proxy_port!=sess->client_myds->proxy_addr.port) { - proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has no matching proxy_port\n", qr->rule_id); - continue; - } - } + } + } + + // match on proxy_port + if (qr->proxy_port>=0) { + if (qr->proxy_port!=sess->client_myds->proxy_addr.port) { + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has no matching proxy_port\n", qr->rule_id); + continue; } } From ec558d58c0f18c42b2687161062af20ee95a2e7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Wed, 9 Nov 2016 19:12:41 +0000 Subject: [PATCH 02/21] SHOW TABLES FROM xxx is now sorted #788 --- lib/ProxySQL_Admin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index ad2f20ad0..9d8ce966e 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -1922,7 +1922,7 @@ void admin_session_handler(MySQL_Session *sess, ProxySQL_Admin *pa, PtrSize_t *p if ((query_no_space_length>17) && (!strncasecmp("SHOW TABLES FROM ", query_no_space, 17))) { strA=query_no_space+17; strAl=strlen(strA); - strB=(char *)"SELECT name AS tables FROM %s.sqlite_master WHERE type='table' AND name NOT IN ('sqlite_sequence')"; + strB=(char *)"SELECT name AS tables FROM %s.sqlite_master WHERE type='table' AND name NOT IN ('sqlite_sequence') ORDER BY name"; strBl=strlen(strB); int l=strBl+strAl-2; char *b=(char *)l_alloc(l+1); From a689a5ce054cfce7fb5060703945c0559bdcae7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Wed, 9 Nov 2016 23:36:52 +0000 Subject: [PATCH 03/21] Fix bug #774 --- lib/mysql_connection.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index 9dbb6ddae..56927c5e2 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -1106,6 +1106,8 @@ int MySQL_Connection::async_query(short event, char *stmt, unsigned long length, (parent->status==MYSQL_SERVER_STATUS_OFFLINE_HARD) // the server is OFFLINE as specific by the user || (parent->status==MYSQL_SERVER_STATUS_SHUNNED && parent->shunned_automatic==true && parent->shunned_and_kill_all_connections==true) // the server is SHUNNED due to a serious issue + || + (parent->status==MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) // slave is lagging! see #774 ) { return -1; } From 1a805cfcf251350778343da0fab0c6d5e076d92e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Thu, 10 Nov 2016 01:14:56 +0000 Subject: [PATCH 04/21] Removed unused variables from Monitor --- lib/MySQL_Monitor.cpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index a6c25c151..6500625e5 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -907,7 +907,6 @@ void * MySQL_Monitor::monitor_connect() { unsigned long long t1; unsigned long long t2; unsigned long long next_loop_at=0; - unsigned long long start_time; while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true) { char *error=NULL; @@ -932,8 +931,6 @@ void * MySQL_Monitor::monitor_connect() { } next_loop_at=t1+1000*mysql_thread___monitor_connect_interval; - start_time=monotonic_time(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); if (error) { @@ -1018,7 +1015,6 @@ void * MySQL_Monitor::monitor_ping() { unsigned long long t1; unsigned long long t2; - unsigned long long start_time; unsigned long long next_loop_at=0; while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true) { @@ -1044,8 +1040,6 @@ void * MySQL_Monitor::monitor_ping() { } next_loop_at=t1+1000*mysql_thread___monitor_ping_interval; - start_time=monotonic_time(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); if (error) { @@ -1241,7 +1235,6 @@ void * MySQL_Monitor::monitor_read_only() { unsigned long long t1; unsigned long long t2; - unsigned long long start_time; unsigned long long next_loop_at=0; while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true) { @@ -1265,7 +1258,6 @@ void * MySQL_Monitor::monitor_read_only() { if (t1 < next_loop_at) { goto __sleep_monitor_read_only; } - start_time=monotonic_time(); next_loop_at=t1+1000*mysql_thread___monitor_read_only_interval; proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); // admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); @@ -1359,7 +1351,6 @@ void * MySQL_Monitor::monitor_replication_lag() { unsigned long long t1; unsigned long long t2; - unsigned long long start_time; unsigned long long next_loop_at=0; while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true) { @@ -1384,7 +1375,6 @@ void * MySQL_Monitor::monitor_replication_lag() { goto __sleep_monitor_replication_lag; } next_loop_at=t1+1000*mysql_thread___monitor_replication_lag_interval; - start_time=t1; proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); // admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); From 274fa25a62d6fbb0851ddb931cd8913e16fab466 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sat, 29 Oct 2016 16:23:21 +0000 Subject: [PATCH 05/21] Fix some race condition during PROXYSQL RESTART --- lib/MySQL_Thread.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index b2c786ac7..c1c8d3b10 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -2092,7 +2092,7 @@ __run_skip_1: if (idle_maintenance_thread==false) { int r=rand()%(GloMTH->num_threads); MySQL_Thread *thr=GloMTH->mysql_threads_idles[r].worker; - if (idle_mysql_sessions->len) { + if (shutdown==0 && thr->shudown==0 idle_mysql_sessions->len) { unsigned int ims=0; //spin_wrlock(&GloMTH->rwlock_idles); pthread_mutex_lock(&thr->myexchange.mutex_idles); @@ -2480,6 +2480,7 @@ __run_skip_2: //spin_wrlock(&GloMTH->rwlock_resumes); pthread_mutex_lock(&thr->myexchange.mutex_resumes); unsigned int ims; + if (shutdown==0 && thr->shudown==0) for (ims=0; imslen; ims++) { MySQL_Session *mysess=(MySQL_Session *)resume_mysql_sessions->remove_index_fast(0); thr->myexchange.resume_mysql_sessions->add(mysess); @@ -2499,7 +2500,7 @@ __run_skip_2: } else { //spin_wrlock(&GloMTH->rwlock_resumes); pthread_mutex_lock(&thr->myexchange.mutex_resumes); - if (thr->myexchange.resume_mysql_sessions->len) { + if (shutdown==0 && thr->shudown==0 && thr->myexchange.resume_mysql_sessions->len) { //unsigned int w=rand()%(GloMTH->num_threads); //w++; unsigned char c=0; From c65866c96fb243608df0006d1c6f0ac58aa96481 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sun, 13 Nov 2016 00:12:00 +0000 Subject: [PATCH 06/21] Access denied reports source #795 --- lib/MySQL_Session.cpp | 89 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 6 deletions(-) diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index 5faf9ba6d..0b5c16fa9 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -2508,8 +2508,35 @@ void MySQL_Session::handler___status_CHANGING_USER_CLIENT___STATE_CLIENT_HANDSHA *wrong_pass=true; // FIXME: this should become close connection client_myds->setDSS_STATE_QUERY_SENT_NET(); - char *_s=(char *)malloc(strlen(client_myds->myconn->userinfo->username)+100); - sprintf(_s,"ProxySQL Error: Access denied for user '%s' (using password: %s)", client_myds->myconn->userinfo->username, (client_myds->myconn->userinfo->password ? "YES" : "NO")); + char *client_addr=NULL; + if (client_myds->client_addr) { + char buf[512]; + switch (client_myds->client_addr->sa_family) { + case AF_INET: { + struct sockaddr_in *ipv4 = (struct sockaddr_in *)client_myds->client_addr; + if (ipv4->sin_port) { + inet_ntop(client_myds->client_addr->sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); + client_addr = strdup(buf); + } else { + client_addr = strdup((char *)"localhost"); + } + break; + } + case AF_INET6: { + struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)client_myds->client_addr; + inet_ntop(client_myds->client_addr->sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); + client_addr = strdup(buf); + break; + } + default: + client_addr = strdup((char *)"localhost"); + break; + } + } else { + client_addr = strdup((char *)""); + } + char *_s=(char *)malloc(strlen(client_myds->myconn->userinfo->username)+100+strlen(client_addr)); + sprintf(_s,"ProxySQL Error: Access denied for user '%s'@'%s' (using password: %s)", client_myds->myconn->userinfo->username, client_addr, (client_myds->myconn->userinfo->password ? "YES" : "NO")); client_myds->myprot.generate_pkt_ERR(true,NULL,NULL,2,1045,(char *)"28000", _s); free(_s); } @@ -2582,8 +2609,35 @@ void MySQL_Session::handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE( *wrong_pass=true; // FIXME: this should become close connection client_myds->setDSS_STATE_QUERY_SENT_NET(); - char *_s=(char *)malloc(strlen(client_myds->myconn->userinfo->username)+100); - sprintf(_s,"ProxySQL Error: Access denied for user '%s' (using password: %s)", client_myds->myconn->userinfo->username, (client_myds->myconn->userinfo->password ? "YES" : "NO")); + char *client_addr=NULL; + if (client_myds->client_addr) { + char buf[512]; + switch (client_myds->client_addr->sa_family) { + case AF_INET: { + struct sockaddr_in *ipv4 = (struct sockaddr_in *)client_myds->client_addr; + if (ipv4->sin_port) { + inet_ntop(client_myds->client_addr->sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); + client_addr = strdup(buf); + } else { + client_addr = strdup((char *)"localhost"); + } + break; + } + case AF_INET6: { + struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)client_myds->client_addr; + inet_ntop(client_myds->client_addr->sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); + client_addr = strdup(buf); + break; + } + default: + client_addr = strdup((char *)"localhost"); + break; + } + } else { + client_addr = strdup((char *)""); + } + char *_s=(char *)malloc(strlen(client_myds->myconn->userinfo->username)+100+strlen(client_addr)); + sprintf(_s,"ProxySQL Error: Access denied for user '%s'@'%s' (using password: %s)", client_myds->myconn->userinfo->username, client_addr, (client_myds->myconn->userinfo->password ? "YES" : "NO")); client_myds->myprot.generate_pkt_ERR(true,NULL,NULL,2,1045,(char *)"28000", _s); __sync_add_and_fetch(&MyHGM->status.client_connections_aborted,1); free(_s); @@ -2846,8 +2900,31 @@ void MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C *wrong_pass=true; // FIXME: this should become close connection client_myds->setDSS_STATE_QUERY_SENT_NET(); - char *_s=(char *)malloc(strlen(client_myds->myconn->userinfo->username)+100); - sprintf(_s,"ProxySQL Error: Access denied for user '%s' (using password: %s)", client_myds->myconn->userinfo->username, (client_myds->myconn->userinfo->password ? "YES" : "NO")); + char *client_addr=NULL; + if (client_myds->client_addr) { + char buf[512]; + switch (client_myds->client_addr->sa_family) { + case AF_INET: { + struct sockaddr_in *ipv4 = (struct sockaddr_in *)client_myds->client_addr; + inet_ntop(client_myds->client_addr->sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); + client_addr = strdup(buf); + break; + } + case AF_INET6: { + struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)client_myds->client_addr; + inet_ntop(client_myds->client_addr->sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); + client_addr = strdup(buf); + break; + } + default: + client_addr = strdup((char *)"localhost"); + break; + } + } else { + client_addr = strdup((char *)""); + } + char *_s=(char *)malloc(strlen(client_myds->myconn->userinfo->username)+100+strlen(client_addr)); + sprintf(_s,"ProxySQL Error: Access denied for user '%s'@'%s' (using password: %s)", client_myds->myconn->userinfo->username, client_addr, (client_myds->myconn->userinfo->password ? "YES" : "NO")); client_myds->myprot.generate_pkt_ERR(true,NULL,NULL,2,1045,(char *)"28000", _s); free(_s); } From af9cb53c5ddab6c3159669c283bbdc632e3e632f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sun, 13 Nov 2016 07:35:51 +0000 Subject: [PATCH 07/21] Several memory sanitizer Conflicts: lib/MySQL_Thread.cpp src/main.cpp --- deps/Makefile | 1 + lib/MySQL_Thread.cpp | 6 +++++- lib/ProxySQL_Admin.cpp | 5 ++++- src/main.cpp | 8 +++++--- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/deps/Makefile b/deps/Makefile index 9f0069454..7eb74c0b5 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -58,6 +58,7 @@ re2/re2/obj/libre2.a: cd re2 && tar -zxf re2-20140304.tgz cd re2/re2 && sed -i -e 's/-O3 -g /-O3 -fPIC /' Makefile # cd re2 && patch re2/util/mutex.h < mutex.h.patch + cd re2/re2 && sed -i -e 's/-O3 /-O3 -DMEMORY_SANITIZER /' Makefile cd re2/re2 && CC=${CC} CXX=${CXX} ${MAKE} re2: re2/re2/obj/libre2.a diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index c1c8d3b10..2ea3b9d1b 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -2499,8 +2499,10 @@ __run_skip_2: } } else { //spin_wrlock(&GloMTH->rwlock_resumes); + VALGRIND_DISABLE_ERROR_REPORTING; pthread_mutex_lock(&thr->myexchange.mutex_resumes); - if (shutdown==0 && thr->shudown==0 && thr->myexchange.resume_mysql_sessions->len) { + VALGRIND_ENABLE_ERROR_REPORTING; + if (shutdown==0 && thr->shutdown==0 && thr->myexchange.resume_mysql_sessions->len) { //unsigned int w=rand()%(GloMTH->num_threads); //w++; unsigned char c=0; @@ -2511,7 +2513,9 @@ __run_skip_2: } } //spin_wrunlock(&GloMTH->rwlock_resumes); + VALGRIND_DISABLE_ERROR_REPORTING; pthread_mutex_unlock(&thr->myexchange.mutex_resumes); + VALGRIND_ENABLE_ERROR_REPORTING; } } else { // iterate through all sessions and process the session logic diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 9d8ce966e..0bc840ebb 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -2409,7 +2409,10 @@ __end_while_pool: char *add=NULL; char *port=NULL; close(fds[i].fd); c_split_2(socket_names[i], ":" , &add, &port); - if (atoi(port)==0) { unlink(socket_names[i]); } + if (atoi(port)==0) { + if (socket_names[i]) + unlink(socket_names[i]); + } } free(arg); return NULL; diff --git a/src/main.cpp b/src/main.cpp index b8a888a0f..a7da9e7c7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -128,9 +128,11 @@ using namespace std; static volatile int load_; //__thread l_sfp *__thr_sfp=NULL; - -//const char *malloc_conf = "xmalloc:true,lg_tcache_max:16,purge:decay"; -const char *malloc_conf = "xmalloc:true,lg_chunk:18,lg_tcache_max:12,purge:ratio"; +//#ifdef DEBUG +//const char *malloc_conf = "xmalloc:true,lg_tcache_max:16,purge:decay,junk:true,tcache:false"; +//#else +const char *malloc_conf = "xmalloc:true,lg_tcache_max:16,purge:decay"; +//#endif /* DEBUG */ //const char *malloc_conf = "prof_leak:true,lg_prof_sample:0,prof_final:true,xmalloc:true,lg_tcache_max:16"; int listen_fd; From 7de3c9e073a2d0e9df1f9c5043eb7a4692656602 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sun, 13 Nov 2016 07:55:30 +0000 Subject: [PATCH 08/21] mysql_servers.comment not loaded at runtime #787 See also #782 --- lib/MySQL_HostGroups_Manager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index c35517d5b..4f6e9f2cc 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -460,8 +460,8 @@ bool MySQL_HostGroups_Manager::commit() { proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing max_latency_ms for server %s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[9] , mysrvc->max_latency_us , atoi(r->fields[18])); mysrvc->max_latency_us=1000*atoi(r->fields[18]); } - if (atoi(r->fields[10])!=atoi(r->fields[19])) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing comment for server %s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[10] , mysrvc->max_latency_us , atoi(r->fields[19])); + if (strcmp(r->fields[10],r->fields[19])) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing comment for server %s:%d (%s:%d) from '%s' to '%s'\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[10], r->fields[19]); free(mysrvc->comment); mysrvc->comment=strdup(r->fields[19]); } From ffa266ed060d0a73c3291c58730568ed0727d0c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Mon, 14 Nov 2016 03:23:36 +0000 Subject: [PATCH 09/21] Added release note for v1.3.0g --- doc/release_notes/ProxySQL_v1.3.0g.md | 97 +++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 doc/release_notes/ProxySQL_v1.3.0g.md diff --git a/doc/release_notes/ProxySQL_v1.3.0g.md b/doc/release_notes/ProxySQL_v1.3.0g.md new file mode 100644 index 000000000..ceed17823 --- /dev/null +++ b/doc/release_notes/ProxySQL_v1.3.0g.md @@ -0,0 +1,97 @@ +# ProxySQL v1.3.0 , v1.3.0f v1.3.0g + +## ProxySQL v1.3.0 + +Release date: 2016-10-19 + +### Performance improvement + +* support for millions of connections. It is also able to handle workloads with hundreds thousands client connections, but only few connections are active. For example, 100k total connections, but only 100 active connections. +* use of pthread_mutex for connection pool +* several performance improvements + + +### Usability improvement + +* Packaging: version number now includes commit hash +* Monitor: limit the number of Monitor thread to 16 + + +### New features + +* EXPERIMENTAL support for proepared statements +* Connection Pool: each MySQL_Thread now has a pair thread, therefore the number of threads executed are mysql-threads x 2 . The second set of MySQL_Threads are responsible to only handle idle connections. + It also introduced two new global variables and a new global status: + * `mysql-session_idle_ms` (default 1000) : when a session is idle for that long, it is moved to a thread responsible to handle idle connections + * `mysql-session_idle_show_processlist` (default false) : specifies if idle connections are displayed on SHOW PROCESSLIST or on any query against `stats_mysql_processlist` + * `Client_Connections_non_idle` : returns the number of client connections that are not idle, therefore handled by the main MySQL_Threads and not moved to the second set of MySQL_Threads responsible to only handle idle connections +* Network: add support for IPv6 [#726](../../../../issues/726) and [#460](../../../../issues/460), thanks to @ton31337 +* Connection Pool: support for SO_REUSEPORT, it can be only enabled on the command line with -r or --reuseport at startup +* Query Processor: added new variable mysql-digests_lowercase to always set digest to lowercase [#725](../../../../issues/725) + + +### Bug fixes + +* Query Processor: rules with only digest were matching everything [#717](../../../../issues/717) +* Query Processor: rules without digest where incorrectly displayed [#719](../../../../issues/719) +* General: Unix Socket Domain file was not removed at shutdown [#714](../../../../issues/714) +* MySQL Protocol: upgrade from mariadb-connector-c 2.1.0 to 2.3.1 due to several bugs +* Connection Pool: server is not shunned if max_user_connections is reached for user [#737](../../../../issues/737) + + +## ProxySQL v1.3.0f + + +Release date: 2016-10-29 +Compared to v1.3.0: + +### New features + +* killed query because of query_timeout returns error 1907 [#750](../../../../issues/750) +* removes any lower limit for query_timeout (before was 300ms) +* added support for `STMT_SEND_LONG_DATA` [#764](../../../../issues/764) +* new customer error codes for `Max connect timeout` [#761](../../../../issues/761) and [wiki](https://github.com/sysown/proxysql/wiki/Error-codes) + +### Bug fixes + +* `mysql_query_rules`.`log` not working [#723](../../../../issues/723) +* clients are disconnected if server is running for less than 8 hours [#744](../../../../issues/744) +* remove annoying error from Admin when running `\s` [#745](../../../../issues/745) +* escaping errors on `INSERT INTO stats_mysql_processlist` [#746](../../../../issues/746) +* kill threads were using wrong credentials due to encrypted passwords +* on slow network, client connections could become stall +* `session_idle_ms` was processed as us instead of ms +* `mysql_close` was trying to close prepared statements on already closed connection [#765](../../../../issues/765) + + + +## ProxySQL v1.3.0g + +Release date: 2016-11-14 +Compared to v1.3.0f + +### New features + +* when a connection attempts receive error 1226, retry mechanism is disabled [#786](../../../../issues/786) +* better support for IPv6 +* more verbose message on connection timeout [#776](../../../../issues/776) +* removed some race conditions during `PROXYSQL RESTART` +* `Access denied` message reports source (client) [#795](../../../../issues/795) +* `SHOW TABLES FROM xxx` is now sorted [#788](../../../../issues/788) +* allows rules on proxy_port without proxy_addr [#712](../../../../issues/712) +* Monitor threads are started without arena cache +* Monitor threads are started with 64KB stack + +### Bug fixes + +* validate `mysql` state before calling `mysql_stmt_free_result()` [#779](../../../../issues/779) +* `SHUNNED_REPLICATION_LAG` does not stop queries [#774](../../../../issues/774) +* `mysql_servers`.`comment` not loaded at runtime [#787](../../../../issues/787) +* fixed two memory leak in Monitor , reported in [#766](../../../../issues/766) and [#796](../../../../issues/796) + +### Others + +* upgraded jemalloc from 4.2.1 to 4.3.1 +* jemalloc is compiled with `--enable-prof` +* added few memory sanitizers + From 56ec512493c1d5d5bbeab4c944f3867583c8c8cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Mon, 14 Nov 2016 04:35:14 +0000 Subject: [PATCH 10/21] Typo from wrong commit --- lib/MySQL_Thread.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index 2ea3b9d1b..afbbf3050 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -2092,7 +2092,7 @@ __run_skip_1: if (idle_maintenance_thread==false) { int r=rand()%(GloMTH->num_threads); MySQL_Thread *thr=GloMTH->mysql_threads_idles[r].worker; - if (shutdown==0 && thr->shudown==0 idle_mysql_sessions->len) { + if (shutdown==0 && thr->shutdown==0 && idle_mysql_sessions->len) { unsigned int ims=0; //spin_wrlock(&GloMTH->rwlock_idles); pthread_mutex_lock(&thr->myexchange.mutex_idles); @@ -2480,7 +2480,7 @@ __run_skip_2: //spin_wrlock(&GloMTH->rwlock_resumes); pthread_mutex_lock(&thr->myexchange.mutex_resumes); unsigned int ims; - if (shutdown==0 && thr->shudown==0) + if (shutdown==0 && thr->shutdown==0) for (ims=0; imslen; ims++) { MySQL_Session *mysess=(MySQL_Session *)resume_mysql_sessions->remove_index_fast(0); thr->myexchange.resume_mysql_sessions->add(mysess); From 244a0521881f166a3542bf42e37685d3ce21cecf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Mon, 14 Nov 2016 04:43:32 +0000 Subject: [PATCH 11/21] Preparation for v1.3.0g --- Makefile | 2 +- docker/images/proxysql/centos67-build/proxysql.spec | 2 +- docker/images/proxysql/centos7-build/proxysql.spec | 2 +- docker/images/proxysql/debian-7.8-build/proxysql.ctl | 2 +- docker/images/proxysql/debian-8.2-build/proxysql.ctl | 2 +- docker/images/proxysql/fedora24-build/proxysql.spec | 2 +- docker/images/proxysql/ubuntu-12.04-build/proxysql.ctl | 2 +- docker/images/proxysql/ubuntu-14.04-build/proxysql.ctl | 2 +- docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 1aa28ce41..84b42bc7f 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ DEBUG=${ALL_DEBUG} #export DEBUG #export OPTZ #export EXTRALINK -CURVER=1.3.0f +CURVER=1.3.0g MAKEOPT="-j 8" DISTRO := $(shell gawk -F= '/^NAME/{print $$2}' /etc/os-release) ifeq ($(wildcard /usr/lib/systemd/systemd), /usr/lib/systemd/systemd) diff --git a/docker/images/proxysql/centos67-build/proxysql.spec b/docker/images/proxysql/centos67-build/proxysql.spec index 8824da08e..e62873b3f 100644 --- a/docker/images/proxysql/centos67-build/proxysql.spec +++ b/docker/images/proxysql/centos67-build/proxysql.spec @@ -7,7 +7,7 @@ Summary: A high-performance MySQL proxy Name: proxysql -Version: 1.3.0f +Version: 1.3.0g Release: 1 License: GPL+ Group: Development/Tools diff --git a/docker/images/proxysql/centos7-build/proxysql.spec b/docker/images/proxysql/centos7-build/proxysql.spec index 8824da08e..e62873b3f 100644 --- a/docker/images/proxysql/centos7-build/proxysql.spec +++ b/docker/images/proxysql/centos7-build/proxysql.spec @@ -7,7 +7,7 @@ Summary: A high-performance MySQL proxy Name: proxysql -Version: 1.3.0f +Version: 1.3.0g Release: 1 License: GPL+ Group: Development/Tools diff --git a/docker/images/proxysql/debian-7.8-build/proxysql.ctl b/docker/images/proxysql/debian-7.8-build/proxysql.ctl index 08519c7c6..2d533f855 100644 --- a/docker/images/proxysql/debian-7.8-build/proxysql.ctl +++ b/docker/images/proxysql/debian-7.8-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.3.0f +Version: 1.3.0g Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md diff --git a/docker/images/proxysql/debian-8.2-build/proxysql.ctl b/docker/images/proxysql/debian-8.2-build/proxysql.ctl index 08519c7c6..2d533f855 100644 --- a/docker/images/proxysql/debian-8.2-build/proxysql.ctl +++ b/docker/images/proxysql/debian-8.2-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.3.0f +Version: 1.3.0g Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md diff --git a/docker/images/proxysql/fedora24-build/proxysql.spec b/docker/images/proxysql/fedora24-build/proxysql.spec index 8824da08e..e62873b3f 100644 --- a/docker/images/proxysql/fedora24-build/proxysql.spec +++ b/docker/images/proxysql/fedora24-build/proxysql.spec @@ -7,7 +7,7 @@ Summary: A high-performance MySQL proxy Name: proxysql -Version: 1.3.0f +Version: 1.3.0g Release: 1 License: GPL+ Group: Development/Tools diff --git a/docker/images/proxysql/ubuntu-12.04-build/proxysql.ctl b/docker/images/proxysql/ubuntu-12.04-build/proxysql.ctl index 08519c7c6..2d533f855 100644 --- a/docker/images/proxysql/ubuntu-12.04-build/proxysql.ctl +++ b/docker/images/proxysql/ubuntu-12.04-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.3.0f +Version: 1.3.0g Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md diff --git a/docker/images/proxysql/ubuntu-14.04-build/proxysql.ctl b/docker/images/proxysql/ubuntu-14.04-build/proxysql.ctl index 08519c7c6..2d533f855 100644 --- a/docker/images/proxysql/ubuntu-14.04-build/proxysql.ctl +++ b/docker/images/proxysql/ubuntu-14.04-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.3.0f +Version: 1.3.0g Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md diff --git a/docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl b/docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl index 08519c7c6..2d533f855 100644 --- a/docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl +++ b/docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.3.0f +Version: 1.3.0g Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md From 8750a34378747dc03dc8c9e403946449c1c792bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Mon, 14 Nov 2016 15:23:44 +0000 Subject: [PATCH 12/21] Crashing bug in connect timeout --- lib/mysql_connection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index 56927c5e2..c3519b26e 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -592,7 +592,7 @@ handler_again: break; case ASYNC_CONNECT_TIMEOUT: //proxy_error("Connect timeout on %s:%d : %llu - %llu = %llu\n", parent->address, parent->port, myds->sess->thread->curtime , myds->wait_until, myds->sess->thread->curtime - myds->wait_until); - proxy_error("Connect timeout on %s:%d : exceeded by %lluus\n", myds->sess->thread->curtime - myds->wait_until); + proxy_error("Connect timeout on %s:%d : exceeded by %lluus\n", parent->address, parent->port, myds->sess->thread->curtime, myds->sess->thread->curtime - myds->wait_until); parent->connect_error(mysql_errno(mysql)); break; case ASYNC_CHANGE_USER_START: From 603239922de638f3b945dc58c1faf2d2db12e4e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Thu, 24 Nov 2016 17:55:30 +0000 Subject: [PATCH 13/21] Fixed wrong message during timeout --- lib/mysql_connection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index c3519b26e..4fa45997f 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -592,7 +592,7 @@ handler_again: break; case ASYNC_CONNECT_TIMEOUT: //proxy_error("Connect timeout on %s:%d : %llu - %llu = %llu\n", parent->address, parent->port, myds->sess->thread->curtime , myds->wait_until, myds->sess->thread->curtime - myds->wait_until); - proxy_error("Connect timeout on %s:%d : exceeded by %lluus\n", parent->address, parent->port, myds->sess->thread->curtime, myds->sess->thread->curtime - myds->wait_until); + proxy_error("Connect timeout on %s:%d : exceeded by %lluus\n", parent->address, parent->port, myds->sess->thread->curtime - myds->wait_until); parent->connect_error(mysql_errno(mysql)); break; case ASYNC_CHANGE_USER_START: From 95ffe7303ff9590e4d2d836ec70fec275fc9c73a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Thu, 24 Nov 2016 18:01:13 +0000 Subject: [PATCH 14/21] Increased mysql-monitor_ping_timeout #803 Increased mysql-monitor_ping_timeout from 100 to 1000 milliseconds. Also increased monitor_read_only_timeout from 100 to 800 milliseconds as read_only timeout is even more important than ping timeout. --- lib/MySQL_Thread.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index afbbf3050..a39787307 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -313,9 +313,9 @@ MySQL_Threads_Handler::MySQL_Threads_Handler() { variables.monitor_connect_timeout=200; variables.monitor_ping_interval=60000; variables.monitor_ping_max_failures=3; - variables.monitor_ping_timeout=100; + variables.monitor_ping_timeout=1000; variables.monitor_read_only_interval=1000; - variables.monitor_read_only_timeout=100; + variables.monitor_read_only_timeout=800; variables.monitor_replication_lag_interval=10000; variables.monitor_replication_lag_timeout=1000; variables.monitor_query_interval=60000; From 0cfb3511903e700ebad64b2a3a7bdf71ff1c35a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Thu, 24 Nov 2016 18:10:54 +0000 Subject: [PATCH 15/21] monitor_read_only_timeout was ignored #805 --- lib/MySQL_Monitor.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 6500625e5..2ea9f1ba3 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -607,9 +607,9 @@ void * monitor_read_only_thread(void *arg) { while (mmsd->async_exit_status) { mmsd->async_exit_status=wait_for_mysql(mmsd->mysql, mmsd->async_exit_status); unsigned long long now=monotonic_time(); - if (now > mmsd->t1 + mysql_thread___monitor_ping_timeout * 1000) { + if (now > mmsd->t1 + mysql_thread___monitor_read_only_timeout * 1000) { mmsd->mysql_error_msg=strdup("timeout check"); - proxy_error("Timeout on read_only check for %s:%d. If the server is overload, increase mysql-monitor_read_only_timeout. Assuming read_only=1\n", mmsd->hostname, mmsd->port); + proxy_error("Timeout on read_only check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_read_only_timeout. Assuming read_only=1\n", mmsd->hostname, mmsd->port, now-mmsd->t1); goto __exit_monitor_read_only_thread; } if (GloMyMon->shutdown==true) { @@ -623,9 +623,9 @@ void * monitor_read_only_thread(void *arg) { while (mmsd->async_exit_status) { mmsd->async_exit_status=wait_for_mysql(mmsd->mysql, mmsd->async_exit_status); unsigned long long now=monotonic_time(); - if (now > mmsd->t1 + mysql_thread___monitor_ping_timeout * 1000) { + if (now > mmsd->t1 + mysql_thread___monitor_read_only_timeout * 1000) { mmsd->mysql_error_msg=strdup("timeout check"); - proxy_error("Timeout on read_only check for %s:%d. If the server is overload, increase mysql-monitor_read_only_timeout. Assuming read_only=1\n", mmsd->hostname, mmsd->port); + proxy_error("Timeout on read_only check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_read_only_timeout. Assuming read_only=1\n", mmsd->hostname, mmsd->port, now-mmsd->t1); goto __exit_monitor_read_only_thread; } if (GloMyMon->shutdown==true) { From 597130e367ceaf2ba3033812eea9f981ac86b69a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Thu, 24 Nov 2016 18:34:29 +0000 Subject: [PATCH 16/21] Integer overflow for mysql-monitor_history #804 --- lib/MySQL_Monitor.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 2ea9f1ba3..fe24dd00d 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -971,7 +971,7 @@ __end_monitor_connect_loop: mysql_thread___monitor_history = mysql_thread___monitor_ping_interval * (mysql_thread___monitor_ping_max_failures + 1 ); } unsigned long long time_now=realtime_time(); - rc=sqlite3_bind_int64(statement, 1, time_now-mysql_thread___monitor_history*1000); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement, 1, time_now-(unsigned long long)mysql_thread___monitor_history*1000); assert(rc==SQLITE_OK); SAFE_SQLITE3_STEP(statement); rc=sqlite3_clear_bindings(statement); assert(rc==SQLITE_OK); rc=sqlite3_reset(statement); assert(rc==SQLITE_OK); @@ -1079,7 +1079,7 @@ __end_monitor_ping_loop: mysql_thread___monitor_history = mysql_thread___monitor_ping_interval * (mysql_thread___monitor_ping_max_failures + 1 ); } unsigned long long time_now=realtime_time(); - rc=sqlite3_bind_int64(statement, 1, time_now-mysql_thread___monitor_history*1000); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement, 1, time_now-(unsigned long long)mysql_thread___monitor_history*1000); assert(rc==SQLITE_OK); SAFE_SQLITE3_STEP(statement); rc=sqlite3_clear_bindings(statement); assert(rc==SQLITE_OK); rc=sqlite3_reset(statement); assert(rc==SQLITE_OK); @@ -1304,7 +1304,7 @@ __end_monitor_read_only_loop: mysql_thread___monitor_history = mysql_thread___monitor_ping_interval * (mysql_thread___monitor_ping_max_failures + 1 ); } unsigned long long time_now=realtime_time(); - rc=sqlite3_bind_int64(statement, 1, time_now-mysql_thread___monitor_history*1000); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement, 1, time_now-(unsigned long long)mysql_thread___monitor_history*1000); assert(rc==SQLITE_OK); SAFE_SQLITE3_STEP(statement); rc=sqlite3_clear_bindings(statement); assert(rc==SQLITE_OK); rc=sqlite3_reset(statement); assert(rc==SQLITE_OK); @@ -1418,7 +1418,7 @@ __end_monitor_replication_lag_loop: mysql_thread___monitor_history = mysql_thread___monitor_ping_interval * (mysql_thread___monitor_ping_max_failures + 1 ); } unsigned long long time_now=realtime_time(); - rc=sqlite3_bind_int64(statement, 1, time_now-mysql_thread___monitor_history*1000); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement, 1, time_now-(unsigned long long)mysql_thread___monitor_history*1000); assert(rc==SQLITE_OK); SAFE_SQLITE3_STEP(statement); rc=sqlite3_clear_bindings(statement); assert(rc==SQLITE_OK); rc=sqlite3_reset(statement); assert(rc==SQLITE_OK); From 909d2d57a39469bba94f287330c0c093906bb98c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Thu, 24 Nov 2016 19:36:48 +0000 Subject: [PATCH 17/21] Fix memory leak #796 --- lib/MySQL_Session.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index 0b5c16fa9..1a1317cd0 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -1656,7 +1656,7 @@ __get_pkts_from_client: mybe->server_myds->wait_until=0; pause_until=0; mybe->server_myds->killed_at=0; - //mybe->server_myds->mysql_real_query.init(&pkt); + mybe->server_myds->mysql_real_query.init(&pkt); // fix memory leak for PREPARE in prepared statements #796 client_myds->setDSS_STATE_QUERY_SENT_NET(); } } From 6b4052e5f9cedd172f9793fe6dabd33a87e71aea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Thu, 24 Nov 2016 22:07:49 +0000 Subject: [PATCH 18/21] Possible fix for race condition of #774 --- include/mysql_connection.h | 1 + lib/MySQL_Session.cpp | 7 +++++-- lib/mysql_connection.cpp | 7 ++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/include/mysql_connection.h b/include/mysql_connection.h index 184a9599a..a94eadc8b 100644 --- a/include/mysql_connection.h +++ b/include/mysql_connection.h @@ -67,6 +67,7 @@ class MySQL_Connection { MySrvC *parent; MySQL_Connection_userinfo *userinfo; MySQL_Data_Stream *myds; + enum MySerStatus server_status; // this to solve a side effect of #774 unsigned long largest_query_length; uint32_t status_flags; int async_exit_status; // exit status of MariaDB Client Library Non blocking API diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index 1a1317cd0..c2b060d5c 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -2140,9 +2140,12 @@ handler_again: } */ if ( - (myconn->parent->status==MYSQL_SERVER_STATUS_OFFLINE_HARD) // the query failed because the server is offline hard + // due to #774 , we now read myconn->server_status instead of myconn->parent->status + (myconn->server_status==MYSQL_SERVER_STATUS_OFFLINE_HARD) // the query failed because the server is offline hard || - (myconn->parent->status==MYSQL_SERVER_STATUS_SHUNNED && myconn->parent->shunned_automatic==true && myconn->parent->shunned_and_kill_all_connections==true) // the query failed because the server is shunned due to a serious failure + (myconn->server_status==MYSQL_SERVER_STATUS_SHUNNED && myconn->parent->shunned_automatic==true && myconn->parent->shunned_and_kill_all_connections==true) // the query failed because the server is shunned due to a serious failure + || + (myconn->server_status==MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) // slave is lagging! see #774 ) { if (mysql_thread___connect_timeout_server_max) { myds->max_connect_time=thread->curtime+mysql_thread___connect_timeout_server_max*1000; diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index 4fa45997f..7347933d2 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -1102,12 +1102,13 @@ int MySQL_Connection::async_query(short event, char *stmt, unsigned long length, PROXY_TRACE(); assert(mysql); assert(ret_mysql); + server_status=parent->status; // we copy it here to avoid race condition. The caller will see this if ( - (parent->status==MYSQL_SERVER_STATUS_OFFLINE_HARD) // the server is OFFLINE as specific by the user + (server_status==MYSQL_SERVER_STATUS_OFFLINE_HARD) // the server is OFFLINE as specific by the user || - (parent->status==MYSQL_SERVER_STATUS_SHUNNED && parent->shunned_automatic==true && parent->shunned_and_kill_all_connections==true) // the server is SHUNNED due to a serious issue + (server_status==MYSQL_SERVER_STATUS_SHUNNED && parent->shunned_automatic==true && parent->shunned_and_kill_all_connections==true) // the server is SHUNNED due to a serious issue || - (parent->status==MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) // slave is lagging! see #774 + (server_status==MYSQL_SERVER_STATUS_SHUNNED_REPLICATION_LAG) // slave is lagging! see #774 ) { return -1; } From 289dd2e14767dc9f8c352d110b54b55d41f79f2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sat, 26 Nov 2016 18:38:19 +0000 Subject: [PATCH 19/21] Setting non-blocking the pipe used for IPC between threads --- lib/MySQL_Thread.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index a39787307..371820d61 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -1854,6 +1854,7 @@ bool MySQL_Thread::init() { GloQPro->init_thread(); refresh_variables(); i=pipe(pipefd); + ioctl_FIONBIO(pipefd[1],1); mypolls.add(POLLIN, pipefd[0], NULL, 0); assert(i==0); return true; @@ -2111,7 +2112,7 @@ __run_skip_1: unsigned char c=1; int fd=thr->pipefd[1]; if (write(fd,&c,1)==-1) { - proxy_error("Error while signaling maintenance thread\n"); + //proxy_error("Error while signaling maintenance thread\n"); } } } @@ -2494,7 +2495,7 @@ __run_skip_2: //MySQL_Thread *thr=GloMTH->mysql_threads[w].worker; int fd=thr->pipefd[1]; if (write(fd,&c,1)==-1) { - proxy_error("Error while signaling maintenance thread\n"); + //proxy_error("Error while signaling maintenance thread\n"); } } } else { @@ -2509,7 +2510,7 @@ __run_skip_2: //MySQL_Thread *thr=GloMTH->mysql_threads[w].worker; int fd=thr->pipefd[1]; if (write(fd,&c,1)==-1) { - proxy_error("Error while signaling maintenance thread\n"); + //proxy_error("Error while signaling maintenance thread\n"); } } //spin_wrunlock(&GloMTH->rwlock_resumes); From b25239c72a8102eaad413d4778c987a5d3149249 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sun, 27 Nov 2016 14:48:16 +0000 Subject: [PATCH 20/21] Improvements related to STMT cache These improvements avoid a race condition in which a prepared statement is removed from the cache before it is marked as used by the client. This race condition was noticed when running benchmark creating 7000 _unique_ prepared statements per second. --- include/MySQL_PreparedStatement.h | 4 ++-- lib/MySQL_PreparedStatement.cpp | 16 ++++++++++------ lib/MySQL_Session.cpp | 4 +++- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/include/MySQL_PreparedStatement.h b/include/MySQL_PreparedStatement.h index 784310495..4ba32d6f9 100644 --- a/include/MySQL_PreparedStatement.h +++ b/include/MySQL_PreparedStatement.h @@ -228,8 +228,8 @@ class MySQL_STMT_Manager { MySQL_STMT_Manager(); ~MySQL_STMT_Manager(); int ref_count(uint32_t statement_id, int cnt, bool lock, bool is_client); - MySQL_STMT_Global_info * add_prepared_statement(unsigned int h, char *u, char *s, char *q, unsigned int ql, MYSQL_STMT *stmt, bool lock=true); - MySQL_STMT_Global_info * add_prepared_statement(unsigned int h, char *u, char *s, char *q, unsigned int ql, MYSQL_STMT *stmt, int _cache_ttl, int _timeout, int _delay, bool lock=true); + MySQL_STMT_Global_info * add_prepared_statement(bool *is_new, unsigned int h, char *u, char *s, char *q, unsigned int ql, MYSQL_STMT *stmt, bool lock=true); + MySQL_STMT_Global_info * add_prepared_statement(bool *is_new, unsigned int h, char *u, char *s, char *q, unsigned int ql, MYSQL_STMT *stmt, int _cache_ttl, int _timeout, int _delay, bool lock=true); MySQL_STMT_Global_info * find_prepared_statement_by_stmt_id(uint32_t id, bool lock=true); MySQL_STMT_Global_info * find_prepared_statement_by_hash(uint64_t hash, bool lock=true); uint32_t total_prepared_statements() { return next_statement_id-1; } diff --git a/lib/MySQL_PreparedStatement.cpp b/lib/MySQL_PreparedStatement.cpp index 31e45397a..b3743914e 100644 --- a/lib/MySQL_PreparedStatement.cpp +++ b/lib/MySQL_PreparedStatement.cpp @@ -161,8 +161,9 @@ int MySQL_STMT_Manager::ref_count(uint32_t statement_id, int cnt, bool lock, boo if (s!=m.end()) { MySQL_STMT_Global_info *a=s->second; if (is_client) { - __sync_fetch_and_add(&a->ref_count_client,cnt); - ret=a->ref_count_client; + ret=__sync_add_and_fetch(&a->ref_count_client,cnt); + //__sync_fetch_and_add(&a->ref_count_client,cnt); + //ret=a->ref_count_client; if (m.size() > (unsigned)mysql_thread___max_stmts_cache) { int max_purge=m.size()/20; // purge up to 5% int i=-1; @@ -170,7 +171,7 @@ int MySQL_STMT_Manager::ref_count(uint32_t statement_id, int cnt, bool lock, boo for (std::map::iterator it=m.begin(); it!=m.end(); ++it) { if (i==(max_purge-1)) continue; MySQL_STMT_Global_info *a=it->second; - if (a->ref_count_client == 0) { + if (__sync_add_and_fetch(&a->ref_count_client,0) == 0) { uint64_t hash=a->hash; auto s2=h.find(hash); if (s2!=h.end()) { @@ -206,11 +207,11 @@ int MySQL_STMT_Manager::ref_count(uint32_t statement_id, int cnt, bool lock, boo return ret; } -MySQL_STMT_Global_info * MySQL_STMT_Manager::add_prepared_statement(unsigned int _h, char *u, char *s, char *q, unsigned int ql, MYSQL_STMT *stmt, bool lock) { - return add_prepared_statement(_h, u, s, q, ql, stmt, -1, -1, -1, lock); +MySQL_STMT_Global_info * MySQL_STMT_Manager::add_prepared_statement(bool *is_new, unsigned int _h, char *u, char *s, char *q, unsigned int ql, MYSQL_STMT *stmt, bool lock) { + return add_prepared_statement(is_new, _h, u, s, q, ql, stmt, -1, -1, -1, lock); } -MySQL_STMT_Global_info * MySQL_STMT_Manager::add_prepared_statement(unsigned int _h, char *u, char *s, char *q, unsigned int ql, MYSQL_STMT *stmt, int _cache_ttl, int _timeout, int _delay, bool lock) { +MySQL_STMT_Global_info * MySQL_STMT_Manager::add_prepared_statement(bool *is_new, unsigned int _h, char *u, char *s, char *q, unsigned int ql, MYSQL_STMT *stmt, int _cache_ttl, int _timeout, int _delay, bool lock) { MySQL_STMT_Global_info *ret=NULL; uint64_t hash=stmt_compute_hash(_h, u, s, q, ql); // this identifies the prepared statement if (lock) { @@ -223,6 +224,7 @@ MySQL_STMT_Global_info * MySQL_STMT_Manager::add_prepared_statement(unsigned int //MySQL_STMT_Global_info *a=f->second; //ret=a->statement_id; ret=f->second; + *is_new=false; } else { // we need to create a new one bool free_id_avail=false; @@ -246,6 +248,8 @@ MySQL_STMT_Global_info * MySQL_STMT_Manager::add_prepared_statement(unsigned int ret=a; //next_statement_id++; // increment it //__sync_fetch_and_add(&ret->ref_count_client,1); // increase reference count + __sync_fetch_and_add(&ret->ref_count_client,1); // increase reference count + *is_new=true; } __sync_fetch_and_add(&add_prepared_statement_calls,1); __sync_fetch_and_add(&ret->ref_count_server,1); // increase reference count diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index c2b060d5c..89e6d53cd 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -2022,9 +2022,10 @@ handler_again: case PROCESSING_STMT_PREPARE: { uint32_t stmid; + bool is_new; MySQL_STMT_Global_info *stmt_info=NULL; // if (mysql_thread___stmt_multiplexing) { - stmt_info=GloMyStmt->add_prepared_statement(current_hostgroup, + stmt_info=GloMyStmt->add_prepared_statement(&is_new, current_hostgroup, (char *)client_myds->myconn->userinfo->username, (char *)client_myds->myconn->userinfo->schemaname, (char *)CurrentQuery.QueryPointer, @@ -2077,6 +2078,7 @@ handler_again: } else { client_myds->myprot.generate_STMT_PREPARE_RESPONSE(client_myds->pkt_sid+1,stmt_info); client_myds->myconn->local_stmts->insert(stmt_info->statement_id,NULL); + if (is_new) __sync_fetch_and_sub(&stmt_info->ref_count_client,1); } } break; From d382c6eabdae2b95a1af2246ca6e446bd09cff6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sun, 27 Nov 2016 14:56:56 +0000 Subject: [PATCH 21/21] Version v1.3.0h --- docker/images/proxysql/centos67-build/proxysql.spec | 2 +- docker/images/proxysql/centos7-build/proxysql.spec | 2 +- docker/images/proxysql/debian-7.8-build/proxysql.ctl | 2 +- docker/images/proxysql/debian-8.2-build/proxysql.ctl | 2 +- docker/images/proxysql/fedora24-build/proxysql.spec | 2 +- docker/images/proxysql/ubuntu-12.04-build/proxysql.ctl | 2 +- docker/images/proxysql/ubuntu-14.04-build/proxysql.ctl | 2 +- docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docker/images/proxysql/centos67-build/proxysql.spec b/docker/images/proxysql/centos67-build/proxysql.spec index e62873b3f..0ca966d7e 100644 --- a/docker/images/proxysql/centos67-build/proxysql.spec +++ b/docker/images/proxysql/centos67-build/proxysql.spec @@ -7,7 +7,7 @@ Summary: A high-performance MySQL proxy Name: proxysql -Version: 1.3.0g +Version: 1.3.0h Release: 1 License: GPL+ Group: Development/Tools diff --git a/docker/images/proxysql/centos7-build/proxysql.spec b/docker/images/proxysql/centos7-build/proxysql.spec index e62873b3f..0ca966d7e 100644 --- a/docker/images/proxysql/centos7-build/proxysql.spec +++ b/docker/images/proxysql/centos7-build/proxysql.spec @@ -7,7 +7,7 @@ Summary: A high-performance MySQL proxy Name: proxysql -Version: 1.3.0g +Version: 1.3.0h Release: 1 License: GPL+ Group: Development/Tools diff --git a/docker/images/proxysql/debian-7.8-build/proxysql.ctl b/docker/images/proxysql/debian-7.8-build/proxysql.ctl index 2d533f855..df1f083bc 100644 --- a/docker/images/proxysql/debian-7.8-build/proxysql.ctl +++ b/docker/images/proxysql/debian-7.8-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.3.0g +Version: 1.3.0h Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md diff --git a/docker/images/proxysql/debian-8.2-build/proxysql.ctl b/docker/images/proxysql/debian-8.2-build/proxysql.ctl index 2d533f855..df1f083bc 100644 --- a/docker/images/proxysql/debian-8.2-build/proxysql.ctl +++ b/docker/images/proxysql/debian-8.2-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.3.0g +Version: 1.3.0h Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md diff --git a/docker/images/proxysql/fedora24-build/proxysql.spec b/docker/images/proxysql/fedora24-build/proxysql.spec index e62873b3f..0ca966d7e 100644 --- a/docker/images/proxysql/fedora24-build/proxysql.spec +++ b/docker/images/proxysql/fedora24-build/proxysql.spec @@ -7,7 +7,7 @@ Summary: A high-performance MySQL proxy Name: proxysql -Version: 1.3.0g +Version: 1.3.0h Release: 1 License: GPL+ Group: Development/Tools diff --git a/docker/images/proxysql/ubuntu-12.04-build/proxysql.ctl b/docker/images/proxysql/ubuntu-12.04-build/proxysql.ctl index 2d533f855..df1f083bc 100644 --- a/docker/images/proxysql/ubuntu-12.04-build/proxysql.ctl +++ b/docker/images/proxysql/ubuntu-12.04-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.3.0g +Version: 1.3.0h Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md diff --git a/docker/images/proxysql/ubuntu-14.04-build/proxysql.ctl b/docker/images/proxysql/ubuntu-14.04-build/proxysql.ctl index 2d533f855..df1f083bc 100644 --- a/docker/images/proxysql/ubuntu-14.04-build/proxysql.ctl +++ b/docker/images/proxysql/ubuntu-14.04-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.3.0g +Version: 1.3.0h Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md diff --git a/docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl b/docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl index 2d533f855..df1f083bc 100644 --- a/docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl +++ b/docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.3.0g +Version: 1.3.0h Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md