From b8109f1572d225a1c6a06e1b118434ec474c6792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sat, 27 Feb 2016 17:41:58 +0000 Subject: [PATCH 1/7] Added mutex in MySQL_Monitor_Connection_Pool() Issue #490 - Access to MySQL_Monitor_Connection_Pool was not thread-safe --- lib/MySQL_Monitor.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index c0f93b7cd..a3043573d 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -70,6 +70,7 @@ struct cmp_str { class MySQL_Monitor_Connection_Pool { private: + pthread_mutex_t mutex; int size; //std::map* > my_connections; std::map* , cmp_str> my_connections; @@ -82,6 +83,7 @@ class MySQL_Monitor_Connection_Pool { MySQL_Monitor_Connection_Pool::MySQL_Monitor_Connection_Pool() { size=0; + pthread_mutex_init(&mutex,NULL); } MySQL_Monitor_Connection_Pool::~MySQL_Monitor_Connection_Pool() { @@ -92,6 +94,7 @@ MYSQL * MySQL_Monitor_Connection_Pool::get_connection(char *hostname, int port) //it = my_connections.find(std::make_pair(hostname,port)); char *buf=(char *)malloc(16+strlen(hostname)); sprintf(buf,"%s:%d",hostname,port); + pthread_mutex_lock(&mutex); it = my_connections.find(buf); free(buf); if (it != my_connections.end()) { @@ -100,9 +103,11 @@ MYSQL * MySQL_Monitor_Connection_Pool::get_connection(char *hostname, int port) MYSQL *ret=lst->front(); lst->pop_front(); size--; + pthread_mutex_unlock(&mutex); return ret; } } + pthread_mutex_unlock(&mutex); return NULL; } @@ -111,6 +116,7 @@ void MySQL_Monitor_Connection_Pool::put_connection(char *hostname, int port, MYS std::map* , cmp_str >::iterator it; char * buf=(char *)malloc(16+strlen(hostname)); sprintf(buf,"%s:%d",hostname,port); + pthread_mutex_lock(&mutex); it = my_connections.find(buf); std::list *lst=NULL; if (it==my_connections.end()) { @@ -121,6 +127,7 @@ void MySQL_Monitor_Connection_Pool::put_connection(char *hostname, int port, MYS lst=it->second; } lst->push_back(my); + pthread_mutex_unlock(&mutex); } enum MySQL_Monitor_State_Data_Task_Type { From 01a5a715f7616ef8a84e7cf419e6352977faea85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sat, 27 Feb 2016 20:36:15 +0000 Subject: [PATCH 2/7] purge removed hosts from MySQL_Monitor_Connection_Pool() Related to #490 and #473 --- lib/MySQL_Monitor.cpp | 69 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index a3043573d..ddfc30a77 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -79,6 +79,7 @@ class MySQL_Monitor_Connection_Pool { ~MySQL_Monitor_Connection_Pool(); MYSQL * get_connection(char *hostname, int port); void put_connection(char *hostname, int port, MYSQL *my); + void purge_missing_servers(SQLite3_result *resultset); }; MySQL_Monitor_Connection_Pool::MySQL_Monitor_Connection_Pool() { @@ -89,6 +90,72 @@ MySQL_Monitor_Connection_Pool::MySQL_Monitor_Connection_Pool() { MySQL_Monitor_Connection_Pool::~MySQL_Monitor_Connection_Pool() { } +void MySQL_Monitor_Connection_Pool::purge_missing_servers(SQLite3_result *resultset) { +#define POLLUTE_LENGTH 8 + char pollute_buf[POLLUTE_LENGTH]; + srand(monotonic_time()); + for (int i=0; i *purge_lst=NULL; + purge_lst=new std::list; + pthread_mutex_lock(&mutex); + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + // for each host configured ... + SQLite3_row *r=*it; + char *buf=(char *)malloc(16+strlen(r->fields[0])); + sprintf(buf,"%s:%s",r->fields[0],r->fields[1]); + std::map* , cmp_str >::iterator it2; + it2 = my_connections.find(buf); // find the host + free(buf); + if (it2 != my_connections.end()) { // if the host exists + std::list *lst=it2->second; + std::list::const_iterator it3; + for (it3 = lst->begin(); it3 != lst->end(); ++it3) { + MYSQL *my=*it3; + memcpy(my->net.buff,pollute_buf,8); // pollute this buffer + // + } + } + } + std::map*>::iterator it; + //std::map>::iterator it_type; + for(it = my_connections.begin(); it != my_connections.end(); it++) { + std::list *lst=it->second; + if (!lst->empty()) { + std::list::const_iterator it3; + it3=lst->begin(); + MYSQL *my=*it3; + if (memcmp(my->net.buff,pollute_buf,8)) { + // the buffer is not polluted, it means it didn't match previously + while(!lst->empty()) { + my=lst->front(); + lst->pop_front(); + purge_lst->push_back(my); + } + } + } + } + pthread_mutex_unlock(&mutex); + char quit_buff[5]; + memset(quit_buff,0,5); + quit_buff[0]=1; + quit_buff[4]=1; + + // close all idle connections + while (!purge_lst->empty()) { + MYSQL *my=purge_lst->front(); + purge_lst->pop_front(); + int fd=my->net.fd; + int wb=write(fd,quit_buff,5); + fd+=wb; // dummy, to make compiler happy + fd-=wb; // dummy, to make compiler happy + mysql_close_no_command(my); + shutdown(fd, SHUT_RDWR); + } + delete purge_lst; +} + MYSQL * MySQL_Monitor_Connection_Pool::get_connection(char *hostname, int port) { std::map* , cmp_str >::iterator it; //it = my_connections.find(std::make_pair(hostname,port)); @@ -104,6 +171,7 @@ MYSQL * MySQL_Monitor_Connection_Pool::get_connection(char *hostname, int port) lst->pop_front(); size--; pthread_mutex_unlock(&mutex); + memset(ret->net.buff,0,8); // reset what was polluted return ret; } } @@ -663,6 +731,7 @@ void * MySQL_Monitor::monitor_connect() { proxy_error("Error on %s : %s\n", query, error); goto __end_monitor_connect_loop; } else { + GloMyMon->My_Conn_Pool->purge_missing_servers(resultset); if (resultset->rows_count==0) { goto __end_monitor_connect_loop; } From 3a1912cf4999638942a66724628cc96387b548cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sat, 27 Feb 2016 20:44:27 +0000 Subject: [PATCH 3/7] drop all connections from MySQL_Monitor_Connection_Pool() when shutting down Issues #490 and #473 --- lib/MySQL_Monitor.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index ddfc30a77..4b25c0eaf 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -88,6 +88,7 @@ MySQL_Monitor_Connection_Pool::MySQL_Monitor_Connection_Pool() { } MySQL_Monitor_Connection_Pool::~MySQL_Monitor_Connection_Pool() { + purge_missing_servers(NULL); } void MySQL_Monitor_Connection_Pool::purge_missing_servers(SQLite3_result *resultset) { @@ -100,6 +101,9 @@ void MySQL_Monitor_Connection_Pool::purge_missing_servers(SQLite3_result *result std::list *purge_lst=NULL; purge_lst=new std::list; pthread_mutex_lock(&mutex); + if (resultset==NULL) { + goto __purge_all; + } for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { // for each host configured ... SQLite3_row *r=*it; @@ -118,6 +122,7 @@ void MySQL_Monitor_Connection_Pool::purge_missing_servers(SQLite3_result *result } } } +__purge_all: std::map*>::iterator it; //std::map>::iterator it_type; for(it = my_connections.begin(); it != my_connections.end(); it++) { @@ -640,6 +645,7 @@ MySQL_Monitor::~MySQL_Monitor() { delete tables_defs_monitor; delete monitordb; delete admindb; + delete My_Conn_Pool; }; From 4c53273427c021e9afa2f86dd4c8e2784b699b67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sat, 27 Feb 2016 21:04:20 +0000 Subject: [PATCH 4/7] LOAD MYSQL GLOBAL VARIABLES resets MySQL Monitor timers And immediately triggers new checks Issue #490 --- lib/MySQL_Monitor.cpp | 60 +++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 4b25c0eaf..4334c95cd 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -711,6 +711,13 @@ void * MySQL_Monitor::monitor_connect() { unsigned int glover; t1=monotonic_time(); + glover=GloMTH->get_global_version(); + if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover ) { + MySQL_Monitor__thread_MySQL_Thread_Variables_version=glover; + mysql_thr->refresh_variables(); + next_loop_at=0; + } + if (t1 < next_loop_at) { goto __sleep_monitor_connect_loop; } @@ -724,13 +731,6 @@ void * MySQL_Monitor::monitor_connect() { // create libevent base libevent_base= event_base_new(); - glover=GloMTH->get_global_version(); - if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover ) { - MySQL_Monitor__thread_MySQL_Thread_Variables_version=glover; - mysql_thr->refresh_variables(); - //proxy_error("%s\n", "MySQL_Monitor - CONNECT - refreshing variables"); - } - proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); if (error) { @@ -836,9 +836,16 @@ void * MySQL_Monitor::monitor_ping() { SQLite3_result *resultset=NULL; MySQL_Monitor_State_Data **sds=NULL; int i=0; - char *query=(char *)"SELECT DISTINCT hostname, port FROM mysql_servers"; + char *query=(char *)"SELECT DISTINCT hostname, port FROM mysql_servers WHERE status!='OFFLINE_HARD'"; t1=monotonic_time(); + glover=GloMTH->get_global_version(); + if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover ) { + MySQL_Monitor__thread_MySQL_Thread_Variables_version=glover; + mysql_thr->refresh_variables(); + next_loop_at=0; + } + if (t1 < next_loop_at) { goto __sleep_monitor_ping_loop; } @@ -852,13 +859,6 @@ void * MySQL_Monitor::monitor_ping() { // create libevent base libevent_base= event_base_new(); - glover=GloMTH->get_global_version(); - if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover ) { - MySQL_Monitor__thread_MySQL_Thread_Variables_version=glover; - mysql_thr->refresh_variables(); - //proxy_error("%s\n","MySQL_Monitor - PING - refreshing variables"); - } - proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); if (error) { @@ -969,9 +969,16 @@ void * MySQL_Monitor::monitor_read_only() { SQLite3_result *resultset=NULL; MySQL_Monitor_State_Data **sds=NULL; int i=0; - char *query=(char *)"SELECT DISTINCT hostname, port FROM mysql_servers JOIN mysql_replication_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup"; + char *query=(char *)"SELECT DISTINCT hostname, port FROM mysql_servers JOIN mysql_replication_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE status!='OFFLINE_HARD'"; t1=monotonic_time(); + glover=GloMTH->get_global_version(); + if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover ) { + MySQL_Monitor__thread_MySQL_Thread_Variables_version=glover; + mysql_thr->refresh_variables(); + next_loop_at=0; + } + if (t1 < next_loop_at) { goto __sleep_monitor_read_only; } @@ -985,13 +992,6 @@ void * MySQL_Monitor::monitor_read_only() { // create libevent base libevent_base= event_base_new(); - glover=GloMTH->get_global_version(); - if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover ) { - MySQL_Monitor__thread_MySQL_Thread_Variables_version=glover; - mysql_thr->refresh_variables(); - //proxy_error("%s\n","MySQL_Monitor - PING - refreshing variables"); - } - proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); // admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); resultset = MyHGM->execute_query(query, &error); @@ -1147,6 +1147,13 @@ void * MySQL_Monitor::monitor_replication_lag() { char *query=(char *)"SELECT hostgroup_id, hostname, port, max_replication_lag FROM mysql_servers WHERE max_replication_lag > 0 AND status NOT LIKE 'OFFLINE%'"; t1=monotonic_time(); + glover=GloMTH->get_global_version(); + if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover ) { + MySQL_Monitor__thread_MySQL_Thread_Variables_version=glover; + mysql_thr->refresh_variables(); + next_loop_at=0; + } + if (t1 < next_loop_at) { goto __sleep_monitor_replication_lag; } @@ -1160,13 +1167,6 @@ void * MySQL_Monitor::monitor_replication_lag() { // create libevent base libevent_base= event_base_new(); - glover=GloMTH->get_global_version(); - if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover ) { - MySQL_Monitor__thread_MySQL_Thread_Variables_version=glover; - mysql_thr->refresh_variables(); - //proxy_error("%s\n","MySQL_Monitor - PING - refreshing variables"); - } - proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); // admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); resultset = MyHGM->execute_query(query, &error); From aa73d523124965a508463e353da5b1e9deb48772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sat, 27 Feb 2016 21:13:13 +0000 Subject: [PATCH 5/7] LOAD MYSQL GLOBAL VARIABLES drop all Monitor conns Issue #490 LOAD MYSQL GLOBAL VARIABLES TO RUNTIME drops all MySQL Monitor connections so that they can be re-established --- lib/MySQL_Monitor.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 4334c95cd..9e3a88ff3 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -1309,6 +1309,7 @@ void * MySQL_Monitor::run() { MySQL_Monitor__thread_MySQL_Thread_Variables_version=glover; mysql_thr->refresh_variables(); //proxy_error("%s\n","MySQL_Monitor refreshing variables"); + My_Conn_Pool->purge_missing_servers(NULL); } usleep(500000); } From cdeb02c571d49d26cf2e3d79305dff5a3618384e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sat, 27 Feb 2016 21:33:18 +0000 Subject: [PATCH 6/7] Empty commit --- lib/MySQL_Monitor.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 9e3a88ff3..e6caab3bb 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -138,6 +138,14 @@ __purge_all: lst->pop_front(); purge_lst->push_back(my); } + } else { + // try to keep maximum 2 free connections + // dropping all the others + while(lst->size() > 2) { + my=lst->front(); + lst->pop_front(); + purge_lst->push_back(my); + } } } } From 7124a809e5aa299e7f45acb5599e00f1a49d39b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sat, 27 Feb 2016 22:03:48 +0000 Subject: [PATCH 7/7] delete MySQL_Thread on delete MySQL_Monitor To make valgrind happy --- lib/MySQL_Monitor.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index e6caab3bb..8569d59ed 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -815,6 +815,10 @@ __sleep_monitor_connect_loop: usleep(st); } } + if (mysql_thr) { + delete mysql_thr; + mysql_thr=NULL; + } return NULL; } @@ -951,6 +955,10 @@ __sleep_monitor_ping_loop: usleep(st); } } + if (mysql_thr) { + delete mysql_thr; + mysql_thr=NULL; + } return NULL; } @@ -1126,6 +1134,10 @@ __sleep_monitor_read_only: usleep(st); } } + if (mysql_thr) { + delete mysql_thr; + mysql_thr=NULL; + } return NULL; } @@ -1297,6 +1309,10 @@ __sleep_monitor_replication_lag: usleep(st); } } + if (mysql_thr) { + delete mysql_thr; + mysql_thr=NULL; + } return NULL; } @@ -1325,5 +1341,9 @@ void * MySQL_Monitor::run() { monitor_ping_thread->join(); monitor_read_only_thread->join(); monitor_replication_lag_thread->join(); + if (mysql_thr) { + delete mysql_thr; + mysql_thr=NULL; + } return NULL; };