From ce815a5a726464984f674d94f10b3f3837787eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Mon, 9 Apr 2018 16:38:32 +0200 Subject: [PATCH 1/6] Initial commit to natively support Galera --- include/MySQL_HostGroups_Manager.h | 38 ++ include/MySQL_Monitor.hpp | 7 + include/MySQL_Thread.h | 2 + include/proxysql_structs.h | 4 + lib/MySQL_HostGroups_Manager.cpp | 594 ++++++++++++++++++++++++++++- lib/MySQL_Monitor.cpp | 330 ++++++++++++++++ lib/MySQL_Thread.cpp | 34 ++ lib/ProxySQL_Admin.cpp | 111 +++++- 8 files changed, 1118 insertions(+), 2 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 85ca5430c..b31336302 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -34,6 +34,8 @@ #define MYHGM_MYSQL_GROUP_REPLICATION_HOSTGROUPS "CREATE TABLE mysql_group_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" +#define MYHGM_MYSQL_GALERA_HOSTGROUPS "CREATE TABLE mysql_galera_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" + class MySrvConnList; class MySrvC; class MySrvList; @@ -401,6 +403,28 @@ class Group_Replication_Info { ~Group_Replication_Info(); }; +class Galera_Info { + public: + int writer_hostgroup; + int backup_writer_hostgroup; + int reader_hostgroup; + int offline_hostgroup; + int max_writers; + int max_transactions_behind; + char *comment; + bool active; + bool writer_is_also_reader; + bool __active; + bool need_converge; // this is set to true on LOAD MYSQL SERVERS TO RUNTIME . This ensure that checks wil take an action + int current_num_writers; + int current_num_backup_writers; + int current_num_readers; + int current_num_offline; + Galera_Info(int w, int b, int r, int o, int mw, int mtb, bool _a, bool _w, char *c); + bool update(int b, int r, int o, int mw, int mtb, bool _a, bool _w, char *c); + ~Galera_Info(); +}; + class MySQL_HostGroups_Manager { private: SQLite3DB *admindb; @@ -421,12 +445,19 @@ class MySQL_HostGroups_Manager { void generate_mysql_servers_table(int *_onlyhg=NULL); void generate_mysql_replication_hostgroups_table(); SQLite3_result *incoming_replication_hostgroups; + void generate_mysql_group_replication_hostgroups_table(); SQLite3_result *incoming_group_replication_hostgroups; pthread_mutex_t Group_Replication_Info_mutex; std::map Group_Replication_Info_Map; + void generate_mysql_galera_hostgroups_table(); + SQLite3_result *incoming_galera_hostgroups; + + pthread_mutex_t Galera_Info_mutex; + std::map Galera_Info_Map; + std::thread *HGCU_thread; std::thread *GTID_syncer_thread; @@ -483,10 +514,12 @@ class MySQL_HostGroups_Manager { void set_incoming_replication_hostgroups(SQLite3_result *); void set_incoming_group_replication_hostgroups(SQLite3_result *); + void set_incoming_galera_hostgroups(SQLite3_result *); SQLite3_result * execute_query(char *query, char **error); SQLite3_result *dump_table_mysql_servers(); SQLite3_result *dump_table_mysql_replication_hostgroups(); SQLite3_result *dump_table_mysql_group_replication_hostgroups(); + SQLite3_result *dump_table_mysql_galera_hostgroups(); MyHGC * MyHGC_lookup(unsigned int); void MyConn_add_to_pool(MySQL_Connection *); @@ -514,6 +547,11 @@ class MySQL_HostGroups_Manager { void update_group_replication_set_writer(char *_hostname, int _port, int _writer_hostgroup); void converge_group_replication_config(int _writer_hostgroup); + void update_galera_set_offline(char *_hostname, int _port, int _writer_hostgroup, char *error); + void update_galera_set_read_only(char *_hostname, int _port, int _writer_hostgroup, char *error); + void update_galera_set_writer(char *_hostname, int _port, int _writer_hostgroup); + void converge_galera_config(int _writer_hostgroup); + SQLite3_result * get_stats_mysql_gtid_executed(); void generate_mysql_gtid_executed_tables(); bool gtid_exists(MySrvC *mysrvc, char * gtid_uuid, uint64_t gtid_trxid); diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 8ef28a388..3344de471 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -20,6 +20,8 @@ #define MONITOR_SQLITE_TABLE_MYSQL_SERVER_GROUP_REPLICATION_LOG "CREATE TABLE mysql_server_group_replication_log (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , time_start_us INT NOT NULL DEFAULT 0 , success_time_us INT DEFAULT 0 , viable_candidate VARCHAR NOT NULL DEFAULT 'NO' , read_only VARCHAR NOT NULL DEFAULT 'YES' , transactions_behind INT DEFAULT 0 , error VARCHAR , PRIMARY KEY (hostname, port, time_start_us))" +#define MONITOR_SQLITE_TABLE_MYSQL_SERVER_GALERA_LOG "CREATE TABLE mysql_server_galera_log (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , time_start_us INT NOT NULL DEFAULT 0 , success_time_us INT DEFAULT 0 , viable_candidate VARCHAR NOT NULL DEFAULT 'NO' , read_only VARCHAR NOT NULL DEFAULT 'YES' , transactions_behind INT DEFAULT 0 , error VARCHAR , PRIMARY KEY (hostname, port, time_start_us))" + /* struct cmp_str { bool operator()(char const *a, char const *b) const @@ -117,9 +119,12 @@ class MySQL_Monitor { void check_and_build_standard_tables(SQLite3DB *db, std::vector *tables_defs); public: pthread_mutex_t group_replication_mutex; // for simplicity, a mutex instead of a rwlock + pthread_mutex_t galera_mutex; // for simplicity, a mutex instead of a rwlock //std::map Group_Replication_Hosts_Map; std::map Group_Replication_Hosts_Map; SQLite3_result *Group_Replication_Hosts_resultset; + std::map Galera_Hosts_Map; + SQLite3_result *Galera_Hosts_resultset; unsigned int num_threads; wqueue queue; MySQL_Monitor_Connection_Pool *My_Conn_Pool; @@ -134,9 +139,11 @@ class MySQL_Monitor { void * monitor_ping(); void * monitor_read_only(); void * monitor_group_replication(); + void * monitor_galera(); void * monitor_replication_lag(); void * run(); void populate_monitor_mysql_server_group_replication_log(); + void populate_monitor_mysql_server_galera_log(); }; #endif /* __CLASS_MYSQL_MONITOR_H */ diff --git a/include/MySQL_Thread.h b/include/MySQL_Thread.h index 619ef383a..b9bac6016 100644 --- a/include/MySQL_Thread.h +++ b/include/MySQL_Thread.h @@ -334,6 +334,8 @@ class MySQL_Threads_Handler int monitor_replication_lag_timeout; int monitor_groupreplication_healthcheck_interval; int monitor_groupreplication_healthcheck_timeout; + int monitor_galera_healthcheck_interval; + int monitor_galera_healthcheck_timeout; int monitor_query_interval; int monitor_query_timeout; int monitor_slave_lag_when_null; diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index 7fa31dd99..ddab3d1cd 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -677,6 +677,8 @@ __thread int mysql_thread___monitor_replication_lag_interval; __thread int mysql_thread___monitor_replication_lag_timeout; __thread int mysql_thread___monitor_groupreplication_healthcheck_interval; __thread int mysql_thread___monitor_groupreplication_healthcheck_timeout; +__thread int mysql_thread___monitor_galera_healthcheck_interval; +__thread int mysql_thread___monitor_galera_healthcheck_timeout; __thread int mysql_thread___monitor_query_interval; __thread int mysql_thread___monitor_query_timeout; __thread int mysql_thread___monitor_slave_lag_when_null; @@ -785,6 +787,8 @@ extern __thread int mysql_thread___monitor_replication_lag_interval; extern __thread int mysql_thread___monitor_replication_lag_timeout; extern __thread int mysql_thread___monitor_groupreplication_healthcheck_interval; extern __thread int mysql_thread___monitor_groupreplication_healthcheck_timeout; +extern __thread int mysql_thread___monitor_galera_healthcheck_interval; +extern __thread int mysql_thread___monitor_galera_healthcheck_timeout; extern __thread int mysql_thread___monitor_query_interval; extern __thread int mysql_thread___monitor_query_timeout; extern __thread int mysql_thread___monitor_slave_lag_when_null; diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 14ab2bf33..e65a3a92a 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -831,10 +831,12 @@ MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() { mydb->execute(MYHGM_MYSQL_SERVERS_INCOMING); mydb->execute(MYHGM_MYSQL_REPLICATION_HOSTGROUPS); mydb->execute(MYHGM_MYSQL_GROUP_REPLICATION_HOSTGROUPS); + mydb->execute(MYHGM_MYSQL_GALERA_HOSTGROUPS); queue = wqueue(); MyHostGroups=new PtrArray(); incoming_replication_hostgroups=NULL; incoming_group_replication_hostgroups=NULL; + incoming_galera_hostgroups=NULL; pthread_rwlock_init(>id_rwlock, NULL); gtid_missing_nodes = false; gtid_ev_async = (struct ev_async *)malloc(sizeof(struct ev_async)); @@ -1227,6 +1229,13 @@ bool MySQL_HostGroups_Manager::commit() { generate_mysql_group_replication_hostgroups_table(); } + // galera + if (incoming_galera_hostgroups) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_galera_hostgroups\n"); + mydb->execute("DELETE FROM mysql_galera_hostgroups"); + generate_mysql_galera_hostgroups_table(); + } + if ( GloAdmin && GloAdmin->checksum_variables.checksum_mysql_servers ) { uint64_t hash1=0, hash2=0; @@ -1339,6 +1348,25 @@ bool MySQL_HostGroups_Manager::commit() { delete resultset; } } + { + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + char *query=(char *)"SELECT * FROM mysql_galera_hostgroups ORDER BY writer_hostgroup"; + mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + if (resultset) { + if (resultset->rows_count) { + if (init == false) { + init = true; + myhash.Init(19,3); + } + uint64_t hash1_ = resultset->raw_checksum(); + myhash.Update(&hash1_, sizeof(hash1_)); + } + delete resultset; + } + } if (init == true) { myhash.Final(&hash1, &hash2); } @@ -1733,6 +1761,101 @@ void MySQL_HostGroups_Manager::generate_mysql_group_replication_hostgroups_table pthread_mutex_unlock(&Group_Replication_Info_mutex); } +void MySQL_HostGroups_Manager::generate_mysql_galera_hostgroups_table() { + if (incoming_galera_hostgroups==NULL) { + return; + } + int rc; + sqlite3_stmt *statement=NULL; + sqlite3 *mydb3=mydb->get_db(); + char *query=(char *)"INSERT INTO mysql_galera_hostgroups(writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers,writer_is_also_reader,max_transactions_behind,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"; + rc=sqlite3_prepare_v2(mydb3, query, -1, &statement, 0); + assert(rc==SQLITE_OK); + proxy_info("New mysql_galera_hostgroups table\n"); + pthread_mutex_lock(&Galera_Info_mutex); + for (std::map::iterator it1 = Galera_Info_Map.begin() ; it1 != Galera_Info_Map.end(); ++it1) { + Galera_Info *info=NULL; + info=it1->second; + info->__active=false; + } + for (std::vector::iterator it = incoming_galera_hostgroups->rows.begin() ; it != incoming_galera_hostgroups->rows.end(); ++it) { + SQLite3_row *r=*it; + int writer_hostgroup=atoi(r->fields[0]); + int backup_writer_hostgroup=atoi(r->fields[1]); + int reader_hostgroup=atoi(r->fields[2]); + int offline_hostgroup=atoi(r->fields[3]); + int active=atoi(r->fields[4]); + int max_writers=atoi(r->fields[5]); + int writer_is_also_reader=atoi(r->fields[6]); + int max_transactions_behind=atoi(r->fields[7]); + proxy_info("Loading MySQL Group Replication info for (%d,%d,%d,%d,%s,%d,%d,%d,\"%s\")\n", writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,(active ? "on" : "off"),max_writers,writer_is_also_reader,max_transactions_behind,r->fields[8]); + rc=sqlite3_bind_int64(statement, 1, writer_hostgroup); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement, 2, backup_writer_hostgroup); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement, 3, reader_hostgroup); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement, 4, offline_hostgroup); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement, 5, active); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement, 6, max_writers); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement, 7, writer_is_also_reader); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement, 8, max_transactions_behind); assert(rc==SQLITE_OK); + rc=sqlite3_bind_text(statement, 9, r->fields[8], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); + + SAFE_SQLITE3_STEP2(statement); + rc=sqlite3_clear_bindings(statement); assert(rc==SQLITE_OK); + rc=sqlite3_reset(statement); assert(rc==SQLITE_OK); + std::map::iterator it2; + it2 = Galera_Info_Map.find(writer_hostgroup); + Galera_Info *info=NULL; + if (it2!=Galera_Info_Map.end()) { + info=it2->second; + bool changed=false; + changed=info->update(backup_writer_hostgroup,reader_hostgroup,offline_hostgroup, max_writers, max_transactions_behind, (bool)active, (bool)writer_is_also_reader, r->fields[8]); + if (changed) { + //info->need_converge=true; + } + } else { + info=new Galera_Info(writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup, max_writers, max_transactions_behind, (bool)active, (bool)writer_is_also_reader, r->fields[8]); + //info->need_converge=true; + Galera_Info_Map.insert(Galera_Info_Map.begin(), std::pair(writer_hostgroup,info)); + } + } + sqlite3_finalize(statement); + delete incoming_galera_hostgroups; + incoming_galera_hostgroups=NULL; + + // remove missing ones + for (auto it3 = Galera_Info_Map.begin(); it3 != Galera_Info_Map.end(); ) { + Galera_Info *info=it3->second; + if (info->__active==false) { + delete info; + it3 = Galera_Info_Map.erase(it3); + } else { + it3++; + } + } + // TODO: it is now time to compute all the changes + + + // it is now time to build a new structure in Monitor + pthread_mutex_lock(&GloMyMon->galera_mutex); + { + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + char *query=(char *)"SELECT writer_hostgroup, hostname, port, MAX(use_ssl) use_ssl , writer_is_also_reader , max_transactions_behind FROM mysql_servers JOIN mysql_galera_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=backup_writer_hostgroup OR hostgroup_id=reader_hostgroup OR hostgroup_id=offline_hostgroup WHERE status NOT IN (2,3) GROUP BY hostname, port"; + mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + if (resultset) { + if (GloMyMon->Galera_Hosts_resultset) { + delete GloMyMon->Galera_Hosts_resultset; + } + GloMyMon->Galera_Hosts_resultset=resultset; + } + } + pthread_mutex_unlock(&GloMyMon->galera_mutex); + + pthread_mutex_unlock(&Galera_Info_mutex); +} + SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql_servers() { wrlock(); @@ -1780,6 +1903,19 @@ SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql_group_replication_ho return resultset; } +SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql_galera_hostgroups() { + wrlock(); + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + char *query=(char *)"SELECT writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers,writer_is_also_reader,max_transactions_behind,comment FROM mysql_galera_hostgroups"; + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); + mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + wrunlock(); + return resultset; +} + MyHGC * MySQL_HostGroups_Manager::MyHGC_create(unsigned int _hid) { MyHGC *myhgc=new MyHGC(_hid); return myhgc; @@ -2317,6 +2453,14 @@ void MySQL_HostGroups_Manager::set_incoming_group_replication_hostgroups(SQLite3 incoming_group_replication_hostgroups=s; } +void MySQL_HostGroups_Manager::set_incoming_galera_hostgroups(SQLite3_result *s) { + if (incoming_galera_hostgroups) { + delete incoming_galera_hostgroups; + incoming_galera_hostgroups = NULL; + } + incoming_galera_hostgroups=s; +} + SQLite3_result * MySQL_HostGroups_Manager::SQL3_Connection_Pool(bool _reset) { const int colnum=14; proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 4, "Dumping Connection Pool\n"); @@ -2885,7 +3029,6 @@ unsigned long long MySQL_HostGroups_Manager::Get_Memory_Stats() { return intsize; } - Group_Replication_Info::Group_Replication_Info(int w, int b, int r, int o, int mw, int mtb, bool _a, bool _w, char *c) { comment=NULL; if (c) { @@ -3335,6 +3478,455 @@ void MySQL_HostGroups_Manager::converge_group_replication_config(int _writer_hos pthread_mutex_unlock(&Group_Replication_Info_mutex); } +Galera_Info::Galera_Info(int w, int b, int r, int o, int mw, int mtb, bool _a, bool _w, char *c) { + comment=NULL; + if (c) { + comment=strdup(c); + } + writer_hostgroup=w; + backup_writer_hostgroup=b; + reader_hostgroup=r; + offline_hostgroup=o; + max_writers=mw; + max_transactions_behind=mtb; + active=_a; + writer_is_also_reader=_w; + current_num_writers=0; + current_num_backup_writers=0; + current_num_readers=0; + current_num_offline=0; + __active=true; + need_converge=true; +} + +Galera_Info::~Galera_Info() { + if (comment) { + free(comment); + comment=NULL; + } +} + +bool Galera_Info::update(int b, int r, int o, int mw, int mtb, bool _a, bool _w, char *c) { + bool ret=false; + __active=true; + if (backup_writer_hostgroup!=b) { + backup_writer_hostgroup=b; + ret=true; + } + if (reader_hostgroup!=r) { + reader_hostgroup=r; + ret=true; + } + if (offline_hostgroup!=o) { + offline_hostgroup=o; + ret=true; + } + if (max_writers!=mw) { + max_writers=mw; + ret=true; + } + if (max_transactions_behind!=mtb) { + max_transactions_behind=mtb; + ret=true; + } + if (active!=_a) { + active=_a; + ret=true; + } + if (writer_is_also_reader!=_w) { + writer_is_also_reader=_w; + ret=true; + } + // for comment we don't change return value + if (comment) { + if (c) { + if (strcmp(comment,c)) { + free(comment); + comment=strdup(c); + } + } else { + free(comment); + comment=NULL; + } + } else { + if (c) { + comment=strdup(c); + } + } + return ret; +} + +void MySQL_HostGroups_Manager::update_galera_set_offline(char *_hostname, int _port, int _writer_hostgroup, char *_error) { + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + char *query=NULL; + char *q=NULL; + char *error=NULL; + q=(char *)"SELECT hostgroup_id FROM mysql_servers JOIN mysql_galera_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=backup_writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE hostname='%s' AND port=%d AND status<>3"; + query=(char *)malloc(strlen(q)+strlen(_hostname)+32); + sprintf(query,q,_hostname,_port); + mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + if (error) { + free(error); + error=NULL; + } + free(query); + if (resultset) { // we lock only if needed + if (resultset->rows_count) { + proxy_warning("Group Replication: setting host %s:%d offline because: %s\n", _hostname, _port, _error); + GloAdmin->mysql_servers_wrlock(); + mydb->execute("DELETE FROM mysql_servers_incoming"); + mydb->execute("INSERT INTO mysql_servers_incoming SELECT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers"); + q=(char *)"UPDATE OR IGNORE mysql_servers_incoming SET hostgroup_id=(SELECT offline_hostgroup FROM mysql_galera_hostgroups WHERE writer_hostgroup=%d) WHERE hostname='%s' AND port=%d AND hostgroup_id<>(SELECT offline_hostgroup FROM mysql_galera_hostgroups WHERE writer_hostgroup=%d)"; + query=(char *)malloc(strlen(q)+strlen(_hostname)+64); + sprintf(query,q,_writer_hostgroup,_hostname,_port,_writer_hostgroup); + mydb->execute(query); + //free(query); + q=(char *)"DELETE FROM mysql_servers_incoming WHERE hostname='%s' AND port=%d AND hostgroup_id<>(SELECT offline_hostgroup FROM mysql_galera_hostgroups WHERE writer_hostgroup=%d)"; + //query=(char *)malloc(strlen(q)+strlen(_hostname)+64); + sprintf(query,q,_hostname,_port,_writer_hostgroup); + mydb->execute(query); + //free(query); + q=(char *)"UPDATE mysql_servers_incoming SET status=0 WHERE hostname='%s' AND port=%d AND hostgroup_id=(SELECT offline_hostgroup FROM mysql_galera_hostgroups WHERE writer_hostgroup=%d)"; + //query=(char *)malloc(strlen(q)+strlen(_hostname)+64); + sprintf(query,q,_hostname,_port,_writer_hostgroup); + mydb->execute(query); + //free(query); + converge_galera_config(_writer_hostgroup); + commit(); + wrlock(); + SQLite3_result *resultset2=NULL; + q=(char *)"SELECT writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup FROM mysql_galera_hostgroups WHERE writer_hostgroup=%d"; + //query=(char *)malloc(strlen(q)+strlen(_hostname)+64); + sprintf(query,q,_port,_writer_hostgroup); + mydb->execute_statement(query, &error, &cols , &affected_rows , &resultset2); + if (resultset2) { + if (resultset2->rows_count) { + for (std::vector::iterator it = resultset2->rows.begin() ; it != resultset2->rows.end(); ++it) { + SQLite3_row *r=*it; + int writer_hostgroup=atoi(r->fields[0]); + int backup_writer_hostgroup=atoi(r->fields[1]); + int reader_hostgroup=atoi(r->fields[2]); + int offline_hostgroup=atoi(r->fields[3]); + q=(char *)"DELETE FROM mysql_servers WHERE hostgroup_id IN (%d , %d , %d , %d)"; + sprintf(query,q,_port,_writer_hostgroup); + mydb->execute(query); + generate_mysql_servers_table(&writer_hostgroup); + generate_mysql_servers_table(&backup_writer_hostgroup); + generate_mysql_servers_table(&reader_hostgroup); + generate_mysql_servers_table(&offline_hostgroup); + } + } + delete resultset2; + resultset2=NULL; + } + wrunlock(); + GloAdmin->mysql_servers_wrunlock(); + free(query); + } + } + if (resultset) { + delete resultset; + resultset=NULL; + } +} + +void MySQL_HostGroups_Manager::update_galera_set_read_only(char *_hostname, int _port, int _writer_hostgroup, char *_error) { + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + char *query=NULL; + char *q=NULL; + char *error=NULL; + q=(char *)"SELECT hostgroup_id FROM mysql_servers JOIN mysql_galera_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=backup_writer_hostgroup OR hostgroup_id=offline_hostgroup WHERE hostname='%s' AND port=%d AND status<>3"; + query=(char *)malloc(strlen(q)+strlen(_hostname)+32); + sprintf(query,q,_hostname,_port); + mydb->execute_statement(query, &error, &cols , &affected_rows , &resultset); + if (error) { + free(error); + error=NULL; + } + free(query); + if (resultset) { // we lock only if needed + if (resultset->rows_count) { + proxy_warning("Group Replication: setting host %s:%d (part of cluster with writer_hostgroup=%d) in read_only because: %s\n", _hostname, _port, _writer_hostgroup, _error); + GloAdmin->mysql_servers_wrlock(); + mydb->execute("DELETE FROM mysql_servers_incoming"); + mydb->execute("INSERT INTO mysql_servers_incoming SELECT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers"); + q=(char *)"UPDATE OR IGNORE mysql_servers_incoming SET hostgroup_id=(SELECT reader_hostgroup FROM mysql_galera_hostgroups WHERE writer_hostgroup=%d) WHERE hostname='%s' AND port=%d AND hostgroup_id<>(SELECT reader_hostgroup FROM mysql_galera_hostgroups WHERE writer_hostgroup=%d)"; + query=(char *)malloc(strlen(q)+strlen(_hostname)+64); + sprintf(query,q,_writer_hostgroup,_hostname,_port,_writer_hostgroup); + mydb->execute(query); + //free(query); + q=(char *)"DELETE FROM mysql_servers_incoming WHERE hostname='%s' AND port=%d AND hostgroup_id<>(SELECT reader_hostgroup FROM mysql_galera_hostgroups WHERE writer_hostgroup=%d)"; + //query=(char *)malloc(strlen(q)+strlen(_hostname)+64); + sprintf(query,q,_hostname,_port,_writer_hostgroup); + mydb->execute(query); + //free(query); + q=(char *)"UPDATE mysql_servers_incoming SET status=0 WHERE hostname='%s' AND port=%d AND hostgroup_id=(SELECT reader_hostgroup FROM mysql_galera_hostgroups WHERE writer_hostgroup=%d)"; + //query=(char *)malloc(strlen(q)+strlen(_hostname)+64); + sprintf(query,q,_hostname,_port,_writer_hostgroup); + mydb->execute(query); + //free(query); + converge_galera_config(_writer_hostgroup); + commit(); + wrlock(); + SQLite3_result *resultset2=NULL; + q=(char *)"SELECT writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup FROM mysql_galera_hostgroups WHERE writer_hostgroup=%d"; + //query=(char *)malloc(strlen(q)+strlen(_hostname)+64); + sprintf(query,q,_writer_hostgroup); + mydb->execute_statement(query, &error, &cols , &affected_rows , &resultset2); + if (resultset2) { + if (resultset2->rows_count) { + for (std::vector::iterator it = resultset2->rows.begin() ; it != resultset2->rows.end(); ++it) { + SQLite3_row *r=*it; + int writer_hostgroup=atoi(r->fields[0]); + int backup_writer_hostgroup=atoi(r->fields[1]); + int reader_hostgroup=atoi(r->fields[2]); + int offline_hostgroup=atoi(r->fields[3]); + q=(char *)"DELETE FROM mysql_servers WHERE hostgroup_id IN (%d , %d , %d , %d)"; + sprintf(query,q,writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup); + mydb->execute(query); + generate_mysql_servers_table(&writer_hostgroup); + generate_mysql_servers_table(&backup_writer_hostgroup); + generate_mysql_servers_table(&reader_hostgroup); + generate_mysql_servers_table(&offline_hostgroup); + } + } + delete resultset2; + resultset2=NULL; + } + wrunlock(); + GloAdmin->mysql_servers_wrunlock(); + free(query); + } + } + if (resultset) { + delete resultset; + resultset=NULL; + } +} + +void MySQL_HostGroups_Manager::update_galera_set_writer(char *_hostname, int _port, int _writer_hostgroup) { + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + char *query=NULL; + char *q=NULL; + char *error=NULL; + q=(char *)"SELECT hostgroup_id FROM mysql_servers JOIN mysql_galera_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup OR hostgroup_id=backup_writer_hostgroup OR hostgroup_id=offline_hostgroup WHERE hostname='%s' AND port=%d AND status<>3"; + query=(char *)malloc(strlen(q)+strlen(_hostname)+32); + sprintf(query,q,_hostname,_port); + mydb->execute_statement(query, &error, &cols , &affected_rows , &resultset); + if (error) { + free(error); + error=NULL; + } + free(query); + + bool writer_is_also_reader=false; + bool found_writer=false; + bool found_reader=false; + int read_HG=-1; + bool need_converge=false; + if (resultset) { + // let's get info about this cluster + pthread_mutex_lock(&Galera_Info_mutex); + std::map::iterator it2; + it2 = Galera_Info_Map.find(_writer_hostgroup); + Galera_Info *info=NULL; + if (it2!=Galera_Info_Map.end()) { + info=it2->second; + writer_is_also_reader=info->writer_is_also_reader; + read_HG=info->reader_hostgroup; + need_converge=info->need_converge; + info->need_converge=false; + } + pthread_mutex_unlock(&Galera_Info_mutex); + + if (resultset->rows_count) { + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + int hostgroup=atoi(r->fields[0]); + if (hostgroup==_writer_hostgroup) { + found_writer=true; + } + if (read_HG>=0) { + if (hostgroup==read_HG) { + found_reader=true; + } + } + } + } + if (need_converge==false) { + if (found_writer) { // maybe no-op + if (writer_is_also_reader==found_reader) { // either both true or both false + delete resultset; + resultset=NULL; + } + } + } + } + + if (resultset) { // if we reach there, there is some action to perform + if (resultset->rows_count) { + need_converge=false; + proxy_warning("Group Replication: setting host %s:%d as writer\n", _hostname, _port); + + GloAdmin->mysql_servers_wrlock(); + mydb->execute("DELETE FROM mysql_servers_incoming"); + mydb->execute("INSERT INTO mysql_servers_incoming SELECT hostgroup_id, hostname, port, gtid_port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers"); + q=(char *)"UPDATE OR IGNORE mysql_servers_incoming SET hostgroup_id=%d WHERE hostname='%s' AND port=%d AND hostgroup_id<>%d"; + query=(char *)malloc(strlen(q)+strlen(_hostname)+256); + sprintf(query,q,_writer_hostgroup,_hostname,_port,_writer_hostgroup); + mydb->execute(query); + //free(query); + q=(char *)"DELETE FROM mysql_servers_incoming WHERE hostname='%s' AND port=%d AND hostgroup_id<>%d"; + //query=(char *)malloc(strlen(q)+strlen(_hostname)+64); + sprintf(query,q,_hostname,_port,_writer_hostgroup); + mydb->execute(query); + //free(query); + q=(char *)"UPDATE mysql_servers_incoming SET status=0 WHERE hostname='%s' AND port=%d AND hostgroup_id=%d"; + //query=(char *)malloc(strlen(q)+strlen(_hostname)+64); + sprintf(query,q,_hostname,_port,_writer_hostgroup); + mydb->execute(query); + //free(query); + if (writer_is_also_reader && read_HG>=0) { + q=(char *)"INSERT OR IGNORE INTO mysql_servers_incoming (hostgroup_id,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment) SELECT %d,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM mysql_servers_incoming WHERE hostgroup_id=%d AND hostname='%s' AND port=%d"; + sprintf(query,q,read_HG,_writer_hostgroup,_hostname,_port); + mydb->execute(query); + } + converge_galera_config(_writer_hostgroup); + commit(); + wrlock(); + SQLite3_result *resultset2=NULL; + q=(char *)"SELECT writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup, max_writers, writer_is_also_reader FROM mysql_galera_hostgroups WHERE writer_hostgroup=%d"; + //query=(char *)malloc(strlen(q)+strlen(_hostname)+64); + sprintf(query,q,_writer_hostgroup); + mydb->execute_statement(query, &error, &cols , &affected_rows , &resultset2); + if (resultset2) { + if (resultset2->rows_count) { + for (std::vector::iterator it = resultset2->rows.begin() ; it != resultset2->rows.end(); ++it) { + SQLite3_row *r=*it; + int writer_hostgroup=atoi(r->fields[0]); + int backup_writer_hostgroup=atoi(r->fields[1]); + int reader_hostgroup=atoi(r->fields[2]); + int offline_hostgroup=atoi(r->fields[3]); +// int max_writers=atoi(r->fields[4]); +// int int_writer_is_also_reader=atoi(r->fields[5]); + q=(char *)"DELETE FROM mysql_servers WHERE hostgroup_id IN (%d , %d , %d , %d)"; + sprintf(query,q,_writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup); + mydb->execute(query); + generate_mysql_servers_table(&writer_hostgroup); + generate_mysql_servers_table(&backup_writer_hostgroup); + generate_mysql_servers_table(&reader_hostgroup); + generate_mysql_servers_table(&offline_hostgroup); + } + } + delete resultset2; + resultset2=NULL; + } + wrunlock(); + GloAdmin->mysql_servers_wrunlock(); + free(query); + } + } + if (resultset) { + delete resultset; + resultset=NULL; + } +} + +// this function completes the tuning of mysql_servers_incoming +// it assumes that before calling converge_galera_config() +// * GloAdmin->mysql_servers_wrlock() was already called +// * mysql_servers_incoming has already entries copied from mysql_servers and ready to be loaded +// at this moment, it is only used to check if there are more than one writer +void MySQL_HostGroups_Manager::converge_galera_config(int _writer_hostgroup) { + + // we first gather info about the cluster + pthread_mutex_lock(&Galera_Info_mutex); + std::map::iterator it2; + it2 = Galera_Info_Map.find(_writer_hostgroup); + Galera_Info *info=NULL; + if (it2!=Galera_Info_Map.end()) { + info=it2->second; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + char *query=NULL; + char *q=NULL; + char *error=NULL; + q=(char *)"SELECT hostgroup_id,hostname,port FROM mysql_servers_incoming WHERE status=0 AND hostgroup_id IN (%d, %d, %d, %d) ORDER BY weight DESC, hostname DESC"; + query=(char *)malloc(strlen(q)+256); + sprintf(query, q, info->writer_hostgroup, info->backup_writer_hostgroup, info->reader_hostgroup, info->offline_hostgroup); + mydb->execute_statement(query, &error, &cols , &affected_rows , &resultset); + free(query); + if (resultset) { + if (resultset->rows_count) { + int num_writers=0; + int num_backup_writers=0; + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + int hostgroup=atoi(r->fields[0]); + if (hostgroup==info->writer_hostgroup) { + num_writers++; + } else { + if (hostgroup==info->backup_writer_hostgroup) { + num_backup_writers++; + } + } + } + if (num_writers > info->max_writers) { // there are more writers than allowed + int to_move=num_writers-info->max_writers; + proxy_info("Group replication: max_writers=%d , moving %d nodes from writer HG %d to backup HG %d\n", info->max_writers, to_move, info->writer_hostgroup, info->backup_writer_hostgroup); + for (std::vector::reverse_iterator it = resultset->rows.rbegin() ; it != resultset->rows.rend(); ++it) { + SQLite3_row *r=*it; + if (to_move) { + int hostgroup=atoi(r->fields[0]); + if (hostgroup==info->writer_hostgroup) { + q=(char *)"UPDATE OR REPLACE mysql_servers_incoming SET status=0, hostgroup_id=%d WHERE hostgroup_id=%d AND hostname='%s' AND port=%d"; + query=(char *)malloc(strlen(q)+strlen(r->fields[1])+128); + sprintf(query,q,info->backup_writer_hostgroup,info->writer_hostgroup,r->fields[1],atoi(r->fields[2])); + mydb->execute(query); + free(query); + to_move--; + } + } + } + } else { + if (num_writers < info->max_writers && num_backup_writers) { // or way too low writer + int to_move= ( (info->max_writers - num_writers) < num_backup_writers ? (info->max_writers - num_writers) : num_backup_writers); + proxy_info("Group replication: max_writers=%d , moving %d nodes from backup HG %d to writer HG %d\n", info->max_writers, to_move, info->backup_writer_hostgroup, info->writer_hostgroup); + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + if (to_move) { + int hostgroup=atoi(r->fields[0]); + if (hostgroup==info->backup_writer_hostgroup) { + q=(char *)"UPDATE OR REPLACE mysql_servers_incoming SET status=0, hostgroup_id=%d WHERE hostgroup_id=%d AND hostname='%s' AND port=%d"; + query=(char *)malloc(strlen(q)+strlen(r->fields[1])+128); + sprintf(query,q,info->writer_hostgroup,info->backup_writer_hostgroup,r->fields[1],atoi(r->fields[2])); + mydb->execute(query); + free(query); + to_move--; + } + } + } + } + } + } + } + if (resultset) { + delete resultset; + resultset=NULL; + } + } else { + // we couldn't find the cluster, exits + } + pthread_mutex_unlock(&Galera_Info_mutex); +} + SQLite3_result * MySQL_HostGroups_Manager::get_stats_mysql_gtid_executed() { const int colnum = 4; SQLite3_result * result = new SQLite3_result(colnum); diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index e021ec6ab..7d8bcdf66 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -263,6 +263,15 @@ void * monitor_group_replication_pthread(void *arg) { return NULL; } +void * monitor_galera_pthread(void *arg) { +#ifndef NOJEM + bool cache=false; + mallctl("thread.tcache.enabled", NULL, NULL, &cache, sizeof(bool)); +#endif + GloMyMon->monitor_galera(); + return NULL; +} + void * monitor_replication_lag_pthread(void *arg) { #ifndef NOJEM bool cache=false; @@ -281,6 +290,9 @@ MySQL_Monitor::MySQL_Monitor() { pthread_mutex_init(&group_replication_mutex,NULL); Group_Replication_Hosts_resultset=NULL; + pthread_mutex_init(&galera_mutex,NULL); + Galera_Hosts_resultset=NULL; + shutdown=false; monitor_enabled=true; // default // create new SQLite datatabase @@ -299,6 +311,7 @@ MySQL_Monitor::MySQL_Monitor() { insert_into_tables_defs(tables_defs_monitor,"mysql_server_read_only_log", MONITOR_SQLITE_TABLE_MYSQL_SERVER_READ_ONLY_LOG); insert_into_tables_defs(tables_defs_monitor,"mysql_server_replication_lag_log", MONITOR_SQLITE_TABLE_MYSQL_SERVER_REPLICATION_LAG_LOG); insert_into_tables_defs(tables_defs_monitor,"mysql_server_group_replication_log", MONITOR_SQLITE_TABLE_MYSQL_SERVER_GROUP_REPLICATION_LOG); + insert_into_tables_defs(tables_defs_monitor,"mysql_server_galera_log", MONITOR_SQLITE_TABLE_MYSQL_SERVER_GALERA_LOG); // create monitoring tables check_and_build_standard_tables(monitordb, tables_defs_monitor); monitordb->execute("CREATE INDEX IF NOT EXISTS idx_connect_log_time_start ON mysql_server_connect_log (time_start_us)"); @@ -306,6 +319,7 @@ MySQL_Monitor::MySQL_Monitor() { monitordb->execute("CREATE INDEX IF NOT EXISTS idx_read_only_log_time_start ON mysql_server_read_only_log (time_start_us)"); monitordb->execute("CREATE INDEX IF NOT EXISTS idx_replication_lag_log_time_start ON mysql_server_replication_lag_log (time_start_us)"); monitordb->execute("CREATE INDEX IF NOT EXISTS idx_group_replication_log_time_start ON mysql_server_group_replication_log (time_start_us)"); + monitordb->execute("CREATE INDEX IF NOT EXISTS idx_galera_log_time_start ON mysql_server_galera_log (time_start_us)"); num_threads=8; if (GloMTH) { @@ -328,6 +342,10 @@ MySQL_Monitor::~MySQL_Monitor() { delete Group_Replication_Hosts_resultset; Group_Replication_Hosts_resultset=NULL; } + if (Galera_Hosts_resultset) { + delete Galera_Hosts_resultset; + Galera_Hosts_resultset=NULL; + } }; @@ -1014,6 +1032,188 @@ __fast_exit_monitor_group_replication_thread: return NULL; } +void * monitor_galera_thread(void *arg) { + MySQL_Monitor_State_Data *mmsd=(MySQL_Monitor_State_Data *)arg; + MySQL_Thread * mysql_thr = new MySQL_Thread(); + mysql_thr->curtime=monotonic_time(); + mysql_thr->refresh_variables(); + if (!GloMTH) return NULL; // quick exit during shutdown/restart + + mmsd->mysql=GloMyMon->My_Conn_Pool->get_connection(mmsd->hostname, mmsd->port); + unsigned long long start_time=mysql_thr->curtime; + + + mmsd->t1=start_time; + + bool crc=false; + if (mmsd->mysql==NULL) { // we don't have a connection, let's create it + bool rc; + rc=mmsd->create_new_connection(); + crc=true; + if (rc==false) { + goto __fast_exit_monitor_galera_thread; + } + } + + mmsd->t1=monotonic_time(); + mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT viable_candidate,read_only,transactions_behind FROM sys.gr_member_routing_candidate_status"); + while (mmsd->async_exit_status) { + mmsd->async_exit_status=wait_for_mysql(mmsd->mysql, mmsd->async_exit_status); + unsigned long long now=monotonic_time(); + if (now > mmsd->t1 + mysql_thread___monitor_galera_healthcheck_timeout * 1000) { + mmsd->mysql_error_msg=strdup("timeout check"); + proxy_error("Timeout on group replication health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_galera_healthcheck_timeout. Assuming viable_candidate=no and read_only=YES\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); + goto __exit_monitor_galera_thread; + } + if (GloMyMon->shutdown==true) { + goto __fast_exit_monitor_galera_thread; // exit immediately + } + if ((mmsd->async_exit_status & MYSQL_WAIT_TIMEOUT) == 0) { + mmsd->async_exit_status=mysql_query_cont(&mmsd->interr, mmsd->mysql, mmsd->async_exit_status); + } + } + mmsd->async_exit_status=mysql_store_result_start(&mmsd->result,mmsd->mysql); + while (mmsd->async_exit_status) { + mmsd->async_exit_status=wait_for_mysql(mmsd->mysql, mmsd->async_exit_status); + unsigned long long now=monotonic_time(); + if (now > mmsd->t1 + mysql_thread___monitor_galera_healthcheck_timeout * 1000) { + mmsd->mysql_error_msg=strdup("timeout check"); + proxy_error("Timeout on group replication health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_galera_healthcheck_timeout. Assuming viable_candidate=no and read_only=YES\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); + goto __exit_monitor_galera_thread; + } + if (GloMyMon->shutdown==true) { + goto __fast_exit_monitor_galera_thread; // exit immediately + } + if ((mmsd->async_exit_status & MYSQL_WAIT_TIMEOUT) == 0) { + mmsd->async_exit_status=mysql_store_result_cont(&mmsd->result, mmsd->mysql, mmsd->async_exit_status); + } + } + if (mmsd->interr) { // ping failed + mmsd->mysql_error_msg=strdup(mysql_error(mmsd->mysql)); + } + +__exit_monitor_galera_thread: + mmsd->t2=monotonic_time(); + { + // TODO : complete this + char buf[128]; + char *s=NULL; + int l=strlen(mmsd->hostname); + if (l<110) { + s=buf; + } else { + s=(char *)malloc(l+16); + } + sprintf(s,"%s:%d",mmsd->hostname,mmsd->port); + bool viable_candidate=false; + bool read_only=true; + long long transactions_behind=-1; + if (mmsd->result) { + int num_fields=0; + int num_rows=0; + num_fields = mysql_num_fields(mmsd->result); + if (num_fields!=3) { + proxy_error("Incorrect number of fields, please report a bug\n"); + goto __end_process_galera_result; + } + num_rows = mysql_num_rows(mmsd->result); + if (num_rows!=1) { + proxy_error("Incorrect number of rows, please report a bug\n"); + goto __end_process_galera_result; + } + MYSQL_ROW row=mysql_fetch_row(mmsd->result); + if (!strcasecmp(row[0],"YES")) { + viable_candidate=true; + } + if (!strcasecmp(row[1],"NO")) { + read_only=false; + } + transactions_behind=atol(row[2]); + mysql_free_result(mmsd->result); + mmsd->result=NULL; + } +__end_process_galera_result: + if (mmsd->mysql_error_msg) { + } + unsigned long long time_now=realtime_time(); + time_now=time_now-(mmsd->t2 - start_time); + pthread_mutex_lock(&GloMyMon->galera_mutex); + //auto it = + // TODO : complete this + std::map::iterator it2; + it2 = GloMyMon->Galera_Hosts_Map.find(s); + MyGR_monitor_node *node=NULL; + if (it2!=GloMyMon->Galera_Hosts_Map.end()) { + node=it2->second; + node->add_entry(time_now, (mmsd->mysql_error_msg ? 0 : mmsd->t2-mmsd->t1) , transactions_behind,viable_candidate,read_only,mmsd->mysql_error_msg); + } else { + node = new MyGR_monitor_node(mmsd->hostname,mmsd->port,mmsd->writer_hostgroup); + node->add_entry(time_now, (mmsd->mysql_error_msg ? 0 : mmsd->t2-mmsd->t1) , transactions_behind,viable_candidate,read_only,mmsd->mysql_error_msg); + GloMyMon->Galera_Hosts_Map.insert(std::make_pair(s,node)); + } + pthread_mutex_unlock(&GloMyMon->galera_mutex); + + // NOTE: we update MyHGM outside the mutex galera_mutex + if (mmsd->mysql_error_msg) { // there was an error checking the status of the server, surely we need to reconfigure GR + MyHGM->update_galera_set_offline(mmsd->hostname, mmsd->port, mmsd->writer_hostgroup, mmsd->mysql_error_msg); + } else { + if (viable_candidate==false) { + MyHGM->update_galera_set_offline(mmsd->hostname, mmsd->port, mmsd->writer_hostgroup, (char *)"viable_candidate=NO"); + } else { + if (read_only==true) { + if (transactions_behind > mmsd->max_transactions_behind) { + MyHGM->update_galera_set_offline(mmsd->hostname, mmsd->port, mmsd->writer_hostgroup, (char *)"slave is lagging"); + } else { + MyHGM->update_galera_set_read_only(mmsd->hostname, mmsd->port, mmsd->writer_hostgroup, (char *)"read_only=YES"); + } + } else { + // the node is a writer + // TODO: for now we don't care about the number of writers + MyHGM->update_galera_set_writer(mmsd->hostname, mmsd->port, mmsd->writer_hostgroup); + } + } + } + + // clean up + if (l<110) { + } else { + free(s); + } + } + if (mmsd->interr) { // check failed + } else { + if (crc==false) { + if (mmsd->mysql) { + GloMyMon->My_Conn_Pool->put_connection(mmsd->hostname,mmsd->port,mmsd->mysql); + mmsd->mysql=NULL; + } + } + } +__fast_exit_monitor_galera_thread: + if (mmsd->mysql) { + // if we reached here we didn't put the connection back + if (mmsd->mysql_error_msg) { + mysql_close(mmsd->mysql); // if we reached here we should destroy it + mmsd->mysql=NULL; + } else { + if (crc) { + bool rc=mmsd->set_wait_timeout(); + if (rc) { + GloMyMon->My_Conn_Pool->put_connection(mmsd->hostname,mmsd->port,mmsd->mysql); + } else { + mysql_close(mmsd->mysql); // set_wait_timeout failed + } + mmsd->mysql=NULL; + } else { // really not sure how we reached here, drop it + mysql_close(mmsd->mysql); + mmsd->mysql=NULL; + } + } + } + delete mysql_thr; + return NULL; +} + void * monitor_replication_lag_thread(void *arg) { MySQL_Monitor_State_Data *mmsd=(MySQL_Monitor_State_Data *)arg; MySQL_Thread * mysql_thr = new MySQL_Thread(); @@ -1756,6 +1956,94 @@ __sleep_monitor_group_replication: } return NULL; } +void * MySQL_Monitor::monitor_galera() { + // initialize the MySQL Thread (note: this is not a real thread, just the structures associated with it) +// struct event_base *libevent_base; + unsigned int latest_table_servers_version=0; + unsigned int MySQL_Monitor__thread_MySQL_Thread_Variables_version; + MySQL_Thread * mysql_thr = new MySQL_Thread(); + mysql_thr->curtime=monotonic_time(); + MySQL_Monitor__thread_MySQL_Thread_Variables_version=GloMTH->get_global_version(); + mysql_thr->refresh_variables(); + if (!GloMTH) return NULL; // quick exit during shutdown/restart + + unsigned long long t1; + unsigned long long t2; + unsigned long long next_loop_at=0; + + while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true) { + + unsigned int glover; + t1=monotonic_time(); + + if (!GloMTH) return NULL; // quick exit during shutdown/restart + glover=GloMTH->get_global_version(); + if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover ) { + MySQL_Monitor__thread_MySQL_Thread_Variables_version=glover; + mysql_thr->refresh_variables(); + next_loop_at=0; + } + + if (t1 < next_loop_at) { + goto __sleep_monitor_galera; + } + next_loop_at=t1+1000*mysql_thread___monitor_galera_healthcheck_interval; + pthread_mutex_lock(&galera_mutex); + if (Galera_Hosts_resultset==NULL) { + goto __end_monitor_galera_loop; + } else { + if (Galera_Hosts_resultset->rows_count==0) { + goto __end_monitor_galera_loop; + } + int us=100; + if (Galera_Hosts_resultset->rows_count) { + us=mysql_thread___monitor_read_only_interval/2/Galera_Hosts_resultset->rows_count; + } + for (std::vector::iterator it = Galera_Hosts_resultset->rows.begin() ; it != Galera_Hosts_resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + MySQL_Monitor_State_Data *mmsd=new MySQL_Monitor_State_Data(r->fields[1],atoi(r->fields[2]), NULL, atoi(r->fields[3])); + mmsd->writer_hostgroup=atoi(r->fields[0]); + mmsd->writer_is_also_reader=atoi(r->fields[4]); + mmsd->max_transactions_behind=atoi(r->fields[5]); + mmsd->mondb=monitordb; + WorkItem* item; + item=new WorkItem(mmsd,monitor_galera_thread); + GloMyMon->queue.add(item); + usleep(us); + if (GloMyMon->shutdown) { + pthread_mutex_unlock(&galera_mutex); + return NULL; + } + } + } + +__end_monitor_galera_loop: + pthread_mutex_unlock(&galera_mutex); + if (mysql_thread___monitor_enabled==true) { + } + + +__sleep_monitor_galera: + t2=monotonic_time(); + if (t2 500000) { + st = 500000; + } + usleep(st); + } + } + if (mysql_thr) { + delete mysql_thr; + mysql_thr=NULL; + } + for (unsigned int i=0;iqueue.add(item); + } + return NULL; +} void * MySQL_Monitor::monitor_replication_lag() { // initialize the MySQL Thread (note: this is not a real thread, just the structures associated with it) @@ -1899,6 +2187,8 @@ __monitor_run: pthread_create(&monitor_read_only_thread, &attr, &monitor_read_only_pthread,NULL); pthread_t monitor_group_replication_thread; pthread_create(&monitor_group_replication_thread, &attr, &monitor_group_replication_pthread,NULL); + pthread_t monitor_galera_thread; + pthread_create(&monitor_galera_thread, &attr, &monitor_galera_pthread,NULL); pthread_t monitor_replication_lag_thread; pthread_create(&monitor_replication_lag_thread, &attr, &monitor_replication_lag_pthread,NULL); while (shutdown==false && mysql_thread___monitor_enabled==true) { @@ -1945,6 +2235,7 @@ __monitor_run: pthread_join(monitor_ping_thread,NULL); pthread_join(monitor_read_only_thread,NULL); pthread_join(monitor_group_replication_thread,NULL); + pthread_join(monitor_galera_thread,NULL); pthread_join(monitor_replication_lag_thread,NULL); while (shutdown==false) { unsigned int glover; @@ -2069,3 +2360,42 @@ void MySQL_Monitor::populate_monitor_mysql_server_group_replication_log() { } pthread_mutex_unlock(&GloMyMon->group_replication_mutex); } + +void MySQL_Monitor::populate_monitor_mysql_server_galera_log() { + sqlite3 *mondb=monitordb->get_db(); + int rc; + //char *query=NULL; + char *query1=NULL; + query1=(char *)"INSERT INTO mysql_server_galera_log VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)"; + sqlite3_stmt *statement1=NULL; + pthread_mutex_lock(&GloMyMon->galera_mutex); + rc=sqlite3_prepare_v2(mondb, query1, -1, &statement1, 0); + assert(rc==SQLITE_OK); + monitordb->execute((char *)"DELETE FROM mysql_server_galera_log"); + std::map::iterator it2; + MyGR_monitor_node *node=NULL; + for (it2=GloMyMon->Galera_Hosts_Map.begin(); it2!=GloMyMon->Galera_Hosts_Map.end(); ++it2) { + std::string s=it2->first; + node=it2->second; + std::size_t found=s.find_last_of(":"); + std::string host=s.substr(0,found); + std::string port=s.substr(found+1); + int i; + for (i=0; ilast_entries[i].start_time) { + rc=sqlite3_bind_text(statement1, 1, host.c_str(), -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement1, 2, atoi(port.c_str())); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement1, 3, node->last_entries[i].start_time ); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement1, 4, node->last_entries[i].check_time ); assert(rc==SQLITE_OK); + rc=sqlite3_bind_text(statement1, 5, ( node->last_entries[i].primary_partition ? (char *)"YES" : (char *)"NO" ) , -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); + rc=sqlite3_bind_text(statement1, 6, ( node->last_entries[i].read_only ? (char *)"YES" : (char *)"NO" ) , -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement1, 7, node->last_entries[i].transactions_behind ); assert(rc==SQLITE_OK); + rc=sqlite3_bind_text(statement1, 8, node->last_entries[i].error , -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); + SAFE_SQLITE3_STEP2(statement1); + rc=sqlite3_clear_bindings(statement1); assert(rc==SQLITE_OK); + rc=sqlite3_reset(statement1); assert(rc==SQLITE_OK); + } + } + } + pthread_mutex_unlock(&GloMyMon->galera_mutex); +} diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index eb11f81e8..e02c361be 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -242,6 +242,8 @@ static char * mysql_thread_variables_names[]= { (char *)"monitor_replication_lag_timeout", (char *)"monitor_groupreplication_healthcheck_interval", (char *)"monitor_groupreplication_healthcheck_timeout", + (char *)"monitor_galera_healthcheck_interval", + (char *)"monitor_galera_healthcheck_timeout", (char *)"monitor_username", (char *)"monitor_password", (char *)"monitor_replication_lag_use_percona_heartbeat", @@ -361,6 +363,8 @@ MySQL_Threads_Handler::MySQL_Threads_Handler() { variables.monitor_replication_lag_timeout=1000; variables.monitor_groupreplication_healthcheck_interval=5000; variables.monitor_groupreplication_healthcheck_timeout=800; + variables.monitor_galera_healthcheck_interval=5000; + variables.monitor_galera_healthcheck_timeout=800; variables.monitor_query_interval=60000; variables.monitor_query_timeout=100; variables.monitor_slave_lag_when_null=60; @@ -617,6 +621,8 @@ int MySQL_Threads_Handler::get_variable_int(char *name) { if (!strcasecmp(name,"monitor_replication_lag_timeout")) return (int)variables.monitor_replication_lag_timeout; if (!strcasecmp(name,"monitor_groupreplication_healthcheck_interval")) return (int)variables.monitor_groupreplication_healthcheck_interval; if (!strcasecmp(name,"monitor_groupreplication_healthcheck_timeout")) return (int)variables.monitor_groupreplication_healthcheck_timeout; + if (!strcasecmp(name,"monitor_galera_healthcheck_interval")) return (int)variables.monitor_galera_healthcheck_interval; + if (!strcasecmp(name,"monitor_galera_healthcheck_timeout")) return (int)variables.monitor_galera_healthcheck_timeout; if (!strcasecmp(name,"monitor_query_interval")) return (int)variables.monitor_query_interval; if (!strcasecmp(name,"monitor_query_timeout")) return (int)variables.monitor_query_timeout; if (!strcasecmp(name,"monitor_slave_lag_when_null")) return (int)variables.monitor_slave_lag_when_null; @@ -814,6 +820,14 @@ char * MySQL_Threads_Handler::get_variable(char *name) { // this is the public f sprintf(intbuf,"%d",variables.monitor_groupreplication_healthcheck_timeout); return strdup(intbuf); } + if (!strcasecmp(name,"monitor_galera_healthcheck_interval")) { + sprintf(intbuf,"%d",variables.monitor_galera_healthcheck_interval); + return strdup(intbuf); + } + if (!strcasecmp(name,"monitor_galera_healthcheck_timeout")) { + sprintf(intbuf,"%d",variables.monitor_galera_healthcheck_timeout); + return strdup(intbuf); + } if (!strcasecmp(name,"monitor_query_interval")) { sprintf(intbuf,"%d",variables.monitor_query_interval); return strdup(intbuf); @@ -1265,6 +1279,24 @@ bool MySQL_Threads_Handler::set_variable(char *name, char *value) { // this is t return false; } } + if (!strcasecmp(name,"monitor_galera_healthcheck_interval")) { + int intv=atoi(value); + if (intv >= 50 && intv <= 7*24*3600*1000) { + variables.monitor_galera_healthcheck_interval=intv; + return true; + } else { + return false; + } + } + if (!strcasecmp(name,"monitor_galera_healthcheck_timeout")) { + int intv=atoi(value); + if (intv >= 50 && intv <= 600*1000) { + variables.monitor_galera_healthcheck_timeout=intv; + return true; + } else { + return false; + } + } if (!strcasecmp(name,"monitor_query_interval")) { int intv=atoi(value); if (intv >= 100 && intv <= 7*24*3600*1000) { @@ -3391,6 +3423,8 @@ void MySQL_Thread::refresh_variables() { mysql_thread___monitor_replication_lag_timeout=GloMTH->get_variable_int((char *)"monitor_replication_lag_timeout"); mysql_thread___monitor_groupreplication_healthcheck_interval=GloMTH->get_variable_int((char *)"monitor_groupreplication_healthcheck_interval"); mysql_thread___monitor_groupreplication_healthcheck_timeout=GloMTH->get_variable_int((char *)"monitor_groupreplication_healthcheck_timeout"); + mysql_thread___monitor_galera_healthcheck_interval=GloMTH->get_variable_int((char *)"monitor_galera_healthcheck_interval"); + mysql_thread___monitor_galera_healthcheck_timeout=GloMTH->get_variable_int((char *)"monitor_galera_healthcheck_timeout"); mysql_thread___monitor_query_interval=GloMTH->get_variable_int((char *)"monitor_query_interval"); mysql_thread___monitor_query_timeout=GloMTH->get_variable_int((char *)"monitor_query_timeout"); mysql_thread___monitor_slave_lag_when_null=GloMTH->get_variable_int((char *)"monitor_slave_lag_when_null"); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 51db65dd7..2ef8ff41a 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -303,6 +303,10 @@ static int http_handler(void *cls, struct MHD_Connection *connection, const char #define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GROUP_REPLICATION_HOSTGROUPS "CREATE TABLE runtime_mysql_group_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" +#define ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS "CREATE TABLE mysql_galera_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" + +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GALERA_HOSTGROUPS "CREATE TABLE runtime_mysql_galera_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , backup_writer_hostgroup INT CHECK (backup_writer_hostgroup>=0 AND backup_writer_hostgroup<>writer_hostgroup) NOT NULL , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND backup_writer_hostgroup<>reader_hostgroup AND reader_hostgroup>0) , offline_hostgroup INT NOT NULL CHECK (offline_hostgroup<>writer_hostgroup AND offline_hostgroup<>reader_hostgroup AND backup_writer_hostgroup<>offline_hostgroup AND offline_hostgroup>=0) , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , max_writers INT NOT NULL CHECK (max_writers >= 0) DEFAULT 1 , writer_is_also_reader INT CHECK (writer_is_also_reader IN (0,1)) NOT NULL DEFAULT 0 , max_transactions_behind INT CHECK (max_transactions_behind>=0) NOT NULL DEFAULT 0 , comment VARCHAR , UNIQUE (reader_hostgroup) , UNIQUE (offline_hostgroup) , UNIQUE (backup_writer_hostgroup))" + // Cluster solution @@ -1898,6 +1902,8 @@ void ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign bool monitor_mysql_server_group_replication_log=false; + bool monitor_mysql_server_galera_log=false; + bool stats_proxysql_servers_checksums = false; bool stats_proxysql_servers_metrics = false; bool stats_proxysql_servers_status = false; @@ -1952,6 +1958,8 @@ void ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign strstr(query_no_space,"runtime_mysql_replication_hostgroups") || strstr(query_no_space,"runtime_mysql_group_replication_hostgroups") + || + strstr(query_no_space,"runtime_mysql_galera_hostgroups") ) { runtime_mysql_servers=true; refresh=true; } @@ -1985,6 +1993,9 @@ void ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign if (strstr(query_no_space,"mysql_server_group_replication_log")) { monitor_mysql_server_group_replication_log=true; refresh=true; } + if (strstr(query_no_space,"mysql_server_galera_log")) { + monitor_mysql_server_galera_log=true; refresh=true; + } // if (stats_mysql_processlist || stats_mysql_connection_pool || stats_mysql_query_digest || stats_mysql_query_digest_reset) { if (refresh==true) { pthread_mutex_lock(&admin_mutex); @@ -2075,6 +2086,11 @@ void ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign GloMyMon->populate_monitor_mysql_server_group_replication_log(); } } + if (monitor_mysql_server_galera_log) { + if (GloMyMon) { + GloMyMon->populate_monitor_mysql_server_galera_log(); + } + } pthread_mutex_unlock(&admin_mutex); } } @@ -2739,6 +2755,16 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); } + if ((strlen(query_no_space)==strlen("CHECKSUM MEMORY MYSQL GALERA HOSTGROUPS") && !strncasecmp("CHECKSUM MEMORY GROUP MYSQL REPLICATION HOSTGROUPS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MEM MYSQL GALERA HOSTGROUPS") && !strncasecmp("CHECKSUM MEM MYSQL GALERA HOSTGROUPS", query_no_space, strlen(query_no_space))) + || + (strlen(query_no_space)==strlen("CHECKSUM MYSQL GALERA HOSTGROUPS") && !strncasecmp("CHECKSUM MYSQL GALERA HOSTGROUPS", query_no_space, strlen(query_no_space)))){ + char *q=(char *)"SELECT * FROM mysql_galera_hostgroups ORDER BY writer_hostgroup"; + tablename=(char *)"MYSQL GALERA HOSTGROUPS"; + SPA->admindb->execute_statement(q, &error, &cols, &affected_rows, &resultset); + } + if (error) { proxy_error("Error: %s\n", error); char buf[1024]; @@ -3487,6 +3513,8 @@ bool ProxySQL_Admin::init() { insert_into_tables_defs(tables_defs_admin,"mysql_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS); insert_into_tables_defs(tables_defs_admin,"mysql_group_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); insert_into_tables_defs(tables_defs_admin,"runtime_mysql_group_replication_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GROUP_REPLICATION_HOSTGROUPS); + insert_into_tables_defs(tables_defs_admin,"mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_GALERA_HOSTGROUPS); insert_into_tables_defs(tables_defs_admin,"mysql_query_rules", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES); insert_into_tables_defs(tables_defs_admin,"mysql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_FAST_ROUTING); insert_into_tables_defs(tables_defs_admin,"runtime_mysql_query_rules", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_QUERY_RULES); @@ -3511,6 +3539,7 @@ bool ProxySQL_Admin::init() { insert_into_tables_defs(tables_defs_config,"mysql_users", ADMIN_SQLITE_TABLE_MYSQL_USERS); insert_into_tables_defs(tables_defs_config,"mysql_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS); insert_into_tables_defs(tables_defs_config,"mysql_group_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GROUP_REPLICATION_HOSTGROUPS); + insert_into_tables_defs(tables_defs_config,"mysql_galera_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_GALERA_HOSTGROUPS); insert_into_tables_defs(tables_defs_config,"mysql_query_rules", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES); insert_into_tables_defs(tables_defs_config,"mysql_query_rules_fast_routing", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_FAST_ROUTING); insert_into_tables_defs(tables_defs_config,"global_variables", ADMIN_SQLITE_TABLE_GLOBAL_VARIABLES); @@ -5753,6 +5782,7 @@ void ProxySQL_Admin::__insert_or_ignore_maintable_select_disktable() { admindb->execute("INSERT OR IGNORE INTO main.mysql_servers SELECT * FROM disk.mysql_servers"); admindb->execute("INSERT OR IGNORE INTO main.mysql_replication_hostgroups SELECT * FROM disk.mysql_replication_hostgroups"); admindb->execute("INSERT OR IGNORE INTO main.mysql_group_replication_hostgroups SELECT * FROM disk.mysql_group_replication_hostgroups"); + admindb->execute("INSERT OR IGNORE INTO main.mysql_galera_hostgroups SELECT * FROM disk.mysql_galera_hostgroups"); admindb->execute("INSERT OR IGNORE INTO main.mysql_users SELECT * FROM disk.mysql_users"); admindb->execute("INSERT OR IGNORE INTO main.mysql_query_rules SELECT * FROM disk.mysql_query_rules"); admindb->execute("INSERT OR IGNORE INTO main.mysql_query_rules_fast_routing SELECT * FROM disk.mysql_query_rules_fast_routing"); @@ -5775,6 +5805,7 @@ void ProxySQL_Admin::__insert_or_replace_maintable_select_disktable() { admindb->execute("INSERT OR REPLACE INTO main.mysql_servers SELECT * FROM disk.mysql_servers"); admindb->execute("INSERT OR REPLACE INTO main.mysql_replication_hostgroups SELECT * FROM disk.mysql_replication_hostgroups"); admindb->execute("INSERT OR REPLACE INTO main.mysql_group_replication_hostgroups SELECT * FROM disk.mysql_group_replication_hostgroups"); + admindb->execute("INSERT OR REPLACE INTO main.mysql_galera_hostgroups SELECT * FROM disk.mysql_galera_hostgroups"); admindb->execute("INSERT OR REPLACE INTO main.mysql_users SELECT * FROM disk.mysql_users"); admindb->execute("INSERT OR REPLACE INTO main.mysql_query_rules SELECT * FROM disk.mysql_query_rules"); admindb->execute("INSERT OR REPLACE INTO main.mysql_query_rules_fast_routing SELECT * FROM disk.mysql_query_rules_fast_routing"); @@ -5815,6 +5846,7 @@ void ProxySQL_Admin::__insert_or_replace_disktable_select_maintable() { admindb->execute("INSERT OR REPLACE INTO disk.mysql_servers SELECT * FROM main.mysql_servers"); admindb->execute("INSERT OR REPLACE INTO disk.mysql_replication_hostgroups SELECT * FROM main.mysql_replication_hostgroups"); admindb->execute("INSERT OR REPLACE INTO disk.mysql_group_replication_hostgroups SELECT * FROM main.mysql_group_replication_hostgroups"); + admindb->execute("INSERT OR REPLACE INTO disk.mysql_galera_hostgroups SELECT * FROM main.mysql_galera_hostgroups"); admindb->execute("INSERT OR REPLACE INTO disk.mysql_query_rules SELECT * FROM main.mysql_query_rules"); admindb->execute("INSERT OR REPLACE INTO disk.mysql_users SELECT * FROM main.mysql_users"); admindb->execute("INSERT OR REPLACE INTO disk.mysql_query_rules_fast_routing SELECT * FROM main.mysql_query_rules_fast_routing"); @@ -5890,9 +5922,11 @@ void ProxySQL_Admin::flush_mysql_servers__from_disk_to_memory() { admindb->execute("DELETE FROM main.mysql_servers"); admindb->execute("DELETE FROM main.mysql_replication_hostgroups"); admindb->execute("DELETE FROM main.mysql_group_replication_hostgroups"); + admindb->execute("DELETE FROM main.mysql_galera_hostgroups"); admindb->execute("INSERT INTO main.mysql_servers SELECT * FROM disk.mysql_servers"); admindb->execute("INSERT INTO main.mysql_replication_hostgroups SELECT * FROM disk.mysql_replication_hostgroups"); admindb->execute("INSERT INTO main.mysql_group_replication_hostgroups SELECT * FROM disk.mysql_group_replication_hostgroups"); + admindb->execute("INSERT INTO main.mysql_galera_hostgroups SELECT * FROM disk.mysql_galera_hostgroups"); admindb->execute("PRAGMA foreign_keys = ON"); admindb->wrunlock(); } @@ -5903,9 +5937,10 @@ void ProxySQL_Admin::flush_mysql_servers__from_memory_to_disk() { admindb->execute("DELETE FROM disk.mysql_servers"); admindb->execute("DELETE FROM disk.mysql_replication_hostgroups"); admindb->execute("DELETE FROM disk.mysql_group_replication_hostgroups"); + admindb->execute("DELETE FROM disk.mysql_galera_hostgroups"); admindb->execute("INSERT INTO disk.mysql_servers SELECT * FROM main.mysql_servers"); admindb->execute("INSERT INTO disk.mysql_replication_hostgroups SELECT * FROM main.mysql_replication_hostgroups"); - admindb->execute("INSERT INTO disk.mysql_group_replication_hostgroups SELECT * FROM main.mysql_group_replication_hostgroups"); + admindb->execute("INSERT INTO disk.mysql_galera_hostgroups SELECT * FROM main.mysql_galera_hostgroups"); admindb->execute("PRAGMA foreign_keys = ON"); admindb->wrunlock(); } @@ -6918,6 +6953,48 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { sqlite3_finalize(statement); } if(resultset) delete resultset; + + // dump mysql_galera_hostgroups + if (_runtime) { + query=(char *)"DELETE FROM main.runtime_mysql_galera_hostgroups"; + } else { + query=(char *)"DELETE FROM main.mysql_galera_hostgroups"; + } + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + admindb->execute(query); + resultset=MyHGM->dump_table_mysql_galera_hostgroups(); + if (resultset) { + int rc; + sqlite3_stmt *statement=NULL; + sqlite3 *mydb3=admindb->get_db(); + char *query=NULL; + if (_runtime) { + query=(char *)"INSERT INTO runtime_mysql_galera_hostgroups(writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers,writer_is_also_reader,max_transactions_behind,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"; + } else { + query=(char *)"INSERT INTO mysql_galera_hostgroups(writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers,writer_is_also_reader,max_transactions_behind,comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"; + } + rc=sqlite3_prepare_v2(mydb3, query, -1, &statement, 0); + assert(rc==SQLITE_OK); + //proxy_info("New mysql_galera_hostgroups table\n"); + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + rc=sqlite3_bind_int64(statement, 1, atoi(r->fields[0])); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement, 2, atoi(r->fields[1])); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement, 3, atoi(r->fields[2])); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement, 4, atoi(r->fields[3])); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement, 5, atoi(r->fields[4])); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement, 6, atoi(r->fields[5])); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement, 7, atoi(r->fields[6])); assert(rc==SQLITE_OK); + rc=sqlite3_bind_int64(statement, 8, atoi(r->fields[7])); assert(rc==SQLITE_OK); + rc=sqlite3_bind_text(statement, 9, r->fields[8], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); + + SAFE_SQLITE3_STEP2(statement); + rc=sqlite3_clear_bindings(statement); assert(rc==SQLITE_OK); + rc=sqlite3_reset(statement); assert(rc==SQLITE_OK); + } + sqlite3_finalize(statement); + } + if(resultset) delete resultset; resultset=NULL; } @@ -6946,6 +7023,7 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime() { SQLite3_result *resultset=NULL; SQLite3_result *resultset_replication=NULL; SQLite3_result *resultset_group_replication=NULL; + SQLite3_result *resultset_galera=NULL; char *query=(char *)"SELECT hostgroup_id,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM main.mysql_servers"; proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); @@ -7013,6 +7091,33 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime() { MyHGM->set_incoming_group_replication_hostgroups(resultset_group_replication); } + // support for Galera, table mysql_galera_hostgroups + + // look for invalid combinations + query=(char *)"SELECT a.* FROM mysql_galera_hostgroups a JOIN mysql_galera_hostgroups b ON a.writer_hostgroup=b.reader_hostgroup WHERE b.reader_hostgroup UNION ALL SELECT a.* FROM mysql_galera_hostgroups a JOIN mysql_galera_hostgroups b ON a.writer_hostgroup=b.backup_writer_hostgroup WHERE b.backup_writer_hostgroup UNION ALL SELECT a.* FROM mysql_galera_hostgroups a JOIN mysql_galera_hostgroups b ON a.writer_hostgroup=b.offline_hostgroup WHERE b.offline_hostgroup"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + if (error) { + proxy_error("Error on %s : %s\n", query, error); + } else { + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + proxy_error("Incompatible entry in mysql_galera_hostgroups will be ignored : ( %s , %s , %s , %s )\n", r->fields[0], r->fields[1], r->fields[2], r->fields[3]); + } + } + if (resultset) delete resultset; + resultset=NULL; + + query=(char *)"SELECT a.* FROM mysql_galera_hostgroups a LEFT JOIN mysql_galera_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup OR a.writer_hostgroup=b.backup_writer_hostgroup OR a.writer_hostgroup=b.offline_hostgroup) WHERE b.reader_hostgroup IS NULL AND b.backup_writer_hostgroup IS NULL AND b.offline_hostgroup IS NULL"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_galera); + if (error) { + proxy_error("Error on %s : %s\n", query, error); + } else { + // Pass the resultset to MyHGM + MyHGM->set_incoming_galera_hostgroups(resultset_galera); + } + // commit all the changes MyHGM->commit(); @@ -7027,6 +7132,10 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime() { delete resultset_replication; resultset_group_replication=NULL; } + if (resultset_galera) { + delete resultset_galera; + resultset_galera=NULL; + } } From 1cb13309dbef53975c35bdc935146ef3d6f820b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Mon, 16 Apr 2018 16:26:59 +0200 Subject: [PATCH 2/6] Adding Galera check --- lib/MySQL_Monitor.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 7d8bcdf66..4b4da6240 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -1056,13 +1056,13 @@ void * monitor_galera_thread(void *arg) { } mmsd->t1=monotonic_time(); - mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT viable_candidate,read_only,transactions_behind FROM sys.gr_member_routing_candidate_status"); + mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT (SELECT IF(VARIABLE_VALUE=4,'YES','NO') FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='WSREP_LOCAL_STATE') wsrep_local_state, (SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME='READ_ONLY') read_only, (SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='WSREP_LOCAL_RECV_QUEUE') wsrep_local_recv_queue"); while (mmsd->async_exit_status) { mmsd->async_exit_status=wait_for_mysql(mmsd->mysql, mmsd->async_exit_status); unsigned long long now=monotonic_time(); if (now > mmsd->t1 + mysql_thread___monitor_galera_healthcheck_timeout * 1000) { mmsd->mysql_error_msg=strdup("timeout check"); - proxy_error("Timeout on group replication health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_galera_healthcheck_timeout. Assuming viable_candidate=no and read_only=YES\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); + proxy_error("Timeout on Galera health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_galera_healthcheck_timeout. Assuming wsrep_local_state is NOT 4 and read_only=YES\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); goto __exit_monitor_galera_thread; } if (GloMyMon->shutdown==true) { @@ -1078,7 +1078,7 @@ void * monitor_galera_thread(void *arg) { unsigned long long now=monotonic_time(); if (now > mmsd->t1 + mysql_thread___monitor_galera_healthcheck_timeout * 1000) { mmsd->mysql_error_msg=strdup("timeout check"); - proxy_error("Timeout on group replication health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_galera_healthcheck_timeout. Assuming viable_candidate=no and read_only=YES\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); + proxy_error("Timeout on Galera health check for %s:%d after %lldms. If the server is overload, increase mysql-monitor_galera_healthcheck_timeout. Assuming wsrep_local_state is NOT 4 and read_only=YES\n", mmsd->hostname, mmsd->port, (now-mmsd->t1)/1000); goto __exit_monitor_galera_thread; } if (GloMyMon->shutdown==true) { From ffffd1324e61a1bfbcf4532009a7efb3e43a61f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 17 Apr 2018 09:18:15 +0200 Subject: [PATCH 3/6] Added missing mutex initialization for Galera --- lib/MySQL_HostGroups_Manager.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index e65a3a92a..9454c8d04 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -815,6 +815,7 @@ MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() { status.frontend_use_db=0; pthread_mutex_init(&readonly_mutex, NULL); pthread_mutex_init(&Group_Replication_Info_mutex, NULL); + pthread_mutex_init(&Galera_Info_mutex, NULL); #ifdef MHM_PTHREAD_MUTEX pthread_mutex_init(&lock, NULL); #else From 9f98174e5c1bbd0e128f0ef40728b226a110ddc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 17 Apr 2018 09:25:10 +0200 Subject: [PATCH 4/6] Remove incorrect delete of resultset from Admin --- lib/ProxySQL_Admin.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 2ef8ff41a..5631d223b 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -7129,11 +7129,11 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime() { resultset_replication=NULL; } if (resultset_group_replication) { - delete resultset_replication; + //delete resultset_replication; // do not delete, resultset is stored in MyHGM resultset_group_replication=NULL; } if (resultset_galera) { - delete resultset_galera; + //delete resultset_galera; // do not delete, resultset is stored in MyHGM resultset_galera=NULL; } } From 9175e5f426830b6530bddb4514d47b8365bd94a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 17 Apr 2018 09:44:29 +0200 Subject: [PATCH 5/6] Fix typo --- lib/MySQL_HostGroups_Manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 9454c8d04..c4a3df474 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1124,7 +1124,7 @@ bool MySQL_HostGroups_Manager::commit() { //fprintf(stderr,"%lld\n", ptr); if (ptr==0) { if (GloMTH->variables.hostgroup_manager_verbose) { - proxy_info("Creating new server in HG %d : %s:%d , gtid_port=%d, weight=%d, status=%s\n", atoi(r->fields[0]), r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), atoi(r->fields[4]), (MySerStatus) atoi(r->fields[5])); + proxy_info("Creating new server in HG %d : %s:%d , gtid_port=%d, weight=%d, status=%d\n", atoi(r->fields[0]), r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), atoi(r->fields[4]), (MySerStatus) atoi(r->fields[5])); } MySrvC *mysrvc=new MySrvC(r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), atoi(r->fields[4]), (MySerStatus) atoi(r->fields[5]), atoi(r->fields[6]), atoi(r->fields[7]), atoi(r->fields[8]), atoi(r->fields[9]), atoi(r->fields[10]), r->fields[11]); // add new fields here if adding more columns in mysql_servers proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Adding new server %s:%d , weight=%d, status=%d, mem_ptr=%p into hostgroup=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), (MySerStatus) atoi(r->fields[4]), mysrvc, atoi(r->fields[0])); From 7b11967fe304e52e74cd57bfdbff6d4a2a2a4439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 17 Apr 2018 09:48:50 +0200 Subject: [PATCH 6/6] Change query check for Galera --- lib/MySQL_Monitor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 4b4da6240..967560ddc 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -1056,7 +1056,7 @@ void * monitor_galera_thread(void *arg) { } mmsd->t1=monotonic_time(); - mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT (SELECT IF(VARIABLE_VALUE=4,'YES','NO') FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='WSREP_LOCAL_STATE') wsrep_local_state, (SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME='READ_ONLY') read_only, (SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='WSREP_LOCAL_RECV_QUEUE') wsrep_local_recv_queue"); + mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT (SELECT IF(VARIABLE_VALUE=4,'YES','NO') FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='WSREP_LOCAL_STATE') wsrep_local_state, (SELECT IF(VARIABLE_VALUE=0 OR VARIABLE_VALUE='OFF','NO','YES') FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME='READ_ONLY') read_only, (SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME='WSREP_LOCAL_RECV_QUEUE') wsrep_local_recv_queue"); while (mmsd->async_exit_status) { mmsd->async_exit_status=wait_for_mysql(mmsd->mysql, mmsd->async_exit_status); unsigned long long now=monotonic_time();