From 308f3a8a90f86640ff63ad6292d6436d9e17c093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Thu, 26 Feb 2015 11:48:29 +0000 Subject: [PATCH] Further development on MySQL Hostgroups Manager Implemented functions: - MySQL_HostGroups_Manager::dump_table_mysql_servers() - MySQL_HostGroups_Manager::generate_mysql_servers_table() - Standard_ProxySQL_Admin::save_mysql_servers_runtime_to_database() (issue #17) Improved Standard_ProxySQL_Admin::load_mysql_servers_to_runtime() to process also status Improved MySQL_HostGroups_Manager::commit() to modified MySrvCs Database in MyHGM is accessible via Admin (issue #212) Enforced debugging if specified at the command line (issue #213) --- include/MySQL_HostGroups_Manager.h | 2 + lib/MySQL_HostGroups_Manager.cpp | 62 +++++++++++++++++++++++++-- lib/MySQL_Session.cpp | 1 + lib/Standard_ProxySQL_Admin.cpp | 67 ++++++++++++++++++++++++++---- 4 files changed, 120 insertions(+), 12 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 909f2aebf..4530b00eb 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -82,6 +82,7 @@ class MySQL_HostGroups_Manager { MyHGC * MyHGC_create(unsigned int); void add(MySrvC *, unsigned int); + void generate_mysql_servers_table(); public: MySQL_HostGroups_Manager(); @@ -93,6 +94,7 @@ class MySQL_HostGroups_Manager { bool server_add(unsigned int hid, char *add, uint16_t p=3306, unsigned int _weight=1, enum MySerStatus status=MYSQL_SERVER_STATUS_ONLINE); bool commit(); + SQLite3_result *dump_table_mysql_servers(); MyHGC * MyHGC_lookup(unsigned int); void MyConn_add_to_pool(MySQL_Connection *); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 5e3463fb5..a7b05cd59 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -211,6 +211,7 @@ void MySQL_HostGroups_Manager::wrunlock() { // we always assume that the calling thread has acquired a rdlock() bool MySQL_HostGroups_Manager::server_add(unsigned int hid, char *add, uint16_t p, unsigned int _weight, enum MySerStatus status) { bool ret; + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Adding in mysql_servers_incoming server %s:%d in hostgroup %u with weight %u and status %u\n", add,p,hid,_weight,status); char *q=(char *)"INSERT INTO mysql_servers_incoming VALUES (%u, \"%s\", %u, %u, %u)"; char *query=(char *)malloc(strlen(q)+strlen(add)+100); sprintf(query,q,hid,add,p,_weight,status); @@ -220,6 +221,7 @@ bool MySQL_HostGroups_Manager::server_add(unsigned int hid, char *add, uint16_t } + bool MySQL_HostGroups_Manager::commit() { char *error=NULL; int cols=0; @@ -232,6 +234,7 @@ bool MySQL_HostGroups_Manager::commit() { if (error) { proxy_error("Error on %s : %s\n", query, error); } else { +// FIXME: this part is for debugging only, needs to be removed/cleaned for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; long long ptr=atoll(r->fields[0]); @@ -240,9 +243,14 @@ bool MySQL_HostGroups_Manager::commit() { } if (resultset) { delete resultset; resultset=NULL; } - mydb->execute("INSERT OR IGNORE INTO mysql_servers(hostgroup_id, hostname, port, weight, status) SELECT hostgroup_id, hostname, port, weight, status FROM mysql_servers_incoming"); +// INSERT OR IGNORE INTO mysql_servers SELECT ... FROM mysql_servers_incoming + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "INSERT OR IGNORE INTO mysql_servers(hostgroup_id, hostname, port, weight, status) SELECT hostgroup_id, hostname, port, weight, status FROM mysql_servers_incoming\n"); + mydb->execute("INSERT OR IGNORE INTO mysql_servers(hostgroup_id, hostname, port, weight, status) SELECT hostgroup_id, hostname, port, weight, status FROM mysql_servers_incoming"); + + // SELECT FROM mysql_servers whatever is not identical in mysql_servers_incoming, or where mem_pointer=0 (where there is no pointer yet) query=(char *)"SELECT t1.*, t2.weight, t2.status FROM mysql_servers t1 JOIN mysql_servers_incoming t2 ON (t1.hostgroup_id=t2.hostgroup_id AND t1.hostname=t2.hostname AND t1.port=t2.port) WHERE mem_pointer=0 OR t1.weight<>t2.weight OR t1.status<>t2.status"; + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); if (error) { proxy_error("Error on %s : %s\n", query, error); @@ -250,23 +258,71 @@ bool MySQL_HostGroups_Manager::commit() { for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; long long ptr=atoll(r->fields[5]); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Server %s:%d , weight=%d, status=%d, mem_pointer=%llu, hostgroup=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), (MySerStatus) atoi(r->fields[4]), ptr, atoi(r->fields[0])); fprintf(stderr,"%lld\n", ptr); if (ptr==0) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Creating new server %s:%d , weight=%d, status=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), (MySerStatus) atoi(r->fields[4])); MySrvC *mysrvc=new MySrvC(r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), (MySerStatus) atoi(r->fields[4])); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Adding new server %s:%d , weight=%d, status=%d, mem_ptr=%p into hostgroup=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), (MySerStatus) atoi(r->fields[4]), mysrvc, atoi(r->fields[0])); add(mysrvc,atoi(r->fields[0])); + } else { + MySrvC *mysrvc=(MySrvC *)ptr; + if (atoi(r->fields[3])!=atoi(r->fields[6])) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing weight for server %s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[3] , mysrvc->weight , atoi(r->fields[6])); + mysrvc->weight=atoi(r->fields[6]); + } + if (atoi(r->fields[4])!=atoi(r->fields[7])) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing status for server %s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[4] , mysrvc->status , atoi(r->fields[7])); + mysrvc->status=(MySerStatus)atoi(r->fields[7]); + } } - } } if (resultset) { delete resultset; resultset=NULL; } - + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers_incoming\n"); mydb->execute("DELETE FROM mysql_servers_incoming"); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers\n"); + mydb->execute("DELETE FROM mysql_servers"); + + // FIXME: scan all servers and recreate mysql_servers + generate_mysql_servers_table(); wrunlock(); return true; } +void MySQL_HostGroups_Manager::generate_mysql_servers_table() { + for (unsigned int i=0; ilen; i++) { + MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i); + MySrvC *mysrvc=NULL; + for (unsigned int j=0; jmysrvs->servers->len; j++) { + mysrvc=myhgc->mysrvs->idx(j); + uintptr_t ptr=(uintptr_t)mysrvc; + char *q=(char *)"INSERT INTO mysql_servers VALUES(%d,\"%s\",%d,%d,%d,%llu)"; + char *query=(char *)malloc(strlen(q)+8+strlen(mysrvc->address)+8+8+8+16+32); + sprintf(query, q, mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, mysrvc->weight, mysrvc->status, ptr); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); + fprintf(stderr,"%s\n",query); + mydb->execute(query); + free(query); + } + } +} + +SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql_servers() { + wrlock(); + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + char *query=(char *)"SELECT hostgroup_id, hostname, port, weight, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" END FROM mysql_servers"; + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); + mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + wrunlock(); + return resultset; +} + MyHGC * MySQL_HostGroups_Manager::MyHGC_create(unsigned int _hid) { MyHGC *myhgc=new MyHGC(_hid); return myhgc; diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index 6f96d71bc..50b5ff621 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -543,6 +543,7 @@ __exit_DSS__STATE_NOT_INITIALIZED: writeout(); + // FIXME: see bug #211 if ( mybe && diff --git a/lib/Standard_ProxySQL_Admin.cpp b/lib/Standard_ProxySQL_Admin.cpp index dc673928b..edb4d4e89 100644 --- a/lib/Standard_ProxySQL_Admin.cpp +++ b/lib/Standard_ProxySQL_Admin.cpp @@ -42,7 +42,8 @@ pthread_mutex_t admin_mutex = PTHREAD_MUTEX_INITIALIZER; //#define ADMIN_SQLITE_TABLE_MYSQL_SERVER_STATUS "CREATE TABLE mysql_server_status ( status INT NOT NULL PRIMARY KEY, status_desc VARCHAR NOT NULL, UNIQuE(status_desc) )" //#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS "CREATE TABLE mysql_servers ( hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status INT NOT NULL DEFAULT 0 REFERENCES server_status(status) , PRIMARY KEY(hostname, port) )" //#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS "CREATE TABLE mysql_servers ( hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status VARCHAR CHECK (status IN ('OFFLINE_HARD', 'OFFLINE_SOFT', 'SHUNNED', 'ONLINE')) NOT NULL DEFAULT 'OFFLINE_HARD', PRIMARY KEY(hostname, port) )" -#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS "CREATE TABLE mysql_servers ( hostgroup_id INT NOT NULL DEFAULT 0, hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306, status VARCHAR CHECK (status IN ('OFFLINE_HARD', 'OFFLINE_SOFT', 'SHUNNED', 'ONLINE')) NOT NULL DEFAULT 'OFFLINE_HARD', weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , PRIMARY KEY (hostgroup_id, hostname, port) )" +//#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS "CREATE TABLE mysql_servers ( hostgroup_id INT NOT NULL DEFAULT 0, hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306, status VARCHAR CHECK (status IN ('OFFLINE_HARD', 'OFFLINE_SOFT', 'SHUNNED', 'ONLINE')) NOT NULL DEFAULT 'OFFLINE_HARD', weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , PRIMARY KEY (hostgroup_id, hostname, port) )" +#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS "CREATE TABLE mysql_servers ( hostgroup_id INT NOT NULL DEFAULT 0, hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306, status VARCHAR CHECK (status IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE', weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , PRIMARY KEY (hostgroup_id, hostname, port) )" //#define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUPS "CREATE TABLE mysql_hostgroups ( hostgroup_id INT NOT NULL , description VARCHAR, PRIMARY KEY(hostgroup_id) )" //#define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ENTRIES "CREATE TABLE mysql_hostgroup_entries ( hostgroup_id INT NOT NULL DEFAULT 0, hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306, weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , FOREIGN KEY (hostname, port) REFERENCES mysql_servers (hostname, port) , FOREIGN KEY (hostgroup_id) REFERENCES mysql_hostgroups (hostgroup_id) , PRIMARY KEY (hostgroup_id, hostname, port) )" //#define ADMIN_SQLITE_TABLE_MYSQL_USERS "CREATE TABLE mysql_users ( username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , use_ssl INT CHECK (use_ssl IN (0,1)) NOT NULL DEFAULT 0, default_hostgroup INT NOT NULL DEFAULT 0, transaction_persistent INT CHECK (transaction_persistent IN (0,1)) NOT NULL DEFAULT 0, backend INT CHECK (backend IN (0,1)) NOT NULL DEFAULT 1, frontend INT CHECK (frontend IN (0,1)) NOT NULL DEFAULT 1, PRIMARY KEY (username, backend), UNIQUE (username, frontend) , FOREIGN KEY (default_hostgroup) REFERENCES mysql_hostgroups (hostgroup_id))" @@ -191,6 +192,7 @@ class Standard_ProxySQL_Admin: public ProxySQL_Admin { // void __attach_configdb_to_admindb(); void __attach_db_to_admindb(SQLite3DB *db, char *alias); + void __add_active_users(enum cred_username_type usertype); void __delete_inactive_users(enum cred_username_type usertype); // void add_default_user(char *, char *); @@ -227,6 +229,7 @@ class Standard_ProxySQL_Admin: public ProxySQL_Admin { virtual void init_mysql_servers(); virtual void init_mysql_query_rules(); void save_mysql_users_runtime_to_database(); + void save_mysql_servers_runtime_to_database(); virtual void admin_shutdown(); bool is_command(std::string); // void SQLite3_to_MySQL(SQLite3_result *result, char *error, int affected_rows, MySQL_Protocol *myprot); @@ -728,7 +731,7 @@ bool admin_handler_command_load_or_save(char *query_no_space, unsigned int query SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); return false; } -/* + if ( (query_no_space_length==strlen("SAVE MYSQL SERVERS TO MEMORY") && !strncasecmp("SAVE MYSQL SERVERS TO MEMORY",query_no_space, query_no_space_length)) || @@ -740,12 +743,11 @@ bool admin_handler_command_load_or_save(char *query_no_space, unsigned int query ) { proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received %s command\n", query_no_space); Standard_ProxySQL_Admin *SPA=(Standard_ProxySQL_Admin *)pa; - SPA->save_mysql_users_runtime_to_database(); - proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved mysql users from RUNTIME\n"); - SPA->send_MySQL_OK(&sess->myprot_client, NULL); + SPA->save_mysql_servers_runtime_to_database(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved mysql servers from RUNTIME\n"); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); return false; } -*/ } if ((query_no_space_length>23) && ( (!strncasecmp("SAVE MYSQL QUERY RULES ", query_no_space, 23)) || (!strncasecmp("LOAD MYSQL QUERY RULES ", query_no_space, 23))) ) { @@ -1457,6 +1459,11 @@ bool Standard_ProxySQL_Admin::init() { //__attach_configdb_to_admindb(); __attach_db_to_admindb(configdb, (char *)"disk"); __attach_db_to_admindb(statsdb, (char *)"stats"); + +#ifdef DEBUG + admindb->execute("ATTACH DATABASE 'file:mem_mydb?mode=memory&cache=shared' AS myhgm"); +#endif /* DEBUG */ + #ifdef DEBUG flush_debug_levels_runtime_to_database(configdb, false); flush_debug_levels_runtime_to_database(admindb, true); @@ -1474,6 +1481,12 @@ bool Standard_ProxySQL_Admin::init() { __insert_or_replace_maintable_select_disktable(); flush_admin_variables___database_to_runtime(admindb,true); +#ifdef DEBUG + if (GloVars.global.gdbg==false && GloVars.__cmd_proxysql_gdbg) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Enabling GloVars.global.gdbg because GloVars.__cmd_proxysql_gdbg==%d\n", GloVars.__cmd_proxysql_gdbg); + GloVars.global.gdbg=true; + } +#endif /* DEBUG */ flush_mysql_variables___database_to_runtime(admindb,true); @@ -2452,6 +2465,25 @@ void Standard_ProxySQL_Admin::add_default_user(char *user, char *password) { void Standard_ProxySQL_Admin::save_mysql_users_runtime_to_database() { } +void Standard_ProxySQL_Admin::save_mysql_servers_runtime_to_database() { + char *query=(char *)"DELETE FROM main.mysql_servers"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + admindb->execute(query); + SQLite3_result *resultset=MyHGM->dump_table_mysql_servers(); + if (!resultset) return; + char *q=(char *)"INSERT INTO mysql_servers VALUES(%s,\"%s\",%s,\"%s\",%s)"; + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + char *query=(char *)malloc(strlen(q)+strlen(r->fields[0])+strlen(r->fields[1])+strlen(r->fields[2])+strlen(r->fields[3])+strlen(r->fields[4])+16); + sprintf(query, q, r->fields[0], r->fields[1], r->fields[2], r->fields[4], r->fields[3]); + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); + fprintf(stderr,"%s\n",query); + admindb->execute(query); + free(query); + } + if(resultset) delete resultset; +} + void Standard_ProxySQL_Admin::load_mysql_servers_to_runtime() { char *error=NULL; @@ -2459,15 +2491,32 @@ void Standard_ProxySQL_Admin::load_mysql_servers_to_runtime() { int affected_rows=0; SQLite3_result *resultset=NULL; //char *query=(char *)"SELECT hostgroup_id,hostname,port,weight FROM main.mysql_hostgroup_entries"; - char *query=(char *)"SELECT hostgroup_id,hostname,port,weight FROM main.mysql_servers"; + char *query=(char *)"SELECT hostgroup_id,hostname,port,status,weight FROM main.mysql_servers"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); //MyHGH->wrlock(); if (error) { proxy_error("Error on %s : %s\n", query, error); } else { for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { - SQLite3_row *r=*it; - MyHGM->server_add(atoi(r->fields[0]), r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), MYSQL_SERVER_STATUS_ONLINE); + SQLite3_row *r=*it; + MySerStatus status=MYSQL_SERVER_STATUS_ONLINE; + if (strcasecmp(r->fields[3],"ONLINE")) { + if (!strcasecmp(r->fields[3],"SHUNNED")) { + status=MYSQL_SERVER_STATUS_SHUNNED; + } else { + if (!strcasecmp(r->fields[3],"OFFLINE_SOFT")) { + status=MYSQL_SERVER_STATUS_OFFLINE_SOFT; + } else { + if (!strcasecmp(r->fields[3],"OFFLINE_HARD")) { + status=MYSQL_SERVER_STATUS_OFFLINE_HARD; + } + } + } + } + //MyHGM->server_add(atoi(r->fields[0]), r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), MYSQL_SERVER_STATUS_ONLINE); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "hid=%d , hostname=%s , port=%d , status=%s , weight=%d\n", atoi(r->fields[0]), r->fields[1], atoi(r->fields[2]), r->fields[3], atoi(r->fields[4])); + MyHGM->server_add(atoi(r->fields[0]), r->fields[1], atoi(r->fields[2]), atoi(r->fields[4]), status); //MyHGH->server_add_hg(atoi(r->fields[0]), r->fields[1], atoi(r->fields[2]), atoi(r->fields[3])); } }