From a9395cd087f4a424a9b9c7677060398bcd743428 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 16 Aug 2016 21:18:35 +0000 Subject: [PATCH 01/13] Adding comment field per issue #643 Added: * mysql_query_rules.comment VARCHAR * mysql_replication_hostgroups.comment VARCHAR * mysql_servers.comment VARCHAR NOT NULL DEFAULT '' All these tables support online upgrade from previous version (without such fields) at startup. Introduced also a parser for escaping strings with single quotes: double quotes are allowed without extra escaping --- .gitignore | 1 + include/MySQL_HostGroups_Manager.h | 11 ++- include/gen_utils.h | 1 + include/query_processor.h | 8 +- lib/MySQL_HostGroups_Manager.cpp | 107 ++++++++++++++-------- lib/ProxySQL_Admin.cpp | 142 +++++++++++++++++++++++------ lib/Query_Processor.cpp | 14 ++- lib/gen_utils.cpp | 31 +++++++ test/PrepStmt/.gitignore | 8 ++ 9 files changed, 244 insertions(+), 79 deletions(-) create mode 100644 test/PrepStmt/.gitignore diff --git a/.gitignore b/.gitignore index e6af4ccae..cd5b27499 100644 --- a/.gitignore +++ b/.gitignore @@ -79,6 +79,7 @@ deps/mariadb-client-library/mariadb_client-2.0.0-Linux-x86/ deps/mariadb-client-library/mariadb_client-2.0.0-Linux-x86_64/ deps/mariadb-client-library/mariadb_client-2.0.0-src/ deps/mariadb-client-library/mariadb-connector-c-2.1.0-src/ +deps/mariadb-client-library/mariadb-connector-c-2.3.1/ #glib deps/glib/glib-2.40.0/ diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index e10f2446d..26d549e21 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -3,9 +3,9 @@ #include "proxysql.h" #include "cpp.h" -#define MYHGM_MYSQL_SERVERS "CREATE TABLE mysql_servers ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , status INT CHECK (status IN (0, 1, 2, 3, 4)) NOT NULL DEFAULT 0 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , mem_pointer INT NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" -#define MYHGM_MYSQL_SERVERS_INCOMING "CREATE TABLE mysql_servers_incoming ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , status INT CHECK (status IN (0, 1, 2, 3, 4)) NOT NULL DEFAULT 0 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port))" -#define MYHGM_MYSQL_REPLICATION_HOSTGROUPS "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , UNIQUE (reader_hostgroup))" +#define MYHGM_MYSQL_SERVERS "CREATE TABLE mysql_servers ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , status INT CHECK (status IN (0, 1, 2, 3, 4)) NOT NULL DEFAULT 0 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , mem_pointer INT NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" +#define MYHGM_MYSQL_SERVERS_INCOMING "CREATE TABLE mysql_servers_incoming ( hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , status INT CHECK (status IN (0, 1, 2, 3, 4)) NOT NULL DEFAULT 0 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port))" +#define MYHGM_MYSQL_REPLICATION_HOSTGROUPS "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , comment VARCHAR , UNIQUE (reader_hostgroup))" class MySrvConnList; class MySrvC; @@ -60,10 +60,11 @@ class MySrvC { // MySQL Server Container bool shunned_automatic; bool shunned_and_kill_all_connections; // if a serious failure is detected, this will cause all connections to die even if the server is just shunned bool use_ssl; + char *comment; //uint8_t charset; MySrvConnList *ConnectionsUsed; MySrvConnList *ConnectionsFree; - MySrvC(char *, uint16_t, unsigned int, enum MySerStatus, unsigned int, unsigned int _max_connections, unsigned int _max_replication_lag, unsigned int _use_ssl, unsigned int _max_latency_ms); + MySrvC(char *, uint16_t, unsigned int, enum MySerStatus, unsigned int, unsigned int _max_connections, unsigned int _max_replication_lag, unsigned int _use_ssl, unsigned int _max_latency_ms, char *_comment); ~MySrvC(); void connect_error(int); void shun_and_killall(); @@ -136,7 +137,7 @@ class MySQL_HostGroups_Manager { void rdunlock(); void wrlock(); void wrunlock(); - bool server_add(unsigned int hid, char *add, uint16_t p=3306, unsigned int _weight=1, enum MySerStatus status=MYSQL_SERVER_STATUS_ONLINE, unsigned int _comp=0, unsigned int _max_connections=100, unsigned int _max_replication_lag=0, unsigned int _use_ssl=0, unsigned int _max_latency_ms=0); + bool server_add(unsigned int hid, char *add, uint16_t p=3306, unsigned int _weight=1, enum MySerStatus status=MYSQL_SERVER_STATUS_ONLINE, unsigned int _comp=0, unsigned int _max_connections=100, unsigned int _max_replication_lag=0, unsigned int _use_ssl=0, unsigned int _max_latency_ms=0, char *comment=NULL); bool commit(); void set_incoming_replication_hostgroups(SQLite3_result *); diff --git a/include/gen_utils.h b/include/gen_utils.h index 6b839448b..f28b2bd0b 100644 --- a/include/gen_utils.h +++ b/include/gen_utils.h @@ -158,6 +158,7 @@ inline unsigned long long realtime_time() { bool Proxy_file_exists(const char *); bool Proxy_file_regular(const char *); +char *escape_string_single_quotes(char *input, bool free_it); int remove_spaces(const char *); char *trim_spaces_in_place(char *str); char *trim_spaces_and_quotes_in_place(char *str); diff --git a/include/query_processor.h b/include/query_processor.h index 1b90a024a..0ab309c10 100644 --- a/include/query_processor.h +++ b/include/query_processor.h @@ -82,6 +82,7 @@ struct _Query_Processor_rule_t { char *error_msg; int log; bool apply; + char *comment; // #643 void *regex_engine1; void *regex_engine2; int hits; @@ -117,6 +118,7 @@ class Query_Processor_Output { int delay; char *error_msg; int log; + char *comment; // #643 std::string *new_query; void * operator new(size_t size) { return l_alloc(size); @@ -138,11 +140,15 @@ class Query_Processor_Output { log=-1; new_query=NULL; error_msg=NULL; + comment=NULL; // #643 } ~Query_Processor_Output() { if (error_msg) { free(error_msg); } + if (comment) { // #643 + free(comment); + } } }; @@ -221,7 +227,7 @@ class Query_Processor { void wrunlock(); // explicit write unlock bool insert(QP_rule_t *qr, bool lock=true); // insert a new rule. Uses a generic void pointer to a structure that may vary depending from the Query Processor // virtual bool insert_locked(QP_rule_t *qr) {return false;}; // call this instead of insert() in case lock was already acquired via wrlock() - QP_rule_t * new_query_rule(int rule_id, bool active, char *username, char *schemaname, int flagIN, char *client_addr, char *proxy_addr, int proxy_port, char *digest, char *match_digest, char *match_pattern, bool negate_match_pattern, int flagOUT, char *replace_pattern, int destination_hostgroup, int cache_ttl, int reconnect, int timeout, int retries, int delay, int mirror_hostgroup, int mirror_flagOUT, char *error_msg, int log, bool apply); // to use a generic query rule struct, this is generated by this function and returned as generic void pointer + QP_rule_t * new_query_rule(int rule_id, bool active, char *username, char *schemaname, int flagIN, char *client_addr, char *proxy_addr, int proxy_port, char *digest, char *match_digest, char *match_pattern, bool negate_match_pattern, int flagOUT, char *replace_pattern, int destination_hostgroup, int cache_ttl, int reconnect, int timeout, int retries, int delay, int mirror_hostgroup, int mirror_flagOUT, char *error_msg, int log, bool apply, char *comment); // to use a generic query rule struct, this is generated by this function and returned as generic void pointer void delete_query_rule(QP_rule_t *qr); // destructor //virtual bool remove(int rule_id, bool lock=true) {return false;}; // FIXME: not implemented yet, should be implemented at all ? // virtual bool remove_locked(int rule_id) {return false;}; // call this instead of remove() in case lock was already acquired via wrlock() diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 80c7d61ad..764fc2a64 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -103,7 +103,7 @@ void MySrvConnList::drop_all_connections() { } -MySrvC::MySrvC(char *add, uint16_t p, unsigned int _weight, enum MySerStatus _status, unsigned int _compression /*, uint8_t _charset */, unsigned int _max_connections, unsigned int _max_replication_lag, unsigned int _use_ssl, unsigned int _max_latency_ms) { +MySrvC::MySrvC(char *add, uint16_t p, unsigned int _weight, enum MySerStatus _status, unsigned int _compression /*, uint8_t _charset */, unsigned int _max_connections, unsigned int _max_replication_lag, unsigned int _use_ssl, unsigned int _max_latency_ms, char *_comment) { address=strdup(add); port=p; weight=_weight; @@ -125,6 +125,7 @@ MySrvC::MySrvC(char *add, uint16_t p, unsigned int _weight, enum MySerStatus _st shunned_and_kill_all_connections=false; // false to default //charset=_charset; myhgc=NULL; + comment=strdup(_comment); ConnectionsUsed=new MySrvConnList(this); ConnectionsFree=new MySrvConnList(this); } @@ -176,6 +177,7 @@ void MySrvC::shun_and_killall() { MySrvC::~MySrvC() { if (address) free(address); + if (comment) free(comment); delete ConnectionsUsed; delete ConnectionsFree; } @@ -308,14 +310,18 @@ unsigned int MySQL_HostGroups_Manager::get_servers_table_version() { // add a new row in mysql_servers_incoming // we always assume that the calling thread has acquired a rdlock() -bool MySQL_HostGroups_Manager::server_add(unsigned int hid, char *add, uint16_t p, unsigned int _weight, enum MySerStatus status, unsigned int _comp /*, uint8_t _charset */, unsigned int _max_connections, unsigned int _max_replication_lag, unsigned int _use_ssl, unsigned int _max_latency_ms ) { +bool MySQL_HostGroups_Manager::server_add(unsigned int hid, char *add, uint16_t p, unsigned int _weight, enum MySerStatus status, unsigned int _comp /*, uint8_t _charset */, unsigned int _max_connections, unsigned int _max_replication_lag, unsigned int _use_ssl, unsigned int _max_latency_ms , char *comment) { bool ret; proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Adding in mysql_servers_incoming server %s:%d in hostgroup %u with weight %u , status %u, %s compression, max_connections %d, max_replication_lag %u, use_ssl=%u, max_latency_ms=%u\n", add,p,hid,_weight,status, (_comp ? "with" : "without") /*, _charset */ , _max_connections, _max_replication_lag, _use_ssl, _max_latency_ms); - char *q=(char *)"INSERT INTO mysql_servers_incoming VALUES (%u, \"%s\", %u, %u, %u, %u, %u, %u, %u, %u)"; - char *query=(char *)malloc(strlen(q)+strlen(add)+128); - sprintf(query,q,hid,add,p,_weight,status,_comp /*,_charset */, _max_connections, _max_replication_lag, _use_ssl, _max_latency_ms); + char *q=(char *)"INSERT INTO mysql_servers_incoming VALUES (%u, \"%s\", %u, %u, %u, %u, %u, %u, %u, %u, '%s')"; + char *o=escape_string_single_quotes(comment,false); + char *query=(char *)malloc(strlen(q)+strlen(add)+128+strlen(o)); + sprintf(query,q,hid,add,p,_weight,status,_comp /*,_charset */, _max_connections, _max_replication_lag, _use_ssl, _max_latency_ms,o); ret=mydb->execute(query); free(query); + if (o!=comment) { // there was a copy + free(o); + } return ret; } @@ -367,11 +373,11 @@ bool MySQL_HostGroups_Manager::commit() { // INSERT OR IGNORE INTO mysql_servers SELECT ... FROM mysql_servers_incoming // proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "INSERT OR IGNORE INTO mysql_servers(hostgroup_id, hostname, port, weight, status, compression, max_connections) SELECT hostgroup_id, hostname, port, weight, status, compression, max_connections FROM mysql_servers_incoming\n"); - mydb->execute("INSERT OR IGNORE INTO mysql_servers(hostgroup_id, hostname, port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms) SELECT hostgroup_id, hostname, port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms FROM mysql_servers_incoming"); + mydb->execute("INSERT OR IGNORE INTO mysql_servers(hostgroup_id, hostname, port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment) SELECT hostgroup_id, hostname, port, weight, status, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers_incoming"); // SELECT FROM mysql_servers whatever is not identical in mysql_servers_incoming, or where mem_pointer=0 (where there is no pointer yet) - query=(char *)"SELECT t1.*, t2.weight, t2.status, t2.compression, t2.max_connections, t2.max_replication_lag, t2.use_ssl, t2.max_latency_ms FROM mysql_servers t1 JOIN mysql_servers_incoming t2 ON (t1.hostgroup_id=t2.hostgroup_id AND t1.hostname=t2.hostname AND t1.port=t2.port) WHERE mem_pointer=0 OR t1.weight<>t2.weight OR t1.status<>t2.status OR t1.compression<>t2.compression OR t1.max_connections<>t2.max_connections OR t1.max_replication_lag<>t2.max_replication_lag OR t1.use_ssl<>t2.use_ssl OR t1.max_latency_ms<>t2.max_latency_ms"; + query=(char *)"SELECT t1.*, t2.weight, t2.status, t2.compression, t2.max_connections, t2.max_replication_lag, t2.use_ssl, t2.max_latency_ms, t2.comment FROM mysql_servers t1 JOIN mysql_servers_incoming t2 ON (t1.hostgroup_id=t2.hostgroup_id AND t1.hostname=t2.hostname AND t1.port=t2.port) WHERE mem_pointer=0 OR t1.weight<>t2.weight OR t1.status<>t2.status OR t1.compression<>t2.compression OR t1.max_connections<>t2.max_connections OR t1.max_replication_lag<>t2.max_replication_lag OR t1.use_ssl<>t2.use_ssl OR t1.max_latency_ms<>t2.max_latency_ms or t1.comment<>t2.comment"; proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); if (error) { @@ -379,47 +385,52 @@ bool MySQL_HostGroups_Manager::commit() { } else { for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; - long long ptr=atoll(r->fields[10]); // increase this index every time a new column is added + long long ptr=atoll(r->fields[11]); // increase this index every time a new column is added proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Server %s:%d , weight=%d, status=%d, mem_pointer=%llu, hostgroup=%d, compression=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), (MySerStatus) atoi(r->fields[4]), ptr, atoi(r->fields[0]), atoi(r->fields[5])); //fprintf(stderr,"%lld\n", ptr); if (ptr==0) { proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Creating new server %s:%d , weight=%d, status=%d, compression=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), (MySerStatus) atoi(r->fields[4]), atoi(r->fields[5]) ); - MySrvC *mysrvc=new MySrvC(r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), (MySerStatus) atoi(r->fields[4]), atoi(r->fields[5]), atoi(r->fields[6]), atoi(r->fields[7]), atoi(r->fields[8]), atoi(r->fields[9])); // add new fields here if adding more columns in mysql_servers + MySrvC *mysrvc=new MySrvC(r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), (MySerStatus) atoi(r->fields[4]), atoi(r->fields[5]), atoi(r->fields[6]), atoi(r->fields[7]), atoi(r->fields[8]), atoi(r->fields[9]), r->fields[10]); // add new fields here if adding more columns in mysql_servers proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Adding new server %s:%d , weight=%d, status=%d, mem_ptr=%p into hostgroup=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), (MySerStatus) atoi(r->fields[4]), mysrvc, atoi(r->fields[0])); add(mysrvc,atoi(r->fields[0])); } else { MySrvC *mysrvc=(MySrvC *)ptr; // carefully increase the 2nd index by 1 for every new column added - if (atoi(r->fields[3])!=atoi(r->fields[11])) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing weight for server %s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[3] , mysrvc->weight , atoi(r->fields[11])); - mysrvc->weight=atoi(r->fields[11]); + if (atoi(r->fields[3])!=atoi(r->fields[12])) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing weight for server %s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[3] , mysrvc->weight , atoi(r->fields[12])); + mysrvc->weight=atoi(r->fields[12]); } - if (atoi(r->fields[4])!=atoi(r->fields[12])) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing status for server %s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[4] , mysrvc->status , atoi(r->fields[12])); - mysrvc->status=(MySerStatus)atoi(r->fields[12]); + if (atoi(r->fields[4])!=atoi(r->fields[13])) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing status for server %s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[4] , mysrvc->status , atoi(r->fields[13])); + mysrvc->status=(MySerStatus)atoi(r->fields[13]); if (mysrvc->status==MYSQL_SERVER_STATUS_SHUNNED) { mysrvc->shunned_automatic=false; } } - if (atoi(r->fields[5])!=atoi(r->fields[13])) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing compression for server %s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[5] , mysrvc->compression , atoi(r->fields[13])); - mysrvc->compression=atoi(r->fields[13]); + if (atoi(r->fields[5])!=atoi(r->fields[14])) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing compression for server %s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[5] , mysrvc->compression , atoi(r->fields[14])); + mysrvc->compression=atoi(r->fields[14]); + } + if (atoi(r->fields[6])!=atoi(r->fields[15])) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing max_connections for server %s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[6] , mysrvc->max_connections , atoi(r->fields[15])); + mysrvc->max_connections=atoi(r->fields[15]); } - if (atoi(r->fields[6])!=atoi(r->fields[14])) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing max_connections for server %s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[6] , mysrvc->max_connections , atoi(r->fields[14])); - mysrvc->max_connections=atoi(r->fields[14]); + if (atoi(r->fields[7])!=atoi(r->fields[16])) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing max_replication_lag for server %s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[7] , mysrvc->max_replication_lag , atoi(r->fields[16])); + mysrvc->max_replication_lag=atoi(r->fields[16]); } - if (atoi(r->fields[7])!=atoi(r->fields[15])) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing max_replication_lag for server %s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[7] , mysrvc->max_replication_lag , atoi(r->fields[15])); - mysrvc->max_replication_lag=atoi(r->fields[15]); + if (atoi(r->fields[8])!=atoi(r->fields[17])) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing use_ssl for server %s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[8] , mysrvc->use_ssl , atoi(r->fields[17])); + mysrvc->use_ssl=atoi(r->fields[17]); } - if (atoi(r->fields[8])!=atoi(r->fields[16])) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing use_ssl for server %s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[8] , mysrvc->use_ssl , atoi(r->fields[16])); - mysrvc->use_ssl=atoi(r->fields[16]); + if (atoi(r->fields[9])!=atoi(r->fields[18])) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing max_latency_ms for server %s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[9] , mysrvc->max_latency_us , atoi(r->fields[18])); + mysrvc->max_latency_us=1000*atoi(r->fields[18]); } - if (atoi(r->fields[9])!=atoi(r->fields[17])) { - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing max_latency_ms for server %s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[9] , mysrvc->max_latency_us , atoi(r->fields[17])); - mysrvc->max_latency_us=1000*atoi(r->fields[17]); + if (atoi(r->fields[10])!=atoi(r->fields[19])) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Changing comment for server %s:%d (%s:%d) from %d (%d) to %d\n" , mysrvc->address, mysrvc->port, r->fields[1], atoi(r->fields[2]), r->fields[10] , mysrvc->max_latency_us , atoi(r->fields[19])); + free(mysrvc->comment); + mysrvc->comment=strdup(r->fields[19]); } } } @@ -470,9 +481,13 @@ void MySQL_HostGroups_Manager::generate_mysql_servers_table() { for (unsigned int j=0; jmysrvs->servers->len; j++) { mysrvc=myhgc->mysrvs->idx(j); uintptr_t ptr=(uintptr_t)mysrvc; - char *q=(char *)"INSERT INTO mysql_servers VALUES(%d,\"%s\",%d,%d,%d,%u,%u,%u,%u,%u,%llu)"; - char *query=(char *)malloc(strlen(q)+8+strlen(mysrvc->address)+8+8+8+8+8+16+8+16+32); - sprintf(query, q, mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, mysrvc->weight, mysrvc->status, mysrvc->compression, mysrvc->max_connections, mysrvc->max_replication_lag, mysrvc->use_ssl, mysrvc->max_latency_us/1000, ptr); + char *q=(char *)"INSERT INTO mysql_servers VALUES(%d,\"%s\",%d,%d,%d,%u,%u,%u,%u,%u,'%s',%llu)"; + char *o=escape_string_single_quotes(mysrvc->comment, false); + char *query=(char *)malloc(strlen(q)+8+strlen(mysrvc->address)+8+8+8+8+8+16+8+16+32+strlen(o)); + sprintf(query, q, mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, mysrvc->weight, mysrvc->status, mysrvc->compression, mysrvc->max_connections, mysrvc->max_replication_lag, mysrvc->use_ssl, mysrvc->max_latency_us/1000, o, ptr); + if (o!=mysrvc->comment) { // there was a copy + free(o); + } char *st; switch (mysrvc->status) { case 0: @@ -490,7 +505,7 @@ void MySQL_HostGroups_Manager::generate_mysql_servers_table() { st=(char *)"SHUNNED"; break; } - fprintf(stderr,"HID: %d , address: %s , port: %d , weight: %d , status: %s , max_connections: %u , max_replication_lag: %u , use_ssl: %u , max_latency_ms: %u\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, mysrvc->weight, st, mysrvc->max_connections, mysrvc->max_replication_lag, mysrvc->use_ssl, mysrvc->max_latency_us*1000); + fprintf(stderr,"HID: %d , address: %s , port: %d , weight: %d , status: %s , max_connections: %u , max_replication_lag: %u , use_ssl: %u , max_latency_ms: %u , comment: %s\n", mysrvc->myhgc->hid, mysrvc->address, mysrvc->port, mysrvc->weight, st, mysrvc->max_connections, mysrvc->max_replication_lag, mysrvc->use_ssl, mysrvc->max_latency_us*1000, mysrvc->comment); proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); //fprintf(stderr,"%s\n",query); mydb->execute(query); @@ -505,10 +520,24 @@ void MySQL_HostGroups_Manager::generate_mysql_replication_hostgroups_table() { proxy_info("New mysql_replication_hostgroups table\n"); for (std::vector::iterator it = incoming_replication_hostgroups->rows.begin() ; it != incoming_replication_hostgroups->rows.end(); ++it) { SQLite3_row *r=*it; - char query[256]; - sprintf(query,"INSERT INTO mysql_replication_hostgroups VALUES(%s,%s)",r->fields[0],r->fields[1]); + char *o=NULL; + int comment_length=0; // #issue #643 + if (r->fields[2]) { // comment is not null + o=escape_string_single_quotes(r->fields[2],false); + comment_length=strlen(o); + } + char *query=(char *)malloc(256+comment_length); + if (r->fields[2]) { // comment is not null + sprintf(query,"INSERT INTO mysql_replication_hostgroups VALUES(%s,%s,'%s')",r->fields[0], r->fields[1], o); + if (o!=r->fields[2]) { // there was a copy + free(o); + } + } else { + sprintf(query,"INSERT INTO mysql_replication_hostgroups VALUES(%s,%s,NULL)",r->fields[0],r->fields[1]); + } mydb->execute(query); - fprintf(stderr,"writer_hostgroup: %s , reader_hostgroup: %s\n", r->fields[0],r->fields[1]); + fprintf(stderr,"writer_hostgroup: %s , reader_hostgroup: %s, %s\n", r->fields[0],r->fields[1], r->fields[2]); + free(query); } incoming_replication_hostgroups=NULL; } @@ -527,7 +556,7 @@ SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql_servers() { int cols=0; int affected_rows=0; SQLite3_result *resultset=NULL; - char *query=(char *)"SELECT hostgroup_id, hostname, port, weight, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms FROM mysql_servers"; + char *query=(char *)"SELECT hostgroup_id, hostname, port, weight, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers"; proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); wrunlock(); @@ -540,7 +569,7 @@ SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql_replication_hostgrou int cols=0; int affected_rows=0; SQLite3_result *resultset=NULL; - char *query=(char *)"SELECT writer_hostgroup, reader_hostgroup FROM mysql_replication_hostgroups"; + char *query=(char *)"SELECT writer_hostgroup, reader_hostgroup, comment FROM mysql_replication_hostgroups"; proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset); wrunlock(); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 6e38dfe06..591578439 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -77,7 +77,7 @@ pthread_mutex_t admin_mutex = PTHREAD_MUTEX_INITIALIZER; #define LINESIZE 2048 -#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS "CREATE TABLE mysql_servers (hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" +#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS "CREATE TABLE mysql_servers (hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" // mysql_servers in v1.1.0 #define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_1_0 "CREATE TABLE mysql_servers (hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" @@ -85,9 +85,10 @@ pthread_mutex_t admin_mutex = PTHREAD_MUTEX_INITIALIZER; // mysql_servers in v1.2.0e #define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_2_0e "CREATE TABLE mysql_servers (hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" +#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_2_2 "CREATE TABLE mysql_servers (hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" #define ADMIN_SQLITE_TABLE_MYSQL_USERS "CREATE TABLE mysql_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , use_ssl INT CHECK (use_ssl IN (0,1)) NOT NULL DEFAULT 0 , default_hostgroup INT NOT NULL DEFAULT 0 , default_schema VARCHAR , schema_locked INT CHECK (schema_locked IN (0,1)) NOT NULL DEFAULT 0 , transaction_persistent INT CHECK (transaction_persistent IN (0,1)) NOT NULL DEFAULT 0 , fast_forward INT CHECK (fast_forward IN (0,1)) NOT NULL DEFAULT 0 , backend INT CHECK (backend IN (0,1)) NOT NULL DEFAULT 1 , frontend INT CHECK (frontend IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , PRIMARY KEY (username, backend) , UNIQUE (username, frontend))" -#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0)" +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" // mysql_query_rules in v1.1.0 #define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_1_0 "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , delay INT UNSIGNED , error_msg VARCHAR , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0)" @@ -98,21 +99,30 @@ pthread_mutex_t admin_mutex = PTHREAD_MUTEX_INITIALIZER; // mysql_query_rules in v1.2.0g #define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_2_0g "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0)" +// mysql_query_rules in v1.2.2 +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_2_2 "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" + #define ADMIN_SQLITE_TABLE_GLOBAL_VARIABLES "CREATE TABLE global_variables (variable_name VARCHAR NOT NULL PRIMARY KEY , variable_value VARCHAR NOT NULL)" #define ADMIN_SQLITE_RUNTIME_GLOBAL_VARIABLES "CREATE TABLE runtime_global_variables (variable_name VARCHAR NOT NULL PRIMARY KEY , variable_value VARCHAR NOT NULL)" -#define ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , UNIQUE (reader_hostgroup))" +#define ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , comment VARCHAR , UNIQUE (reader_hostgroup))" + +// mysql_replication_hostgroups in v1.0 +#define ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V1_0 "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , UNIQUE (reader_hostgroup))" + +// mysql_replication_hostgroups in v1.2.2 +#define ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V1_2_2 "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , comment VARCHAR , UNIQUE (reader_hostgroup))" #define ADMIN_SQLITE_TABLE_MYSQL_COLLATIONS "CREATE TABLE mysql_collations (Id INTEGER NOT NULL PRIMARY KEY , Collation VARCHAR NOT NULL , Charset VARCHAR NOT NULL , `Default` VARCHAR NOT NULL)" #define ADMIN_SQLITE_TABLE_SCHEDULER "CREATE TABLE scheduler (id INTEGER NOT NULL , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , PRIMARY KEY(id))" -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS "CREATE TABLE runtime_mysql_servers (hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )" +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS "CREATE TABLE runtime_mysql_servers (hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_REPLICATION_HOSTGROUPS "CREATE TABLE runtime_mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , UNIQUE (reader_hostgroup))" +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_REPLICATION_HOSTGROUPS "CREATE TABLE runtime_mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , comment VARCHAR , UNIQUE (reader_hostgroup))" -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_QUERY_RULES "CREATE TABLE runtime_mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0)" +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_QUERY_RULES "CREATE TABLE runtime_mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" #define ADMIN_SQLITE_TABLE_RUNTIME_SCHEDULER "CREATE TABLE runtime_scheduler (id INTEGER NOT NULL , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , PRIMARY KEY(id))" @@ -3302,20 +3312,24 @@ void ProxySQL_Admin::save_mysql_query_rules_from_runtime(bool _runtime) { //char *a=(char *)"INSERT INTO mysql_query_rules VALUES (\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\")"; char *a=NULL; if (_runtime) { - a=(char *)"INSERT INTO runtime_mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, retries, delay, mirror_flagOUT, mirror_hostgroup, error_msg, log, apply) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"; + a=(char *)"INSERT INTO runtime_mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, retries, delay, mirror_flagOUT, mirror_hostgroup, error_msg, log, apply, comment) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"; } else { - a=(char *)"INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, retries, delay, mirror_flagOUT, mirror_hostgroup, error_msg, log, apply) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"; + a=(char *)"INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, retries, delay, mirror_flagOUT, mirror_hostgroup, error_msg, log, apply, comment) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"; } for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; int arg_len=0; - char *buffs[25]; - for (int i=0; i<25; i++) { + char *buffs[26]; + for (int i=0; i<26; i++) { if (r->fields[i]) { - int l=strlen(r->fields[i])+4; + char *o=escape_string_single_quotes(r->fields[i],false); + int l=strlen(o)+4; arg_len+=l; buffs[i]=(char *)malloc(l); - sprintf(buffs[i],"\"%s\"",r->fields[i]); + sprintf(buffs[i],"'%s'",o); + if (o!=r->fields[i]) { // there was a copy + free(o); + } } else { int l=9; arg_len+=l; @@ -3350,11 +3364,12 @@ void ProxySQL_Admin::save_mysql_query_rules_from_runtime(bool _runtime) { ( strcmp(r->fields[21],"-1")==0 ? "NULL" : r->fields[21] ), // mirror_hostgroup buffs[22], // error_msg ( strcmp(r->fields[23],"-1")==0 ? "NULL" : r->fields[23] ), // log - ( strcmp(r->fields[24],"-1")==0 ? "NULL" : r->fields[24] ) // apply + ( strcmp(r->fields[24],"-1")==0 ? "NULL" : r->fields[24] ), // apply + buffs[25] // error_msg ); //fprintf(stderr,"%s\n",query); admindb->execute(query); - for (int i=0; i<25; i++) { + for (int i=0; i<26; i++) { free(buffs[i]); } free(query); @@ -3860,23 +3875,29 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { if (resultset) { char *q=NULL; if (_runtime) { - q=(char *)"INSERT INTO runtime_mysql_servers VALUES(%s,\"%s\",%s,\"%s\",%s,%s,%s,%s,%s,%s)"; + q=(char *)"INSERT INTO runtime_mysql_servers VALUES(%s,\"%s\",%s,\"%s\",%s,%s,%s,%s,%s,%s,'%s')"; } else { - q=(char *)"INSERT INTO mysql_servers VALUES(%s,\"%s\",%s,\"%s\",%s,%s,%s,%s,%s,%s)"; + q=(char *)"INSERT INTO mysql_servers VALUES(%s,\"%s\",%s,\"%s\",%s,%s,%s,%s,%s,%s,'%s')"; } for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; + char *o=escape_string_single_quotes(r->fields[10],false); char *query=(char *)malloc(strlen(q)+strlen(r->fields[0])+strlen(r->fields[1])+strlen(r->fields[2])+strlen(r->fields[3])+strlen(r->fields[4])+strlen(r->fields[5])+strlen(r->fields[6])+strlen(r->fields[7])+ strlen(r->fields[8])+ // use_ssl strlen(r->fields[9])+ // max_latency_ms + strlen(o)+ // comment 16); // padding // if the backend is shunned, save_mysql_servers_runtime_to_database() should set to ONLINE if _runtime==false sprintf(query, q, r->fields[0], r->fields[1], r->fields[2], ( _runtime ? r->fields[4] : ( strcmp(r->fields[4],"SHUNNED")==0 ? "ONLINE" : r->fields[4] ) ), r->fields[3], r->fields[5], r->fields[6], r->fields[7], r->fields[8], // use_ssl - r->fields[9]); // max_latency_ms + r->fields[9], // max_latency_ms + o); // comment proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); admindb->execute(query); free(query); + if (o!=r->fields[10]) { + free(o); + } } } if(resultset) delete resultset; @@ -3892,16 +3913,34 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) { admindb->execute(query); resultset=MyHGM->dump_table_mysql_replication_hostgroups(); if (resultset) { - char *q=NULL; - if (_runtime) { - q=(char *)"INSERT INTO runtime_mysql_replication_hostgroups VALUES(%s,%s)"; - } else { - q=(char *)"INSERT INTO mysql_replication_hostgroups VALUES(%s,%s)"; - } for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; - char *query=(char *)malloc(strlen(q)+strlen(r->fields[0])+strlen(r->fields[1])+16); - sprintf(query, q, r->fields[0], r->fields[1]); + int l=0; + if (r->fields[2]) l=strlen(r->fields[2]); + char *q=NULL; + if (_runtime) { + if (r->fields[2]) { // comment is not null, #643 + q=(char *)"INSERT INTO runtime_mysql_replication_hostgroups VALUES(%s,%s,'%s')"; + } else { + q=(char *)"INSERT INTO runtime_mysql_replication_hostgroups VALUES(%s,%s,NULL)"; + } + } else { + if (r->fields[2]) { // comment is not null, #643 + q=(char *)"INSERT INTO mysql_replication_hostgroups VALUES(%s,%s,'%s')"; + } else { + q=(char *)"INSERT INTO mysql_replication_hostgroups VALUES(%s,%s,NULL)"; + } + } + char *query=(char *)malloc(strlen(q)+strlen(r->fields[0])+strlen(r->fields[1])+16+l); + if (r->fields[2]) { + char *o=escape_string_single_quotes(r->fields[2],false); + sprintf(query, q, r->fields[0], r->fields[1], o); + if (o!=r->fields[2]) { // there was a copy + free(o); + } + } else { + sprintf(query, q, r->fields[0], r->fields[1]); + } proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "%s\n", query); admindb->execute(query); free(query); @@ -3934,7 +3973,7 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime() { int cols=0; int affected_rows=0; SQLite3_result *resultset=NULL; - char *query=(char *)"SELECT hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms FROM main.mysql_servers"; + char *query=(char *)"SELECT hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM main.mysql_servers"; proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); //MyHGH->wrlock(); @@ -3964,7 +4003,8 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime() { ); MyHGM->server_add(atoi(r->fields[0]), r->fields[1], atoi(r->fields[2]), atoi(r->fields[4]), status, atoi(r->fields[5]), atoi(r->fields[6]), atoi(r->fields[7]), atoi(r->fields[8]), // use_ssl - atoi(r->fields[9]) // max_latency_ms + atoi(r->fields[9]), // max_latency_ms + r->fields[10] // comment ); //MyHGH->server_add_hg(atoi(r->fields[0]), r->fields[1], atoi(r->fields[2]), atoi(r->fields[3])); } @@ -4007,7 +4047,7 @@ char * ProxySQL_Admin::load_mysql_query_rules_to_runtime() { int affected_rows=0; if (GloQPro==NULL) return (char *)"Global Query Processor not started: command impossible to run"; SQLite3_result *resultset=NULL; - char *query=(char *)"SELECT rule_id, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, retries, delay, mirror_flagOUT, mirror_hostgroup, error_msg, log, apply FROM main.mysql_query_rules WHERE active=1"; + char *query=(char *)"SELECT rule_id, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, retries, delay, mirror_flagOUT, mirror_hostgroup, error_msg, log, apply, comment FROM main.mysql_query_rules WHERE active=1"; admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); if (error) { proxy_error("Error on %s : %s\n", query, error); @@ -4042,7 +4082,8 @@ char * ProxySQL_Admin::load_mysql_query_rules_to_runtime() { (r->fields[20]==NULL ? -1 : atol(r->fields[20])), // mirror_hostgroup r->fields[21], // error_msg (r->fields[22]==NULL ? -1 : atol(r->fields[22])), // log - (atoi(r->fields[23])==1 ? true : false) + (atoi(r->fields[23])==1 ? true : false), + r->fields[24] // comment ); GloQPro->insert(nqpr, false); } @@ -4443,6 +4484,21 @@ void ProxySQL_Admin::disk_upgrade_mysql_query_rules() { // copy fields from old table configdb->execute("INSERT INTO mysql_query_rules (rule_id,active,username,schemaname,flagIN,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,delay,error_msg,mirror_flagOUT,mirror_hostgroup,apply) SELECT rule_id,active,username,schemaname,flagIN,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,delay,error_msg,mirror_flagOUT,mirror_hostgroup,apply FROM mysql_query_rules_v120a"); } + // upgrade related to issue #643 , adding comment in mysql_query_rules table + rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_2_0g); + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.2.0g of table mysql_query_rules\n"); + proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); + // drop any existing table with suffix _v110 + configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_v120g"); + // rename current table to add suffix _v110 + configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v120g"); + // create new table + configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_query_rules (rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,log,apply) SELECT rule_id,active,username,schemaname,flagIN,client_addr,proxy_addr,proxy_port,digest,match_digest,match_pattern,negate_match_pattern,flagOUT,replace_pattern,destination_hostgroup,cache_ttl,reconnect,timeout,retries,delay,mirror_flagOUT,mirror_hostgroup,error_msg,log,apply FROM mysql_query_rules_v120g"); + } configdb->execute("PRAGMA foreign_keys = ON"); } @@ -4465,6 +4521,34 @@ void ProxySQL_Admin::disk_upgrade_mysql_servers() { // copy fields from old table configdb->execute("INSERT INTO mysql_servers (hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag) SELECT hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag FROM mysql_servers_v110"); } + rci=configdb->check_table_structure((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_2_0e); + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.2.0 of table mysql_servers\n"); + proxy_warning("ONLINE UPGRADE of table mysql_servers in progress\n"); + // drop any existing table with suffix _v110 + configdb->execute("DROP TABLE IF EXISTS mysql_servers_v120"); + // rename current table to add suffix _v110 + configdb->execute("ALTER TABLE mysql_servers RENAME TO mysql_servers_v120"); + // create new table + configdb->build_table((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_servers (hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms) SELECT hostgroup_id,hostname,port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms FROM mysql_servers_v120"); + } + rci=configdb->check_table_structure((char *)"mysql_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V1_0); // isseu #643 + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.0 of table mysql_replication_hostgroups\n"); + proxy_warning("ONLINE UPGRADE of table mysql_replication_hostgroups in progress\n"); + // drop any existing table with suffix _v110 + configdb->execute("DROP TABLE IF EXISTS mysql_replication_hostgroups_v100"); + // rename current table to add suffix _v110 + configdb->execute("ALTER TABLE mysql_replication_hostgroups RENAME TO mysql_replication_hostgroups_v100"); + // create new table + configdb->build_table((char *)"mysql_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V1_2_2,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_replication_hostgroups (writer_hostgroup,reader_hostgroup) SELECT writer_hostgroup , reader_hostgroup FROM mysql_replication_hostgroups_v100"); + } configdb->execute("PRAGMA foreign_keys = ON"); } diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 6f40f3851..eb565152a 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -37,7 +37,7 @@ class QP_rule_text { char **pta; int num_fields; QP_rule_text(QP_rule_t *QPr) { - num_fields=26; + num_fields=27; pta=NULL; pta=(char **)malloc(sizeof(char *)*num_fields); itostr(pta[0], (long long)QPr->rule_id); @@ -73,7 +73,8 @@ class QP_rule_text { pta[22]=strdup_null(QPr->error_msg); itostr(pta[23], (long long)QPr->log); itostr(pta[24], (long long)QPr->apply); - itostr(pta[25], (long long)QPr->hits); + pta[25]=strdup_null(QPr->comment); // issue #643 + itostr(pta[26], (long long)QPr->hits); } ~QP_rule_text() { for(int i=0; irule_id=rule_id; newQR->active=active; @@ -400,6 +401,7 @@ QP_rule_t * Query_Processor::new_query_rule(int rule_id, bool active, char *user newQR->mirror_hostgroup=mirror_hostgroup; newQR->error_msg=(error_msg ? strdup(error_msg) : NULL); newQR->apply=apply; + newQR->comment=(comment ? strdup(comment) : NULL); // see issue #643 newQR->regex_engine1=NULL; newQR->regex_engine2=NULL; newQR->hits=0; @@ -505,7 +507,7 @@ SQLite3_result * Query_Processor::get_stats_query_rules() { SQLite3_result * Query_Processor::get_current_query_rules() { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping current query rules, using Global version %d\n", version); - SQLite3_result *result=new SQLite3_result(26); + SQLite3_result *result=new SQLite3_result(27); spin_rdlock(&rwlock); QP_rule_t *qr1; result->add_column_definition(SQLITE_TEXT,"rule_id"); @@ -533,6 +535,7 @@ SQLite3_result * Query_Processor::get_current_query_rules() { result->add_column_definition(SQLITE_TEXT,"error_msg"); result->add_column_definition(SQLITE_TEXT,"log"); result->add_column_definition(SQLITE_TEXT,"apply"); + result->add_column_definition(SQLITE_TEXT,"comment"); // issue #643 result->add_column_definition(SQLITE_TEXT,"hits"); for (std::vector::iterator it=rules.begin(); it!=rules.end(); ++it) { qr1=*it; @@ -637,7 +640,8 @@ Query_Processor_Output * Query_Processor::process_mysql_query(MySQL_Session *ses qr1->match_digest, qr1->match_pattern, qr1->negate_match_pattern, qr1->flagOUT, qr1->replace_pattern, qr1->destination_hostgroup, qr1->cache_ttl, qr1->reconnect, qr1->timeout, qr1->retries, qr1->delay, qr1->mirror_flagOUT, qr1->mirror_hostgroup, - qr1->error_msg, qr1->log, qr1->apply); + qr1->error_msg, qr1->log, qr1->apply, + qr1->comment); qr2->parent=qr1; // pointer to parent to speed up parent update (hits) if (qr2->match_digest) { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Compiling regex for rule_id: %d, match_digest: \n", qr2->rule_id, qr2->match_digest); diff --git a/lib/gen_utils.cpp b/lib/gen_utils.cpp index 47e4d5256..049953c12 100644 --- a/lib/gen_utils.cpp +++ b/lib/gen_utils.cpp @@ -2,6 +2,37 @@ #include "cpp.h" +char *escape_string_single_quotes(char *input, bool free_it) { + int i,j,l; + char *o=NULL; // output string, if any + l=strlen(input); + j=0; + for (i=0;i Date: Tue, 16 Aug 2016 21:41:37 +0000 Subject: [PATCH 02/13] Copy comment in read_only_action Issue #643 --- lib/MySQL_HostGroups_Manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 764fc2a64..a9d6ce283 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1084,7 +1084,7 @@ void MySQL_HostGroups_Manager::read_only_action(char *hostname, int port, int re //const char *Q1=(char *)"SELECT hostgroup_id FROM mysql_servers join mysql_replication_hostgroups ON hostgroup_id=writer_hostgroup WHERE hostname='%s' AND port=%d AND status=0"; // this query run against myhgm DB . status is an INTEGER const char *Q1=(char *)"SELECT hostgroup_id,status FROM mysql_replication_hostgroups JOIN mysql_servers ON hostgroup_id=writer_hostgroup AND hostname='%s' AND port=%d"; const char *Q2=(char *)"UPDATE OR IGNORE mysql_servers SET hostgroup_id=(SELECT writer_hostgroup FROM mysql_replication_hostgroups WHERE reader_hostgroup=mysql_servers.hostgroup_id) WHERE hostname='%s' AND port=%d AND hostgroup_id IN (SELECT reader_hostgroup FROM mysql_replication_hostgroups WHERE reader_hostgroup=mysql_servers.hostgroup_id)"; - const char *Q3A=(char *)"INSERT OR IGNORE INTO mysql_servers(hostgroup_id, hostname, port, status, weight, max_connections, max_replication_lag, use_ssl, max_latency_ms) SELECT reader_hostgroup, hostname, port, status, weight, max_connections, max_replication_lag, use_ssl, max_latency_ms FROM mysql_servers JOIN mysql_replication_hostgroups ON mysql_servers.hostgroup_id=mysql_replication_hostgroups.writer_hostgroup WHERE hostname='%s' AND port=%d"; + const char *Q3A=(char *)"INSERT OR IGNORE INTO mysql_servers(hostgroup_id, hostname, port, status, weight, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment) SELECT reader_hostgroup, hostname, port, status, weight, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers JOIN mysql_replication_hostgroups ON mysql_servers.hostgroup_id=mysql_replication_hostgroups.writer_hostgroup WHERE hostname='%s' AND port=%d"; const char *Q3B=(char *)"DELETE FROM mysql_servers WHERE hostname='%s' AND port=%d AND hostgroup_id IN (SELECT reader_hostgroup FROM mysql_replication_hostgroups WHERE reader_hostgroup=mysql_servers.hostgroup_id)"; const char *Q4=(char *)"UPDATE OR IGNORE mysql_servers SET hostgroup_id=(SELECT reader_hostgroup FROM mysql_replication_hostgroups WHERE writer_hostgroup=mysql_servers.hostgroup_id) WHERE hostname='%s' AND port=%d AND hostgroup_id IN (SELECT writer_hostgroup FROM mysql_replication_hostgroups WHERE writer_hostgroup=mysql_servers.hostgroup_id)"; const char *Q5=(char *)"DELETE FROM mysql_servers WHERE hostname='%s' AND port=%d AND hostgroup_id IN (SELECT writer_hostgroup FROM mysql_replication_hostgroups WHERE writer_hostgroup=mysql_servers.hostgroup_id)"; From fe39671edf0554aea6fc11643cb230964183f8b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 16 Aug 2016 22:17:56 +0000 Subject: [PATCH 03/13] Adding comment field per issue #643 Added: * scheduler.comment VARCHAR NOT NULL DEFAULT '' Like the other tables, scheduler supports online upgrade from previous version (without such field) at startup. --- include/proxysql_admin.h | 4 ++- lib/ProxySQL_Admin.cpp | 60 ++++++++++++++++++++++++++++++++-------- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 651c1a642..d49db498f 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -21,7 +21,8 @@ class Scheduler_Row { // char *arg4; // char *arg5; char **args; - Scheduler_Row(unsigned int _id, unsigned int _in, char *_f, char *a1, char *a2, char *a3, char *a4, char *a5); + char *comment; + Scheduler_Row(unsigned int _id, unsigned int _in, char *_f, char *a1, char *a2, char *a3, char *a4, char *a5, char *_comment); ~Scheduler_Row(); }; @@ -113,6 +114,7 @@ class ProxySQL_Admin { void flush_admin_variables___runtime_to_database(SQLite3DB *db, bool replace, bool del, bool onlyifempty, bool runtime=false); void disk_upgrade_mysql_query_rules(); void disk_upgrade_mysql_servers(); + void disk_upgrade_scheduler(); #ifdef DEBUG void add_credentials(char *type, char *credentials, int hostgroup_id); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 591578439..a62368b78 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -116,7 +116,11 @@ pthread_mutex_t admin_mutex = PTHREAD_MUTEX_INITIALIZER; #define ADMIN_SQLITE_TABLE_MYSQL_COLLATIONS "CREATE TABLE mysql_collations (Id INTEGER NOT NULL PRIMARY KEY , Collation VARCHAR NOT NULL , Charset VARCHAR NOT NULL , `Default` VARCHAR NOT NULL)" -#define ADMIN_SQLITE_TABLE_SCHEDULER "CREATE TABLE scheduler (id INTEGER NOT NULL , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , PRIMARY KEY(id))" +#define ADMIN_SQLITE_TABLE_SCHEDULER "CREATE TABLE scheduler (id INTEGER NOT NULL , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY(id))" + +#define ADMIN_SQLITE_TABLE_SCHEDULER_V1_2_0 "CREATE TABLE scheduler (id INTEGER NOT NULL , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , PRIMARY KEY(id))" + +#define ADMIN_SQLITE_TABLE_SCHEDULER_V1_2_2 "CREATE TABLE scheduler (id INTEGER NOT NULL , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY(id))" #define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS "CREATE TABLE runtime_mysql_servers (hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" @@ -124,7 +128,7 @@ pthread_mutex_t admin_mutex = PTHREAD_MUTEX_INITIALIZER; #define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_QUERY_RULES "CREATE TABLE runtime_mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" -#define ADMIN_SQLITE_TABLE_RUNTIME_SCHEDULER "CREATE TABLE runtime_scheduler (id INTEGER NOT NULL , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , PRIMARY KEY(id))" +#define ADMIN_SQLITE_TABLE_RUNTIME_SCHEDULER "CREATE TABLE runtime_scheduler (id INTEGER NOT NULL , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY(id))" #define STATS_SQLITE_TABLE_MYSQL_QUERY_RULES "CREATE TABLE stats_mysql_query_rules (rule_id INTEGER PRIMARY KEY , hits INT NOT NULL)" #define STATS_SQLITE_TABLE_MYSQL_COMMANDS_COUNTERS "CREATE TABLE stats_mysql_commands_counters (Command VARCHAR NOT NULL PRIMARY KEY , Total_Time_us INT NOT NULL , Total_cnt INT NOT NULL , cnt_100us INT NOT NULL , cnt_500us INT NOT NULL , cnt_1ms INT NOT NULL , cnt_5ms INT NOT NULL , cnt_10ms INT NOT NULL , cnt_50ms INT NOT NULL , cnt_100ms INT NOT NULL , cnt_500ms INT NOT NULL , cnt_1s INT NOT NULL , cnt_5s INT NOT NULL , cnt_10s INT NOT NULL , cnt_INFs)" @@ -2504,6 +2508,9 @@ bool ProxySQL_Admin::init() { // upgrade mysql_query_rules if needed (upgrade from previous version) disk_upgrade_mysql_query_rules(); + // upgrade scheduler if needed (upgrade from previous version) + disk_upgrade_scheduler(); + check_and_build_standard_tables(admindb, tables_defs_admin); check_and_build_standard_tables(configdb, tables_defs_config); check_and_build_standard_tables(statsdb, tables_defs_stats); @@ -3808,9 +3815,9 @@ void ProxySQL_Admin::save_scheduler_runtime_to_database(bool _runtime) { spin_rdlock(&scheduler->rwlock); char *q=NULL; if (_runtime) { - q=(char *)"INSERT INTO runtime_scheduler VALUES(%lu,%lu,\"%s\" ,%s,%s,%s,%s,%s)"; + q=(char *)"INSERT INTO runtime_scheduler VALUES(%lu,%lu,\"%s\" ,%s,%s,%s,%s,%s,'%s')"; } else { - q=(char *)"INSERT INTO scheduler VALUES(%lu,%lu,\"%s\" ,%s,%s,%s,%s,%s)"; + q=(char *)"INSERT INTO scheduler VALUES(%lu,%lu,\"%s\" ,%s,%s,%s,%s,%s,'%s')"; } for (std::vector::iterator it = scheduler->Scheduler_Rows.begin() ; it != scheduler->Scheduler_Rows.end(); ++it) { Scheduler_Row *sr=*it; @@ -3828,7 +3835,8 @@ void ProxySQL_Admin::save_scheduler_runtime_to_database(bool _runtime) { } l+=strlen(args[i]); } - + char *o=escape_string_single_quotes(sr->comment,false); // issue #643 + l+=strlen(o); l+=32; //padding char *query=(char *)malloc(l); @@ -3837,8 +3845,12 @@ void ProxySQL_Admin::save_scheduler_runtime_to_database(bool _runtime) { sr->id, sr->interval_ms, sr->filename, args[0], args[1], args[2], - args[3], args[4] + args[3], args[4], + o ); + if (o!=sr->comment) { + free(o); + } for (i=0; i<5; i++) { if (sr->args[i]) { @@ -4502,6 +4514,27 @@ void ProxySQL_Admin::disk_upgrade_mysql_query_rules() { configdb->execute("PRAGMA foreign_keys = ON"); } +void ProxySQL_Admin::disk_upgrade_scheduler() { + // this function is called only for configdb table + // it is responsible to upgrade table scheduler if its structure is from a previous version + int rci; + configdb->execute("PRAGMA foreign_keys = OFF"); + rci=configdb->check_table_structure((char *)"scheduler",(char *)ADMIN_SQLITE_TABLE_SCHEDULER_V1_2_0); + if (rci) { + // upgrade is required + proxy_warning("Detected version v1.2.0 of table scheduler\n"); + proxy_warning("ONLINE UPGRADE of table scheduler in progress\n"); + // drop any existing table with suffix _v120 + configdb->execute("DROP TABLE IF EXISTS scheduler_v120"); + // rename current table to add suffix _v120 + configdb->execute("ALTER TABLE scheduler RENAME TO scheduler_v120"); + // create new table + configdb->build_table((char *)"scheduler",(char *)ADMIN_SQLITE_TABLE_SCHEDULER,false); + // copy fields from old table + configdb->execute("INSERT INTO scheduler (id,interval_ms,filename,arg1,arg2,arg3,arg4,arg5) SELECT id,interval_ms,filename,arg1,arg2,arg3,arg4,arg5 FROM scheduler_v120"); + } +} + void ProxySQL_Admin::disk_upgrade_mysql_servers() { // this function is called only for configdb table // it is responsible to upgrade table mysql_servers if its structure is from a previous version @@ -4526,9 +4559,9 @@ void ProxySQL_Admin::disk_upgrade_mysql_servers() { // upgrade is required proxy_warning("Detected version v1.2.0 of table mysql_servers\n"); proxy_warning("ONLINE UPGRADE of table mysql_servers in progress\n"); - // drop any existing table with suffix _v110 + // drop any existing table with suffix _v120 configdb->execute("DROP TABLE IF EXISTS mysql_servers_v120"); - // rename current table to add suffix _v110 + // rename current table to add suffix _v120 configdb->execute("ALTER TABLE mysql_servers RENAME TO mysql_servers_v120"); // create new table configdb->build_table((char *)"mysql_servers",(char *)ADMIN_SQLITE_TABLE_MYSQL_SERVERS,false); @@ -4540,9 +4573,9 @@ void ProxySQL_Admin::disk_upgrade_mysql_servers() { // upgrade is required proxy_warning("Detected version v1.0 of table mysql_replication_hostgroups\n"); proxy_warning("ONLINE UPGRADE of table mysql_replication_hostgroups in progress\n"); - // drop any existing table with suffix _v110 + // drop any existing table with suffix _v100 configdb->execute("DROP TABLE IF EXISTS mysql_replication_hostgroups_v100"); - // rename current table to add suffix _v110 + // rename current table to add suffix _v100 configdb->execute("ALTER TABLE mysql_replication_hostgroups RENAME TO mysql_replication_hostgroups_v100"); // create new table configdb->build_table((char *)"mysql_replication_hostgroups",(char *)ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS_V1_2_2,false); @@ -4556,7 +4589,7 @@ void ProxySQL_Admin::disk_upgrade_mysql_servers() { -Scheduler_Row::Scheduler_Row(unsigned int _id, unsigned int _in, char *_f, char *a1, char *a2, char *a3, char *a4, char *a5) { +Scheduler_Row::Scheduler_Row(unsigned int _id, unsigned int _in, char *_f, char *a1, char *a2, char *a3, char *a4, char *a5, char *_comment) { int i; id=_id; interval_ms=_in; @@ -4581,6 +4614,7 @@ Scheduler_Row::Scheduler_Row(unsigned int _id, unsigned int _in, char *_f, char } } } + comment=strdup(_comment); } @@ -4593,6 +4627,7 @@ Scheduler_Row::~Scheduler_Row() { args[i]=NULL; } free(args); + free(comment); args=NULL; } @@ -4624,7 +4659,8 @@ void ProxySQL_External_Scheduler::update_table(SQLite3_result *resultset) { Scheduler_Row *sr=new Scheduler_Row(id, interval_ms, r->fields[2], r->fields[3], r->fields[4], r->fields[5], - r->fields[6], r->fields[7] + r->fields[6], r->fields[7], + r->fields[8] // comment, issue #643 ); Scheduler_Rows.push_back(sr); } From c710f549872fde0028ccaf7edff06773751e3365 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Fri, 12 Aug 2016 15:31:29 +0000 Subject: [PATCH 04/13] Compiling optimization * converted MySQL_Authentication to use unsorted_map instead of btree_map * Query_Cache now uses pointers to KV_BtreeArray * removed some unnecesary headers --- include/MySQL_Authentication.hpp | 8 +++--- include/cpp.h | 20 +++++++-------- include/proxysql.h | 18 ++++++++++++-- include/proxysql_structs.h | 4 +-- include/query_cache.hpp | 8 +++--- lib/MySQL_Authentication.cpp | 21 ++++++++++------ lib/Query_Cache.cpp | 42 ++++++++++++++++++++++++++------ lib/configfile.cpp | 3 ++- 8 files changed, 89 insertions(+), 35 deletions(-) diff --git a/include/MySQL_Authentication.hpp b/include/MySQL_Authentication.hpp index e26b5b6e4..7089d98b4 100644 --- a/include/MySQL_Authentication.hpp +++ b/include/MySQL_Authentication.hpp @@ -1,7 +1,7 @@ #ifndef __CLASS_MYSQL_AUTHENTICATION_H #define __CLASS_MYSQL_AUTHENTICATION_H -#include "btree_map.h" +//#include "btree_map.h" #include "proxysql.h" #include "cpp.h" @@ -36,13 +36,15 @@ typedef struct _account_details_t { */ -typedef btree::btree_map BtMap_auth; +//typedef btree::btree_map BtMap_auth; +typedef std::unordered_map umap_auth; class PtrArray; typedef struct _creds_group_t { rwlock_t lock; - BtMap_auth bt_map; + //BtMap_auth bt_map; + umap_auth bt_map; PtrArray *cred_array; } creds_group_t; diff --git a/include/cpp.h b/include/cpp.h index 041b526cd..bad6a39e1 100644 --- a/include/cpp.h +++ b/include/cpp.h @@ -21,17 +21,17 @@ #undef swap #undef min #undef max -#include +//#include #include -#include -#include -#include -#include +//#include +//#include +//#include +//#include #include #include -#include -#include -#include -#include -#include +//#include +//#include +//#include +//#include +//#include //#endif /* __cplusplus */ diff --git a/include/proxysql.h b/include/proxysql.h index 506a7fe3c..80dc724ce 100644 --- a/include/proxysql.h +++ b/include/proxysql.h @@ -1,7 +1,15 @@ #ifdef __cplusplus #include #include -#include "btree_map.h" + +#include +//#include +//#include +//#include +//#include + + +//#include "btree_map.h" #ifndef EZOPTION //#include "ezOptionParser.hpp" #define EZOPTION @@ -70,8 +78,14 @@ #include "jemalloc.h" - +#ifdef DEBUG +//#define VALGRIND_ENABLE_ERROR_REPORTING +//#define VALGRIND_DISABLE_ERROR_REPORTING #include "valgrind.h" +#else +#define VALGRIND_ENABLE_ERROR_REPORTING +#define VALGRIND_DISABLE_ERROR_REPORTING +#endif /* DEBUG */ #include "sqlite3.h" diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index d9197893e..b21ca81b4 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -270,8 +270,8 @@ class SQLite3DB; class SimpleKV; class AdvancedKV; class ProxySQL_Poll; -//class Query_Cache; -class Shared_Query_Cache; +class Query_Cache; +//class Shared_Query_Cache; class MySQL_Authentication; class MySQL_Connection; class MySQL_Protocol; diff --git a/include/query_cache.hpp b/include/query_cache.hpp index fc01f2d9f..216bdbd88 100644 --- a/include/query_cache.hpp +++ b/include/query_cache.hpp @@ -28,7 +28,7 @@ struct __QC_entry_t { unsigned long long access_ms; // when the entry was read last , monotonic , millisecond granularity uint32_t ref_count; // reference counter }; - +/* typedef btree::btree_map BtMap_cache; @@ -52,10 +52,12 @@ class KV_BtreeArray { QC_entry_t *lookup(uint64_t key); void empty(); }; - +*/ +class KV_BtreeArray; class Query_Cache { private: - KV_BtreeArray KVs[SHARED_QUERY_CACHE_HASH_TABLES]; + //KV_BtreeArray KVs[SHARED_QUERY_CACHE_HASH_TABLES]; + KV_BtreeArray * KVs[SHARED_QUERY_CACHE_HASH_TABLES]; uint64_t get_data_size_total(); unsigned int current_used_memory_pct(); public: diff --git a/lib/MySQL_Authentication.cpp b/lib/MySQL_Authentication.cpp index 42980d4e7..9082447ed 100644 --- a/lib/MySQL_Authentication.cpp +++ b/lib/MySQL_Authentication.cpp @@ -136,7 +136,8 @@ bool MySQL_Authentication::add(char * username, char * password, enum cred_usern void *sha1_pass=NULL; char *oldpass=NULL; spin_wrlock(&cg.lock); - btree::btree_map::iterator lookup; + //btree::btree_map::iterator lookup; + std::unordered_map::iterator lookup; lookup = cg.bt_map.find(hash1); if (lookup != cg.bt_map.end()) { account_details_t *ad=lookup->second; @@ -264,7 +265,8 @@ int MySQL_Authentication::increase_frontend_user_connections(char *username) { creds_group_t &cg=creds_frontends; int ret=0; spin_wrlock(&cg.lock); - btree::btree_map::iterator it; + //btree::btree_map::iterator it; + std::unordered_map::iterator it; it = cg.bt_map.find(hash1); if (it != cg.bt_map.end()) { account_details_t *ad=it->second; @@ -284,7 +286,8 @@ void MySQL_Authentication::decrease_frontend_user_connections(char *username) { delete myhash; creds_group_t &cg=creds_frontends; spin_wrlock(&cg.lock); - btree::btree_map::iterator it; + //btree::btree_map::iterator it; + std::unordered_map::iterator it; it = cg.bt_map.find(hash1); if (it != cg.bt_map.end()) { account_details_t *ad=it->second; @@ -308,7 +311,8 @@ bool MySQL_Authentication::del(char * username, enum cred_username_type usertype if (set_lock) spin_wrlock(&cg.lock); - btree::btree_map::iterator lookup; + //btree::btree_map::iterator lookup; + std::unordered_map::iterator lookup; lookup = cg.bt_map.find(hash1); if (lookup != cg.bt_map.end()) { account_details_t *ad=lookup->second; @@ -339,7 +343,8 @@ bool MySQL_Authentication::set_SHA1(char * username, enum cred_username_type use creds_group_t &cg=(usertype==USERNAME_BACKEND ? creds_backends : creds_frontends); spin_wrlock(&cg.lock); - btree::btree_map::iterator lookup; + //btree::btree_map::iterator lookup; + std::unordered_map::iterator lookup; lookup = cg.bt_map.find(hash1); if (lookup != cg.bt_map.end()) { account_details_t *ad=lookup->second; @@ -368,7 +373,8 @@ char * MySQL_Authentication::lookup(char * username, enum cred_username_type use creds_group_t &cg=(usertype==USERNAME_BACKEND ? creds_backends : creds_frontends); spin_rdlock(&cg.lock); - btree::btree_map::iterator lookup; + //btree::btree_map::iterator lookup; + std::unordered_map::iterator lookup; lookup = cg.bt_map.find(hash1); if (lookup != cg.bt_map.end()) { account_details_t *ad=lookup->second; @@ -396,7 +402,8 @@ bool MySQL_Authentication::_reset(enum cred_username_type usertype) { creds_group_t &cg=(usertype==USERNAME_BACKEND ? creds_backends : creds_frontends); spin_wrlock(&cg.lock); - btree::btree_map::iterator lookup; + //btree::btree_map::iterator lookup; + std::unordered_map::iterator lookup; while (cg.bt_map.size()) { lookup = cg.bt_map.begin(); diff --git a/lib/Query_Cache.cpp b/lib/Query_Cache.cpp index f747a7f7d..9c7dd1e71 100644 --- a/lib/Query_Cache.cpp +++ b/lib/Query_Cache.cpp @@ -1,6 +1,7 @@ #include "btree_map.h" #include "proxysql.h" #include "cpp.h" +#include "query_cache.hpp" #include "proxysql_atomic.h" #include "SpookyV2.h" @@ -28,6 +29,30 @@ #endif /* DEBUG */ #define QUERY_CACHE_VERSION "0.2.0902" DEB + +typedef btree::btree_map BtMap_cache; + +class KV_BtreeArray { + private: + rwlock_t lock; + BtMap_cache bt_map; + PtrArray *ptrArray; + uint64_t purgeChunkSize; + uint64_t purgeIdx; + bool __insert(uint64_t, void *); + uint64_t freeable_memory; + public: + uint64_t tottopurge; + KV_BtreeArray(); + ~KV_BtreeArray(); + uint64_t get_data_size(); + void purge_some(unsigned long long); + int cnt(); + bool replace(uint64_t key, QC_entry_t *entry); + QC_entry_t *lookup(uint64_t key); + void empty(); +}; + __thread uint64_t __thr_cntSet=0; __thread uint64_t __thr_cntGet=0; __thread uint64_t __thr_cntGetOK=0; @@ -210,7 +235,7 @@ uint64_t Query_Cache::get_data_size_total() { int r=0; int i; for (i=0; iget_data_size(); } return r; }; @@ -236,6 +261,9 @@ Query_Cache::Query_Cache() { perror("Incompatible debagging version"); exit(EXIT_FAILURE); } + for (int i=0; ilookup(hk); if (entry!=NULL) { unsigned long long t=curtime_ms; @@ -294,7 +322,7 @@ bool Query_Cache::set(uint64_t user_hash, const unsigned char *kp, uint32_t kl, uint64_t hk=SpookyHash::Hash64(kp, kl, user_hash); unsigned char i=hk%SHARED_QUERY_CACHE_HASH_TABLES; entry->key=hk; - KVs[i].replace(hk, entry); + KVs[i]->replace(hk, entry); return true; } @@ -303,8 +331,8 @@ uint64_t Query_Cache::flush() { int i; uint64_t total_count=0; for (i=0; icnt(); + KVs[i]->empty(); } return total_count; }; @@ -319,7 +347,7 @@ void * Query_Cache::purgeHash_thread(void *) { if (current_used_memory_pct() < purge_threshold_pct_min ) continue; for (i=0; ipurge_some(QCnow_ms); } } return NULL; diff --git a/lib/configfile.cpp b/lib/configfile.cpp index 8f0096479..a18db874d 100644 --- a/lib/configfile.cpp +++ b/lib/configfile.cpp @@ -1,7 +1,8 @@ #include "proxysql.h" - #include "cpp.h" +#include + #include #include #include From d276262ebe811b7beca5966091665b41c0462f5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 16 Aug 2016 23:19:19 +0000 Subject: [PATCH 05/13] Drafting configuration of QC Related to issue #171 Added variable mysql-query_cache_size_MB Also removed compiling of simple_kv.cpp --- include/MySQL_Thread.h | 1 + include/cpp.h | 2 +- include/proxysql_structs.h | 8 +++++++- lib/Makefile | 2 +- lib/MySQL_Thread.cpp | 17 +++++++++++++++++ 5 files changed, 27 insertions(+), 3 deletions(-) diff --git a/include/MySQL_Thread.h b/include/MySQL_Thread.h index d196e0eb4..bae6d3901 100644 --- a/include/MySQL_Thread.h +++ b/include/MySQL_Thread.h @@ -314,6 +314,7 @@ class MySQL_Threads_Handler char * ssl_p2s_cert; char * ssl_p2s_key; char * ssl_p2s_cipher; + int query_cache_size_MB; } variables; PtrArray *bind_fds; MySQL_Listeners_Manager *MLM; diff --git a/include/cpp.h b/include/cpp.h index bad6a39e1..bf5b3cd4a 100644 --- a/include/cpp.h +++ b/include/cpp.h @@ -6,7 +6,7 @@ #include "query_cache.hpp" #include "mysql_connection.h" #include "sqlite3db.h" -#include "simple_kv.h" +//#include "simple_kv.h" #include "gen_utils.h" #include "StatCounters.h" #include "MySQL_Monitor.hpp" diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index b21ca81b4..f840636f0 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -507,7 +507,7 @@ struct _global_variables_t { unsigned int mysql_query_cache_default_timeout; unsigned long long mysql_wait_timeout; - unsigned long long mysql_query_cache_size; + //unsigned long long mysql_query_cache_size; unsigned long long mysql_max_resultset_size; int mysql_max_query_size; @@ -737,6 +737,9 @@ __thread bool mysql_thread___query_digests; __thread bool mysql_thread___default_reconnect; __thread bool mysql_thread___sessions_sort; +/* variables used for Query Cache */ +__thread int mysql_thread___query_cache_size_MB; + /* variables used for SSL , from proxy to server (p2s) */ __thread char * mysql_thread___ssl_p2s_ca; __thread char * mysql_thread___ssl_p2s_cert; @@ -812,6 +815,9 @@ extern __thread bool mysql_thread___query_digests; extern __thread bool mysql_thread___default_reconnect; extern __thread bool mysql_thread___sessions_sort; +/* variables used for Query Cache */ +extern __thread int mysql_thread___query_cache_size_MB; + /* variables used for SSL , from proxy to server (p2s) */ extern __thread char * mysql_thread___ssl_p2s_ca; extern __thread char * mysql_thread___ssl_p2s_cert; diff --git a/lib/Makefile b/lib/Makefile index 2b04a94cb..d415b8abb 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -47,7 +47,7 @@ default: libproxysql.a _OBJ = c_tokenizer.o OBJ = $(patsubst %,$(ODIR)/%,$(_OBJ)) -_OBJ_CPP = ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Query_Cache.oo SpookyV2.oo MySQL_Authentication.oo gen_utils.oo simple_kv.oo sqlite3db.oo global_variables.oo mysql_connection.oo MySQL_HostGroups_Manager.oo mysql_data_stream.oo MySQL_Thread.oo MySQL_Session.oo MySQL_Protocol.oo mysql_backend.oo Query_Processor.oo ProxySQL_Admin.oo MySQL_Monitor.oo MySQL_Logger.oo thread.oo +_OBJ_CPP = ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Query_Cache.oo SpookyV2.oo MySQL_Authentication.oo gen_utils.oo sqlite3db.oo global_variables.oo mysql_connection.oo MySQL_HostGroups_Manager.oo mysql_data_stream.oo MySQL_Thread.oo MySQL_Session.oo MySQL_Protocol.oo mysql_backend.oo Query_Processor.oo ProxySQL_Admin.oo MySQL_Monitor.oo MySQL_Logger.oo thread.oo OBJ_CPP = $(patsubst %,$(ODIR)/%,$(_OBJ_CPP)) %.ko: %.cpp diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index 832ae6869..330fcadda 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -191,6 +191,7 @@ static char * mysql_thread_variables_names[]= { (char *)"default_query_delay", (char *)"default_query_timeout", (char *)"long_query_time", + (char *)"query_cache_size_MB", (char *)"ping_interval_server_msec", (char *)"ping_timeout_server", (char *)"default_schema", @@ -266,6 +267,7 @@ MySQL_Threads_Handler::MySQL_Threads_Handler() { variables.default_query_delay=0; variables.default_query_timeout=24*3600*1000; variables.long_query_time=1000; + variables.query_cache_size_MB=256; variables.init_connect=NULL; variables.ping_interval_server_msec=10000; variables.ping_timeout_server=200; @@ -471,6 +473,7 @@ int MySQL_Threads_Handler::get_variable_int(char *name) { if (!strcasecmp(name,"default_query_timeout")) return (int)variables.default_query_timeout; if (!strcasecmp(name,"default_max_latency_ms")) return (int)variables.default_max_latency_ms; if (!strcasecmp(name,"long_query_time")) return (int)variables.long_query_time; + if (!strcasecmp(name,"query_cache_size_MB")) return (int)variables.query_cache_size_MB; if (!strcasecmp(name,"free_connections_pct")) return (int)variables.free_connections_pct; if (!strcasecmp(name,"ping_interval_server_msec")) return (int)variables.ping_interval_server_msec; if (!strcasecmp(name,"ping_timeout_server")) return (int)variables.ping_timeout_server; @@ -687,6 +690,10 @@ char * MySQL_Threads_Handler::get_variable(char *name) { // this is the public f sprintf(intbuf,"%d",variables.long_query_time); return strdup(intbuf); } + if (!strcasecmp(name,"query_cache_size_MB")) { + sprintf(intbuf,"%d",variables.query_cache_size_MB); + return strdup(intbuf); + } if (!strcasecmp(name,"ping_interval_server_msec")) { sprintf(intbuf,"%d",variables.ping_interval_server_msec); return strdup(intbuf); @@ -1010,6 +1017,15 @@ bool MySQL_Threads_Handler::set_variable(char *name, char *value) { // this is t return false; } } + if (!strcasecmp(name,"query_cache_size_MB")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 1024*10240) { + variables.query_cache_size_MB=intv; + return true; + } else { + return false; + } + } if (!strcasecmp(name,"ping_interval_server_msec")) { int intv=atoi(value); if (intv >= 1000 && intv <= 7*24*3600*1000) { @@ -1995,6 +2011,7 @@ void MySQL_Thread::refresh_variables() { mysql_thread___default_query_timeout=GloMTH->get_variable_int((char *)"default_query_timeout"); mysql_thread___default_max_latency_ms=GloMTH->get_variable_int((char *)"default_max_latency_ms"); mysql_thread___long_query_time=GloMTH->get_variable_int((char *)"long_query_time"); + mysql_thread___query_cache_size_MB=GloMTH->get_variable_int((char *)"query_cache_size_MB"); mysql_thread___ping_interval_server_msec=GloMTH->get_variable_int((char *)"ping_interval_server_msec"); mysql_thread___ping_timeout_server=GloMTH->get_variable_int((char *)"ping_timeout_server"); mysql_thread___shun_on_failures=GloMTH->get_variable_int((char *)"shun_on_failures"); From 41acb178eeab30353cfa6f198f811a1778a16aea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 16 Aug 2016 23:38:30 +0000 Subject: [PATCH 06/13] mysql_thread___query_cache_size_MB refreshed in QC Issue #171 --- lib/Query_Cache.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/Query_Cache.cpp b/lib/Query_Cache.cpp index 9c7dd1e71..7dea95995 100644 --- a/lib/Query_Cache.cpp +++ b/lib/Query_Cache.cpp @@ -27,8 +27,9 @@ #else #define DEB "" #endif /* DEBUG */ -#define QUERY_CACHE_VERSION "0.2.0902" DEB +#define QUERY_CACHE_VERSION "1.2.0817" DEB +extern MySQL_Threads_Handler *GloMTH; typedef btree::btree_map BtMap_cache; @@ -340,16 +341,25 @@ uint64_t Query_Cache::flush() { void * Query_Cache::purgeHash_thread(void *) { unsigned int i; + unsigned int MySQL_Monitor__thread_MySQL_Thread_Variables_version; + MySQL_Thread * mysql_thr = new MySQL_Thread(); + MySQL_Monitor__thread_MySQL_Thread_Variables_version=GloMTH->get_global_version(); while (shutdown==0) { usleep(purge_loop_time); unsigned long long t=monotonic_time()/1000; QCnow_ms=t; - + unsigned int glover=GloMTH->get_global_version(); + if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover ) { + MySQL_Monitor__thread_MySQL_Thread_Variables_version=glover; + mysql_thr->refresh_variables(); + max_memory_size=mysql_thread___query_cache_size_MB*1024*1024; + } if (current_used_memory_pct() < purge_threshold_pct_min ) continue; for (i=0; ipurge_some(QCnow_ms); } } + delete mysql_thr; return NULL; }; From f50e95f1f1d9ab417dc78993374012b35fe35ac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Wed, 17 Aug 2016 00:16:26 +0000 Subject: [PATCH 07/13] Exporting QC metrics via SQL3_getStats , #140 Still a draft --- include/query_cache.hpp | 1 + lib/ProxySQL_Admin.cpp | 20 +++++++++++++++++++- lib/Query_Cache.cpp | 19 ++++++++++++++++++- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/include/query_cache.hpp b/include/query_cache.hpp index fc01f2d9f..09985d2c3 100644 --- a/include/query_cache.hpp +++ b/include/query_cache.hpp @@ -75,6 +75,7 @@ class Query_Cache { bool set(uint64_t , const unsigned char *, uint32_t, unsigned char *, uint32_t, unsigned long long, unsigned long long); unsigned char * get(uint64_t , const unsigned char *, const uint32_t, uint32_t *, unsigned long long); uint64_t flush(); + SQLite3_result * SQL3_getStats(); }; #endif /* __CLASS_QUERY_CACHE_H */ diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index a62368b78..0fc17eb83 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -62,6 +62,7 @@ static int __admin_refresh_interval=0; static bool proxysql_mysql_paused=false; static int old_wait_timeout; +extern Query_Cache *GloQC; extern MySQL_Authentication *GloMyAuth; extern ProxySQL_Admin *GloAdmin; extern Query_Processor *GloQPro; @@ -3134,6 +3135,8 @@ void ProxySQL_Admin::stats___mysql_global() { statsdb->execute(query); free(query); } + delete resultset; + resultset=NULL; int highwater; int current; sqlite3_status(SQLITE_STATUS_MEMORY_USED, ¤t, &highwater, 0); @@ -3155,8 +3158,23 @@ void ProxySQL_Admin::stats___mysql_global() { statsdb->execute(query); free(query); + resultset=GloQC->SQL3_getStats(); + if (resultset) { + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + int arg_len=0; + for (int i=0; i<2; i++) { + arg_len+=strlen(r->fields[i]); + } + char *query=(char *)malloc(strlen(a)+arg_len+32); + sprintf(query,a,r->fields[0],r->fields[1]); + statsdb->execute(query); + free(query); + } + delete resultset; + resultset=NULL; + } statsdb->execute("COMMIT"); - delete resultset; } void ProxySQL_Admin::stats___mysql_processlist() { diff --git a/lib/Query_Cache.cpp b/lib/Query_Cache.cpp index f747a7f7d..77d0baebd 100644 --- a/lib/Query_Cache.cpp +++ b/lib/Query_Cache.cpp @@ -325,4 +325,21 @@ void * Query_Cache::purgeHash_thread(void *) { return NULL; }; - +SQLite3_result * Query_Cache::SQL3_getStats() { + const int colnum=2; + char buf[256]; + char **pta=(char **)malloc(sizeof(char *)*colnum); + //Get_Memory_Stats(); + SQLite3_result *result=new SQLite3_result(colnum); + result->add_column_definition(SQLITE_TEXT,"Variable_Name"); + result->add_column_definition(SQLITE_TEXT,"Variable_Value"); + // NOTE: as there is no string copy, we do NOT free pta[0] and pta[1] + { // Used Memoery + pta[0]=(char *)"Query_Cache_Memory_MB"; + sprintf(buf,"%lu", get_data_size_total()); + pta[1]=buf; + result->add_row(pta); + } + free(pta); + return result; +} From 6563ae3d224628d1f4d9ec4de6e7fe0108df1c0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Wed, 17 Aug 2016 01:05:24 +0000 Subject: [PATCH 08/13] Export Query Cache metrics , #140 New available metrics: * Query_Cache_Memory_bytes : memory allocated in QC * Query_Cache_bytes_IN : amount of data inserted into QC * Query_Cache_bytes_OUT : amount of data retrieved from QC * Query_Cache_count_GET : total GET from QC * Query_Cache_count_GET_OK : successful GET from QC * Query_Cache_count_SET : total SET into QC Important note on `Query_Cache_Memory_bytes` : Query Cache thread purge doesn't initialize any purging if memory allocated in QC is below 3% of maximum memory. --- lib/Query_Cache.cpp | 64 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 12 deletions(-) diff --git a/lib/Query_Cache.cpp b/lib/Query_Cache.cpp index b0a64db09..0e7be0a9e 100644 --- a/lib/Query_Cache.cpp +++ b/lib/Query_Cache.cpp @@ -178,9 +178,12 @@ int KV_BtreeArray::cnt() { bool KV_BtreeArray::replace(uint64_t key, QC_entry_t *entry) { spin_wrlock(&lock); - THR_UPDATE_CNT(__thr_cntSet,Glo_cntSet,1,100); - THR_UPDATE_CNT(__thr_size_values,Glo_size_values,entry->length,100); - THR_UPDATE_CNT(__thr_dataIN,Glo_dataIN,entry->length,100); + //THR_UPDATE_CNT(__thr_cntSet,Glo_cntSet,1,100); + //THR_UPDATE_CNT(__thr_size_values,Glo_size_values,entry->length,100); + //THR_UPDATE_CNT(__thr_dataIN,Glo_dataIN,entry->length,100); + THR_UPDATE_CNT(__thr_cntSet,Glo_cntSet,1,1); + THR_UPDATE_CNT(__thr_size_values,Glo_size_values,entry->length,1); + THR_UPDATE_CNT(__thr_dataIN,Glo_dataIN,entry->length,1); THR_UPDATE_CNT(__thr_num_entries,Glo_num_entries,1,1); entry->ref_count=1; @@ -200,14 +203,15 @@ bool KV_BtreeArray::replace(uint64_t key, QC_entry_t *entry) { QC_entry_t * KV_BtreeArray::lookup(uint64_t key) { QC_entry_t *entry=NULL; spin_rdlock(&lock); - THR_UPDATE_CNT(__thr_cntGet,Glo_cntGet,1,100); + //THR_UPDATE_CNT(__thr_cntGet,Glo_cntGet,1,100); + THR_UPDATE_CNT(__thr_cntGet,Glo_cntGet,1,1); btree::btree_map::iterator lookup; lookup = bt_map.find(key); if (lookup != bt_map.end()) { entry=lookup->second; __sync_fetch_and_add(&entry->ref_count,1); - THR_UPDATE_CNT(__thr_cntGetOK,Glo_cntGetOK,1,100); - THR_UPDATE_CNT(__thr_dataOUT,Glo_dataOUT,entry->length,10000); + //THR_UPDATE_CNT(__thr_cntGetOK,Glo_cntGetOK,1,100); + //THR_UPDATE_CNT(__thr_dataOUT,Glo_dataOUT,entry->length,10000); } spin_rdunlock(&lock); return entry; @@ -299,6 +303,8 @@ unsigned char * Query_Cache::get(uint64_t user_hash, const unsigned char *kp, co if (entry!=NULL) { unsigned long long t=curtime_ms; if (entry->expire_ms > t) { + THR_UPDATE_CNT(__thr_cntGetOK,Glo_cntGetOK,1,1); + THR_UPDATE_CNT(__thr_dataOUT,Glo_dataOUT,entry->length,1); result=(unsigned char *)malloc(entry->length); memcpy(result,entry->value,entry->length); *lv=entry->length; @@ -344,16 +350,20 @@ void * Query_Cache::purgeHash_thread(void *) { unsigned int MySQL_Monitor__thread_MySQL_Thread_Variables_version; MySQL_Thread * mysql_thr = new MySQL_Thread(); MySQL_Monitor__thread_MySQL_Thread_Variables_version=GloMTH->get_global_version(); + mysql_thr->refresh_variables(); + max_memory_size=mysql_thread___query_cache_size_MB*1024*1024; while (shutdown==0) { usleep(purge_loop_time); unsigned long long t=monotonic_time()/1000; QCnow_ms=t; unsigned int glover=GloMTH->get_global_version(); - if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover ) { - MySQL_Monitor__thread_MySQL_Thread_Variables_version=glover; - mysql_thr->refresh_variables(); - max_memory_size=mysql_thread___query_cache_size_MB*1024*1024; - } + if (GloMTH) { + if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover ) { + MySQL_Monitor__thread_MySQL_Thread_Variables_version=glover; + mysql_thr->refresh_variables(); + max_memory_size=mysql_thread___query_cache_size_MB*1024*1024; + } + } if (current_used_memory_pct() < purge_threshold_pct_min ) continue; for (i=0; ipurge_some(QCnow_ms); @@ -373,11 +383,41 @@ SQLite3_result * Query_Cache::SQL3_getStats() { result->add_column_definition(SQLITE_TEXT,"Variable_Value"); // NOTE: as there is no string copy, we do NOT free pta[0] and pta[1] { // Used Memoery - pta[0]=(char *)"Query_Cache_Memory_MB"; + pta[0]=(char *)"Query_Cache_Memory_bytes"; sprintf(buf,"%lu", get_data_size_total()); pta[1]=buf; result->add_row(pta); } + { // Glo_cntGet + pta[0]=(char *)"Query_Cache_count_GET"; + sprintf(buf,"%lu", Glo_cntGet); + pta[1]=buf; + result->add_row(pta); + } + { // Glo_cntGetOK + pta[0]=(char *)"Query_Cache_count_GET_OK"; + sprintf(buf,"%lu", Glo_cntGetOK); + pta[1]=buf; + result->add_row(pta); + } + { // Glo_cntSet + pta[0]=(char *)"Query_Cache_count_SET"; + sprintf(buf,"%lu", Glo_cntSet); + pta[1]=buf; + result->add_row(pta); + } + { // Glo_dataIN + pta[0]=(char *)"Query_Cache_bytes_IN"; + sprintf(buf,"%lu", Glo_dataIN); + pta[1]=buf; + result->add_row(pta); + } + { // Glo_dataOUT + pta[0]=(char *)"Query_Cache_bytes_OUT"; + sprintf(buf,"%lu", Glo_dataOUT); + pta[1]=buf; + result->add_row(pta); + } free(pta); return result; } From b151da09716edff4c98cc33a398fd0bc573942d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Wed, 17 Aug 2016 06:59:24 +0000 Subject: [PATCH 09/13] Further metrics for Query Cache #140 * Query_Cache_Entries : number of entries currently in the cache (including inactive) * Query_Cache_Purged : number of entries removed from the cache --- lib/Query_Cache.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/Query_Cache.cpp b/lib/Query_Cache.cpp index 0e7be0a9e..06eacd5d9 100644 --- a/lib/Query_Cache.cpp +++ b/lib/Query_Cache.cpp @@ -418,6 +418,18 @@ SQLite3_result * Query_Cache::SQL3_getStats() { pta[1]=buf; result->add_row(pta); } + { // Glo_cntPurge + pta[0]=(char *)"Query_Cache_Purged"; + sprintf(buf,"%lu", Glo_cntPurge); + pta[1]=buf; + result->add_row(pta); + } + { // Glo_num_entries + pta[0]=(char *)"Query_Cache_Entries"; + sprintf(buf,"%lu", Glo_num_entries); + pta[1]=buf; + result->add_row(pta); + } free(pta); return result; } From 6439cedef86077707c016c98232f2bd9ebe4b281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Wed, 17 Aug 2016 11:05:06 +0000 Subject: [PATCH 10/13] Changed version to 1.2.2-RC To avoid confusion --- include/proxysql.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/proxysql.h b/include/proxysql.h index 80dc724ce..b1c7d81f5 100644 --- a/include/proxysql.h +++ b/include/proxysql.h @@ -99,7 +99,7 @@ #else #define DEB "" #endif /* DEBUG */ -#define PROXYSQL_VERSION "1.2.1" DEB +#define PROXYSQL_VERSION "1.2.2-RC" DEB #define PROXYSQL_CODENAME "Truls" #ifndef PROXYSQL_FUNC_DEFS From 66626c13adce02a2f345d3517216ba681c12f809 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Wed, 17 Aug 2016 22:12:35 +0000 Subject: [PATCH 11/13] mysql_replication_hostgroups in configfile , #648 Now Admin supports the configuration of mysql_replication_hostgroups from configfile. That means that also LOAD MYSQL SERVERS FROM CONFIG is responsible for configuring mysql_replication_hostgroups. This commit also allows comments in config file for `mysql_servers` and `mysql_replication_hostgroups` , related to #643 --- lib/ProxySQL_Admin.cpp | 113 ++++++++++++++++++++++++++--------------- src/proxysql.cfg | 15 ++++++ 2 files changed, 87 insertions(+), 41 deletions(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 0fc17eb83..40ff14ac6 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -4404,50 +4404,81 @@ int ProxySQL_Admin::Read_MySQL_Query_Rules_from_configfile() { int ProxySQL_Admin::Read_MySQL_Servers_from_configfile() { const Setting& root = GloVars.confFile->cfg->getRoot(); - if (root.exists("mysql_servers")==false) return 0; - const Setting &mysql_servers = root["mysql_servers"]; - int count = mysql_servers.getLength(); - //fprintf(stderr, "Found %d servers\n",count); int i; int rows=0; admindb->execute("PRAGMA foreign_keys = OFF"); - char *q=(char *)"INSERT OR REPLACE INTO mysql_servers (hostname, port, hostgroup_id, compression, weight, status, max_connections, max_replication_lag, use_ssl, max_latency_ms) VALUES (\"%s\", %d, %d, %d, %d, \"%s\", %d, %d, %d, %d)"; - for (i=0; i< count; i++) { - const Setting &server = mysql_servers[i]; - std::string address; - std::string status="ONLINE"; - int port; - int hostgroup; - int weight=1; - int compression=0; - int max_connections=1000; // default - int max_replication_lag=0; // default - int use_ssl=0; - int max_latency_ms=0; - if (server.lookupValue("address", address)==false) continue; - if (server.lookupValue("port", port)==false) continue; - if (server.lookupValue("hostgroup", hostgroup)==false) continue; - server.lookupValue("status", status); - if ( - (strcasecmp(status.c_str(),(char *)"ONLINE")) - && (strcasecmp(status.c_str(),(char *)"SHUNNED")) - && (strcasecmp(status.c_str(),(char *)"OFFLINE_SOFT")) - && (strcasecmp(status.c_str(),(char *)"OFFLINE_HARD")) - ) { - status="ONLINE"; - } - server.lookupValue("compression", compression); - server.lookupValue("weight", weight); - server.lookupValue("max_connections", max_connections); - server.lookupValue("max_replication_lag", max_replication_lag); - server.lookupValue("use_ssl", use_ssl); - server.lookupValue("max_latency_ms", max_latency_ms); - char *query=(char *)malloc(strlen(q)+strlen(status.c_str())+strlen(address.c_str())+128); - sprintf(query,q, address.c_str(), port, hostgroup, compression, weight, status.c_str(), max_connections, max_replication_lag, use_ssl, max_latency_ms); - //fprintf(stderr, "%s\n", query); - admindb->execute(query); - free(query); - rows++; + if (root.exists("mysql_servers")==true) { + const Setting &mysql_servers = root["mysql_servers"]; + int count = mysql_servers.getLength(); + //fprintf(stderr, "Found %d servers\n",count); + char *q=(char *)"INSERT OR REPLACE INTO mysql_servers (hostname, port, hostgroup_id, compression, weight, status, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment) VALUES (\"%s\", %d, %d, %d, %d, \"%s\", %d, %d, %d, %d, '%s')"; + for (i=0; i< count; i++) { + const Setting &server = mysql_servers[i]; + std::string address; + std::string status="ONLINE"; + int port; + int hostgroup; + int weight=1; + int compression=0; + int max_connections=1000; // default + int max_replication_lag=0; // default + int use_ssl=0; + int max_latency_ms=0; + std::string comment=""; + if (server.lookupValue("address", address)==false) continue; + if (server.lookupValue("port", port)==false) continue; + if (server.lookupValue("hostgroup", hostgroup)==false) continue; + server.lookupValue("status", status); + if ( + (strcasecmp(status.c_str(),(char *)"ONLINE")) + && (strcasecmp(status.c_str(),(char *)"SHUNNED")) + && (strcasecmp(status.c_str(),(char *)"OFFLINE_SOFT")) + && (strcasecmp(status.c_str(),(char *)"OFFLINE_HARD")) + ) { + status="ONLINE"; + } + server.lookupValue("compression", compression); + server.lookupValue("weight", weight); + server.lookupValue("max_connections", max_connections); + server.lookupValue("max_replication_lag", max_replication_lag); + server.lookupValue("use_ssl", use_ssl); + server.lookupValue("max_latency_ms", max_latency_ms); + server.lookupValue("comment", comment); + char *o1=strdup(comment.c_str()); + char *o=escape_string_single_quotes(o1, false); + char *query=(char *)malloc(strlen(q)+strlen(status.c_str())+strlen(address.c_str())+strlen(o)+128); + sprintf(query,q, address.c_str(), port, hostgroup, compression, weight, status.c_str(), max_connections, max_replication_lag, use_ssl, max_latency_ms, o); + //fprintf(stderr, "%s\n", query); + admindb->execute(query); + if (o!=o1) free(o); + free(o1); + free(query); + rows++; + } + } + if (root.exists("mysql_replication_hostgroups")==true) { + const Setting &mysql_replication_hostgroups = root["mysql_replication_hostgroups"]; + int count = mysql_replication_hostgroups.getLength(); + char *q=(char *)"INSERT OR REPLACE INTO mysql_replication_hostgroups (writer_hostgroup, reader_hostgroup, comment) VALUES (%d, %d, '%s')"; + for (i=0; i< count; i++) { + const Setting &line = mysql_replication_hostgroups[i]; + int writer_hostgroup; + int reader_hostgroup; + std::string comment=""; + if (line.lookupValue("writer_hostgroup", writer_hostgroup)==false) continue; + if (line.lookupValue("reader_hostgroup", reader_hostgroup)==false) continue; + line.lookupValue("comment", comment); + char *o1=strdup(comment.c_str()); + char *o=escape_string_single_quotes(o1, false); + char *query=(char *)malloc(strlen(q)+strlen(o)+32); + sprintf(query,q, writer_hostgroup, reader_hostgroup, o); + //fprintf(stderr, "%s\n", query); + admindb->execute(query); + if (o!=o1) free(o); + free(o1); + free(query); + rows++; + } } admindb->execute("PRAGMA foreign_keys = ON"); return rows; diff --git a/src/proxysql.cfg b/src/proxysql.cfg index bc4bb9bd2..6b65d04ca 100644 --- a/src/proxysql.cfg +++ b/src/proxysql.cfg @@ -46,9 +46,24 @@ mysql_servers = port=3306 hostgroup=0 max_connections=200 + comment="test server" } ) +mysql_replication_hostgroups= +( + { + writer_hostgroup=30 + reader_hostgroup=40 + comment="test repl 1" + }, + { + writer_hostgroup=50 + reader_hostgroup=60 + comment="test repl 2" + } +) + mysql_users: ( { From 7df5fd4ec4b5a3daf39e015dd8b9b1f21e129968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Wed, 17 Aug 2016 22:34:39 +0000 Subject: [PATCH 12/13] Load config file if configdb is missing #617 The check if the configdb was not existing wasn't present anymore. This probably happened when introducing the new LOAD FROM CONFIG commands. Now the functionality is back. --- lib/ProxySQL_Admin.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 0fc17eb83..77441ae52 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -2457,6 +2457,10 @@ bool ProxySQL_Admin::init() { admindb->open((char *)"file:mem_admindb?mode=memory&cache=shared", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); statsdb=new SQLite3DB(); statsdb->open((char *)"file:mem_statsdb?mode=memory&cache=shared", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); + + // check if file exists , see #617 + bool admindb_file_exists=Proxy_file_exists(GloVars.admindb); + configdb=new SQLite3DB(); configdb->open((char *)GloVars.admindb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX); @@ -2548,7 +2552,7 @@ bool ProxySQL_Admin::init() { } #endif /* DEBUG */ - if (GloVars.__cmd_proxysql_reload || GloVars.__cmd_proxysql_initial) { + if (GloVars.__cmd_proxysql_reload || GloVars.__cmd_proxysql_initial || admindb_file_exists==false) { // see #617 if (GloVars.configfile_open) { if (GloVars.confFile->cfg) { Read_MySQL_Servers_from_configfile(); From 7af38425085fc5b7ab80e3bc9c9641f8367f81a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Wed, 17 Aug 2016 23:24:31 +0000 Subject: [PATCH 13/13] Disable miltiplexing for FTWRL #613 --- lib/mysql_connection.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index a4aaecda7..21331a9aa 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -1205,6 +1205,11 @@ void MySQL_Connection::ProcessQueryAndSetStatusFlags(char *query_digest_text) { set_status_lock_tables(true); } } + if (get_status_lock_tables()==false) { // we search for lock tables only if not already set + if (!strncasecmp(query_digest_text,"FLUSH TABLES WITH READ LOCK", strlen("FLUSH TABLES WITH READ LOCK"))) { // issue 613 + set_status_lock_tables(true); + } + } if (get_status_lock_tables()==true) { if (!strncasecmp(query_digest_text,"UNLOCK TABLES", strlen("UNLOCK TABLES"))) { set_status_lock_tables(false);