From 72724b22e06ee3daf0ea820ebf4068f5c87ceced Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Wed, 31 Oct 2018 23:54:21 +0100 Subject: [PATCH] Preparing for support of cache_timeout Adding column mysql_query_rules.cache_timeout . Functionality is not implemented yet, only the column is added. --- include/query_processor.h | 5 +- lib/ProxySQL_Admin.cpp | 100 +++++++++++++++++++++++--------------- lib/ProxySQL_Cluster.cpp | 37 +++++++------- lib/Query_Processor.cpp | 49 +++++++++++-------- 4 files changed, 113 insertions(+), 78 deletions(-) diff --git a/include/query_processor.h b/include/query_processor.h index 9b66ed753..7ae3ab70f 100644 --- a/include/query_processor.h +++ b/include/query_processor.h @@ -27,6 +27,7 @@ struct _Query_Processor_rule_t { int destination_hostgroup; int cache_ttl; int cache_empty_result; + int cache_timeout; int reconnect; int timeout; int retries; @@ -60,6 +61,7 @@ class Query_Processor_Output { int next_query_flagIN; int cache_ttl; int cache_empty_result; + int cache_timeout; int reconnect; int timeout; int retries; @@ -93,6 +95,7 @@ class Query_Processor_Output { next_query_flagIN=-1; cache_ttl=-1; cache_empty_result=1; + cache_timeout=-1; reconnect=-1; timeout=-1; retries=-1; @@ -203,7 +206,7 @@ class Query_Processor { void wrlock(); // explicit write lock, to be used in multi-isert void wrunlock(); // explicit write unlock bool insert(QP_rule_t *qr, bool lock=true); // insert a new rule. Uses a generic void pointer to a structure that may vary depending from the Query Processor - QP_rule_t * new_query_rule(int rule_id, bool active, char *username, char *schemaname, int flagIN, char *client_addr, char *proxy_addr, int proxy_port, char *digest, char *match_digest, char *match_pattern, bool negate_match_pattern, char *re_modifiers, int flagOUT, char *replace_pattern, int destination_hostgroup, int cache_ttl, int cache_empty_result, int reconnect, int timeout, int retries, int delay, int next_query_flagIN, int mirror_hostgroup, int mirror_flagOUT, char *error_msg, char *OK_msg, int sticky_conn, int multiplex, int gtid_from_hostgroup, int log, bool apply, char *comment); // to use a generic query rule struct, this is generated by this function and returned as generic void pointer + QP_rule_t * new_query_rule(int rule_id, bool active, char *username, char *schemaname, int flagIN, char *client_addr, char *proxy_addr, int proxy_port, char *digest, char *match_digest, char *match_pattern, bool negate_match_pattern, char *re_modifiers, int flagOUT, char *replace_pattern, int destination_hostgroup, int cache_ttl, int cache_empty_result, int cache_timeout, int reconnect, int timeout, int retries, int delay, int next_query_flagIN, int mirror_hostgroup, int mirror_flagOUT, char *error_msg, char *OK_msg, int sticky_conn, int multiplex, int gtid_from_hostgroup, int log, bool apply, char *comment); // to use a generic query rule struct, this is generated by this function and returned as generic void pointer void delete_query_rule(QP_rule_t *qr); // destructor Query_Processor_Output * process_mysql_query(MySQL_Session *sess, void *ptr, unsigned int size, Query_Info *qi); void delete_QP_out(Query_Processor_Output *o); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 90c06c40e..7bde73ad0 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -241,7 +241,9 @@ static int http_handler(void *cls, struct MHD_Connection *connection, const char #define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0d "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT CHECK (flagIN >= 0) NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT CHECK (proxy_port >= 0 AND proxy_port <= 65535), digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT CHECK (flagOUT >= 0), replace_pattern VARCHAR CHECK(CASE WHEN replace_pattern IS NULL THEN 1 WHEN replace_pattern IS NOT NULL AND match_pattern IS NOT NULL THEN 1 ELSE 0 END) , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , cache_empty_result INT CHECK (cache_empty_result IN (0,1)) DEFAULT NULL , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED CHECK (timeout >= 0) , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED CHECK (delay >=0) , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , gtid_from_hostgroup INT UNSIGNED , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" -#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0d +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0e "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT CHECK (flagIN >= 0) NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT CHECK (proxy_port >= 0 AND proxy_port <= 65535), digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT CHECK (flagOUT >= 0), replace_pattern VARCHAR CHECK(CASE WHEN replace_pattern IS NULL THEN 1 WHEN replace_pattern IS NOT NULL AND match_pattern IS NOT NULL THEN 1 ELSE 0 END) , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , cache_empty_result INT CHECK (cache_empty_result IN (0,1)) DEFAULT NULL , cache_timeout INT CHECK(cache_timeout >= 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED CHECK (timeout >= 0) , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED CHECK (delay >=0) , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , gtid_from_hostgroup INT UNSIGNED , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" + +#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0e //#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V1_4_0b @@ -284,7 +286,7 @@ static int http_handler(void *cls, struct MHD_Connection *connection, const char #define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_REPLICATION_HOSTGROUPS "CREATE TABLE runtime_mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>=0) , check_type VARCHAR CHECK (LOWER(check_type) IN ('read_only','innodb_read_only','super_read_only')) NOT NULL DEFAULT 'read_only' , comment VARCHAR NOT NULL DEFAULT '' , UNIQUE (reader_hostgroup))" -#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_QUERY_RULES "CREATE TABLE runtime_mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT CHECK (flagIN >= 0) NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT CHECK (proxy_port >= 0 AND proxy_port <= 65535), digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT CHECK (flagOUT >= 0), replace_pattern VARCHAR CHECK(CASE WHEN replace_pattern IS NULL THEN 1 WHEN replace_pattern IS NOT NULL AND match_pattern IS NOT NULL THEN 1 ELSE 0 END) , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , cache_empty_result INT CHECK (cache_empty_result IN (0,1)) DEFAULT NULL , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED CHECK (timeout >= 0) , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED CHECK (delay >=0) , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , gtid_from_hostgroup INT UNSIGNED , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" +#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_QUERY_RULES "CREATE TABLE runtime_mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT CHECK (flagIN >= 0) NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT CHECK (proxy_port >= 0 AND proxy_port <= 65535), digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , re_modifiers VARCHAR DEFAULT 'CASELESS' , flagOUT INT CHECK (flagOUT >= 0), replace_pattern VARCHAR CHECK(CASE WHEN replace_pattern IS NULL THEN 1 WHEN replace_pattern IS NOT NULL AND match_pattern IS NOT NULL THEN 1 ELSE 0 END) , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , cache_empty_result INT CHECK (cache_empty_result IN (0,1)) DEFAULT NULL , cache_timeout INT CHECK(cache_timeout >= 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED CHECK (timeout >= 0) , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED CHECK (delay >=0) , next_query_flagIN INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , OK_msg VARCHAR , sticky_conn INT CHECK (sticky_conn IN (0,1)) , multiplex INT CHECK (multiplex IN (0,1,2)) , gtid_from_hostgroup INT UNSIGNED , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" #define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_QUERY_RULES_FAST_ROUTING "CREATE TABLE runtime_mysql_query_rules_fast_routing (username VARCHAR NOT NULL , schemaname VARCHAR NOT NULL , flagIN INT NOT NULL DEFAULT 0 , destination_hostgroup INT CHECK (destination_hostgroup >= 0) NOT NULL , comment VARCHAR NOT NULL , PRIMARY KEY (username, schemaname, flagIN) )" @@ -5775,15 +5777,15 @@ void ProxySQL_Admin::save_mysql_query_rules_from_runtime(bool _runtime) { //char *a=(char *)"INSERT INTO mysql_query_rules VALUES (\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\")"; char *a=NULL; if (_runtime) { - a=(char *)"INSERT INTO runtime_mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"; + a=(char *)"INSERT INTO runtime_mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, cache_timeout, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"; } else { - a=(char *)"INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"; + a=(char *)"INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, cache_timeout, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"; } for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; int arg_len=0; - char *buffs[33]; // number of fields - for (int i=0; i<33; i++) { + char *buffs[34]; // number of fields + for (int i=0; i<34; i++) { if (r->fields[i]) { char *o=escape_string_single_quotes(r->fields[i],false); int l=strlen(o)+4; @@ -5821,25 +5823,26 @@ void ProxySQL_Admin::save_mysql_query_rules_from_runtime(bool _runtime) { ( strcmp(r->fields[15],"-1")==0 ? "NULL" : r->fields[15] ), // destination_hostgroup ( strcmp(r->fields[16],"-1")==0 ? "NULL" : r->fields[16] ), // cache_ttl ( strcmp(r->fields[17],"-1")==0 ? "NULL" : r->fields[17] ), // cache_empty_result - ( strcmp(r->fields[18],"-1")==0 ? "NULL" : r->fields[18] ), // reconnect - ( strcmp(r->fields[19],"-1")==0 ? "NULL" : r->fields[19] ), // timeout - ( strcmp(r->fields[20],"-1")==0 ? "NULL" : r->fields[20] ), // retries - ( strcmp(r->fields[21],"-1")==0 ? "NULL" : r->fields[21] ), // delay - ( strcmp(r->fields[22],"-1")==0 ? "NULL" : r->fields[22] ), // next_query_flagIN - ( strcmp(r->fields[23],"-1")==0 ? "NULL" : r->fields[23] ), // mirror_flagOUT - ( strcmp(r->fields[24],"-1")==0 ? "NULL" : r->fields[24] ), // mirror_hostgroup - buffs[25], // error_msg - buffs[26], // OK_msg - ( strcmp(r->fields[27],"-1")==0 ? "NULL" : r->fields[27] ), // sticky_conn - ( strcmp(r->fields[28],"-1")==0 ? "NULL" : r->fields[28] ), // multiplex - ( strcmp(r->fields[29],"-1")==0 ? "NULL" : r->fields[29] ), // gtid_from_hostgroup - ( strcmp(r->fields[30],"-1")==0 ? "NULL" : r->fields[30] ), // log - ( strcmp(r->fields[31],"-1")==0 ? "NULL" : r->fields[31] ), // apply - buffs[32] // comment + ( strcmp(r->fields[18],"-1")==0 ? "NULL" : r->fields[18] ), // cache_timeout + ( strcmp(r->fields[19],"-1")==0 ? "NULL" : r->fields[19] ), // reconnect + ( strcmp(r->fields[20],"-1")==0 ? "NULL" : r->fields[20] ), // timeout + ( strcmp(r->fields[21],"-1")==0 ? "NULL" : r->fields[21] ), // retries + ( strcmp(r->fields[22],"-1")==0 ? "NULL" : r->fields[22] ), // delay + ( strcmp(r->fields[23],"-1")==0 ? "NULL" : r->fields[23] ), // next_query_flagIN + ( strcmp(r->fields[24],"-1")==0 ? "NULL" : r->fields[24] ), // mirror_flagOUT + ( strcmp(r->fields[25],"-1")==0 ? "NULL" : r->fields[25] ), // mirror_hostgroup + buffs[26], // error_msg + buffs[27], // OK_msg + ( strcmp(r->fields[28],"-1")==0 ? "NULL" : r->fields[28] ), // sticky_conn + ( strcmp(r->fields[29],"-1")==0 ? "NULL" : r->fields[29] ), // multiplex + ( strcmp(r->fields[30],"-1")==0 ? "NULL" : r->fields[30] ), // gtid_from_hostgroup + ( strcmp(r->fields[31],"-1")==0 ? "NULL" : r->fields[31] ), // log + ( strcmp(r->fields[32],"-1")==0 ? "NULL" : r->fields[32] ), // apply + buffs[33] // comment ); //fprintf(stderr,"%s\n",query); admindb->execute(query); - for (int i=0; i<33; i++) { + for (int i=0; i<34; i++) { free(buffs[i]); } free(query); @@ -7347,7 +7350,7 @@ char * ProxySQL_Admin::load_mysql_query_rules_to_runtime() { int affected_rows=0; if (GloQPro==NULL) return (char *)"Global Query Processor not started: command impossible to run"; SQLite3_result *resultset=NULL; - char *query=(char *)"SELECT rule_id, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, ok_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment FROM main.mysql_query_rules WHERE active=1 ORDER BY rule_id"; + char *query=(char *)"SELECT rule_id, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, cache_timeout, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, ok_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment FROM main.mysql_query_rules WHERE active=1 ORDER BY rule_id"; admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); char *error2 = NULL; int cols2 = 0; @@ -7418,21 +7421,22 @@ char * ProxySQL_Admin::load_mysql_query_rules_to_runtime() { (r->fields[14]==NULL ? -1 : atoi(r->fields[14])), // destination_hostgroup (r->fields[15]==NULL ? -1 : atol(r->fields[15])), // cache_ttl (r->fields[16]==NULL ? -1 : atol(r->fields[16])), // cache_empty_result - (r->fields[17]==NULL ? -1 : atol(r->fields[17])), // reconnect - (r->fields[18]==NULL ? -1 : atol(r->fields[18])), // timeout - (r->fields[19]==NULL ? -1 : atol(r->fields[19])), // retries - (r->fields[20]==NULL ? -1 : atol(r->fields[20])), // delay - (r->fields[21]==NULL ? -1 : atol(r->fields[21])), // next_query_flagIN - (r->fields[22]==NULL ? -1 : atol(r->fields[22])), // mirror_flagOUT - (r->fields[23]==NULL ? -1 : atol(r->fields[23])), // mirror_hostgroup - r->fields[24], // error_msg - r->fields[25], // OK_msg - (r->fields[26]==NULL ? -1 : atol(r->fields[26])), // sticky_conn - (r->fields[27]==NULL ? -1 : atol(r->fields[27])), // multiplex - (r->fields[28]==NULL ? -1 : atol(r->fields[28])), // gtid_from_hostgroup - (r->fields[29]==NULL ? -1 : atol(r->fields[29])), // log - (atoi(r->fields[30])==1 ? true : false), - r->fields[31] // comment + (r->fields[17]==NULL ? -1 : atol(r->fields[17])), // cache_timeout + (r->fields[18]==NULL ? -1 : atol(r->fields[18])), // reconnect + (r->fields[19]==NULL ? -1 : atol(r->fields[19])), // timeout + (r->fields[20]==NULL ? -1 : atol(r->fields[20])), // retries + (r->fields[21]==NULL ? -1 : atol(r->fields[21])), // delay + (r->fields[22]==NULL ? -1 : atol(r->fields[22])), // next_query_flagIN + (r->fields[23]==NULL ? -1 : atol(r->fields[23])), // mirror_flagOUT + (r->fields[24]==NULL ? -1 : atol(r->fields[24])), // mirror_hostgroup + r->fields[25], // error_msg + r->fields[26], // OK_msg + (r->fields[27]==NULL ? -1 : atol(r->fields[27])), // sticky_conn + (r->fields[28]==NULL ? -1 : atol(r->fields[28])), // multiplex + (r->fields[29]==NULL ? -1 : atol(r->fields[29])), // gtid_from_hostgroup + (r->fields[30]==NULL ? -1 : atol(r->fields[30])), // log + (atoi(r->fields[31])==1 ? true : false), + r->fields[32] // comment ); GloQPro->insert(nqpr, false); } @@ -7643,7 +7647,7 @@ int ProxySQL_Admin::Read_MySQL_Query_Rules_from_configfile() { int i; int rows=0; admindb->execute("PRAGMA foreign_keys = OFF"); - char *q=(char *)"INSERT OR REPLACE INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, ok_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) VALUES (%d, %d, %s, %s, %s, %s, %s, %s, %s, %s, %s, %d, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %d, %s)"; + char *q=(char *)"INSERT OR REPLACE INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, cache_timeout, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, ok_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) VALUES (%d, %d, %s, %s, %s, %s, %s, %s, %s, %s, %s, %d, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %d, %s)"; for (i=0; i< count; i++) { const Setting &rule = mysql_query_rules[i]; int rule_id; @@ -7688,6 +7692,7 @@ int ProxySQL_Admin::Read_MySQL_Query_Rules_from_configfile() { int mirror_hostgroup=-1; int cache_ttl=-1; int cache_empty_result=-1; + int cache_timeout=-1; int reconnect=-1; int timeout=-1; int retries=-1; @@ -7740,6 +7745,7 @@ int ProxySQL_Admin::Read_MySQL_Query_Rules_from_configfile() { rule.lookupValue("mirror_hostgroup", mirror_hostgroup); rule.lookupValue("cache_ttl", cache_ttl); rule.lookupValue("cache_empty_result", cache_empty_result); + rule.lookupValue("cache_timeout", cache_timeout); rule.lookupValue("reconnect", reconnect); rule.lookupValue("timeout", timeout); rule.lookupValue("retries", retries); @@ -7780,6 +7786,7 @@ int ProxySQL_Admin::Read_MySQL_Query_Rules_from_configfile() { strlen(std::to_string(destination_hostgroup).c_str()) + 4 + strlen(std::to_string(cache_ttl).c_str()) + 4 + strlen(std::to_string(cache_empty_result).c_str()) + 4 + + strlen(std::to_string(cache_timeout).c_str()) + 4 + strlen(std::to_string(reconnect).c_str()) + 4 + strlen(std::to_string(timeout).c_str()) + 4 + strlen(std::to_string(next_query_flagIN).c_str()) + 4 + @@ -7867,6 +7874,7 @@ int ProxySQL_Admin::Read_MySQL_Query_Rules_from_configfile() { ( destination_hostgroup >= 0 ? std::to_string(destination_hostgroup).c_str() : "NULL") , ( cache_ttl >= 0 ? std::to_string(cache_ttl).c_str() : "NULL") , ( cache_empty_result >= 0 ? std::to_string(cache_empty_result).c_str() : "NULL") , + ( cache_timeout >= 0 ? std::to_string(cache_timeout).c_str() : "NULL") , ( reconnect >= 0 ? std::to_string(reconnect).c_str() : "NULL") , ( timeout >= 0 ? std::to_string(timeout).c_str() : "NULL") , ( retries >= 0 ? std::to_string(retries).c_str() : "NULL") , @@ -8304,6 +8312,20 @@ void ProxySQL_Admin::disk_upgrade_mysql_query_rules() { // copy fields from old table configdb->execute("INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) SELECT rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment FROM mysql_query_rules_v200c"); } + rci=configdb->check_table_structure((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES_V2_0_0d); + if (rci) { + // upgrade is required + proxy_warning("Detected version v2.0.0d of table mysql_query_rules\n"); + proxy_warning("ONLINE UPGRADE of table mysql_query_rules in progress\n"); + // drop any existing table with suffix _v200d + configdb->execute("DROP TABLE IF EXISTS mysql_query_rules_200d"); + // rename current table to add suffix _v200d + configdb->execute("ALTER TABLE mysql_query_rules RENAME TO mysql_query_rules_v200d"); + // create new table + configdb->build_table((char *)"mysql_query_rules",(char *)ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES,false); + // copy fields from old table + configdb->execute("INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) SELECT rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, OK_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment FROM mysql_query_rules_v200d"); + } configdb->execute("PRAGMA foreign_keys = ON"); } diff --git a/lib/ProxySQL_Cluster.cpp b/lib/ProxySQL_Cluster.cpp index 76583b05a..f19fd6145 100644 --- a/lib/ProxySQL_Cluster.cpp +++ b/lib/ProxySQL_Cluster.cpp @@ -618,7 +618,7 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer() { if (rc_conn) { MYSQL_RES *result1 = NULL; MYSQL_RES *result2 = NULL; - rc_query = mysql_query(conn,"SELECT rule_id, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, ok_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment FROM runtime_mysql_query_rules"); + rc_query = mysql_query(conn,"SELECT rule_id, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, cache_timeout, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, ok_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment FROM runtime_mysql_query_rules"); if ( rc_query == 0 ) { MYSQL_RES *result1 = mysql_store_result(conn); rc_query = mysql_query(conn,"SELECT username, schemaname, flagIN, destination_hostgroup, comment FROM runtime_mysql_query_rules_fast_routing"); @@ -629,7 +629,7 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer() { GloAdmin->admindb->execute("DELETE FROM mysql_query_rules"); GloAdmin->admindb->execute("DELETE FROM mysql_query_rules_fast_routing"); MYSQL_ROW row; - char *q = (char *)"INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, ok_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) VALUES (?1 , ?2 , ?3 , ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21, ?22, ?23, ?24, ?25, ?26, ?27, ?28, ?29, ?30, ?31, ?32, ?33)"; + char *q = (char *)"INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, cache_timeout, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, ok_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, comment) VALUES (?1 , ?2 , ?3 , ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21, ?22, ?23, ?24, ?25, ?26, ?27, ?28, ?29, ?30, ?31, ?32, ?33, ?34)"; sqlite3_stmt *statement1 = NULL; sqlite3 *mydb3 = GloAdmin->admindb->get_db(); rc=sqlite3_prepare_v2(mydb3, q, -1, &statement1, 0); @@ -652,22 +652,23 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer() { rc=sqlite3_bind_text(statement1, 15, row[13], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // replace_pattern rc=sqlite3_bind_text(statement1, 16, row[14], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // destination_hostgroup rc=sqlite3_bind_text(statement1, 17, row[15], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // cache_ttl - rc=sqlite3_bind_text(statement1, 18, row[16], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // cache_empty_result - rc=sqlite3_bind_text(statement1, 19, row[17], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // reconnect - rc=sqlite3_bind_text(statement1, 20, row[18], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // timeout - rc=sqlite3_bind_text(statement1, 21, row[19], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // retries - rc=sqlite3_bind_text(statement1, 22, row[20], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // delay - rc=sqlite3_bind_text(statement1, 23, row[21], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // next_query_flagIN - rc=sqlite3_bind_text(statement1, 24, row[22], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // mirror_flagOUT - rc=sqlite3_bind_text(statement1, 25, row[23], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // mirror_hostgroup - rc=sqlite3_bind_text(statement1, 26, row[24], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // error_msg - rc=sqlite3_bind_text(statement1, 27, row[25], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // OK_msg - rc=sqlite3_bind_text(statement1, 28, row[26], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // sticky_conn - rc=sqlite3_bind_text(statement1, 29, row[27], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // multiplex - rc=sqlite3_bind_text(statement1, 30, row[28], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // gtid_from_hostgroup - rc=sqlite3_bind_text(statement1, 31, row[29], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // log - rc=sqlite3_bind_text(statement1, 32, row[30], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // apply - rc=sqlite3_bind_text(statement1, 33, row[31], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // comment + rc=sqlite3_bind_text(statement1, 18, row[16], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // cache_empty_result + rc=sqlite3_bind_text(statement1, 19, row[17], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // cache_timeout + rc=sqlite3_bind_text(statement1, 20, row[18], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // reconnect + rc=sqlite3_bind_text(statement1, 21, row[19], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // timeout + rc=sqlite3_bind_text(statement1, 22, row[20], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // retries + rc=sqlite3_bind_text(statement1, 23, row[21], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // delay + rc=sqlite3_bind_text(statement1, 24, row[22], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // next_query_flagIN + rc=sqlite3_bind_text(statement1, 25, row[23], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // mirror_flagOUT + rc=sqlite3_bind_text(statement1, 26, row[24], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // mirror_hostgroup + rc=sqlite3_bind_text(statement1, 27, row[25], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // error_msg + rc=sqlite3_bind_text(statement1, 28, row[26], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // OK_msg + rc=sqlite3_bind_text(statement1, 29, row[27], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // sticky_conn + rc=sqlite3_bind_text(statement1, 30, row[28], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // multiplex + rc=sqlite3_bind_text(statement1, 31, row[29], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // gtid_from_hostgroup + rc=sqlite3_bind_text(statement1, 32, row[30], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // log + rc=sqlite3_bind_text(statement1, 33, row[31], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // apply + rc=sqlite3_bind_text(statement1, 34, row[32], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK); // comment SAFE_SQLITE3_STEP2(statement1); rc=sqlite3_clear_bindings(statement1); assert(rc==SQLITE_OK); rc=sqlite3_reset(statement1); assert(rc==SQLITE_OK); diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 3f7f40371..708778773 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -15,7 +15,7 @@ #else #define DEB "" #endif /* DEBUG */ -#define QUERY_PROCESSOR_VERSION "0.4.0926" DEB +#define QUERY_PROCESSOR_VERSION "0.4.1031" DEB #define QP_RE_MOD_CASELESS 1 #define QP_RE_MOD_GLOBAL 2 @@ -42,7 +42,7 @@ class QP_rule_text { char **pta; int num_fields; QP_rule_text(QP_rule_t *QPr) { - num_fields=34; // this count the number of fields + num_fields=35; // this count the number of fields pta=NULL; pta=(char **)malloc(sizeof(char *)*num_fields); itostr(pta[0], (long long)QPr->rule_id); @@ -81,22 +81,23 @@ class QP_rule_text { itostr(pta[15], (long long)QPr->destination_hostgroup); itostr(pta[16], (long long)QPr->cache_ttl); itostr(pta[17], (long long)QPr->cache_empty_result); - itostr(pta[18], (long long)QPr->reconnect); - itostr(pta[19], (long long)QPr->timeout); - itostr(pta[20], (long long)QPr->retries); - itostr(pta[21], (long long)QPr->delay); - itostr(pta[22], (long long)QPr->next_query_flagIN); - itostr(pta[23], (long long)QPr->mirror_flagOUT); - itostr(pta[24], (long long)QPr->mirror_hostgroup); - pta[25]=strdup_null(QPr->error_msg); - pta[26]=strdup_null(QPr->OK_msg); - itostr(pta[27], (long long)QPr->sticky_conn); - itostr(pta[28], (long long)QPr->multiplex); - itostr(pta[29], (long long)QPr->gtid_from_hostgroup); - itostr(pta[30], (long long)QPr->log); - itostr(pta[31], (long long)QPr->apply); - pta[32]=strdup_null(QPr->comment); // issue #643 - itostr(pta[33], (long long)QPr->hits); + itostr(pta[18], (long long)QPr->cache_timeout); + itostr(pta[19], (long long)QPr->reconnect); + itostr(pta[20], (long long)QPr->timeout); + itostr(pta[21], (long long)QPr->retries); + itostr(pta[22], (long long)QPr->delay); + itostr(pta[23], (long long)QPr->next_query_flagIN); + itostr(pta[24], (long long)QPr->mirror_flagOUT); + itostr(pta[25], (long long)QPr->mirror_hostgroup); + pta[26]=strdup_null(QPr->error_msg); + pta[27]=strdup_null(QPr->OK_msg); + itostr(pta[28], (long long)QPr->sticky_conn); + itostr(pta[29], (long long)QPr->multiplex); + itostr(pta[30], (long long)QPr->gtid_from_hostgroup); + itostr(pta[31], (long long)QPr->log); + itostr(pta[32], (long long)QPr->apply); + pta[33]=strdup_null(QPr->comment); // issue #643 + itostr(pta[34], (long long)QPr->hits); } ~QP_rule_text() { for(int i=0; irule_id=rule_id; newQR->active=active; @@ -479,6 +480,7 @@ QP_rule_t * Query_Processor::new_query_rule(int rule_id, bool active, char *user newQR->destination_hostgroup=destination_hostgroup; newQR->cache_ttl=cache_ttl; newQR->cache_empty_result=cache_empty_result; + newQR->cache_timeout=cache_timeout; newQR->reconnect=reconnect; newQR->timeout=timeout; newQR->retries=retries; @@ -684,6 +686,7 @@ SQLite3_result * Query_Processor::get_current_query_rules() { result->add_column_definition(SQLITE_TEXT,"destination_hostgroup"); result->add_column_definition(SQLITE_TEXT,"cache_ttl"); result->add_column_definition(SQLITE_TEXT,"cache_empty_result"); + result->add_column_definition(SQLITE_TEXT,"cache_timeout"); result->add_column_definition(SQLITE_TEXT,"reconnect"); result->add_column_definition(SQLITE_TEXT,"timeout"); result->add_column_definition(SQLITE_TEXT,"retries"); @@ -903,7 +906,8 @@ Query_Processor_Output * Query_Processor::process_mysql_query(MySQL_Session *ses ( qr1->digest ? buf : NULL ) , qr1->match_digest, qr1->match_pattern, qr1->negate_match_pattern, (char *)re_mod.c_str(), qr1->flagOUT, qr1->replace_pattern, qr1->destination_hostgroup, - qr1->cache_ttl, qr1->cache_empty_result, qr1->reconnect, qr1->timeout, qr1->retries, qr1->delay, + qr1->cache_ttl, qr1->cache_empty_result, qr1->cache_timeout, + qr1->reconnect, qr1->timeout, qr1->retries, qr1->delay, qr1->next_query_flagIN, qr1->mirror_flagOUT, qr1->mirror_hostgroup, qr1->error_msg, qr1->OK_msg, qr1->sticky_conn, qr1->multiplex, qr1->gtid_from_hostgroup, @@ -1129,6 +1133,11 @@ __internal_loop: proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set cache_empty_result: %d. Query with empty result will%s hit the cache\n", qr->rule_id, qr->cache_empty_result, (qr->cache_empty_result == 0 ? " NOT" : "" )); ret->cache_empty_result=qr->cache_empty_result; } + if (qr->cache_timeout >= 0) { + // Note: negative value means this rule doesn't change + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set cache_timeout: %dms. Query will wait up resulset to be avaiable in query cache before running on backend\n", qr->rule_id, qr->cache_timeout); + ret->cache_timeout=qr->cache_timeout; + } if (qr->sticky_conn >= 0) { // Note: negative sticky_conn means this rule doesn't change proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set sticky_conn: %d. Connection will%s stick\n", qr->rule_id, qr->sticky_conn, (qr->sticky_conn == 0 ? " NOT" : "" ));