Query filtering #418

Minor fixes
pull/420/head
René Cannaò 11 years ago
parent a00b927c06
commit 43656f781e

@ -26,11 +26,11 @@ debug:
.PHONY: deps
deps:
cd deps && make -j 20
cd deps && make -j 5
.PHONY: lib
lib:
cd lib && make -j 20
cd lib && make -j 5
.PHONY: src
src:

@ -71,6 +71,7 @@ struct _Query_Processor_rule_t {
int reconnect;
int timeout;
int delay;
char *error_msg;
bool apply;
void *regex_engine1;
void *regex_engine2;
@ -102,6 +103,7 @@ class Query_Processor_Output {
int reconnect;
int timeout;
int delay;
char *error_msg;
std::string *new_query;
void * operator new(size_t size) {
return l_alloc(size);
@ -118,6 +120,12 @@ class Query_Processor_Output {
timeout=-1;
delay=-1;
new_query=NULL;
error_msg=NULL;
}
~Query_Processor_Output() {
if (error_msg) {
free(error_msg);
}
}
};
@ -197,7 +205,7 @@ class Query_Processor {
void wrunlock(); // explicit write unlock
bool insert(QP_rule_t *qr, bool lock=true); // insert a new rule. Uses a generic void pointer to a structure that may vary depending from the Query Processor
// virtual bool insert_locked(QP_rule_t *qr) {return false;}; // call this instead of insert() in case lock was already acquired via wrlock()
QP_rule_t * new_query_rule(int rule_id, bool active, char *username, char *schemaname, int flagIN, char *match_digest, char *match_pattern, bool negate_match_pattern, int flagOUT, char *replace_pattern, int destination_hostgroup, int cache_ttl, int reconnect, int timeout, int delay, bool apply); // to use a generic query rule struct, this is generated by this function and returned as generic void pointer
QP_rule_t * new_query_rule(int rule_id, bool active, char *username, char *schemaname, int flagIN, char *match_digest, char *match_pattern, bool negate_match_pattern, int flagOUT, char *replace_pattern, int destination_hostgroup, int cache_ttl, int reconnect, int timeout, int delay, char *error_msg, bool apply); // to use a generic query rule struct, this is generated by this function and returned as generic void pointer
void delete_query_rule(QP_rule_t *qr); // destructor
//virtual bool remove(int rule_id, bool lock=true) {return false;}; // FIXME: not implemented yet, should be implemented at all ?
// virtual bool remove_locked(int rule_id) {return false;}; // call this instead of remove() in case lock was already acquired via wrlock()

@ -1700,6 +1700,17 @@ void MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C
bool MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_COM_QUERY_qpo(PtrSize_t *pkt) {
if (qpo->error_msg) {
client_myds->DSS=STATE_QUERY_SENT_NET;
client_myds->myprot.generate_pkt_ERR(true,NULL,NULL,1,1148,(char *)"#42000",qpo->error_msg);
client_myds->DSS=STATE_SLEEP;
status=WAITING_CLIENT_DATA;
l_free(pkt->size,pkt->ptr);
CurrentQuery.end();
GloQPro->delete_QP_out(qpo);
qpo=NULL;
return true;
}
if (qpo->new_query) {
// the query was rewritten
l_free(pkt->size,pkt->ptr); // free old pkt

@ -54,7 +54,7 @@ pthread_mutex_t admin_mutex = PTHREAD_MUTEX_INITIALIZER;
#define ADMIN_SQLITE_TABLE_MYSQL_SERVERS "CREATE TABLE mysql_servers (hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , PRIMARY KEY (hostgroup_id, hostname, port) )"
#define ADMIN_SQLITE_TABLE_MYSQL_USERS "CREATE TABLE mysql_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , use_ssl INT CHECK (use_ssl IN (0,1)) NOT NULL DEFAULT 0 , default_hostgroup INT NOT NULL DEFAULT 0 , default_schema VARCHAR , schema_locked INT CHECK (schema_locked IN (0,1)) NOT NULL DEFAULT 0 , transaction_persistent INT CHECK (transaction_persistent IN (0,1)) NOT NULL DEFAULT 0 , fast_forward INT CHECK (fast_forward IN (0,1)) NOT NULL DEFAULT 0 , backend INT CHECK (backend IN (0,1)) NOT NULL DEFAULT 1 , frontend INT CHECK (frontend IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , PRIMARY KEY (username, backend) , UNIQUE (username, frontend))"
#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , delay INT UNSIGNED , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0)"
#define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , delay INT UNSIGNED , error_msg VARCHAR , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0)"
#define ADMIN_SQLITE_TABLE_GLOBAL_VARIABLES "CREATE TABLE global_variables (variable_name VARCHAR NOT NULL PRIMARY KEY , variable_value VARCHAR NOT NULL)"
#define ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS "CREATE TABLE mysql_replication_hostgroups (writer_hostgroup INT CHECK (writer_hostgroup>=0) NOT NULL PRIMARY KEY , reader_hostgroup INT NOT NULL CHECK (reader_hostgroup<>writer_hostgroup AND reader_hostgroup>0) , UNIQUE (reader_hostgroup))"
@ -2642,12 +2642,12 @@ void ProxySQL_Admin::save_mysql_query_rules_from_runtime() {
if (resultset==NULL) return;
admindb->execute("DELETE FROM mysql_query_rules");
//char *a=(char *)"INSERT INTO mysql_query_rules VALUES (\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\")";
char *a=(char *)"INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, match_digest, match_pattern, negate_match_pattern, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, delay, apply) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)";
char *a=(char *)"INSERT INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, match_digest, match_pattern, negate_match_pattern, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, delay, error_msg, apply) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)";
for (std::vector<SQLite3_row *>::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) {
SQLite3_row *r=*it;
int arg_len=0;
char *buffs[16];
for (int i=0; i<16; i++) {
char *buffs[17];
for (int i=0; i<17; i++) {
if (r->fields[i]) {
int l=strlen(r->fields[i])+4;
arg_len+=l;
@ -2678,7 +2678,8 @@ void ProxySQL_Admin::save_mysql_query_rules_from_runtime() {
( strcmp(r->fields[12],"-1")==0 ? "NULL" : r->fields[12] ), // reconnect
( strcmp(r->fields[13],"-1")==0 ? "NULL" : r->fields[13] ), // timeout
( strcmp(r->fields[14],"-1")==0 ? "NULL" : r->fields[14] ), // delay
( strcmp(r->fields[15],"-1")==0 ? "NULL" : r->fields[15] ) // apply
buffs[15], // error_msg
( strcmp(r->fields[16],"-1")==0 ? "NULL" : r->fields[16] ) // apply
);
//fprintf(stderr,"%s\n",query);
admindb->execute(query);
@ -3180,7 +3181,7 @@ char * ProxySQL_Admin::load_mysql_query_rules_to_runtime() {
int affected_rows=0;
if (GloQPro==NULL) return (char *)"Global Query Processor not started: command impossible to run";
SQLite3_result *resultset=NULL;
char *query=(char *)"SELECT rule_id, username, schemaname, flagIN, match_digest, match_pattern, negate_match_pattern, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, delay, apply FROM main.mysql_query_rules WHERE active=1";
char *query=(char *)"SELECT rule_id, username, schemaname, flagIN, match_digest, match_pattern, negate_match_pattern, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, delay, error_msg, apply FROM main.mysql_query_rules WHERE active=1";
admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset);
if (error) {
proxy_error("Error on %s : %s\n", query, error);
@ -3206,7 +3207,8 @@ char * ProxySQL_Admin::load_mysql_query_rules_to_runtime() {
(r->fields[11]==NULL ? -1 : atol(r->fields[11])),
(r->fields[12]==NULL ? -1 : atol(r->fields[12])),
(r->fields[13]==NULL ? -1 : atol(r->fields[13])),
(atoi(r->fields[14])==1 ? true : false)
r->fields[14], // error_msg
(atoi(r->fields[15])==1 ? true : false)
);
GloQPro->insert(nqpr, false);
}
@ -3310,7 +3312,7 @@ int ProxySQL_Admin::Read_MySQL_Query_Rules_from_configfile() {
int i;
int rows=0;
admindb->execute("PRAGMA foreign_keys = OFF");
char *q=(char *)"INSERT OR REPLACE INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, march_digest, match_pattern, negate_match_pattern, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, delay, apply) VALUES (%d, %d, %s, %s, %s, %s, %s, %d, %s, %s, %s, %s, %s, %s, %s, %d)";
char *q=(char *)"INSERT OR REPLACE INTO mysql_query_rules (rule_id, active, username, schemaname, flagIN, march_digest, match_pattern, negate_match_pattern, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, reconnect, timeout, delay, error_msg, apply) VALUES (%d, %d, %s, %s, %s, %s, %s, %d, %s, %s, %s, %s, %s, %s, %s, %s, %d)";
for (i=0; i< count; i++) {
const Setting &rule = mysql_query_rules[i];
int rule_id;
@ -3333,6 +3335,8 @@ int ProxySQL_Admin::Read_MySQL_Query_Rules_from_configfile() {
int reconnect=-1;
int timeout=-1;
int delay=-1;
bool error_msg_exists=false;
std::string error_msg;
int apply=0;
if (rule.lookupValue("rule_id", rule_id)==false) continue;
rule.lookupValue("active", active);
@ -3349,6 +3353,7 @@ int ProxySQL_Admin::Read_MySQL_Query_Rules_from_configfile() {
rule.lookupValue("reconnect", reconnect);
rule.lookupValue("timeout", timeout);
rule.lookupValue("delay", delay);
if (rule.lookupValue("error_msg", username)) error_msg_exists=true;
rule.lookupValue("apply", apply);
//if (user.lookupValue("default_schema", default_schema)==false) default_schema="";
int query_len=0;
@ -3368,6 +3373,7 @@ int ProxySQL_Admin::Read_MySQL_Query_Rules_from_configfile() {
strlen(std::to_string(reconnect).c_str()) + 4 +
strlen(std::to_string(timeout).c_str()) + 4 +
strlen(std::to_string(delay).c_str()) + 4 +
( error_msg_exists ? strlen(error_msg.c_str()) : 0 ) + 4 +
strlen(std::to_string(apply).c_str()) + 4 +
40;
char *query=(char *)malloc(query_len);
@ -3391,6 +3397,10 @@ int ProxySQL_Admin::Read_MySQL_Query_Rules_from_configfile() {
replace_pattern="\"" + replace_pattern + "\"";
else
replace_pattern = "NULL";
if (error_msg_exists)
error_msg="\"" + error_msg + "\"";
else
error_msg = "NULL";
sprintf(query, q,
rule_id, active,
username.c_str(),
@ -3406,6 +3416,7 @@ int ProxySQL_Admin::Read_MySQL_Query_Rules_from_configfile() {
( reconnect >= 0 ? std::to_string(reconnect).c_str() : "NULL") ,
( timeout >= 0 ? std::to_string(timeout).c_str() : "NULL") ,
( delay >= 0 ? std::to_string(delay).c_str() : "NULL") ,
error_msg.c_str(),
( apply == 0 ? 0 : 1)
);
//fprintf(stderr, "%s\n", query);

@ -37,7 +37,7 @@ class QP_rule_text_hitsonly {
class QP_rule_text {
public:
char **pta;
const int num_fields=17;
const int num_fields=18;
QP_rule_text(QP_rule_t *QPr) {
pta=NULL;
pta=(char **)malloc(sizeof(char *)*num_fields);
@ -56,8 +56,9 @@ class QP_rule_text {
itostr(pta[12], (long long)QPr->reconnect);
itostr(pta[13], (long long)QPr->timeout);
itostr(pta[14], (long long)QPr->delay);
itostr(pta[15], (long long)QPr->apply);
itostr(pta[16], (long long)QPr->hits);
pta[15]=strdup_null(QPr->error_msg);
itostr(pta[16], (long long)QPr->apply);
itostr(pta[17], (long long)QPr->hits);
}
~QP_rule_text() {
for(int i=0; i<num_fields; i++) {
@ -222,6 +223,8 @@ static void __delete_query_rule(QP_rule_t *qr) {
free(qr->match_pattern);
if (qr->replace_pattern)
free(qr->replace_pattern);
if (qr->error_msg)
free(qr->error_msg);
if (qr->regex_engine1) {
re2_t *r=(re2_t *)qr->regex_engine1;
delete r->opt;
@ -416,7 +419,7 @@ void Query_Processor::wrunlock() {
QP_rule_t * Query_Processor::new_query_rule(int rule_id, bool active, char *username, char *schemaname, int flagIN, char *match_digest, char *match_pattern, bool negate_match_pattern, int flagOUT, char *replace_pattern, int destination_hostgroup, int cache_ttl, int reconnect, int timeout, int delay, bool apply) {
QP_rule_t * Query_Processor::new_query_rule(int rule_id, bool active, char *username, char *schemaname, int flagIN, char *match_digest, char *match_pattern, bool negate_match_pattern, int flagOUT, char *replace_pattern, int destination_hostgroup, int cache_ttl, int reconnect, int timeout, int delay, char *error_msg, bool apply) {
QP_rule_t * newQR=(QP_rule_t *)malloc(sizeof(QP_rule_t));
newQR->rule_id=rule_id;
newQR->active=active;
@ -433,6 +436,7 @@ QP_rule_t * Query_Processor::new_query_rule(int rule_id, bool active, char *user
newQR->reconnect=reconnect;
newQR->timeout=timeout;
newQR->delay=delay;
newQR->error_msg=(error_msg ? strdup(error_msg) : NULL);
newQR->apply=apply;
newQR->regex_engine1=NULL;
newQR->regex_engine2=NULL;
@ -525,7 +529,7 @@ SQLite3_result * Query_Processor::get_stats_query_rules() {
SQLite3_result * Query_Processor::get_current_query_rules() {
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping current query rules, using Global version %d\n", version);
SQLite3_result *result=new SQLite3_result(17);
SQLite3_result *result=new SQLite3_result(18);
spin_rdlock(&rwlock);
QP_rule_t *qr1;
result->add_column_definition(SQLITE_TEXT,"rule_id");
@ -543,6 +547,7 @@ SQLite3_result * Query_Processor::get_current_query_rules() {
result->add_column_definition(SQLITE_TEXT,"reconnect");
result->add_column_definition(SQLITE_TEXT,"timeout");
result->add_column_definition(SQLITE_TEXT,"delay");
result->add_column_definition(SQLITE_TEXT,"error_msg");
result->add_column_definition(SQLITE_TEXT,"apply");
result->add_column_definition(SQLITE_TEXT,"hits");
for (std::vector<QP_rule_t *>::iterator it=rules.begin(); it!=rules.end(); ++it) {
@ -630,7 +635,7 @@ Query_Processor_Output * Query_Processor::process_mysql_query(MySQL_Session *ses
qr1=*it;
if (qr1->active) {
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Copying Query Rule id: %d\n", qr1->rule_id);
qr2=new_query_rule(qr1->rule_id, qr1->active, qr1->username, qr1->schemaname, qr1->flagIN, qr1->match_digest, qr1->match_pattern, qr1->negate_match_pattern, qr1->flagOUT, qr1->replace_pattern, qr1->destination_hostgroup, qr1->cache_ttl, qr1->reconnect, qr1->timeout, qr1->delay, qr1->apply);
qr2=new_query_rule(qr1->rule_id, qr1->active, qr1->username, qr1->schemaname, qr1->flagIN, qr1->match_digest, qr1->match_pattern, qr1->negate_match_pattern, qr1->flagOUT, qr1->replace_pattern, qr1->destination_hostgroup, qr1->cache_ttl, qr1->reconnect, qr1->timeout, qr1->delay, qr1->error_msg, qr1->apply);
qr2->parent=qr1; // pointer to parent to speed up parent update (hits)
if (qr2->match_digest) {
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Compiling regex for rule_id: %d, match_digest: \n", qr2->rule_id, qr2->match_digest);
@ -745,6 +750,11 @@ Query_Processor_Output * Query_Processor::process_mysql_query(MySQL_Session *ses
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set delay: %d. Session will%s be paused for %dms\n", qr->rule_id, qr->delay, (qr->delay == 0 ? " NOT" : "" ) , qr->delay);
ret->delay=qr->delay;
}
if (qr->error_msg) {
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set error_msg: %s\n", qr->rule_id, qr->error_msg);
proxy_warning("User %s has issued query that has been fintered: %s \n " , sess->client_myds->myconn->userinfo->username, query);
ret->error_msg=strdup(qr->error_msg);
}
if (qr->cache_ttl >= 0) {
// Note: negative TTL means this rule doesn't change
proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "query rule %d has set cache_ttl: %d. Query will%s hit the cache\n", qr->rule_id, qr->cache_ttl, (qr->cache_ttl == 0 ? " NOT" : "" ));

@ -16,7 +16,9 @@ RUN apt-get update && apt-get install -y\
libssl-dev\
libtool
RUN cd /opt; git clone https://github.com/sysown/proxysql-0.2.git proxysql
RUN apt-get install -y wget unzip
#RUN cd /opt; git clone https://github.com/sysown/proxysql-0.2.git proxysql
RUN cd /opt ; wget https://github.com/sysown/proxysql-0.2/archive/1.0.zip ; unzip 1.0.zip ; mv proxysql-0.2-1.0 proxysql
RUN cd /opt/proxysql; make clean && make -j
ADD ./proxysql.ctl /opt/proxysql/proxysql.ctl

Loading…
Cancel
Save