From a85bc447157d0f8192e71062ff14c71a5dca2387 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Wed, 21 Jan 2015 09:05:39 +0000 Subject: [PATCH] Implemented table stats_mysql_commands_counters : issue #168 Implemented get_stats_commands_counters() in Query_Processor Implemented stats___mysql_commands_counters() in ProxySQL_Admin Added table mysql_commands_counters Added MYSQL_COM_QUERY_SHOW_TABLE_STATUS in MYSQL_COM_QUERY_command Standard_Query_Processor has the commands description in commands_counters_desc Introduced per thread storage variable _thr_commands_counters query_parser_command_type() is now a wrapper to __query_parser_command_type() so to update counters based on its return value Bug fixed: Added transactions in Standard_ProxySQL_Admin::stats___mysql_query_rules() Added SHOW command in libinjection --- deps/libinjection/libinjection_sqli_data.h | 2 +- deps/libinjection/sqlparse_data.json | 2 +- deps/libinjection/sqlparse_map.py | 3 +- include/query_processor.h | 3 +- lib/Standard_ProxySQL_Admin.cpp | 30 +++++- lib/Standard_Query_Processor.cpp | 101 +++++++++++++++++++++ lib/mysql_session.cpp | 4 +- 7 files changed, 138 insertions(+), 7 deletions(-) diff --git a/deps/libinjection/libinjection_sqli_data.h b/deps/libinjection/libinjection_sqli_data.h index 1adaea84c..bff90409b 100644 --- a/deps/libinjection/libinjection_sqli_data.h +++ b/deps/libinjection/libinjection_sqli_data.h @@ -9093,7 +9093,7 @@ static const keyword_t sql_keywords[] = { {"SHA", 'f'}, {"SHA1", 'f'}, {"SHA2", 'f'}, - {"SHOW", 'n'}, + {"SHOW", 'k'}, {"SHUTDOWN", 'T'}, {"SIGN", 'f'}, {"SIGNAL", 'k'}, diff --git a/deps/libinjection/sqlparse_data.json b/deps/libinjection/sqlparse_data.json index f5343d9de..e62427a2c 100644 --- a/deps/libinjection/sqlparse_data.json +++ b/deps/libinjection/sqlparse_data.json @@ -9059,7 +9059,7 @@ "SHA": "f", "SHA1": "f", "SHA2": "f", - "SHOW": "n", + "SHOW": "k", "SHUTDOWN": "T", "SIGN": "f", "SIGNAL": "k", diff --git a/deps/libinjection/sqlparse_map.py b/deps/libinjection/sqlparse_map.py index a2d31709a..d79530b07 100755 --- a/deps/libinjection/sqlparse_map.py +++ b/deps/libinjection/sqlparse_map.py @@ -855,7 +855,8 @@ KEYWORDS = { 'SHA' : 'f', 'SHA1' : 'f', 'SHA2' : 'f', -'SHOW' : 'n', +#'SHOW' : 'n', +'SHOW' : 'k', 'SHUTDOWN' : 'T', 'SIGN' : 'f', 'SIGNBYASMKEY' : 'f', diff --git a/include/query_processor.h b/include/query_processor.h index bd3f47c84..321393175 100644 --- a/include/query_processor.h +++ b/include/query_processor.h @@ -42,6 +42,7 @@ enum MYSQL_COM_QUERY_command { MYSQL_COM_QUERY_SELECT, MYSQL_COM_QUERY_SELECT_FOR_UPDATE, MYSQL_COM_QUERY_SET, + MYSQL_COM_QUERY_SHOW_TABLE_STATUS, MYSQL_COM_QUERY_START_TRANSACTION, MYSQL_COM_QUERY_UNLOCK_TABLES, MYSQL_COM_QUERY_UPDATE, @@ -116,7 +117,7 @@ class Query_Processor { virtual enum MYSQL_COM_QUERY_command query_parser_command_type(void *args) {return MYSQL_COM_QUERY___UNKNOWN;} virtual char * query_parser_first_comment(void *args) { return NULL; } virtual void query_parser_free(void *args) {}; - + virtual SQLite3_result * get_stats_commands_counters() {return NULL;}; }; diff --git a/lib/Standard_ProxySQL_Admin.cpp b/lib/Standard_ProxySQL_Admin.cpp index ac2e65a4c..1a702da1a 100644 --- a/lib/Standard_ProxySQL_Admin.cpp +++ b/lib/Standard_ProxySQL_Admin.cpp @@ -47,6 +47,7 @@ pthread_mutex_t sock_mutex = PTHREAD_MUTEX_INITIALIZER; #define STATS_SQLITE_TABLE_MYSQL_QUERY_RULES "CREATE TABLE stats_mysql_query_rules (rule_id INTEGER PRIMARY KEY, hits INT NOT NULL)" +#define STATS_SQLITE_TABLE_MYSQL_COMMANDS_COUNTERS "CREATE TABLE stats_mysql_commands_counters ( Command VARCHAR NOT NULL PRIMARY KEY, Counter INT NOT NULL)" @@ -245,7 +246,7 @@ class Standard_ProxySQL_Admin: public ProxySQL_Admin { void stats___mysql_query_rules(); - + void stats___mysql_commands_counters(); }; static Standard_ProxySQL_Admin *SPA=NULL; @@ -1090,6 +1091,7 @@ void *child_mysql(void *arg) { oldtime=curtime; Standard_ProxySQL_Admin *SPA=(Standard_ProxySQL_Admin *)GloAdmin; SPA->stats___mysql_query_rules(); + SPA->stats___mysql_commands_counters(); } } if (rc == -1) { @@ -1409,6 +1411,7 @@ bool Standard_ProxySQL_Admin::init() { insert_into_tables_defs(tables_defs_stats,"mysql_query_rules", STATS_SQLITE_TABLE_MYSQL_QUERY_RULES); + insert_into_tables_defs(tables_defs_stats,"mysql_commands_counters", STATS_SQLITE_TABLE_MYSQL_COMMANDS_COUNTERS); check_and_build_standard_tables(admindb, tables_defs_admin); @@ -1918,11 +1921,33 @@ bool Standard_ProxySQL_Admin::set_variable(char *name, char *value) { // this i - +void Standard_ProxySQL_Admin::stats___mysql_commands_counters() { + SQLite3_result * resultset=GloQPro->get_stats_commands_counters(); + if (resultset==NULL) return; +// fprintf(stderr,"Number of columns: %d, rows: %d\n", result->columns, result->rows_count); + statsdb->execute("BEGIN"); + statsdb->execute("DELETE FROM stats_mysql_commands_counters"); + char *a=(char *)"INSERT INTO stats_mysql_commands_counters VALUES (\"%s\",\"%s\")"; + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + int arg_len=0; + for (int i=0; i<2; i++) { + arg_len+=strlen(r->fields[i]); + } + char *query=(char *)malloc(strlen(a)+arg_len+32); + sprintf(query,a,r->fields[0],r->fields[1]); + //fprintf(stderr,"%s\n",query); + statsdb->execute(query); + free(query); + } + statsdb->execute("COMMIT"); + delete resultset; +} void Standard_ProxySQL_Admin::stats___mysql_query_rules() { SQLite3_result * resultset=GloQPro->get_stats_query_rules(); if (resultset==NULL) return; // fprintf(stderr,"Number of columns: %d, rows: %d\n", result->columns, result->rows_count); + statsdb->execute("BEGIN"); statsdb->execute("DELETE FROM stats_mysql_query_rules"); char *a=(char *)"INSERT INTO stats_mysql_query_rules VALUES (\"%s\",\"%s\")"; for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { @@ -1937,6 +1962,7 @@ void Standard_ProxySQL_Admin::stats___mysql_query_rules() { statsdb->execute(query); free(query); } + statsdb->execute("COMMIT"); delete resultset; } diff --git a/lib/Standard_Query_Processor.cpp b/lib/Standard_Query_Processor.cpp index 64383d74b..3d858abd4 100644 --- a/lib/Standard_Query_Processor.cpp +++ b/lib/Standard_Query_Processor.cpp @@ -115,6 +115,9 @@ struct __SQP_query_parser_t { typedef struct __SQP_query_parser_t SQP_par_t; +static char *commands_counters_desc[MYSQL_COM_QUERY___UNKNOWN+1]; + + struct __RE2_objects_t { re2::RE2::Options *opt; @@ -170,12 +173,15 @@ static void __reset_rules(std::vector * qrs) { // per thread variables __thread unsigned int _thr_SQP_version; __thread std::vector * _thr_SQP_rules; +__thread unsigned int _thr_commands_counters[MYSQL_COM_QUERY___UNKNOWN+1]; + class Standard_Query_Processor: public Query_Processor { private: rwlock_t rwlock; std::vector rules; +unsigned int commands_counters[MYSQL_COM_QUERY___UNKNOWN+1]; volatile unsigned int version; protected: @@ -185,6 +191,51 @@ Standard_Query_Processor() { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Initializing Query Processor with version=0\n"); spinlock_rwlock_init(&rwlock); version=0; + for (int i=0; i<=MYSQL_COM_QUERY___UNKNOWN; i++) commands_counters[i]=0; + + commands_counters_desc[MYSQL_COM_QUERY_ALTER_TABLE]=(char *)"ALTER_TABLE"; + commands_counters_desc[MYSQL_COM_QUERY_ANALYZE_TABLE]=(char *)"ANALYZE_TABLE"; + commands_counters_desc[MYSQL_COM_QUERY_BEGIN]=(char *)"BEGIN"; + commands_counters_desc[MYSQL_COM_QUERY_CHANGE_MASTER]=(char *)"CHANGE_MASTER"; + commands_counters_desc[MYSQL_COM_QUERY_CREATE_DATABASE]=(char *)"CREATE_DATABASE"; + commands_counters_desc[MYSQL_COM_QUERY_CREATE_INDEX]=(char *)"CREATE_INDEX"; + commands_counters_desc[MYSQL_COM_QUERY_CREATE_TABLE]=(char *)"CREATE_TABLE"; + commands_counters_desc[MYSQL_COM_QUERY_CREATE_TEMPORARY]=(char *)"CREATE_TEMPORARY"; + commands_counters_desc[MYSQL_COM_QUERY_CREATE_TRIGGER]=(char *)"CREATE_TRIGGER"; + commands_counters_desc[MYSQL_COM_QUERY_CREATE_USER]=(char *)"CREATE_USER"; + commands_counters_desc[MYSQL_COM_QUERY_DELETE]=(char *)"DELETE"; + commands_counters_desc[MYSQL_COM_QUERY_DESCRIBE]=(char *)"DESCRIBE"; + commands_counters_desc[MYSQL_COM_QUERY_DROP_DATABASE]=(char *)"DROP_DATABASE"; + commands_counters_desc[MYSQL_COM_QUERY_DROP_INDEX]=(char *)"DROP_INDEX"; + commands_counters_desc[MYSQL_COM_QUERY_DROP_TABLE]=(char *)"DROP_TABLE"; + commands_counters_desc[MYSQL_COM_QUERY_DROP_TRIGGER]=(char *)"DROP_TRIGGER"; + commands_counters_desc[MYSQL_COM_QUERY_DROP_USER]=(char *)"DROP_USER"; + commands_counters_desc[MYSQL_COM_QUERY_GRANT]=(char *)"GRANT"; + commands_counters_desc[MYSQL_COM_QUERY_EXPLAIN]=(char *)"EXPLAIN"; + commands_counters_desc[MYSQL_COM_QUERY_FLUSH]=(char *)"FLUSH"; + commands_counters_desc[MYSQL_COM_QUERY_INSERT]=(char *)"INSERT"; + commands_counters_desc[MYSQL_COM_QUERY_KILL]=(char *)"KILL"; + commands_counters_desc[MYSQL_COM_QUERY_LOAD]=(char *)"LOAD"; + commands_counters_desc[MYSQL_COM_QUERY_LOCK_TABLE]=(char *)"LOCK_TABLE"; + commands_counters_desc[MYSQL_COM_QUERY_OPTIMIZE]=(char *)"OPTIMIZE"; + commands_counters_desc[MYSQL_COM_QUERY_PREPARE]=(char *)"PREPARE"; + commands_counters_desc[MYSQL_COM_QUERY_PURGE]=(char *)"PURGE"; + commands_counters_desc[MYSQL_COM_QUERY_RENAME_TABLE]=(char *)"RENAME_TABLE"; + commands_counters_desc[MYSQL_COM_QUERY_RESET_MASTER]=(char *)"RESET_MASTER"; + commands_counters_desc[MYSQL_COM_QUERY_RESET_SLAVE]=(char *)"RESET_SLAVE"; + commands_counters_desc[MYSQL_COM_QUERY_REPLACE]=(char *)"REPLACE"; + commands_counters_desc[MYSQL_COM_QUERY_REVOKE]=(char *)"REVOKE"; + commands_counters_desc[MYSQL_COM_QUERY_ROLLBACK]=(char *)"ROLLBACK"; + commands_counters_desc[MYSQL_COM_QUERY_SAVEPOINT]=(char *)"SAVEPOINT"; + commands_counters_desc[MYSQL_COM_QUERY_SELECT]=(char *)"SELECT"; + commands_counters_desc[MYSQL_COM_QUERY_SELECT_FOR_UPDATE]=(char *)"SELECT_FOR_UPDATE"; + commands_counters_desc[MYSQL_COM_QUERY_SET]=(char *)"SET"; + commands_counters_desc[MYSQL_COM_QUERY_SHOW_TABLE_STATUS]=(char *)"SHOW_TABLE_STATUS"; + commands_counters_desc[MYSQL_COM_QUERY_START_TRANSACTION]=(char *)"START_TRANSACTION"; + commands_counters_desc[MYSQL_COM_QUERY_UNLOCK_TABLES]=(char *)"UNLOCK_TABLES"; + commands_counters_desc[MYSQL_COM_QUERY_UPDATE]=(char *)"UPDATE"; + commands_counters_desc[MYSQL_COM_QUERY_USE]=(char *)"USE"; + commands_counters_desc[MYSQL_COM_QUERY___UNKNOWN]=(char *)"UNKNOWN"; }; virtual ~Standard_Query_Processor() { @@ -196,6 +247,7 @@ virtual void init_thread() { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Initializing Per-Thread Query Processor Table with version=0\n"); _thr_SQP_version=0; _thr_SQP_rules=new std::vector; + for (int i=0; i<=MYSQL_COM_QUERY___UNKNOWN; i++) _thr_commands_counters[i]=0; }; @@ -295,6 +347,21 @@ virtual void commit() { }; +virtual SQLite3_result * get_stats_commands_counters() { + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping commands counters%d\n"); + SQLite3_result *result=new SQLite3_result(2); + result->add_column_definition(SQLITE_TEXT,"Command"); + result->add_column_definition(SQLITE_TEXT,"Counter"); + for (int i=0;i<=MYSQL_COM_QUERY___UNKNOWN;i++) { + char **pta=(char **)malloc(sizeof(char *)*2); + pta[0]=commands_counters_desc[i]; + itostr(pta[1], commands_counters[i]); + result->add_row(pta); + free(pta[1]); + free(pta); + } + return result; +} virtual SQLite3_result * get_stats_query_rules() { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping query rules statistics, using Global version %d\n", version); SQLite3_result *result=new SQLite3_result(2); @@ -513,6 +580,10 @@ virtual void update_query_processor_stats() { } } spin_rdunlock(&rwlock); + for (int i=0; i<=MYSQL_COM_QUERY___UNKNOWN; i++) { + __sync_fetch_and_add(&commands_counters[i],_thr_commands_counters[i]); + _thr_commands_counters[i]=0; + } }; virtual void * query_parser_init(char *query, int query_length, int flags) { @@ -522,6 +593,13 @@ virtual void * query_parser_init(char *query, int query_length, int flags) { }; virtual enum MYSQL_COM_QUERY_command query_parser_command_type(void *args) { + enum MYSQL_COM_QUERY_command ret=__query_parser_command_type(args); + _thr_commands_counters[ret]++; + return ret; +} + + +enum MYSQL_COM_QUERY_command __query_parser_command_type(void *args) { SQP_par_t *qp=(SQP_par_t *)args; while (libinjection_sqli_tokenize(&qp->sf)) { if (qp->sf.current->type=='E' || qp->sf.current->type=='k' || qp->sf.current->type=='T') { @@ -591,6 +669,29 @@ virtual enum MYSQL_COM_QUERY_command query_parser_command_type(void *args) { if (!strcasecmp("SET",qp->sf.current->val)) { // SET return MYSQL_COM_QUERY_SET; } + if (!strcasecmp("SHOW",qp->sf.current->val)) { // SHOW + while (libinjection_sqli_tokenize(&qp->sf)) { + if (qp->sf.current->type=='c') continue; +/* + if (qp->sf.current->type=='n') { + if (!strcasecmp("OFFLINE",qp->sf.current->val)) continue; + if (!strcasecmp("ONLINE",qp->sf.current->val)) continue; + } +*/ + if (qp->sf.current->type=='k') { + if (!strcasecmp("TABLE",qp->sf.current->val)) { + while (libinjection_sqli_tokenize(&qp->sf)) { + if (qp->sf.current->type=='c') continue; + if (qp->sf.current->type=='n') { + if (!strcasecmp("STATUS",qp->sf.current->val)) + return MYSQL_COM_QUERY_SHOW_TABLE_STATUS; + } + } + } + } + return MYSQL_COM_QUERY___UNKNOWN; + } + } return MYSQL_COM_QUERY___UNKNOWN; break; case 'U': diff --git a/lib/mysql_session.cpp b/lib/mysql_session.cpp index a22f131b4..e33d25b15 100644 --- a/lib/mysql_session.cpp +++ b/lib/mysql_session.cpp @@ -344,10 +344,12 @@ int MySQL_Session::handler() { switch ((enum_mysql_command)c) { case _MYSQL_COM_QUERY: if (admin==false) { +/**/ query_parser_args=GloQPro->query_parser_init((char *)pkt.ptr+5, pkt.size-5, 0); enum MYSQL_COM_QUERY_command mcqc=GloQPro->query_parser_command_type(query_parser_args); - fprintf(stderr,"Command=%d, query=%s\n", mcqc, (char *)pkt.ptr+5); +// if (mcqc==42) fprintf(stderr,"Command=%d, query=%s\n", mcqc, (char *)pkt.ptr+5); GloQPro->query_parser_free(query_parser_args); +/**/ myprot_client.process_pkt_COM_QUERY((unsigned char *)pkt.ptr,pkt.size); qpo=GloQPro->process_mysql_query(this,pkt.ptr,pkt.size,false); if (qpo) {