From b13c320210ef50d1ce69f824a1c9febd5ae0e168 Mon Sep 17 00:00:00 2001 From: Valentin Rakush Date: Thu, 26 Sep 2019 18:51:11 +0000 Subject: [PATCH 1/2] digests number: converting numbers to ? for digest calculation --- include/MySQL_Thread.h | 1 + include/proxysql_structs.h | 2 ++ lib/MySQL_Thread.cpp | 19 +++++++++++++++++++ lib/Query_Processor.cpp | 1 + lib/c_tokenizer.c | 28 +++++++++++++++++++++++++--- 5 files changed, 48 insertions(+), 3 deletions(-) diff --git a/include/MySQL_Thread.h b/include/MySQL_Thread.h index 982dcf1e0..49c6d3cfe 100644 --- a/include/MySQL_Thread.h +++ b/include/MySQL_Thread.h @@ -409,6 +409,7 @@ class MySQL_Threads_Handler bool query_digests; bool query_digests_lowercase; bool query_digests_replace_null; + bool query_digests_no_digits; bool query_digests_normalize_digest_text; bool query_digests_track_hostname; bool default_reconnect; diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index 515155800..9b53a2e3a 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -689,6 +689,7 @@ __thread bool mysql_thread___commands_stats; __thread bool mysql_thread___query_digests; __thread bool mysql_thread___query_digests_lowercase; __thread bool mysql_thread___query_digests_replace_null; +__thread bool mysql_thread___query_digests_no_digits; __thread bool mysql_thread___query_digests_normalize_digest_text; __thread bool mysql_thread___query_digests_track_hostname; __thread int mysql_thread___query_digests_max_digest_length; @@ -836,6 +837,7 @@ extern __thread bool mysql_thread___servers_stats; extern __thread bool mysql_thread___commands_stats; extern __thread bool mysql_thread___query_digests; extern __thread bool mysql_thread___query_digests_lowercase; +extern __thread bool mysql_thread___query_digests_no_digits; extern __thread bool mysql_thread___query_digests_replace_null; extern __thread bool mysql_thread___query_digests_normalize_digest_text; extern __thread bool mysql_thread___query_digests_track_hostname; diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index 6090f8aea..87b65345b 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -316,6 +316,7 @@ static char * mysql_thread_variables_names[]= { (char *)"query_digests", (char *)"query_digests_lowercase", (char *)"query_digests_replace_null", + (char *)"query_digests_no_digits", (char *)"query_digests_normalize_digest_text", (char *)"query_digests_track_hostname", (char *)"servers_stats", @@ -487,6 +488,7 @@ MySQL_Threads_Handler::MySQL_Threads_Handler() { variables.query_digests=true; variables.query_digests_lowercase=false; variables.query_digests_replace_null=false; + variables.query_digests_no_digits=false; variables.query_digests_normalize_digest_text=false; variables.query_digests_track_hostname=false; variables.connpoll_reset_queue_length = 50; @@ -843,6 +845,7 @@ int MySQL_Threads_Handler::get_variable_int(const char *name) { if (!strcmp(name,"query_digests")) return (int)variables.query_digests; if (!strcmp(name,"query_digests_lowercase")) return (int)variables.query_digests_lowercase; if (!strcmp(name,"query_digests_replace_null")) return (int)variables.query_digests_replace_null; + if (!strcmp(name,"query_digests_no_digits")) return (int)variables.query_digests_no_digits; if (!strcmp(name,"query_digests_normalize_digest_text")) return (int)variables.query_digests_normalize_digest_text; if (!strcmp(name,"query_digests_track_hostname")) return (int)variables.query_digests_track_hostname; } @@ -902,6 +905,7 @@ int MySQL_Threads_Handler::get_variable_int(const char *name) { if (!strcmp(name,"query_digests")) return (int)variables.query_digests; if (!strcmp(name,"query_digests_lowercase")) return (int)variables.query_digests_lowercase; if (!strcmp(name,"query_digests_replace_null")) return (int)variables.query_digests_replace_null; + if (!strcmp(name,"query_digests_no_digits")) return (int)variables.query_digests_no_digits; if (!strcmp(name,"query_digests_normalize_digest_text")) return (int)variables.query_digests_normalize_digest_text; if (!strcmp(name,"query_digests_track_hostname")) return (int)variables.query_digests_track_hostname; if (!strcmp(name,"connpoll_reset_queue_length")) return (int)variables.connpoll_reset_queue_length; @@ -1446,6 +1450,9 @@ char * MySQL_Threads_Handler::get_variable(char *name) { // this is the public f if (!strcasecmp(name,"query_digests_replace_null")) { return strdup((variables.query_digests_replace_null ? "true" : "false")); } + if (!strcasecmp(name,"query_digests_no_digits")) { + return strdup((variables.query_digests_no_digits ? "true" : "false")); + } if (!strcasecmp(name,"query_digests_normalize_digest_text")) { return strdup((variables.query_digests_normalize_digest_text ? "true" : "false")); } @@ -2793,6 +2800,17 @@ bool MySQL_Threads_Handler::set_variable(char *name, char *value) { // this is t } return false; } + if (!strcasecmp(name,"query_digests_no_digits")) { + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.query_digests_no_digits=true; + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.query_digests_no_digits=false; + return true; + } + return false; + } if (!strcasecmp(name,"query_digests_normalize_digest_text")) { if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { variables.query_digests_normalize_digest_text=true; @@ -4475,6 +4493,7 @@ void MySQL_Thread::refresh_variables() { mysql_thread___query_digests=(bool)GloMTH->get_variable_int((char *)"query_digests"); mysql_thread___query_digests_lowercase=(bool)GloMTH->get_variable_int((char *)"query_digests_lowercase"); mysql_thread___query_digests_replace_null=(bool)GloMTH->get_variable_int((char *)"query_digests_replace_null"); + mysql_thread___query_digests_no_digits=(bool)GloMTH->get_variable_int((char *)"query_digests_no_digits"); mysql_thread___query_digests_normalize_digest_text=(bool)GloMTH->get_variable_int((char *)"query_digests_normalize_digest_text"); mysql_thread___query_digests_track_hostname=(bool)GloMTH->get_variable_int((char *)"query_digests_track_hostname"); variables.min_num_servers_lantency_awareness=GloMTH->get_variable_int((char *)"min_num_servers_lantency_awareness"); diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 257fcc449..c655c329e 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -1982,6 +1982,7 @@ void Query_Processor::update_query_processor_stats() { void Query_Processor::query_parser_init(SQP_par_t *qp, char *query, int query_length, int flags) { // trying to get rid of libinjection // instead of initializing qp->sf , we copy query info later in this function + proxy_warning("TRACE : query_parser_init\n"); qp->digest_text=NULL; qp->first_comment=NULL; qp->query_prefix=NULL; diff --git a/lib/c_tokenizer.c b/lib/c_tokenizer.c index 2d75e340e..60048cc42 100644 --- a/lib/c_tokenizer.c +++ b/lib/c_tokenizer.c @@ -1,6 +1,7 @@ /* c_tokenizer.c */ // Borrowed from http://www.cplusplus.com/faq/sequences/strings/split/ +#include #include #include @@ -12,6 +13,7 @@ extern __thread int mysql_thread___query_digests_max_query_length; #define bool char extern __thread bool mysql_thread___query_digests_lowercase; extern __thread bool mysql_thread___query_digests_replace_null; +extern __thread bool mysql_thread___query_digests_no_digits; void tokenizer(tokenizer_t *result, const char* s, const char* delimiters, int empties ) { @@ -209,8 +211,10 @@ char *mysql_query_digest_and_first_comment(char *s, int _len, char **first_comme bool lowercase=0; bool replace_null=0; + bool replace_number=0; lowercase=mysql_thread___query_digests_lowercase; replace_null = mysql_thread___query_digests_replace_null; + replace_number = mysql_thread___query_digests_no_digits; while(i < len) { @@ -253,9 +257,18 @@ char *mysql_query_digest_and_first_comment(char *s, int _len, char **first_comme // may be digit - start with digit else if(is_token_char(prev_char) && is_digit_char(*s)) { - flag = 5; - if(len == i+1) - continue; + if (replace_number) { + *p_r++ = '?'; + while(*s != '\0' && is_digit_char(*s)) { + s++; + i++; + } + } + else { + flag = 5; + if(len == i+1) + continue; + } } // not above case - remove duplicated space char @@ -275,6 +288,15 @@ char *mysql_query_digest_and_first_comment(char *s, int _len, char **first_comme i++; continue; } + if (replace_number) { + if (!is_digit_char(prev_char) && is_digit_char(*s)) { + *p_r++ = '?'; + while(*s != '\0' && is_digit_char(*s)) { + s++; + i++; + } + } + } if (replace_null) { if (*s == 'n' || *s == 'N') { // we search for NULL , #2171 if (i && is_token_char(prev_char)) { From bea3223e420ed4be15dc00d8afad1e99c7f27344 Mon Sep 17 00:00:00 2001 From: Valentin Rakush Date: Fri, 27 Sep 2019 13:28:35 +0000 Subject: [PATCH 2/2] digests numbers: cleanup debugging --- lib/Query_Processor.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index c655c329e..257fcc449 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -1982,7 +1982,6 @@ void Query_Processor::update_query_processor_stats() { void Query_Processor::query_parser_init(SQP_par_t *qp, char *query, int query_length, int flags) { // trying to get rid of libinjection // instead of initializing qp->sf , we copy query info later in this function - proxy_warning("TRACE : query_parser_init\n"); qp->digest_text=NULL; qp->first_comment=NULL; qp->query_prefix=NULL;