From 7a4a2092a58ed1d4dbdd1c7d271518beb84be447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Fri, 1 Jan 2016 10:09:28 +0000 Subject: [PATCH 1/4] MySQL_Logger implementation - Re-enabled MySQL_Logger - handling special case in which mysql-commands_stats and mysql-query_digests are disabled - specifies the length of the query - Query_Info is always initialized --- lib/MySQL_Logger.cpp | 10 ++++++++-- lib/MySQL_Session.cpp | 3 ++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/MySQL_Logger.cpp b/lib/MySQL_Logger.cpp index 1e997e29d..86c042917 100644 --- a/lib/MySQL_Logger.cpp +++ b/lib/MySQL_Logger.cpp @@ -38,7 +38,7 @@ void MySQL_Logger::flush_log() { wrunlock(); } void MySQL_Logger::flush_log_unlocked() { - return; // FIXME: this line prevents the logger from starting. Here for testing + //return; // FIXME: this line prevents the logger from starting. Here for testing if (logfile) { logfile->flush(); logfile->close(); @@ -83,7 +83,13 @@ void MySQL_Logger::log_request(MySQL_Session *sess) { ev.set_start_time(sess->CurrentQuery.start_time); ev.set_end_time(sess->CurrentQuery.end_time); ev.set_query_digest(GloQPro->get_digest(sess->CurrentQuery.QueryParserArgs)); - ev.set_query((const char *)sess->CurrentQuery.QueryPointer); + char *c=(char *)sess->CurrentQuery.QueryPointer; + if (c) { + // FIXME: NEEDS LENGTH . Use Bytes + ev.set_query(c,sess->CurrentQuery.QueryLength); + } else { + ev.set_query(""); + } ev.set_server(""); ev.set_client(""); wrlock(); diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index 1941264d7..29fafd28d 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -92,8 +92,8 @@ void Query_Info::begin(unsigned char *_p, int len, bool mysql_header) { QueryLength=0; QueryParserArgs=NULL; start_time=sess->thread->curtime; + init(_p, len, mysql_header); if (mysql_thread___commands_stats || mysql_thread___query_digests) { - init(_p, len, mysql_header); query_parser_init(); if (mysql_thread___commands_stats) query_parser_command_type(); @@ -136,6 +136,7 @@ void Query_Info::query_parser_free() { unsigned long long Query_Info::query_parser_update_counters() { if (MyComQueryCmd==MYSQL_COM_QUERY___NONE) return 0; // this means that it was never initialized + if (MyComQueryCmd==MYSQL_COM_QUERY_UNKNOWN) return 0; // this means that it was never initialized unsigned long long ret=GloQPro->query_parser_update_counters(sess, MyComQueryCmd, QueryParserArgs, end_time-start_time); MyComQueryCmd=MYSQL_COM_QUERY___NONE; //l_free(QueryLength+1,QueryPointer); From a63b7cf951776bf7d83c970110678685270f505b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sat, 2 Jan 2016 18:27:34 +0000 Subject: [PATCH 2/4] MySQL_Logger configuration - added global variable eventslog_filename - added global variable eventslog_filesize --- include/MySQL_Logger.hpp | 4 +++ include/MySQL_Thread.h | 2 ++ include/proxysql_structs.h | 8 ++++++ lib/MySQL_Logger.cpp | 58 ++++++++++++++++++++++++++++++++------ lib/MySQL_Thread.cpp | 36 +++++++++++++++++++++++ 5 files changed, 100 insertions(+), 8 deletions(-) diff --git a/include/MySQL_Logger.hpp b/include/MySQL_Logger.hpp index 49db93247..188ce5b6b 100644 --- a/include/MySQL_Logger.hpp +++ b/include/MySQL_Logger.hpp @@ -5,6 +5,7 @@ class MySQL_Logger { private: + bool enabled; char *base_filename; char *datadir; unsigned int log_file_id; @@ -13,6 +14,8 @@ class MySQL_Logger { void wrlock(); void wrunlock(); std::fstream *logfile; + void close_log_unlocked(); + void open_log_unlocked(); public: MySQL_Logger(); ~MySQL_Logger(); @@ -20,6 +23,7 @@ class MySQL_Logger { void flush_log_unlocked(); unsigned int find_next_id(); void set_datadir(char *); + void set_base_filename(); void log_request(MySQL_Session *); void flush(); }; diff --git a/include/MySQL_Thread.h b/include/MySQL_Thread.h index ef37e953c..c65c17202 100644 --- a/include/MySQL_Thread.h +++ b/include/MySQL_Thread.h @@ -295,6 +295,8 @@ class MySQL_Threads_Handler uint16_t server_capabilities; int poll_timeout; int poll_timeout_on_failure; + char *eventslog_filename; + int eventslog_filesize; } variables; PtrArray *bind_fds; MySQL_Listeners_Manager *MLM; diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index 016389d76..8540c4c34 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -706,6 +706,10 @@ __thread bool mysql_thread___query_digests; __thread bool mysql_thread___default_reconnect; __thread bool mysql_thread___sessions_sort; +/* variables used by events log */ +__thread char * mysql_thread___eventslog_filename; +__thread int mysql_thread___eventslog_filesize; + /* variables used by the monitoring module */ __thread int mysql_thread___monitor_history; __thread int mysql_thread___monitor_connect_interval; @@ -765,6 +769,10 @@ extern __thread bool mysql_thread___query_digests; extern __thread bool mysql_thread___default_reconnect; extern __thread bool mysql_thread___sessions_sort; +/* variables used by events log */ +extern __thread char * mysql_thread___eventslog_filename; +extern __thread int mysql_thread___eventslog_filesize; + /* variables used by the monitoring module */ extern __thread int mysql_thread___monitor_history; extern __thread int mysql_thread___monitor_connect_interval; diff --git a/lib/MySQL_Logger.cpp b/lib/MySQL_Logger.cpp index 86c042917..d2827b775 100644 --- a/lib/MySQL_Logger.cpp +++ b/lib/MySQL_Logger.cpp @@ -8,20 +8,21 @@ extern Query_Processor *GloQPro; MySQL_Logger::MySQL_Logger() { + enabled=false; base_filename=NULL; datadir=NULL; - // FIXME, temp - base_filename=(char *)"mysql-log"; + base_filename=strdup((char *)""); spinlock_rwlock_init(&rwlock); logfile=NULL; log_file_id=0; - max_log_file_size=1024*1024; + max_log_file_size=100*1024*1024; }; MySQL_Logger::~MySQL_Logger() { if (datadir) { free(datadir); } + free(base_filename); }; void MySQL_Logger::wrlock() { @@ -37,21 +38,38 @@ void MySQL_Logger::flush_log() { flush_log_unlocked(); wrunlock(); } -void MySQL_Logger::flush_log_unlocked() { - //return; // FIXME: this line prevents the logger from starting. Here for testing + + +void MySQL_Logger::close_log_unlocked() { if (logfile) { logfile->flush(); logfile->close(); delete logfile; logfile=NULL; } +} + +void MySQL_Logger::flush_log_unlocked() { + if (enabled==false) return; + close_log_unlocked(); + open_log_unlocked(); +} + + +void MySQL_Logger::open_log_unlocked() { if (log_file_id==0) { log_file_id=find_next_id()+1; } else { log_file_id++; } - char *filen=(char *)malloc(strlen(datadir)+strlen(base_filename)+10); - sprintf(filen,"%s/%s.%06d",datadir,base_filename,log_file_id); + char *filen=NULL; + if (base_filename[0]=='/') { // absolute path + filen=(char *)malloc(strlen(base_filename)+10); + sprintf(filen,"/%s.%08d",base_filename,log_file_id); + } else { // relative path + filen=(char *)malloc(strlen(datadir)+strlen(base_filename)+10); + sprintf(filen,"%s/%s.%08d",datadir,base_filename,log_file_id); + } logfile=new std::fstream(); logfile->exceptions ( std::ofstream::failbit | std::ofstream::badbit ); try { @@ -68,12 +86,36 @@ void MySQL_Logger::flush_log_unlocked() { //close(fd); }; +void MySQL_Logger::set_base_filename() { + // if filename is the same, return + wrlock(); + max_log_file_size=mysql_thread___eventslog_filesize; + if (strcmp(base_filename,mysql_thread___eventslog_filename)==0) { + wrunlock(); + return; + } + // close current log + close_log_unlocked(); + // set file id to 0 , so that find_next_id() will be called + log_file_id=0; + free(base_filename); + base_filename=strdup(mysql_thread___eventslog_filename); + if (strlen(base_filename)) { + enabled=true; + open_log_unlocked(); + } else { + enabled=false; + } + wrunlock(); +} + void MySQL_Logger::set_datadir(char *s) { datadir=strdup(s); flush_log(); }; void MySQL_Logger::log_request(MySQL_Session *sess) { + if (enabled==false) return; if (logfile==NULL) return; mysql_logger::event ev; ev.set_thread_id(sess->thread_session_id); @@ -119,7 +161,7 @@ unsigned int MySQL_Logger::find_next_id() { size_t bfl=strlen(base_filename); if ((dir = opendir(datadir)) != NULL) { while ((ent = readdir (dir)) != NULL) { - if (strlen(ent->d_name)==bfl+7) { + if (strlen(ent->d_name)==bfl+9) { if (strncmp(ent->d_name,base_filename,bfl)==0) { if (ent->d_name[bfl]=='.') { int idx=atoi(ent->d_name+bfl+1); diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index 871ae1fc1..d69608978 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -140,6 +140,8 @@ static char * mysql_thread_variables_names[]= { (char *)"connect_retries_delay", (char *)"connect_timeout_server", (char *)"connect_timeout_server_max", + (char *)"eventslog_filename", + (char *)"eventslog_filesize", (char *)"default_charset", (char *)"free_connections_pct", (char *)"have_compress", @@ -243,6 +245,8 @@ MySQL_Threads_Handler::MySQL_Threads_Handler() { variables.default_charset=33; variables.interfaces=strdup((char *)""); variables.server_version=strdup((char *)"5.1.30"); + variables.eventslog_filename=strdup((char *)""); // proxysql-mysql-eventslog is recommended + variables.eventslog_filesize=100*1024*1024; variables.server_capabilities=CLIENT_FOUND_ROWS | CLIENT_PROTOCOL_41 | CLIENT_IGNORE_SIGPIPE | CLIENT_TRANSACTIONS | CLIENT_SECURE_CONNECTION | CLIENT_CONNECT_WITH_DB | CLIENT_SSL; variables.poll_timeout=2000; variables.poll_timeout_on_failure=100; @@ -337,6 +341,7 @@ char * MySQL_Threads_Handler::get_variable_string(char *name) { if (!strcasecmp(name,"monitor_query_status")) return strdup(variables.monitor_query_status); } if (!strcasecmp(name,"server_version")) return strdup(variables.server_version); + if (!strcasecmp(name,"eventslog_filename")) return strdup(variables.eventslog_filename); if (!strcasecmp(name,"default_schema")) return strdup(variables.default_schema); if (!strcasecmp(name,"interfaces")) return strdup(variables.interfaces); proxy_error("Not existing variable: %s\n", name); assert(0); @@ -380,6 +385,7 @@ int MySQL_Threads_Handler::get_variable_int(char *name) { if (!strcasecmp(name,"connect_timeout_server")) return (int)variables.connect_timeout_server; if (!strcasecmp(name,"connect_timeout_server_max")) return (int)variables.connect_timeout_server_max; if (!strcasecmp(name,"connect_retries_delay")) return (int)variables.connect_retries_delay; + if (!strcasecmp(name,"eventslog_filesize")) return (int)variables.eventslog_filesize; if (!strcasecmp(name,"max_transaction_time")) return (int)variables.max_transaction_time; if (!strcasecmp(name,"threshold_query_length")) return (int)variables.threshold_query_length; if (!strcasecmp(name,"threshold_resultset_size")) return (int)variables.threshold_resultset_size; @@ -409,6 +415,7 @@ char * MySQL_Threads_Handler::get_variable(char *name) { // this is the public f #define INTBUFSIZE 4096 char intbuf[INTBUFSIZE]; if (!strcasecmp(name,"server_version")) return strdup(variables.server_version); + if (!strcasecmp(name,"eventslog_filename")) return strdup(variables.eventslog_filename); if (!strcasecmp(name,"default_schema")) return strdup(variables.default_schema); if (!strcasecmp(name,"interfaces")) return strdup(variables.interfaces); if (!strcasecmp(name,"server_capabilities")) { @@ -509,6 +516,10 @@ char * MySQL_Threads_Handler::get_variable(char *name) { // this is the public f sprintf(intbuf,"%d",variables.connect_retries_delay); return strdup(intbuf); } + if (!strcasecmp(name,"eventslog_filesize")) { + sprintf(intbuf,"%d",variables.eventslog_filesize); + return strdup(intbuf); + } if (!strcasecmp(name,"max_transaction_time")) { sprintf(intbuf,"%d",variables.max_transaction_time); return strdup(intbuf); @@ -921,6 +932,15 @@ bool MySQL_Threads_Handler::set_variable(char *name, char *value) { // this is t return false; } } + if (!strcasecmp(name,"eventslog_filesize")) { + int intv=atoi(value); + if (intv >= 1024*1024 && intv <= 1*1024*1024*1024) { + variables.eventslog_filesize=intv; + return true; + } else { + return false; + } + } if (!strcasecmp(name,"default_schema")) { if (vallen) { free(variables.default_schema); @@ -952,6 +972,15 @@ bool MySQL_Threads_Handler::set_variable(char *name, char *value) { // this is t return false; } } + if (!strcasecmp(name,"eventslog_filename")) { + if (vallen) { + free(variables.eventslog_filename); + variables.eventslog_filename=strdup(value); + return true; + } else { + return false; + } + } if (!strcasecmp(name,"server_capabilities")) { int intv=atoi(value); if (intv > 10 && intv <= 65535) { @@ -1183,6 +1212,7 @@ MySQL_Threads_Handler::~MySQL_Threads_Handler() { if (variables.default_schema) free(variables.default_schema); if (variables.interfaces) free(variables.interfaces); if (variables.server_version) free(variables.server_version); + if (variables.eventslog_filename) free(variables.eventslog_filename); free(mysql_threads); mysql_threads=NULL; delete MLM; @@ -1222,6 +1252,7 @@ MySQL_Thread::~MySQL_Thread() { if (mysql_thread___default_schema) { free(mysql_thread___default_schema); mysql_thread___default_schema=NULL; } if (mysql_thread___server_version) { free(mysql_thread___server_version); mysql_thread___server_version=NULL; } + if (mysql_thread___eventslog_filename) { free(mysql_thread___eventslog_filename); mysql_thread___eventslog_filename=NULL; } } @@ -1732,6 +1763,10 @@ void MySQL_Thread::refresh_variables() { if (mysql_thread___server_version) free(mysql_thread___server_version); mysql_thread___server_version=GloMTH->get_variable_string((char *)"server_version"); + if (mysql_thread___eventslog_filename) free(mysql_thread___eventslog_filename); + mysql_thread___eventslog_filesize=GloMTH->get_variable_int((char *)"eventslog_filesize"); + mysql_thread___eventslog_filename=GloMTH->get_variable_string((char *)"eventslog_filename"); + GloMyLogger->set_base_filename(); // both filename and filesize are set here if (mysql_thread___default_schema) free(mysql_thread___default_schema); mysql_thread___default_schema=GloMTH->get_variable_string((char *)"default_schema"); mysql_thread___server_capabilities=GloMTH->get_variable_uint16((char *)"server_capabilities"); @@ -1765,6 +1800,7 @@ MySQL_Thread::MySQL_Thread() { mysql_sessions_connections_handler=NULL; __thread_MySQL_Thread_Variables_version=0; mysql_thread___server_version=NULL; + mysql_thread___eventslog_filename=NULL; status_variables.queries=0; status_variables.queries_slow=0; From 396332edf1a75e148a5f86676b1f7bc78e0ab7e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sat, 2 Jan 2016 18:42:19 +0000 Subject: [PATCH 3/4] Moving GloVars.start_time to the daemon code Instead of the parent code --- src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.cpp b/src/main.cpp index c08aa919c..55548c645 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -317,7 +317,6 @@ void ProxySQL_Main_init() { #else glovars.has_debug=false; #endif /* DEBUG */ - GloVars.global.start_time=monotonic_time(); __thr_sfp=l_mem_init(); { @@ -508,6 +507,7 @@ int main(int argc, const char * argv[]) { } else { /* The daemon */ + GloVars.global.start_time=monotonic_time(); if (ProxySQL_daemonize_phase2()==false) { goto finish; } From 57b45120402f66640f75a4a2027448fd688699d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sat, 2 Jan 2016 21:56:36 +0000 Subject: [PATCH 4/4] try to generate a core dump signaling again the thread --- lib/ProxySQL_GloVars.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/ProxySQL_GloVars.cpp b/lib/ProxySQL_GloVars.cpp index af68b8177..9a1945a2a 100644 --- a/lib/ProxySQL_GloVars.cpp +++ b/lib/ProxySQL_GloVars.cpp @@ -23,6 +23,13 @@ void crash_handler(int sig) { fprintf(stderr, "Error: signal %d:\n", sig); backtrace_symbols_fd(arr, s, STDERR_FILENO); +#ifdef SYS_gettid + // try to generate a core dump signaling again the thread + signal(sig, SIG_DFL); + pid_t tid; + tid = syscall(SYS_gettid); + kill(tid, sig); +#endif /* SYS_gettid */ exit(EXIT_FAILURE); }