From c710f549872fde0028ccaf7edff06773751e3365 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Fri, 12 Aug 2016 15:31:29 +0000 Subject: [PATCH 1/3] Compiling optimization * converted MySQL_Authentication to use unsorted_map instead of btree_map * Query_Cache now uses pointers to KV_BtreeArray * removed some unnecesary headers --- include/MySQL_Authentication.hpp | 8 +++--- include/cpp.h | 20 +++++++-------- include/proxysql.h | 18 ++++++++++++-- include/proxysql_structs.h | 4 +-- include/query_cache.hpp | 8 +++--- lib/MySQL_Authentication.cpp | 21 ++++++++++------ lib/Query_Cache.cpp | 42 ++++++++++++++++++++++++++------ lib/configfile.cpp | 3 ++- 8 files changed, 89 insertions(+), 35 deletions(-) diff --git a/include/MySQL_Authentication.hpp b/include/MySQL_Authentication.hpp index e26b5b6e4..7089d98b4 100644 --- a/include/MySQL_Authentication.hpp +++ b/include/MySQL_Authentication.hpp @@ -1,7 +1,7 @@ #ifndef __CLASS_MYSQL_AUTHENTICATION_H #define __CLASS_MYSQL_AUTHENTICATION_H -#include "btree_map.h" +//#include "btree_map.h" #include "proxysql.h" #include "cpp.h" @@ -36,13 +36,15 @@ typedef struct _account_details_t { */ -typedef btree::btree_map BtMap_auth; +//typedef btree::btree_map BtMap_auth; +typedef std::unordered_map umap_auth; class PtrArray; typedef struct _creds_group_t { rwlock_t lock; - BtMap_auth bt_map; + //BtMap_auth bt_map; + umap_auth bt_map; PtrArray *cred_array; } creds_group_t; diff --git a/include/cpp.h b/include/cpp.h index 041b526cd..bad6a39e1 100644 --- a/include/cpp.h +++ b/include/cpp.h @@ -21,17 +21,17 @@ #undef swap #undef min #undef max -#include +//#include #include -#include -#include -#include -#include +//#include +//#include +//#include +//#include #include #include -#include -#include -#include -#include -#include +//#include +//#include +//#include +//#include +//#include //#endif /* __cplusplus */ diff --git a/include/proxysql.h b/include/proxysql.h index 506a7fe3c..80dc724ce 100644 --- a/include/proxysql.h +++ b/include/proxysql.h @@ -1,7 +1,15 @@ #ifdef __cplusplus #include #include -#include "btree_map.h" + +#include +//#include +//#include +//#include +//#include + + +//#include "btree_map.h" #ifndef EZOPTION //#include "ezOptionParser.hpp" #define EZOPTION @@ -70,8 +78,14 @@ #include "jemalloc.h" - +#ifdef DEBUG +//#define VALGRIND_ENABLE_ERROR_REPORTING +//#define VALGRIND_DISABLE_ERROR_REPORTING #include "valgrind.h" +#else +#define VALGRIND_ENABLE_ERROR_REPORTING +#define VALGRIND_DISABLE_ERROR_REPORTING +#endif /* DEBUG */ #include "sqlite3.h" diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index d9197893e..b21ca81b4 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -270,8 +270,8 @@ class SQLite3DB; class SimpleKV; class AdvancedKV; class ProxySQL_Poll; -//class Query_Cache; -class Shared_Query_Cache; +class Query_Cache; +//class Shared_Query_Cache; class MySQL_Authentication; class MySQL_Connection; class MySQL_Protocol; diff --git a/include/query_cache.hpp b/include/query_cache.hpp index fc01f2d9f..216bdbd88 100644 --- a/include/query_cache.hpp +++ b/include/query_cache.hpp @@ -28,7 +28,7 @@ struct __QC_entry_t { unsigned long long access_ms; // when the entry was read last , monotonic , millisecond granularity uint32_t ref_count; // reference counter }; - +/* typedef btree::btree_map BtMap_cache; @@ -52,10 +52,12 @@ class KV_BtreeArray { QC_entry_t *lookup(uint64_t key); void empty(); }; - +*/ +class KV_BtreeArray; class Query_Cache { private: - KV_BtreeArray KVs[SHARED_QUERY_CACHE_HASH_TABLES]; + //KV_BtreeArray KVs[SHARED_QUERY_CACHE_HASH_TABLES]; + KV_BtreeArray * KVs[SHARED_QUERY_CACHE_HASH_TABLES]; uint64_t get_data_size_total(); unsigned int current_used_memory_pct(); public: diff --git a/lib/MySQL_Authentication.cpp b/lib/MySQL_Authentication.cpp index 42980d4e7..9082447ed 100644 --- a/lib/MySQL_Authentication.cpp +++ b/lib/MySQL_Authentication.cpp @@ -136,7 +136,8 @@ bool MySQL_Authentication::add(char * username, char * password, enum cred_usern void *sha1_pass=NULL; char *oldpass=NULL; spin_wrlock(&cg.lock); - btree::btree_map::iterator lookup; + //btree::btree_map::iterator lookup; + std::unordered_map::iterator lookup; lookup = cg.bt_map.find(hash1); if (lookup != cg.bt_map.end()) { account_details_t *ad=lookup->second; @@ -264,7 +265,8 @@ int MySQL_Authentication::increase_frontend_user_connections(char *username) { creds_group_t &cg=creds_frontends; int ret=0; spin_wrlock(&cg.lock); - btree::btree_map::iterator it; + //btree::btree_map::iterator it; + std::unordered_map::iterator it; it = cg.bt_map.find(hash1); if (it != cg.bt_map.end()) { account_details_t *ad=it->second; @@ -284,7 +286,8 @@ void MySQL_Authentication::decrease_frontend_user_connections(char *username) { delete myhash; creds_group_t &cg=creds_frontends; spin_wrlock(&cg.lock); - btree::btree_map::iterator it; + //btree::btree_map::iterator it; + std::unordered_map::iterator it; it = cg.bt_map.find(hash1); if (it != cg.bt_map.end()) { account_details_t *ad=it->second; @@ -308,7 +311,8 @@ bool MySQL_Authentication::del(char * username, enum cred_username_type usertype if (set_lock) spin_wrlock(&cg.lock); - btree::btree_map::iterator lookup; + //btree::btree_map::iterator lookup; + std::unordered_map::iterator lookup; lookup = cg.bt_map.find(hash1); if (lookup != cg.bt_map.end()) { account_details_t *ad=lookup->second; @@ -339,7 +343,8 @@ bool MySQL_Authentication::set_SHA1(char * username, enum cred_username_type use creds_group_t &cg=(usertype==USERNAME_BACKEND ? creds_backends : creds_frontends); spin_wrlock(&cg.lock); - btree::btree_map::iterator lookup; + //btree::btree_map::iterator lookup; + std::unordered_map::iterator lookup; lookup = cg.bt_map.find(hash1); if (lookup != cg.bt_map.end()) { account_details_t *ad=lookup->second; @@ -368,7 +373,8 @@ char * MySQL_Authentication::lookup(char * username, enum cred_username_type use creds_group_t &cg=(usertype==USERNAME_BACKEND ? creds_backends : creds_frontends); spin_rdlock(&cg.lock); - btree::btree_map::iterator lookup; + //btree::btree_map::iterator lookup; + std::unordered_map::iterator lookup; lookup = cg.bt_map.find(hash1); if (lookup != cg.bt_map.end()) { account_details_t *ad=lookup->second; @@ -396,7 +402,8 @@ bool MySQL_Authentication::_reset(enum cred_username_type usertype) { creds_group_t &cg=(usertype==USERNAME_BACKEND ? creds_backends : creds_frontends); spin_wrlock(&cg.lock); - btree::btree_map::iterator lookup; + //btree::btree_map::iterator lookup; + std::unordered_map::iterator lookup; while (cg.bt_map.size()) { lookup = cg.bt_map.begin(); diff --git a/lib/Query_Cache.cpp b/lib/Query_Cache.cpp index f747a7f7d..9c7dd1e71 100644 --- a/lib/Query_Cache.cpp +++ b/lib/Query_Cache.cpp @@ -1,6 +1,7 @@ #include "btree_map.h" #include "proxysql.h" #include "cpp.h" +#include "query_cache.hpp" #include "proxysql_atomic.h" #include "SpookyV2.h" @@ -28,6 +29,30 @@ #endif /* DEBUG */ #define QUERY_CACHE_VERSION "0.2.0902" DEB + +typedef btree::btree_map BtMap_cache; + +class KV_BtreeArray { + private: + rwlock_t lock; + BtMap_cache bt_map; + PtrArray *ptrArray; + uint64_t purgeChunkSize; + uint64_t purgeIdx; + bool __insert(uint64_t, void *); + uint64_t freeable_memory; + public: + uint64_t tottopurge; + KV_BtreeArray(); + ~KV_BtreeArray(); + uint64_t get_data_size(); + void purge_some(unsigned long long); + int cnt(); + bool replace(uint64_t key, QC_entry_t *entry); + QC_entry_t *lookup(uint64_t key); + void empty(); +}; + __thread uint64_t __thr_cntSet=0; __thread uint64_t __thr_cntGet=0; __thread uint64_t __thr_cntGetOK=0; @@ -210,7 +235,7 @@ uint64_t Query_Cache::get_data_size_total() { int r=0; int i; for (i=0; iget_data_size(); } return r; }; @@ -236,6 +261,9 @@ Query_Cache::Query_Cache() { perror("Incompatible debagging version"); exit(EXIT_FAILURE); } + for (int i=0; ilookup(hk); if (entry!=NULL) { unsigned long long t=curtime_ms; @@ -294,7 +322,7 @@ bool Query_Cache::set(uint64_t user_hash, const unsigned char *kp, uint32_t kl, uint64_t hk=SpookyHash::Hash64(kp, kl, user_hash); unsigned char i=hk%SHARED_QUERY_CACHE_HASH_TABLES; entry->key=hk; - KVs[i].replace(hk, entry); + KVs[i]->replace(hk, entry); return true; } @@ -303,8 +331,8 @@ uint64_t Query_Cache::flush() { int i; uint64_t total_count=0; for (i=0; icnt(); + KVs[i]->empty(); } return total_count; }; @@ -319,7 +347,7 @@ void * Query_Cache::purgeHash_thread(void *) { if (current_used_memory_pct() < purge_threshold_pct_min ) continue; for (i=0; ipurge_some(QCnow_ms); } } return NULL; diff --git a/lib/configfile.cpp b/lib/configfile.cpp index 8f0096479..a18db874d 100644 --- a/lib/configfile.cpp +++ b/lib/configfile.cpp @@ -1,7 +1,8 @@ #include "proxysql.h" - #include "cpp.h" +#include + #include #include #include From d276262ebe811b7beca5966091665b41c0462f5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 16 Aug 2016 23:19:19 +0000 Subject: [PATCH 2/3] Drafting configuration of QC Related to issue #171 Added variable mysql-query_cache_size_MB Also removed compiling of simple_kv.cpp --- include/MySQL_Thread.h | 1 + include/cpp.h | 2 +- include/proxysql_structs.h | 8 +++++++- lib/Makefile | 2 +- lib/MySQL_Thread.cpp | 17 +++++++++++++++++ 5 files changed, 27 insertions(+), 3 deletions(-) diff --git a/include/MySQL_Thread.h b/include/MySQL_Thread.h index d196e0eb4..bae6d3901 100644 --- a/include/MySQL_Thread.h +++ b/include/MySQL_Thread.h @@ -314,6 +314,7 @@ class MySQL_Threads_Handler char * ssl_p2s_cert; char * ssl_p2s_key; char * ssl_p2s_cipher; + int query_cache_size_MB; } variables; PtrArray *bind_fds; MySQL_Listeners_Manager *MLM; diff --git a/include/cpp.h b/include/cpp.h index bad6a39e1..bf5b3cd4a 100644 --- a/include/cpp.h +++ b/include/cpp.h @@ -6,7 +6,7 @@ #include "query_cache.hpp" #include "mysql_connection.h" #include "sqlite3db.h" -#include "simple_kv.h" +//#include "simple_kv.h" #include "gen_utils.h" #include "StatCounters.h" #include "MySQL_Monitor.hpp" diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index b21ca81b4..f840636f0 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -507,7 +507,7 @@ struct _global_variables_t { unsigned int mysql_query_cache_default_timeout; unsigned long long mysql_wait_timeout; - unsigned long long mysql_query_cache_size; + //unsigned long long mysql_query_cache_size; unsigned long long mysql_max_resultset_size; int mysql_max_query_size; @@ -737,6 +737,9 @@ __thread bool mysql_thread___query_digests; __thread bool mysql_thread___default_reconnect; __thread bool mysql_thread___sessions_sort; +/* variables used for Query Cache */ +__thread int mysql_thread___query_cache_size_MB; + /* variables used for SSL , from proxy to server (p2s) */ __thread char * mysql_thread___ssl_p2s_ca; __thread char * mysql_thread___ssl_p2s_cert; @@ -812,6 +815,9 @@ extern __thread bool mysql_thread___query_digests; extern __thread bool mysql_thread___default_reconnect; extern __thread bool mysql_thread___sessions_sort; +/* variables used for Query Cache */ +extern __thread int mysql_thread___query_cache_size_MB; + /* variables used for SSL , from proxy to server (p2s) */ extern __thread char * mysql_thread___ssl_p2s_ca; extern __thread char * mysql_thread___ssl_p2s_cert; diff --git a/lib/Makefile b/lib/Makefile index 2b04a94cb..d415b8abb 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -47,7 +47,7 @@ default: libproxysql.a _OBJ = c_tokenizer.o OBJ = $(patsubst %,$(ODIR)/%,$(_OBJ)) -_OBJ_CPP = ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Query_Cache.oo SpookyV2.oo MySQL_Authentication.oo gen_utils.oo simple_kv.oo sqlite3db.oo global_variables.oo mysql_connection.oo MySQL_HostGroups_Manager.oo mysql_data_stream.oo MySQL_Thread.oo MySQL_Session.oo MySQL_Protocol.oo mysql_backend.oo Query_Processor.oo ProxySQL_Admin.oo MySQL_Monitor.oo MySQL_Logger.oo thread.oo +_OBJ_CPP = ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Query_Cache.oo SpookyV2.oo MySQL_Authentication.oo gen_utils.oo sqlite3db.oo global_variables.oo mysql_connection.oo MySQL_HostGroups_Manager.oo mysql_data_stream.oo MySQL_Thread.oo MySQL_Session.oo MySQL_Protocol.oo mysql_backend.oo Query_Processor.oo ProxySQL_Admin.oo MySQL_Monitor.oo MySQL_Logger.oo thread.oo OBJ_CPP = $(patsubst %,$(ODIR)/%,$(_OBJ_CPP)) %.ko: %.cpp diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index 832ae6869..330fcadda 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -191,6 +191,7 @@ static char * mysql_thread_variables_names[]= { (char *)"default_query_delay", (char *)"default_query_timeout", (char *)"long_query_time", + (char *)"query_cache_size_MB", (char *)"ping_interval_server_msec", (char *)"ping_timeout_server", (char *)"default_schema", @@ -266,6 +267,7 @@ MySQL_Threads_Handler::MySQL_Threads_Handler() { variables.default_query_delay=0; variables.default_query_timeout=24*3600*1000; variables.long_query_time=1000; + variables.query_cache_size_MB=256; variables.init_connect=NULL; variables.ping_interval_server_msec=10000; variables.ping_timeout_server=200; @@ -471,6 +473,7 @@ int MySQL_Threads_Handler::get_variable_int(char *name) { if (!strcasecmp(name,"default_query_timeout")) return (int)variables.default_query_timeout; if (!strcasecmp(name,"default_max_latency_ms")) return (int)variables.default_max_latency_ms; if (!strcasecmp(name,"long_query_time")) return (int)variables.long_query_time; + if (!strcasecmp(name,"query_cache_size_MB")) return (int)variables.query_cache_size_MB; if (!strcasecmp(name,"free_connections_pct")) return (int)variables.free_connections_pct; if (!strcasecmp(name,"ping_interval_server_msec")) return (int)variables.ping_interval_server_msec; if (!strcasecmp(name,"ping_timeout_server")) return (int)variables.ping_timeout_server; @@ -687,6 +690,10 @@ char * MySQL_Threads_Handler::get_variable(char *name) { // this is the public f sprintf(intbuf,"%d",variables.long_query_time); return strdup(intbuf); } + if (!strcasecmp(name,"query_cache_size_MB")) { + sprintf(intbuf,"%d",variables.query_cache_size_MB); + return strdup(intbuf); + } if (!strcasecmp(name,"ping_interval_server_msec")) { sprintf(intbuf,"%d",variables.ping_interval_server_msec); return strdup(intbuf); @@ -1010,6 +1017,15 @@ bool MySQL_Threads_Handler::set_variable(char *name, char *value) { // this is t return false; } } + if (!strcasecmp(name,"query_cache_size_MB")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 1024*10240) { + variables.query_cache_size_MB=intv; + return true; + } else { + return false; + } + } if (!strcasecmp(name,"ping_interval_server_msec")) { int intv=atoi(value); if (intv >= 1000 && intv <= 7*24*3600*1000) { @@ -1995,6 +2011,7 @@ void MySQL_Thread::refresh_variables() { mysql_thread___default_query_timeout=GloMTH->get_variable_int((char *)"default_query_timeout"); mysql_thread___default_max_latency_ms=GloMTH->get_variable_int((char *)"default_max_latency_ms"); mysql_thread___long_query_time=GloMTH->get_variable_int((char *)"long_query_time"); + mysql_thread___query_cache_size_MB=GloMTH->get_variable_int((char *)"query_cache_size_MB"); mysql_thread___ping_interval_server_msec=GloMTH->get_variable_int((char *)"ping_interval_server_msec"); mysql_thread___ping_timeout_server=GloMTH->get_variable_int((char *)"ping_timeout_server"); mysql_thread___shun_on_failures=GloMTH->get_variable_int((char *)"shun_on_failures"); From 41acb178eeab30353cfa6f198f811a1778a16aea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 16 Aug 2016 23:38:30 +0000 Subject: [PATCH 3/3] mysql_thread___query_cache_size_MB refreshed in QC Issue #171 --- lib/Query_Cache.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/Query_Cache.cpp b/lib/Query_Cache.cpp index 9c7dd1e71..7dea95995 100644 --- a/lib/Query_Cache.cpp +++ b/lib/Query_Cache.cpp @@ -27,8 +27,9 @@ #else #define DEB "" #endif /* DEBUG */ -#define QUERY_CACHE_VERSION "0.2.0902" DEB +#define QUERY_CACHE_VERSION "1.2.0817" DEB +extern MySQL_Threads_Handler *GloMTH; typedef btree::btree_map BtMap_cache; @@ -340,16 +341,25 @@ uint64_t Query_Cache::flush() { void * Query_Cache::purgeHash_thread(void *) { unsigned int i; + unsigned int MySQL_Monitor__thread_MySQL_Thread_Variables_version; + MySQL_Thread * mysql_thr = new MySQL_Thread(); + MySQL_Monitor__thread_MySQL_Thread_Variables_version=GloMTH->get_global_version(); while (shutdown==0) { usleep(purge_loop_time); unsigned long long t=monotonic_time()/1000; QCnow_ms=t; - + unsigned int glover=GloMTH->get_global_version(); + if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover ) { + MySQL_Monitor__thread_MySQL_Thread_Variables_version=glover; + mysql_thr->refresh_variables(); + max_memory_size=mysql_thread___query_cache_size_MB*1024*1024; + } if (current_used_memory_pct() < purge_threshold_pct_min ) continue; for (i=0; ipurge_some(QCnow_ms); } } + delete mysql_thr; return NULL; };