From d9a2df7039be97d535fe87bd47ece4d461507ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Wed, 13 May 2015 07:18:53 +0000 Subject: [PATCH] Query Cache is not a plugin anymore --- include/query_cache.hpp | 122 +++++------- lib/Makefile | 8 +- lib/MySQL_Authentication.cpp | 2 + lib/Query_Cache.cpp | 332 +++++++++++++++++++++++++++++++ lib/Standard_Query_Cache.cpp | 374 ----------------------------------- src/main.cpp | 7 +- 6 files changed, 388 insertions(+), 457 deletions(-) create mode 100644 lib/Query_Cache.cpp delete mode 100644 lib/Standard_Query_Cache.cpp diff --git a/include/query_cache.hpp b/include/query_cache.hpp index eee577117..09bc78485 100644 --- a/include/query_cache.hpp +++ b/include/query_cache.hpp @@ -4,107 +4,77 @@ #include "cpp.h" -/* -#define EXPIRE_DROPIT 0 -#define SHARED_QUERY_CACHE_HASH_TABLES 16 -#define HASH_EXPIRE_MAX 3600*24*365*10 -#define DEFAULT_purge_loop_time 400000 +#define EXPIRE_DROPIT 0 +#define SHARED_QUERY_CACHE_HASH_TABLES 32 +#define HASH_EXPIRE_MAX 3600*24*365*10 +#define DEFAULT_purge_loop_time 500000 #define DEFAULT_purge_total_time 10000000 -#define DEFAULT_purge_threshold_pct_min 50 -#define DEFAULT_purge_threshold_pct_max 90 - -*/ -//#define SHARED_QUERY_CACHE_HASH_TABLES 16 -//#define DEFAULT_SQC_size 64*1024*1024 -/* -typedef struct __fdb_hash_t fdb_hash_t; -typedef struct __fdb_hash_entry fdb_hash_entry; -*/ -/* -struct __fdb_hash_t { - pthread_rwlock_t lock; - GHashTable *hash; - GPtrArray *ptrArray; - uint64_t dataSize; - uint64_t purgeChunkSize; - uint64_t purgeIdx; -}; +#define DEFAULT_purge_threshold_pct_min 3 +#define DEFAULT_purge_threshold_pct_max 90 +class KV_BtreeArray; +typedef struct __QC_entry_t QC_entry_t; -struct __fdb_hash_entry { - unsigned char *key; - unsigned char *value; - fdb_hash_t *hash; - struct __fdb_hash_entry *self; +struct __QC_entry_t { + uint64_t key; + char *value; + KV_BtreeArray *kv; + QC_entry_t *self; uint32_t klen; uint32_t length; time_t expire; time_t access; uint32_t ref_count; }; -*/ +typedef btree::btree_map BtMap_cache; +class KV_BtreeArray { + private: + rwlock_t lock; + BtMap_cache bt_map; + PtrArray *ptrArray; + uint64_t purgeChunkSize; + uint64_t purgeIdx; + bool __insert(uint64_t, void *); + uint64_t freeable_memory; + public: + uint64_t tottopurge; + KV_BtreeArray(); + ~KV_BtreeArray(); + uint64_t get_data_size(); + void purge_some(time_t QCnow); + int cnt(); + bool replace(uint64_t key, QC_entry_t *entry); + QC_entry_t *lookup(uint64_t key); + void empty(); +}; + class Query_Cache { -// protected: -// int test; -// int whatever; + private: + KV_BtreeArray KVs[SHARED_QUERY_CACHE_HASH_TABLES]; + uint64_t get_data_size_total(); + unsigned int current_used_memory_pct(); public: - virtual void * purgeHash_thread(void *) { return NULL; } ; + void * purgeHash_thread(void *); int size; int shutdown; time_t QCnow; pthread_t purge_thread_id; - //fdb_hash_t fdb_hashes[SHARED_QUERY_CACHE_HASH_TABLES]; - //fdb_hash_t fdb_hashes[]; unsigned int purge_loop_time; unsigned int purge_total_time; unsigned int purge_threshold_pct_min; unsigned int purge_threshold_pct_max; -// unsigned int hash_expire_default; uint64_t max_memory_size; - //uint64_t cntDel; - //uint64_t cntGet; - //uint64_t cntGetOK; - //uint64_t cntSet; - -// uint64_t cntSetERR; -// uint64_t cntPurge; -// uint64_t size_keys; -// uint64_t size_values; -// uint64_t size_metas; -// uint64_t dataIN; -// uint64_t dataOUT; - //Shared_Query_Cache() {} - //Query_Cache(uint64_t _max_memory_size=DEFAULT_SQC_size); - //Query_Cache() {max_memory_size=0;} - //Query_Cache() { whatever=0; } - Query_Cache() {}; - virtual ~Query_Cache() {}; - virtual const char *version() {return NULL;}; - virtual void print_version() {}; - virtual bool set(unsigned char *, uint32_t, unsigned char *, uint32_t, time_t) { return true;}; - virtual unsigned char * get(const unsigned char *, uint32_t *) { return 0;}; - virtual uint64_t flush() {return 0;}; - virtual uint64_t empty() {return 0;}; -// virtual uint64_t current_free_memory() {return 0;}; -// virtual unsigned int current_used_memory_pct() {return 0;} -// void set_side_length(double side_length) { -// test = side_length; -// } -// virtual double area() {return 0;}; -// int aa; -// void register_LQC(Local_Query_Cache *); -// void unregister_LQC(Local_Query_Cache *); + Query_Cache(); + ~Query_Cache(); + void print_version(); + bool set(unsigned char *, uint32_t, unsigned char *, uint32_t, time_t); + unsigned char * get(const unsigned char *, uint32_t *); + uint64_t flush(); }; - - -// the types of the class factories -typedef Query_Cache* create_QC_t(); -typedef void destroy_QC_t(Query_Cache*); - #endif /* __CLASS_QUERY_CACHE_H */ diff --git a/lib/Makefile b/lib/Makefile index 8121f2f3c..bc95e3e45 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -60,7 +60,7 @@ OBJ = $(patsubst %,$(ODIR)/%,$(_OBJ)) #_OBJ_CPP = ProxySQL_GloVars.oo network.oo debug.oo proxysql_global.oo configfile.oo Standard_Query_Cache.oo SpookyV2.oo Standard_MySQL_Thread.oo Standard_MySQL_Authentication.oo gen_utils.oo mysql_connection_pool.oo simple_kv.oo advanced_kv.oo sqlite3db.oo mysql_connection.oo global_variables.oo proxysql_mem.oo MySQL_Protocol.oo mysql_data_stream.oo mysql_session.oo mysql_backend.oo #_OBJ_CPP = ProxySQL_GloVars.oo network.oo debug.oo proxysql_global.oo configfile.oo Standard_Query_Cache.oo SpookyV2.oo Standard_MySQL_Authentication.oo gen_utils.oo mysql_connection_pool.oo simple_kv.oo advanced_kv.oo sqlite3db.oo mysql_connection.oo global_variables.oo proxysql_mem.oo MySQL_Protocol.oo mysql_data_stream.oo mysql_session.oo mysql_backend.oo #_OBJ_CPP = ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Standard_Query_Cache.oo SpookyV2.oo Standard_MySQL_Authentication.oo gen_utils.oo mysql_connection_pool.oo simple_kv.oo advanced_kv.oo sqlite3db.oo mysql_connection.oo global_variables.oo proxysql_mem.oo MySQL_Protocol.oo mysql_data_stream.oo MySQL_Session.oo mysql_backend.oo MySQL_HostGroups_Manager.oo -_OBJ_CPP = ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Standard_Query_Cache.oo SpookyV2.oo MySQL_Authentication.oo gen_utils.oo simple_kv.oo sqlite3db.oo global_variables.oo proxysql_mem.oo mysql_connection.oo MySQL_HostGroups_Manager.oo mysql_data_stream.oo MySQL_Thread.oo MySQL_Session.oo MySQL_Protocol.oo mysql_backend.oo Query_Processor.oo +_OBJ_CPP = ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Query_Cache.oo SpookyV2.oo MySQL_Authentication.oo gen_utils.oo simple_kv.oo sqlite3db.oo global_variables.oo proxysql_mem.oo mysql_connection.oo MySQL_HostGroups_Manager.oo mysql_data_stream.oo MySQL_Thread.oo MySQL_Session.oo MySQL_Protocol.oo mysql_backend.oo Query_Processor.oo OBJ_CPP = $(patsubst %,$(ODIR)/%,$(_OBJ_CPP)) #_SHARED_OBJ= Standard_Query_Cache.ko Standard_MySQL_Thread.ko @@ -81,15 +81,15 @@ Standard_ProxySQL_Admin.so: Standard_ProxySQL_Admin.cpp libproxysql.a #Standard_MySQL_Authentication.so: Standard_MySQL_Authentication.ko libproxysql.a # $(CPP) -shared -fPIC -o $@ $(CPPFLAGS) Standard_MySQL_Authentication.ko libproxysql.a -lcrypto $(LIBS) -Standard_Query_Cache.so: Standard_Query_Cache.ko libproxysql.a - $(CPP) -shared -fPIC -o $@ $(CPPFLAGS) Standard_Query_Cache.ko libproxysql.a -lcrypto -linjection $(LIBS) +#Standard_Query_Cache.so: Standard_Query_Cache.ko libproxysql.a +# $(CPP) -shared -fPIC -o $@ $(CPPFLAGS) Standard_Query_Cache.ko libproxysql.a -lcrypto -linjection $(LIBS) # $(CC) -shared -fPIC -o $@ $(CPPFLAGS) $(ODIR)/Standard_Query_Cache.oo $(ODIR)/SpookyV2.oo $(ODIR)/gen_utils.oo #Standard_MySQL_Thread.so: Standard_MySQL_Thread.ko libproxysql.a # $(CPP) -shared -fPIC -o $@ $(CPPFLAGS) Standard_MySQL_Thread.ko libproxysql.a -lcrypto $(LIBS) # $(CC) -shared -fPIC -o $@ $(CPPFLAGS) $(ODIR)/Standard_MySQL_Thread.oo $(ODIR)/mysql_session.oo -LOADABLE_LIB = Standard_ProxySQL_Admin.so Standard_Query_Cache.so +LOADABLE_LIB = Standard_ProxySQL_Admin.so # Standard_Query_Cache.so DYN_LIB: $(LOADABLE_LIB) .PHONY: DYN_LIB diff --git a/lib/MySQL_Authentication.cpp b/lib/MySQL_Authentication.cpp index 9512db696..ed0073b3a 100644 --- a/lib/MySQL_Authentication.cpp +++ b/lib/MySQL_Authentication.cpp @@ -52,6 +52,8 @@ MySQL_Authentication::MySQL_Authentication() { MySQL_Authentication::~MySQL_Authentication() { reset(); + delete creds_backends.cred_array; + delete creds_frontends.cred_array; }; void MySQL_Authentication::print_version() { diff --git a/lib/Query_Cache.cpp b/lib/Query_Cache.cpp new file mode 100644 index 000000000..74ddaa6e6 --- /dev/null +++ b/lib/Query_Cache.cpp @@ -0,0 +1,332 @@ +#include "btree_map.h" +#include "proxysql.h" +#include "cpp.h" +#include "proxysql_atomic.h" +#include "SpookyV2.h" + +#define THR_UPDATE_CNT(__a, __b, __c, __d) \ + do {\ + __a+=__c; \ + if (__a>=__d) { \ + __sync_fetch_and_add(&__b, __a - __a % __d); __a = __a % __d; \ + } \ + } while(0) + +#define THR_DECREASE_CNT(__a, __b, __c, __d) \ + do {\ + __a+=__c; \ + if (__a>=__d) { \ + __sync_fetch_and_sub(&__b, __a - __a % __d); __a = __a % __d; \ + } \ + } while(0) + + +#ifdef DEBUG +#define DEB "_DEBUG" +#else +#define DEB "" +#endif /* DEBUG */ +#define QUERY_CACHE_VERSION "0.1.0629" DEB + +__thread uint64_t __thr_cntSet=0; +__thread uint64_t __thr_cntGet=0; +__thread uint64_t __thr_cntGetOK=0; +__thread uint64_t __thr_dataIN=0; +__thread uint64_t __thr_dataOUT=0; +__thread uint64_t __thr_num_entries=0; +__thread uint64_t __thr_num_deleted=0; +__thread uint64_t __thr_size_values=0; +//__thread uint64_t __thr_freeable_memory=0; + +#define DEFAULT_SQC_size 4*1024*1024 + + +static uint64_t Glo_cntSet=0; +static uint64_t Glo_cntGet=0; +static uint64_t Glo_cntGetOK=0; +static uint64_t Glo_num_entries=0; +static uint64_t Glo_dataIN=0; +static uint64_t Glo_dataOUT=0; +static uint64_t Glo_cntPurge=0; +static uint64_t Glo_size_values=0; +static uint64_t Glo_total_freed_memory; + + +/* +class KV_BtreeArray; + +typedef struct __QC_entry_t QC_entry_t; + +struct __QC_entry_t { + uint64_t key; + char *value; + KV_BtreeArray *kv; + QC_entry_t *self; + uint32_t klen; + uint32_t length; + time_t expire; + time_t access; + uint32_t ref_count; +}; + +typedef btree::btree_map BtMap; +*/ + + + + + KV_BtreeArray::KV_BtreeArray() { + freeable_memory=0; + tottopurge=0; + spinlock_rwlock_init(&lock); + ptrArray = new PtrArray; +}; + +KV_BtreeArray::~KV_BtreeArray() { + proxy_debug(PROXY_DEBUG_QUERY_CACHE, 3, "Size of KVBtreeArray:%d , ptrArray:%llu\n", cnt() , ptrArray->len); + empty(); + QC_entry_t *qce=NULL; + while (ptrArray->len) { + qce=(QC_entry_t *)ptrArray->remove_index_fast(0); + free(qce->value); + free(qce); + } + delete ptrArray; +}; + + +uint64_t KV_BtreeArray::get_data_size() { + uint64_t r = __sync_fetch_and_add(&Glo_num_entries,0) * (sizeof(QC_entry_t)+sizeof(QC_entry_t *)*2+sizeof(uint64_t)*2) + __sync_fetch_and_add(&Glo_size_values,0) ; + return r; +}; + +void KV_BtreeArray::purge_some(time_t QCnow) { + uint64_t ret=0, i, _size=0; + QC_entry_t *qce; + spin_rdlock(&lock); + for (i=0; ilen;i++) { + qce=(QC_entry_t *)ptrArray->index(i); + if (qce->expire==EXPIRE_DROPIT || qce->expirelength; + } + } + freeable_memory=_size; + spin_rdunlock(&lock); + if ( (freeable_memory + ret * (sizeof(QC_entry_t)+sizeof(QC_entry_t *)*2+sizeof(uint64_t)*2) ) > get_data_size()*0.01) { + uint64_t removed_entries=0; + uint64_t freed_memory=0; + spin_wrlock(&lock); + for (i=0; ilen;i++) { + qce=(QC_entry_t *)ptrArray->index(i); + if ((qce->expire==EXPIRE_DROPIT || qce->expireref_count,0)<=1)) { + qce=(QC_entry_t *)ptrArray->remove_index_fast(i); + + btree::btree_map::iterator lookup; + lookup = bt_map.find(qce->key); + if (lookup != bt_map.end()) { + bt_map.erase(lookup); + } + i--; + freed_memory+=qce->length; + removed_entries++; + free(qce->value); + free(qce); + } + } + spin_wrunlock(&lock); + THR_DECREASE_CNT(__thr_num_deleted,Glo_num_entries,removed_entries,1); + if (removed_entries) { + __sync_fetch_and_add(&Glo_total_freed_memory,freed_memory); + __sync_fetch_and_sub(&Glo_size_values,freed_memory); + __sync_fetch_and_add(&Glo_cntPurge,removed_entries); +// if (removed_entries) fprintf(stderr,"Removed: %lu, total: %lu, arraylen: %d\n", removed_entries, __sync_fetch_and_sub(&Glo_num_entries,0), ptrArray.len); +// if (removed_entries) firintf(stderr,"Size of KVBtreeArray:%d , freed_memory:%lu, Glo_cntGet:%lu, Glo_cntGetOK:%lu, Glo_cntSet:%lu, cntPurge:%lu, dataIN:%lu, dataOUT:%lu\n", cnt() , Glo_total_freed_memory, Glo_cntGet, Glo_cntGetOK, Glo_cntSet, Glo_cntPurge, Glo_dataIN, Glo_dataOUT); + } + } +}; + +int KV_BtreeArray::cnt() { + return bt_map.size(); +}; + +bool KV_BtreeArray::replace(uint64_t key, QC_entry_t *entry) { + spin_wrlock(&lock); + THR_UPDATE_CNT(__thr_cntSet,Glo_cntSet,1,100); + THR_UPDATE_CNT(__thr_size_values,Glo_size_values,entry->length,100); + THR_UPDATE_CNT(__thr_dataIN,Glo_dataIN,entry->length,100); + THR_UPDATE_CNT(__thr_num_entries,Glo_num_entries,1,1); + + entry->ref_count=1; + ptrArray->add(entry); + btree::btree_map::iterator lookup; + lookup = bt_map.find(key); + if (lookup != bt_map.end()) { + lookup->second->expire=EXPIRE_DROPIT; + __sync_fetch_and_sub(&lookup->second->ref_count,1); + bt_map.erase(lookup); + } + bt_map.insert(std::make_pair(key,entry)); + spin_wrunlock(&lock); + return true; +} + +QC_entry_t * KV_BtreeArray::lookup(uint64_t key) { + QC_entry_t *entry=NULL; + spin_rdlock(&lock); + THR_UPDATE_CNT(__thr_cntGet,Glo_cntGet,1,100); + btree::btree_map::iterator lookup; + lookup = bt_map.find(key); + if (lookup != bt_map.end()) { + entry=lookup->second; + __sync_fetch_and_add(&entry->ref_count,1); + THR_UPDATE_CNT(__thr_cntGetOK,Glo_cntGetOK,1,100); + THR_UPDATE_CNT(__thr_dataOUT,Glo_dataOUT,entry->length,10000); + } + spin_rdunlock(&lock); + return entry; +}; + +void KV_BtreeArray::empty() { + spin_wrlock(&lock); + + btree::btree_map::iterator lookup; + + while (bt_map.size()) { + lookup = bt_map.begin(); + if ( lookup != bt_map.end() ) { + lookup->second->expire=EXPIRE_DROPIT; + //const char *f=lookup->first; + bt_map.erase(lookup); + } + } + spin_wrunlock(&lock); +}; + + + + +uint64_t Query_Cache::get_data_size_total() { + int r=0; + int i; + for (i=0; i 100) return 100; + int pct=pctf; + return pct; +} + + + +Query_Cache::Query_Cache() { +#ifdef DEBUG + if (glovars.has_debug==false) { +#else + if (glovars.has_debug==true) { +#endif /* DEBUG */ + perror("Incompatible debagging version"); + exit(EXIT_FAILURE); + } + QCnow=time(NULL); + size=SHARED_QUERY_CACHE_HASH_TABLES; + shutdown=0; + purge_loop_time=DEFAULT_purge_loop_time; + purge_total_time=DEFAULT_purge_total_time; + purge_threshold_pct_min=DEFAULT_purge_threshold_pct_min; + purge_threshold_pct_max=DEFAULT_purge_threshold_pct_max; + max_memory_size=DEFAULT_SQC_size; +}; + +void Query_Cache::print_version() { + fprintf(stderr,"In memory Standard Query Cache (SQC) rev. %s -- %s -- %s\n", QUERY_CACHE_VERSION, __FILE__, __TIMESTAMP__); +}; + +Query_Cache::~Query_Cache() { + unsigned int i; + for (i=0; iexpire > t) { + result=(unsigned char *)malloc(entry->length); + memcpy(result,entry->value,entry->length); + *lv=entry->length; + if (t > entry->access) entry->access=t; + } + __sync_fetch_and_sub(&entry->ref_count,1); + } + return result; +} + +bool Query_Cache::set(unsigned char *kp, uint32_t kl, unsigned char *vp, uint32_t vl, time_t expire) { + QC_entry_t *entry = (QC_entry_t *)malloc(sizeof(QC_entry_t)); + entry->klen=kl; + entry->length=vl; + entry->ref_count=0; + + entry->value=(char *)malloc(vl); + memcpy(entry->value,vp,vl); + entry->self=entry; + entry->access=QCnow; + if (expire > HASH_EXPIRE_MAX) { + entry->expire=expire; // expire is a unix timestamp + } else { + entry->expire=QCnow+expire; // expire is seconds + } + uint64_t hk=SpookyHash::Hash64(kp,strlen((const char *)kp),0); + unsigned char i=hk%SHARED_QUERY_CACHE_HASH_TABLES; + entry->key=hk; + KVs[i].replace(hk, entry); + + return true; +} + +uint64_t Query_Cache::flush() { + int i; + uint64_t total_count=0; + for (i=0; i=__d) { \ - __sync_fetch_and_add(&__b, __a - __a % __d); __a = __a % __d; \ - } \ - } while(0) - -#define THR_DECREASE_CNT(__a, __b, __c, __d) \ - do {\ - __a+=__c; \ - if (__a>=__d) { \ - __sync_fetch_and_sub(&__b, __a - __a % __d); __a = __a % __d; \ - } \ - } while(0) - - -#ifdef DEBUG -#define DEB "_DEBUG" -#else -#define DEB "" -#endif /* DEBUG */ -#define QUERY_CACHE_VERSION "0.1.0629" DEB - -__thread uint64_t __thr_cntSet=0; -__thread uint64_t __thr_cntGet=0; -__thread uint64_t __thr_cntGetOK=0; -__thread uint64_t __thr_dataIN=0; -__thread uint64_t __thr_dataOUT=0; -__thread uint64_t __thr_num_entries=0; -__thread uint64_t __thr_num_deleted=0; -__thread uint64_t __thr_size_values=0; -//__thread uint64_t __thr_freeable_memory=0; - -#define DEFAULT_SQC_size 4*1024*1024 - - -static uint64_t Glo_cntSet=0; -static uint64_t Glo_cntGet=0; -static uint64_t Glo_cntGetOK=0; -static uint64_t Glo_num_entries=0; -static uint64_t Glo_dataIN=0; -static uint64_t Glo_dataOUT=0; -static uint64_t Glo_cntPurge=0; -static uint64_t Glo_size_values=0; -static uint64_t Glo_total_freed_memory; - -class KV_BtreeArray; - -typedef struct __QC_entry_t QC_entry_t; - -struct __QC_entry_t { - uint64_t key; - char *value; - KV_BtreeArray *kv; - QC_entry_t *self; - uint32_t klen; - uint32_t length; - time_t expire; - time_t access; - uint32_t ref_count; -}; - -typedef btree::btree_map BtMap; - - - - -class KV_BtreeArray { - - private: - rwlock_t lock; - BtMap bt_map; - PtrArray ptrArray; - uint64_t purgeChunkSize; - uint64_t purgeIdx; - bool __insert(uint64_t, void *); - uint64_t freeable_memory; - public: - uint64_t tottopurge; - KV_BtreeArray() { - freeable_memory=0; - tottopurge=0; - spinlock_rwlock_init(&lock); - }; - - ~KV_BtreeArray() { - proxy_debug(PROXY_DEBUG_QUERY_CACHE, 3, "Size of KVBtreeArray:%d , ptrArray:%llu\n", cnt() , ptrArray.len); - empty(); - QC_entry_t *qce=NULL; - while (ptrArray.len) { - qce=(QC_entry_t *)ptrArray.remove_index_fast(0); - free(qce->value); - free(qce); - } - }; - - - uint64_t get_data_size() { - uint64_t r = __sync_fetch_and_add(&Glo_num_entries,0) * (sizeof(QC_entry_t)+sizeof(QC_entry_t *)*2+sizeof(uint64_t)*2) + __sync_fetch_and_add(&Glo_size_values,0) ; - return r; - }; - - void purge_some(time_t QCnow) { - uint64_t ret=0, i, _size=0; - QC_entry_t *qce; - spin_rdlock(&lock); - for (i=0; iexpire==EXPIRE_DROPIT || qce->expirelength; - } - } - freeable_memory=_size; - spin_rdunlock(&lock); - if ( (freeable_memory + ret * (sizeof(QC_entry_t)+sizeof(QC_entry_t *)*2+sizeof(uint64_t)*2) ) > get_data_size()*0.01) { - uint64_t removed_entries=0; - uint64_t freed_memory=0; - spin_wrlock(&lock); - for (i=0; iexpire==EXPIRE_DROPIT || qce->expireref_count,0)<=1)) { - qce=(QC_entry_t *)ptrArray.remove_index_fast(i); - - btree::btree_map::iterator lookup; - lookup = bt_map.find(qce->key); - if (lookup != bt_map.end()) { - bt_map.erase(lookup); - } - i--; - freed_memory+=qce->length; - removed_entries++; - free(qce->value); - free(qce); - } - } - spin_wrunlock(&lock); - THR_DECREASE_CNT(__thr_num_deleted,Glo_num_entries,removed_entries,1); - if (removed_entries) { - __sync_fetch_and_add(&Glo_total_freed_memory,freed_memory); - __sync_fetch_and_sub(&Glo_size_values,freed_memory); - __sync_fetch_and_add(&Glo_cntPurge,removed_entries); -// if (removed_entries) fprintf(stderr,"Removed: %lu, total: %lu, arraylen: %d\n", removed_entries, __sync_fetch_and_sub(&Glo_num_entries,0), ptrArray.len); -// if (removed_entries) firintf(stderr,"Size of KVBtreeArray:%d , freed_memory:%lu, Glo_cntGet:%lu, Glo_cntGetOK:%lu, Glo_cntSet:%lu, cntPurge:%lu, dataIN:%lu, dataOUT:%lu\n", cnt() , Glo_total_freed_memory, Glo_cntGet, Glo_cntGetOK, Glo_cntSet, Glo_cntPurge, Glo_dataIN, Glo_dataOUT); - } - } - }; - - int cnt() { - return bt_map.size(); - }; - - bool replace(uint64_t key, QC_entry_t *entry) { - spin_wrlock(&lock); - THR_UPDATE_CNT(__thr_cntSet,Glo_cntSet,1,100); - THR_UPDATE_CNT(__thr_size_values,Glo_size_values,entry->length,100); - THR_UPDATE_CNT(__thr_dataIN,Glo_dataIN,entry->length,100); - THR_UPDATE_CNT(__thr_num_entries,Glo_num_entries,1,1); - - entry->ref_count=1; - ptrArray.add(entry); - btree::btree_map::iterator lookup; - lookup = bt_map.find(key); - if (lookup != bt_map.end()) { - lookup->second->expire=EXPIRE_DROPIT; - __sync_fetch_and_sub(&lookup->second->ref_count,1); - bt_map.erase(lookup); - } - bt_map.insert(std::make_pair(key,entry)); - spin_wrunlock(&lock); - return true; - } - - QC_entry_t *lookup(uint64_t key) { - QC_entry_t *entry=NULL; - spin_rdlock(&lock); - THR_UPDATE_CNT(__thr_cntGet,Glo_cntGet,1,100); - btree::btree_map::iterator lookup; - lookup = bt_map.find(key); - if (lookup != bt_map.end()) { - entry=lookup->second; - __sync_fetch_and_add(&entry->ref_count,1); - THR_UPDATE_CNT(__thr_cntGetOK,Glo_cntGetOK,1,100); - THR_UPDATE_CNT(__thr_dataOUT,Glo_dataOUT,entry->length,10000); - } - spin_rdunlock(&lock); - return entry; - }; - - void empty() { - spin_wrlock(&lock); - - btree::btree_map::iterator lookup; - - while (bt_map.size()) { - lookup = bt_map.begin(); - if ( lookup != bt_map.end() ) { - lookup->second->expire=EXPIRE_DROPIT; - //const char *f=lookup->first; - bt_map.erase(lookup); - } - } - spin_wrunlock(&lock); - }; - -}; - - - -class Standard_Query_Cache: public Query_Cache { - - -private: -KV_BtreeArray KVs[SHARED_QUERY_CACHE_HASH_TABLES]; - - - -uint64_t get_data_size_total() { - int r=0; - int i; - for (i=0; i 100) return 100; - int pct=pctf; - return pct; -} - - -public: -//Standard_Query_Cache(uint64_t _max_memory_size) { - -virtual double area() const { - return max_memory_size*rand(); -}; - -Standard_Query_Cache() { -#ifdef DEBUG - if (glovars.has_debug==false) { -#else - if (glovars.has_debug==true) { -#endif /* DEBUG */ - perror("Incompatible debagging version"); - exit(EXIT_FAILURE); - } - QCnow=time(NULL); - size=SHARED_QUERY_CACHE_HASH_TABLES; - shutdown=0; - purge_loop_time=DEFAULT_purge_loop_time; - purge_total_time=DEFAULT_purge_total_time; - purge_threshold_pct_min=DEFAULT_purge_threshold_pct_min; - purge_threshold_pct_max=DEFAULT_purge_threshold_pct_max; - max_memory_size=DEFAULT_SQC_size; -}; - -virtual void print_version() { - fprintf(stderr,"In memory Standard Query Cache (SQC) rev. %s -- %s -- %s\n", QUERY_CACHE_VERSION, __FILE__, __TIMESTAMP__); -}; - -virtual ~Standard_Query_Cache() { - - unsigned int i; - - - for (i=0; iexpire > t) { - result=(unsigned char *)malloc(entry->length); - memcpy(result,entry->value,entry->length); - *lv=entry->length; - if (t > entry->access) entry->access=t; - } - __sync_fetch_and_sub(&entry->ref_count,1); - } - return result; -} - -virtual bool set(unsigned char *kp, uint32_t kl, unsigned char *vp, uint32_t vl, time_t expire) { - QC_entry_t *entry = (QC_entry_t *)malloc(sizeof(QC_entry_t)); - entry->klen=kl; - entry->length=vl; - entry->ref_count=0; - - entry->value=(char *)malloc(vl); - memcpy(entry->value,vp,vl); - entry->self=entry; - entry->access=QCnow; - if (expire > HASH_EXPIRE_MAX) { - entry->expire=expire; // expire is a unix timestamp - } else { - entry->expire=QCnow+expire; // expire is seconds - } - uint64_t hk=SpookyHash::Hash64(kp,strlen((const char *)kp),0); - unsigned char i=hk%SHARED_QUERY_CACHE_HASH_TABLES; - entry->key=hk; - KVs[i].replace(hk, entry); - - return true; -} - -virtual uint64_t flush() { - int i; - uint64_t total_count=0; - for (i=0; iprint_version(); pthread_create(&GloQC->purge_thread_id, NULL, mysql_shared_query_cache_funct , NULL); //void *(*__f)(void *) = (void* (*)(void*))&SQC->purgeHash_thread;