aherman
/
proxysql
mirror of https://github.com/sysown/proxysql
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'v1.2.2-issue140' into v1.2.2

Browse Source
pull/674/head
René Cannaò 10 years ago
parent f50e95f1f1 41acb178ee
commit b5fb363d1c
11 changed files with 128 additions and 40 deletions
Show Stats Download Patch File Download Diff File

8
include/MySQL_Authentication.hpp
Unescape View File

@ -1,7 +1,7 @@
#ifndef __CLASS_MYSQL_AUTHENTICATION_H
#define __CLASS_MYSQL_AUTHENTICATION_H
#include "btree_map.h"
//#include "btree_map.h"
#include "proxysql.h"
#include "cpp.h"
@ -36,13 +36,15 @@ typedef struct _account_details_t {
*/
typedef btree::btree_map<uint64_t, account_details_t *> BtMap_auth;
//typedef btree::btree_map<uint64_t, account_details_t *> BtMap_auth;
typedef std::unordered_map<uint64_t, account_details_t *> umap_auth;
class PtrArray;
typedef struct _creds_group_t {
rwlock_t lock;
BtMap_auth bt_map;
//BtMap_auth bt_map;
umap_auth bt_map;
PtrArray *cred_array;
} creds_group_t;

1
include/MySQL_Thread.h
Unescape View File

@ -314,6 +314,7 @@ class MySQL_Threads_Handler
char * ssl_p2s_cert;
char * ssl_p2s_key;
char * ssl_p2s_cipher;
int query_cache_size_MB;
} variables;
PtrArray *bind_fds;
MySQL_Listeners_Manager *MLM;

22
include/cpp.h
Unescape View File

@ -6,7 +6,7 @@
#include "query_cache.hpp"
#include "mysql_connection.h"
#include "sqlite3db.h"
#include "simple_kv.h"
//#include "simple_kv.h"
#include "gen_utils.h"
#include "StatCounters.h"
#include "MySQL_Monitor.hpp"
@ -21,17 +21,17 @@
#undef swap
#undef min
#undef max
#include <memory>
//#include <memory>
#include <stdio.h>
#include <algorithm>
#include <functional>
#include <type_traits>
#include <iosfwd>
//#include <algorithm>
//#include <functional>
//#include <type_traits>
//#include <iosfwd>
#include <map>
#include <unordered_map>
#include <set>
#include <sstream>
#include <string>
#include <utility>
#include <vector>
//#include <set>
//#include <sstream>
//#include <string>
//#include <utility>
//#include <vector>
//#endif /* __cplusplus */

18
include/proxysql.h
Unescape View File

@ -1,7 +1,15 @@
#ifdef __cplusplus
#include <string>
#include <stack>
#include "btree_map.h"
#include <algorithm>
//#include <functional>
//#include <memory>
//#include <string>
//#include <utility>
//#include "btree_map.h"
#ifndef EZOPTION
//#include "ezOptionParser.hpp"
#define EZOPTION
@ -70,8 +78,14 @@
#include "jemalloc.h"
#ifdef DEBUG
//#define VALGRIND_ENABLE_ERROR_REPORTING
//#define VALGRIND_DISABLE_ERROR_REPORTING
#include "valgrind.h"
#else
#define VALGRIND_ENABLE_ERROR_REPORTING
#define VALGRIND_DISABLE_ERROR_REPORTING
#endif /* DEBUG */
#include "sqlite3.h"

12
include/proxysql_structs.h
Unescape View File

@ -270,8 +270,8 @@ class SQLite3DB;
class SimpleKV;
class AdvancedKV;
class ProxySQL_Poll;
//class Query_Cache;
class Shared_Query_Cache;
class Query_Cache;
//class Shared_Query_Cache;
class MySQL_Authentication;
class MySQL_Connection;
class MySQL_Protocol;
@ -507,7 +507,7 @@ struct _global_variables_t {
unsigned int mysql_query_cache_default_timeout;
unsigned long long mysql_wait_timeout;
unsigned long long mysql_query_cache_size;
//unsigned long long mysql_query_cache_size;
unsigned long long mysql_max_resultset_size;
int mysql_max_query_size;
@ -737,6 +737,9 @@ __thread bool mysql_thread___query_digests;
__thread bool mysql_thread___default_reconnect;
__thread bool mysql_thread___sessions_sort;
/* variables used for Query Cache */
__thread int mysql_thread___query_cache_size_MB;
/* variables used for SSL , from proxy to server (p2s) */
__thread char * mysql_thread___ssl_p2s_ca;
__thread char * mysql_thread___ssl_p2s_cert;
@ -812,6 +815,9 @@ extern __thread bool mysql_thread___query_digests;
extern __thread bool mysql_thread___default_reconnect;
extern __thread bool mysql_thread___sessions_sort;
/* variables used for Query Cache */
extern __thread int mysql_thread___query_cache_size_MB;
/* variables used for SSL , from proxy to server (p2s) */
extern __thread char * mysql_thread___ssl_p2s_ca;
extern __thread char * mysql_thread___ssl_p2s_cert;

8
include/query_cache.hpp
Unescape View File

@ -28,7 +28,7 @@ struct __QC_entry_t {
unsigned long long access_ms; // when the entry was read last , monotonic , millisecond granularity
uint32_t ref_count; // reference counter
};
/*
typedef btree::btree_map<uint64_t, QC_entry_t *> BtMap_cache;
@ -52,10 +52,12 @@ class KV_BtreeArray {
QC_entry_t *lookup(uint64_t key);
void empty();
};
*/
class KV_BtreeArray;
class Query_Cache {
private:
KV_BtreeArray KVs[SHARED_QUERY_CACHE_HASH_TABLES];
//KV_BtreeArray KVs[SHARED_QUERY_CACHE_HASH_TABLES];
KV_BtreeArray * KVs[SHARED_QUERY_CACHE_HASH_TABLES];
uint64_t get_data_size_total();
unsigned int current_used_memory_pct();
public:

2
lib/Makefile
Unescape View File

@ -47,7 +47,7 @@ default: libproxysql.a
_OBJ = c_tokenizer.o
OBJ = $(patsubst %,$(ODIR)/%,$(_OBJ))
_OBJ_CPP = ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Query_Cache.oo SpookyV2.oo MySQL_Authentication.oo gen_utils.oo simple_kv.oo sqlite3db.oo global_variables.oo mysql_connection.oo MySQL_HostGroups_Manager.oo mysql_data_stream.oo MySQL_Thread.oo MySQL_Session.oo MySQL_Protocol.oo mysql_backend.oo Query_Processor.oo ProxySQL_Admin.oo MySQL_Monitor.oo MySQL_Logger.oo thread.oo
_OBJ_CPP = ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Query_Cache.oo SpookyV2.oo MySQL_Authentication.oo gen_utils.oo sqlite3db.oo global_variables.oo mysql_connection.oo MySQL_HostGroups_Manager.oo mysql_data_stream.oo MySQL_Thread.oo MySQL_Session.oo MySQL_Protocol.oo mysql_backend.oo Query_Processor.oo ProxySQL_Admin.oo MySQL_Monitor.oo MySQL_Logger.oo thread.oo
OBJ_CPP = $(patsubst %,$(ODIR)/%,$(_OBJ_CPP))
%.ko: %.cpp

21
lib/MySQL_Authentication.cpp
Unescape View File

@ -136,7 +136,8 @@ bool MySQL_Authentication::add(char * username, char * password, enum cred_usern
void *sha1_pass=NULL;
char *oldpass=NULL;
spin_wrlock(&cg.lock);
btree::btree_map<uint64_t, account_details_t *>::iterator lookup;
//btree::btree_map<uint64_t, account_details_t *>::iterator lookup;
std::unordered_map<uint64_t, account_details_t *>::iterator lookup;
lookup = cg.bt_map.find(hash1);
if (lookup != cg.bt_map.end()) {
account_details_t *ad=lookup->second;
@ -264,7 +265,8 @@ int MySQL_Authentication::increase_frontend_user_connections(char *username) {
creds_group_t &cg=creds_frontends;
int ret=0;
spin_wrlock(&cg.lock);
btree::btree_map<uint64_t, account_details_t *>::iterator it;
//btree::btree_map<uint64_t, account_details_t *>::iterator it;
std::unordered_map<uint64_t, account_details_t *>::iterator it;
it = cg.bt_map.find(hash1);
if (it != cg.bt_map.end()) {
account_details_t *ad=it->second;
@ -284,7 +286,8 @@ void MySQL_Authentication::decrease_frontend_user_connections(char *username) {
delete myhash;
creds_group_t &cg=creds_frontends;
spin_wrlock(&cg.lock);
btree::btree_map<uint64_t, account_details_t *>::iterator it;
//btree::btree_map<uint64_t, account_details_t *>::iterator it;
std::unordered_map<uint64_t, account_details_t *>::iterator it;
it = cg.bt_map.find(hash1);
if (it != cg.bt_map.end()) {
account_details_t *ad=it->second;
@ -308,7 +311,8 @@ bool MySQL_Authentication::del(char * username, enum cred_username_type usertype
if (set_lock)
spin_wrlock(&cg.lock);
btree::btree_map<uint64_t, account_details_t *>::iterator lookup;
//btree::btree_map<uint64_t, account_details_t *>::iterator lookup;
std::unordered_map<uint64_t, account_details_t *>::iterator lookup;
lookup = cg.bt_map.find(hash1);
if (lookup != cg.bt_map.end()) {
account_details_t *ad=lookup->second;
@ -339,7 +343,8 @@ bool MySQL_Authentication::set_SHA1(char * username, enum cred_username_type use
creds_group_t &cg=(usertype==USERNAME_BACKEND ? creds_backends : creds_frontends);
spin_wrlock(&cg.lock);
btree::btree_map<uint64_t, account_details_t *>::iterator lookup;
//btree::btree_map<uint64_t, account_details_t *>::iterator lookup;
std::unordered_map<uint64_t, account_details_t *>::iterator lookup;
lookup = cg.bt_map.find(hash1);
if (lookup != cg.bt_map.end()) {
account_details_t *ad=lookup->second;
@ -368,7 +373,8 @@ char * MySQL_Authentication::lookup(char * username, enum cred_username_type use
creds_group_t &cg=(usertype==USERNAME_BACKEND ? creds_backends : creds_frontends);
spin_rdlock(&cg.lock);
btree::btree_map<uint64_t, account_details_t *>::iterator lookup;
//btree::btree_map<uint64_t, account_details_t *>::iterator lookup;
std::unordered_map<uint64_t, account_details_t *>::iterator lookup;
lookup = cg.bt_map.find(hash1);
if (lookup != cg.bt_map.end()) {
account_details_t *ad=lookup->second;
@ -396,7 +402,8 @@ bool MySQL_Authentication::_reset(enum cred_username_type usertype) {
creds_group_t &cg=(usertype==USERNAME_BACKEND ? creds_backends : creds_frontends);
spin_wrlock(&cg.lock);
btree::btree_map<uint64_t, account_details_t *>::iterator lookup;
//btree::btree_map<uint64_t, account_details_t *>::iterator lookup;
std::unordered_map<uint64_t, account_details_t *>::iterator lookup;
while (cg.bt_map.size()) {
lookup = cg.bt_map.begin();

17
lib/MySQL_Thread.cpp
Unescape View File

@ -191,6 +191,7 @@ static char * mysql_thread_variables_names[]= {
(char *)"default_query_delay",
(char *)"default_query_timeout",
(char *)"long_query_time",
(char *)"query_cache_size_MB",
(char *)"ping_interval_server_msec",
(char *)"ping_timeout_server",
(char *)"default_schema",
@ -266,6 +267,7 @@ MySQL_Threads_Handler::MySQL_Threads_Handler() {
variables.default_query_delay=0;
variables.default_query_timeout=24*3600*1000;
variables.long_query_time=1000;
variables.query_cache_size_MB=256;
variables.init_connect=NULL;
variables.ping_interval_server_msec=10000;
variables.ping_timeout_server=200;
@ -471,6 +473,7 @@ int MySQL_Threads_Handler::get_variable_int(char *name) {
if (!strcasecmp(name,"default_query_timeout")) return (int)variables.default_query_timeout;
if (!strcasecmp(name,"default_max_latency_ms")) return (int)variables.default_max_latency_ms;
if (!strcasecmp(name,"long_query_time")) return (int)variables.long_query_time;
if (!strcasecmp(name,"query_cache_size_MB")) return (int)variables.query_cache_size_MB;
if (!strcasecmp(name,"free_connections_pct")) return (int)variables.free_connections_pct;
if (!strcasecmp(name,"ping_interval_server_msec")) return (int)variables.ping_interval_server_msec;
if (!strcasecmp(name,"ping_timeout_server")) return (int)variables.ping_timeout_server;
@ -687,6 +690,10 @@ char * MySQL_Threads_Handler::get_variable(char *name) { // this is the public f
sprintf(intbuf,"%d",variables.long_query_time);
return strdup(intbuf);
}
if (!strcasecmp(name,"query_cache_size_MB")) {
sprintf(intbuf,"%d",variables.query_cache_size_MB);
return strdup(intbuf);
}
if (!strcasecmp(name,"ping_interval_server_msec")) {
sprintf(intbuf,"%d",variables.ping_interval_server_msec);
return strdup(intbuf);
@ -1010,6 +1017,15 @@ bool MySQL_Threads_Handler::set_variable(char *name, char *value) { // this is t
return false;
}
}
if (!strcasecmp(name,"query_cache_size_MB")) {
int intv=atoi(value);
if (intv >= 0 && intv <= 1024*10240) {
variables.query_cache_size_MB=intv;
return true;
} else {
return false;
}
}
if (!strcasecmp(name,"ping_interval_server_msec")) {
int intv=atoi(value);
if (intv >= 1000 && intv <= 7*24*3600*1000) {
@ -1995,6 +2011,7 @@ void MySQL_Thread::refresh_variables() {
mysql_thread___default_query_timeout=GloMTH->get_variable_int((char *)"default_query_timeout");
mysql_thread___default_max_latency_ms=GloMTH->get_variable_int((char *)"default_max_latency_ms");
mysql_thread___long_query_time=GloMTH->get_variable_int((char *)"long_query_time");
mysql_thread___query_cache_size_MB=GloMTH->get_variable_int((char *)"query_cache_size_MB");
mysql_thread___ping_interval_server_msec=GloMTH->get_variable_int((char *)"ping_interval_server_msec");
mysql_thread___ping_timeout_server=GloMTH->get_variable_int((char *)"ping_timeout_server");
mysql_thread___shun_on_failures=GloMTH->get_variable_int((char *)"shun_on_failures");

56
lib/Query_Cache.cpp
Unescape View File

@ -1,6 +1,7 @@
#include "btree_map.h"
#include "proxysql.h"
#include "cpp.h"
#include "query_cache.hpp"
#include "proxysql_atomic.h"
#include "SpookyV2.h"
@ -26,7 +27,32 @@
#else
#define DEB ""
#endif /* DEBUG */
#define QUERY_CACHE_VERSION "0.2.0902" DEB
#define QUERY_CACHE_VERSION "1.2.0817" DEB
extern MySQL_Threads_Handler *GloMTH;
typedef btree::btree_map<uint64_t, QC_entry_t *> BtMap_cache;
class KV_BtreeArray {
private:
rwlock_t lock;
BtMap_cache bt_map;
PtrArray *ptrArray;
uint64_t purgeChunkSize;
uint64_t purgeIdx;
bool __insert(uint64_t, void *);
uint64_t freeable_memory;
public:
uint64_t tottopurge;
KV_BtreeArray();
~KV_BtreeArray();
uint64_t get_data_size();
void purge_some(unsigned long long);
int cnt();
bool replace(uint64_t key, QC_entry_t *entry);
QC_entry_t *lookup(uint64_t key);
void empty();
};
__thread uint64_t __thr_cntSet=0;
__thread uint64_t __thr_cntGet=0;
@ -210,7 +236,7 @@ uint64_t Query_Cache::get_data_size_total() {
int r=0;
int i;
for (i=0; i<SHARED_QUERY_CACHE_HASH_TABLES; i++) {
r+=KVs[i].get_data_size();
r+=KVs[i]->get_data_size();
}
return r;
};
@ -236,6 +262,9 @@ Query_Cache::Query_Cache() {
perror("Incompatible debagging version");
exit(EXIT_FAILURE);
}
for (int i=0; i<SHARED_QUERY_CACHE_HASH_TABLES; i++) {
KVs[i]=new KV_BtreeArray();
}
QCnow_ms=monotonic_time()/1000;
size=SHARED_QUERY_CACHE_HASH_TABLES;
shutdown=0;
@ -253,7 +282,7 @@ void Query_Cache::print_version() {
Query_Cache::~Query_Cache() {
unsigned int i;
for (i=0; i<SHARED_QUERY_CACHE_HASH_TABLES; i++) {
// FIXME: what here? I think KV_BtreeArray are automatically destroyed
delete KVs[i];
}
};
@ -265,7 +294,7 @@ unsigned char * Query_Cache::get(uint64_t user_hash, const unsigned char *kp, co
uint64_t hk=SpookyHash::Hash64(kp, kl, user_hash);
unsigned char i=hk%SHARED_QUERY_CACHE_HASH_TABLES;
QC_entry_t *entry=KVs[i].lookup(hk);
QC_entry_t *entry=KVs[i]->lookup(hk);
if (entry!=NULL) {
unsigned long long t=curtime_ms;
@ -294,7 +323,7 @@ bool Query_Cache::set(uint64_t user_hash, const unsigned char *kp, uint32_t kl,
uint64_t hk=SpookyHash::Hash64(kp, kl, user_hash);
unsigned char i=hk%SHARED_QUERY_CACHE_HASH_TABLES;
entry->key=hk;
KVs[i].replace(hk, entry);
KVs[i]->replace(hk, entry);
return true;
}
@ -303,8 +332,8 @@ uint64_t Query_Cache::flush() {
int i;
uint64_t total_count=0;
for (i=0; i<SHARED_QUERY_CACHE_HASH_TABLES; i++) {
total_count+=KVs[i].cnt();
KVs[i].empty();
total_count+=KVs[i]->cnt();
KVs[i]->empty();
}
return total_count;
};
@ -312,16 +341,25 @@ uint64_t Query_Cache::flush() {
void * Query_Cache::purgeHash_thread(void *) {
unsigned int i;
unsigned int MySQL_Monitor__thread_MySQL_Thread_Variables_version;
MySQL_Thread * mysql_thr = new MySQL_Thread();
MySQL_Monitor__thread_MySQL_Thread_Variables_version=GloMTH->get_global_version();
while (shutdown==0) {
usleep(purge_loop_time);
unsigned long long t=monotonic_time()/1000;
QCnow_ms=t;
unsigned int glover=GloMTH->get_global_version();
if (MySQL_Monitor__thread_MySQL_Thread_Variables_version < glover ) {
MySQL_Monitor__thread_MySQL_Thread_Variables_version=glover;
mysql_thr->refresh_variables();
max_memory_size=mysql_thread___query_cache_size_MB*1024*1024;
}
if (current_used_memory_pct() < purge_threshold_pct_min ) continue;
for (i=0; i<SHARED_QUERY_CACHE_HASH_TABLES; i++) {
KVs[i].purge_some(QCnow_ms);
KVs[i]->purge_some(QCnow_ms);
}
}
delete mysql_thr;
return NULL;
};

3
lib/configfile.cpp
Unescape View File

@ -1,7 +1,8 @@
#include "proxysql.h"
#include "cpp.h"
#include <iostream>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>

Write Preview
Loading…
Cancel
Save