From 21fdb12e0b624f4af4d1d96fa292f5f285fc53ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 6 Sep 2016 00:13:35 +0000 Subject: [PATCH 01/10] Support for PREPARE (text protocol) #684 Added queries EXECUTE and DEALLOCATE PREPARE in Query Processor PREPARE automatically disables multiplexing --- include/query_processor.h | 2 ++ lib/Query_Processor.cpp | 24 ++++++++++++++++++++++++ lib/mysql_connection.cpp | 5 +++++ 3 files changed, 31 insertions(+) diff --git a/include/query_processor.h b/include/query_processor.h index 0ab309c10..4abe7019c 100644 --- a/include/query_processor.h +++ b/include/query_processor.h @@ -19,6 +19,7 @@ enum MYSQL_COM_QUERY_command { MYSQL_COM_QUERY_CREATE_TEMPORARY, MYSQL_COM_QUERY_CREATE_TRIGGER, MYSQL_COM_QUERY_CREATE_USER, + MYSQL_COM_QUERY_DEALLOCATE, MYSQL_COM_QUERY_DELETE, MYSQL_COM_QUERY_DESCRIBE, MYSQL_COM_QUERY_DROP_DATABASE, @@ -27,6 +28,7 @@ enum MYSQL_COM_QUERY_command { MYSQL_COM_QUERY_DROP_TRIGGER, MYSQL_COM_QUERY_DROP_USER, MYSQL_COM_QUERY_GRANT, + MYSQL_COM_QUERY_EXECUTE, MYSQL_COM_QUERY_EXPLAIN, MYSQL_COM_QUERY_FLUSH, MYSQL_COM_QUERY_INSERT, diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 77986456c..9353876ef 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -308,6 +308,7 @@ Query_Processor::Query_Processor() { commands_counters_desc[MYSQL_COM_QUERY_CREATE_TEMPORARY]=(char *)"CREATE_TEMPORARY"; commands_counters_desc[MYSQL_COM_QUERY_CREATE_TRIGGER]=(char *)"CREATE_TRIGGER"; commands_counters_desc[MYSQL_COM_QUERY_CREATE_USER]=(char *)"CREATE_USER"; + commands_counters_desc[MYSQL_COM_QUERY_DEALLOCATE]=(char *)"DEALLOCATE"; commands_counters_desc[MYSQL_COM_QUERY_DELETE]=(char *)"DELETE"; commands_counters_desc[MYSQL_COM_QUERY_DESCRIBE]=(char *)"DESCRIBE"; commands_counters_desc[MYSQL_COM_QUERY_DROP_DATABASE]=(char *)"DROP_DATABASE"; @@ -316,6 +317,7 @@ Query_Processor::Query_Processor() { commands_counters_desc[MYSQL_COM_QUERY_DROP_TRIGGER]=(char *)"DROP_TRIGGER"; commands_counters_desc[MYSQL_COM_QUERY_DROP_USER]=(char *)"DROP_USER"; commands_counters_desc[MYSQL_COM_QUERY_GRANT]=(char *)"GRANT"; + commands_counters_desc[MYSQL_COM_QUERY_EXECUTE]=(char *)"EXECUTE"; commands_counters_desc[MYSQL_COM_QUERY_EXPLAIN]=(char *)"EXPLAIN"; commands_counters_desc[MYSQL_COM_QUERY_FLUSH]=(char *)"FLUSH"; commands_counters_desc[MYSQL_COM_QUERY_INSERT]=(char *)"INSERT"; @@ -1060,8 +1062,23 @@ enum MYSQL_COM_QUERY_command Query_Processor::__query_parser_command_type(SQP_pa break; case 'd': case 'D': + if (!strcasecmp("DEALLOCATE",token)) { // DEALLOCATE PREPARE + token=(char *)tokenize(&tok); + if (token==NULL) break; + if (!strcasecmp("PREPARE",token)) { + ret=MYSQL_COM_QUERY_DEALLOCATE; + break; + } + } if (!strcasecmp("DELETE",token)) { // DELETE ret=MYSQL_COM_QUERY_DELETE; + break; + } + break; + case 'e': + case 'E': + if (!strcasecmp("EXECUTE",token)) { // INSERT + ret=MYSQL_COM_QUERY_EXECUTE; } break; case 'i': @@ -1070,6 +1087,13 @@ enum MYSQL_COM_QUERY_command Query_Processor::__query_parser_command_type(SQP_pa ret=MYSQL_COM_QUERY_INSERT; } break; + case 'p': + case 'P': + if (!strcasecmp("PREPARE",token)) { // ROLLBACK + ret=MYSQL_COM_QUERY_PREPARE; + break; + } + break; case 'r': case 'R': if (!strcasecmp("REPLACE",token)) { // ROLLBACK diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index 0cf48dbd8..cf554b6cb 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -1196,6 +1196,11 @@ void MySQL_Connection::ProcessQueryAndSetStatusFlags(char *query_digest_text) { set_status_user_variable(true); } } + if (get_status_prepared_statement()==false) { // we search if prepared was already executed + if (!strncasecmp(query_digest_text,"PREPARE ", strlen("PREPARE "))) { + set_status_prepared_statement(true); + } + } if (get_status_temporary_table()==false) { // we search for temporary if not already set if (!strncasecmp(query_digest_text,"CREATE TEMPORARY TABLE ", strlen("CREATE TEMPORARY TABLE "))) { set_status_temporary_table(true); From e29f70976ab620ba17c36447e6ddadbdbc46cf8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 6 Sep 2016 23:25:30 +0000 Subject: [PATCH 02/10] Implemented aggressive query cache purging #690 Aggressive purging is triggered when 90% of the memory limit is reached Fixed also a bug in the computation of memory used --- lib/Query_Cache.cpp | 69 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 11 deletions(-) diff --git a/lib/Query_Cache.cpp b/lib/Query_Cache.cpp index 06eacd5d9..40a3e1689 100644 --- a/lib/Query_Cache.cpp +++ b/lib/Query_Cache.cpp @@ -27,7 +27,7 @@ #else #define DEB "" #endif /* DEBUG */ -#define QUERY_CACHE_VERSION "1.2.0817" DEB +#define QUERY_CACHE_VERSION "1.2.0905" DEB extern MySQL_Threads_Handler *GloMTH; @@ -47,7 +47,7 @@ class KV_BtreeArray { KV_BtreeArray(); ~KV_BtreeArray(); uint64_t get_data_size(); - void purge_some(unsigned long long); + void purge_some(unsigned long long, bool); int cnt(); bool replace(uint64_t key, QC_entry_t *entry); QC_entry_t *lookup(uint64_t key); @@ -122,30 +122,75 @@ KV_BtreeArray::~KV_BtreeArray() { uint64_t KV_BtreeArray::get_data_size() { - uint64_t r = __sync_fetch_and_add(&Glo_num_entries,0) * (sizeof(QC_entry_t)+sizeof(QC_entry_t *)*2+sizeof(uint64_t)*2) + __sync_fetch_and_add(&Glo_size_values,0) ; + uint64_t r = __sync_fetch_and_add(&Glo_num_entries,0) * (sizeof(QC_entry_t)+sizeof(QC_entry_t *)*2+sizeof(uint64_t)*2); // + __sync_fetch_and_add(&Glo_size_values,0) ; return r; }; -void KV_BtreeArray::purge_some(unsigned long long QCnow_ms) { +void KV_BtreeArray::purge_some(unsigned long long QCnow_ms, bool aggressive) { uint64_t ret=0, i, _size=0; QC_entry_t *qce; + unsigned long long access_ms_min=0; + unsigned long long access_ms_max=0; spin_rdlock(&lock); for (i=0; ilen;i++) { qce=(QC_entry_t *)ptrArray->index(i); - if (qce->expire_ms==EXPIRE_DROPIT || qce->expire_mslength; + if (aggressive) { // we have been asked to do aggressive purging + if (access_ms_min==0) { + access_ms_min = qce->access_ms; + } else { + if (access_ms_min > qce->access_ms) { + access_ms_min = qce->access_ms; + } + } + if (access_ms_max==0) { + access_ms_max = qce->access_ms; + } else { + if (access_ms_max < qce->access_ms) { + access_ms_max = qce->access_ms; + } + } + } else { // no aggresssive purging , legacy algorithm + if (qce->expire_ms==EXPIRE_DROPIT || qce->expire_mslength; + } } } freeable_memory=_size; spin_rdunlock(&lock); - if ( (freeable_memory + ret * (sizeof(QC_entry_t)+sizeof(QC_entry_t *)*2+sizeof(uint64_t)*2) ) > get_data_size()*0.01) { + bool cond_freeable_memory=false; + if (aggressive==false) { + uint64_t total_freeable_memory=0; + total_freeable_memory=freeable_memory + ret * (sizeof(QC_entry_t)+sizeof(QC_entry_t *)*2+sizeof(uint64_t)*2); + if ( total_freeable_memory > get_data_size()*0.01 ) { + cond_freeable_memory=true; // there is memory that can be freed + } + } + //if ( freeable_memory + ret * (sizeof(QC_entry_t) > get_data_size()*0.01) { + if ( aggressive || cond_freeable_memory ) { uint64_t removed_entries=0; uint64_t freed_memory=0; + unsigned long long access_ms_lower_mark=0; + if (aggressive) { + access_ms_lower_mark=access_ms_min+(access_ms_max-access_ms_min)*0.1; // hardcoded for now. Remove the entries with access time in the 10% range closest to access_ms_min + } spin_wrlock(&lock); for (i=0; ilen;i++) { qce=(QC_entry_t *)ptrArray->index(i); - if ((qce->expire_ms==EXPIRE_DROPIT || qce->expire_msref_count,0)<=1)) { + bool drop_entry=false; + if (__sync_fetch_and_add(&qce->ref_count,0)<=1) { // currently not in use + if (qce->expire_ms==EXPIRE_DROPIT || qce->expire_msaccess_ms < access_ms_lower_mark) { + drop_entry=true; + } + } + } + } + if (drop_entry) { qce=(QC_entry_t *)ptrArray->remove_index_fast(i); btree::btree_map::iterator lookup; @@ -242,6 +287,7 @@ uint64_t Query_Cache::get_data_size_total() { for (i=0; iget_data_size(); } + r += __sync_fetch_and_add(&Glo_size_values,0); return r; }; @@ -364,9 +410,10 @@ void * Query_Cache::purgeHash_thread(void *) { max_memory_size=mysql_thread___query_cache_size_MB*1024*1024; } } - if (current_used_memory_pct() < purge_threshold_pct_min ) continue; + unsigned int curr_pct=current_used_memory_pct(); + if (curr_pct < purge_threshold_pct_min ) continue; for (i=0; ipurge_some(QCnow_ms); + KVs[i]->purge_some(QCnow_ms, (curr_pct > purge_threshold_pct_max)); } } delete mysql_thr; From 99abb48c2120c4a27512e409d682f9d2ec77bc64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Fri, 9 Sep 2016 08:28:48 +0000 Subject: [PATCH 03/10] Implementation of runtime_mysql_users #691 --- include/proxysql_admin.h | 2 +- lib/ProxySQL_Admin.cpp | 49 +++++++++++++++++++++++++++++++--------- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index f1f645e4b..54be571e3 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -147,7 +147,7 @@ class ProxySQL_Admin { void init_users(); void init_mysql_servers(); void init_mysql_query_rules(); - void save_mysql_users_runtime_to_database(); + void save_mysql_users_runtime_to_database(bool _runtime); void save_mysql_servers_runtime_to_database(bool); void admin_shutdown(); bool is_command(std::string); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 4c68587c3..74aac25ab 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -89,6 +89,9 @@ pthread_mutex_t admin_mutex = PTHREAD_MUTEX_INITIALIZER; #define ADMIN_SQLITE_TABLE_MYSQL_SERVERS_V1_2_2 "CREATE TABLE mysql_servers (hostgroup_id INT NOT NULL DEFAULT 0 , hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , status VARCHAR CHECK (UPPER(status) IN ('ONLINE','SHUNNED','OFFLINE_SOFT', 'OFFLINE_HARD')) NOT NULL DEFAULT 'ONLINE' , weight INT CHECK (weight >= 0) NOT NULL DEFAULT 1 , compression INT CHECK (compression >=0 AND compression <= 102400) NOT NULL DEFAULT 0 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 1000 , max_replication_lag INT CHECK (max_replication_lag >= 0 AND max_replication_lag <= 126144000) NOT NULL DEFAULT 0 , use_ssl INT CHECK (use_ssl IN(0,1)) NOT NULL DEFAULT 0 , max_latency_ms INT UNSIGNED CHECK (max_latency_ms>=0) NOT NULL DEFAULT 0 , comment VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup_id, hostname, port) )" #define ADMIN_SQLITE_TABLE_MYSQL_USERS "CREATE TABLE mysql_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , use_ssl INT CHECK (use_ssl IN (0,1)) NOT NULL DEFAULT 0 , default_hostgroup INT NOT NULL DEFAULT 0 , default_schema VARCHAR , schema_locked INT CHECK (schema_locked IN (0,1)) NOT NULL DEFAULT 0 , transaction_persistent INT CHECK (transaction_persistent IN (0,1)) NOT NULL DEFAULT 0 , fast_forward INT CHECK (fast_forward IN (0,1)) NOT NULL DEFAULT 0 , backend INT CHECK (backend IN (0,1)) NOT NULL DEFAULT 1 , frontend INT CHECK (frontend IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , PRIMARY KEY (username, backend) , UNIQUE (username, frontend))" + +#define ADMIN_SQLITE_RUNTIME_MYSQL_USERS "CREATE TABLE runtime_mysql_users (username VARCHAR NOT NULL , password VARCHAR , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , use_ssl INT CHECK (use_ssl IN (0,1)) NOT NULL DEFAULT 0 , default_hostgroup INT NOT NULL DEFAULT 0 , default_schema VARCHAR , schema_locked INT CHECK (schema_locked IN (0,1)) NOT NULL DEFAULT 0 , transaction_persistent INT CHECK (transaction_persistent IN (0,1)) NOT NULL DEFAULT 0 , fast_forward INT CHECK (fast_forward IN (0,1)) NOT NULL DEFAULT 0 , backend INT CHECK (backend IN (0,1)) NOT NULL DEFAULT 1 , frontend INT CHECK (frontend IN (0,1)) NOT NULL DEFAULT 1 , max_connections INT CHECK (max_connections >=0) NOT NULL DEFAULT 10000 , PRIMARY KEY (username, backend) , UNIQUE (username, frontend))" + #define ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES "CREATE TABLE mysql_query_rules (rule_id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 0 , username VARCHAR , schemaname VARCHAR , flagIN INT NOT NULL DEFAULT 0 , client_addr VARCHAR , proxy_addr VARCHAR , proxy_port INT , digest VARCHAR , match_digest VARCHAR , match_pattern VARCHAR , negate_match_pattern INT CHECK (negate_match_pattern IN (0,1)) NOT NULL DEFAULT 0 , flagOUT INT , replace_pattern VARCHAR , destination_hostgroup INT DEFAULT NULL , cache_ttl INT CHECK(cache_ttl > 0) , reconnect INT CHECK (reconnect IN (0,1)) DEFAULT NULL , timeout INT UNSIGNED , retries INT CHECK (retries>=0 AND retries <=1000) , delay INT UNSIGNED , mirror_flagOUT INT UNSIGNED , mirror_hostgroup INT UNSIGNED , error_msg VARCHAR , log INT CHECK (log IN (0,1)) , apply INT CHECK(apply IN (0,1)) NOT NULL DEFAULT 0 , comment VARCHAR)" // mysql_query_rules in v1.1.0 @@ -825,7 +828,7 @@ bool admin_handler_command_load_or_save(char *query_no_space, unsigned int query ) { proxy_info("Received %s command\n", query_no_space); ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->save_mysql_users_runtime_to_database(); + SPA->save_mysql_users_runtime_to_database(false); proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved mysql users from RUNTIME\n"); SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); return false; @@ -1206,6 +1209,7 @@ void ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign bool dump_global_variables=false; bool runtime_scheduler=false; + bool runtime_mysql_users=false; bool runtime_mysql_servers=false; bool runtime_mysql_query_rules=false; @@ -1238,6 +1242,9 @@ void ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign ) { runtime_mysql_servers=true; refresh=true; } + if (strstr(query_no_space,"runtime_mysql_users")) { + runtime_mysql_users=true; refresh=true; + } if (strstr(query_no_space,"runtime_mysql_query_rules")) { runtime_mysql_query_rules=true; refresh=true; } @@ -1275,6 +1282,9 @@ void ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign save_mysql_servers_runtime_to_database(true); mysql_servers_wrunlock(); } + if (runtime_mysql_users) { + save_mysql_users_runtime_to_database(true); + } if (runtime_mysql_query_rules) { save_mysql_query_rules_from_runtime(true); } @@ -2477,6 +2487,7 @@ bool ProxySQL_Admin::init() { insert_into_tables_defs(tables_defs_admin,"mysql_servers", ADMIN_SQLITE_TABLE_MYSQL_SERVERS); insert_into_tables_defs(tables_defs_admin,"runtime_mysql_servers", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_SERVERS); insert_into_tables_defs(tables_defs_admin,"mysql_users", ADMIN_SQLITE_TABLE_MYSQL_USERS); + insert_into_tables_defs(tables_defs_admin,"runtime_mysql_users", ADMIN_SQLITE_RUNTIME_MYSQL_USERS); insert_into_tables_defs(tables_defs_admin,"runtime_mysql_replication_hostgroups", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_REPLICATION_HOSTGROUPS); insert_into_tables_defs(tables_defs_admin,"mysql_replication_hostgroups", ADMIN_SQLITE_TABLE_MYSQL_REPLICATION_HOSTGROUPS); insert_into_tables_defs(tables_defs_admin,"mysql_query_rules", ADMIN_SQLITE_TABLE_MYSQL_QUERY_RULES); @@ -3773,7 +3784,7 @@ void ProxySQL_Admin::__add_active_users(enum cred_username_type usertype) { } -void ProxySQL_Admin::save_mysql_users_runtime_to_database() { +void ProxySQL_Admin::save_mysql_users_runtime_to_database(bool _runtime) { /* char *error=NULL; int cols=0; @@ -3788,32 +3799,48 @@ void ProxySQL_Admin::save_mysql_users_runtime_to_database() { } if(resultset) delete resultset; */ - char *qd=(char *)"UPDATE mysql_users SET active=0"; - proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", qd); - admindb->execute(qd); + char *query=NULL; + if (_runtime) { + query=(char *)"DELETE FROM main.runtime_mysql_users"; + admindb->execute(query); + } else { + char *qd=(char *)"UPDATE mysql_users SET active=0"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", qd); + admindb->execute(qd); + } account_details_t **ads=NULL; int num_users; int i; char *qf=(char *)"REPLACE INTO mysql_users(username,password,active,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,backend,frontend,max_connections) VALUES(\"%s\",\"%s\",1,%d,%d,\"%s\",%d,%d,%d,COALESCE((SELECT backend FROM mysql_users WHERE username=\"%s\" AND frontend=1),0),1,%d)"; char *qb=(char *)"REPLACE INTO mysql_users(username,password,active,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,backend,frontend,max_connections) VALUES(\"%s\",\"%s\",1,%d,%d,\"%s\",%d,%d,%d,1,COALESCE((SELECT frontend FROM mysql_users WHERE username=\"%s\" AND backend=1),0),%d)"; + char *qfr=(char *)"REPLACE INTO runtime_mysql_users(username,password,active,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,backend,frontend,max_connections) VALUES(\"%s\",\"%s\",1,%d,%d,\"%s\",%d,%d,%d,COALESCE((SELECT backend FROM runtime_mysql_users WHERE username=\"%s\" AND frontend=1),0),1,%d)"; + char *qbr=(char *)"REPLACE INTO runtime_mysql_users(username,password,active,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,backend,frontend,max_connections) VALUES(\"%s\",\"%s\",1,%d,%d,\"%s\",%d,%d,%d,1,COALESCE((SELECT frontend FROM runtime_mysql_users WHERE username=\"%s\" AND backend=1),0),%d)"; num_users=GloMyAuth->dump_all_users(&ads); if (num_users==0) return; for (i=0; iusername, ads[i]->default_hostgroup); account_details_t *ad=ads[i]; if (ads[i]->default_hostgroup >= 0) { - char *query; - char *q; - if (ad->__frontend) { - q=qf; - } else { - q=qb; + char *q=NULL; + if (_runtime==false) { + if (ad->__frontend) { + q=qf; + } else { + q=qb; + } + } else { // _runtime==true + if (ad->__frontend) { + q=qfr; + } else { + q=qbr; + } } query=(char *)malloc(strlen(q)+strlen(ad->username)*2+strlen(ad->password)+strlen(ad->default_schema)+256); sprintf(query, q, ad->username, ad->password, ad->use_ssl, ad->default_hostgroup, ad->default_schema, ad->schema_locked, ad->transaction_persistent, ad->fast_forward, ad->username, ad->max_connections); //fprintf(stderr,"%s\n",query); proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); admindb->execute(query); + free(query); } free(ad->username); free(ad->password); From 128ff3dedaa8472c88fd95d8a453d2ebbe76c4fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sun, 11 Sep 2016 22:54:53 +0000 Subject: [PATCH 04/10] Introduce new variable admin-hash_passwords #676 New variable admin-hash_passwords , default true When set, passwords in mysql_users are automatically hashed in runtime when running LOAD MYSQL USERS TO RUNTIME. For backward compatibility, passwords in mysql_users in memory and disk aren't automatically hashed. Although, they can be easily hashed running SAVE MYSQL USERS TO MEMORY (and eventually SAVE MYSQL USERS TO DISK) immediately after running LOAD MYSQL USERS TO RUNTIME --- include/proxysql_admin.h | 1 + lib/ProxySQL_Admin.cpp | 65 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index f1f645e4b..3dcf4f9ab 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -71,6 +71,7 @@ class ProxySQL_Admin { char *telnet_admin_ifaces; char *telnet_stats_ifaces; bool admin_read_only; + bool hash_passwords; char * admin_version; #ifdef DEBUG bool debug; diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 4c68587c3..f569ce0f4 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -54,6 +54,21 @@ char *s_strdup(char *s) { return ret; } + +static char *sha1_pass_hex(char *sha1_pass) { // copied from MySQL_Protocol.cpp + if (sha1_pass==NULL) return NULL; + char *buff=(char *)malloc(SHA_DIGEST_LENGTH*2+2); + buff[0]='*'; + buff[SHA_DIGEST_LENGTH*2+1]='\0'; + int i; + uint8_t a; + for (i=0;i::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; + char *password=NULL; + if (variables.hash_passwords) { // We must use hashed password. See issue #676 + // Admin needs to hash the password + if (r->fields[1] && strlen(r->fields[1])) { + if (r->fields[1][0]=='*') { // the password is already hashed + password=strdup(r->fields[1]); + } else { // we must hash it + uint8 hash_stage1[SHA_DIGEST_LENGTH]; + uint8 hash_stage2[SHA_DIGEST_LENGTH]; + SHA_CTX sha1_context; + SHA1_Init(&sha1_context); + SHA1_Update(&sha1_context, r->fields[1], strlen(r->fields[1])); + SHA1_Final(hash_stage1, &sha1_context); + SHA1_Init(&sha1_context); + SHA1_Update(&sha1_context,hash_stage1,SHA_DIGEST_LENGTH); + SHA1_Final(hash_stage2, &sha1_context); + password=sha1_pass_hex((char *)hash_stage2); // note that sha1_pass_hex() returns a new buffer + } + } else { + password=strdup((char *)""); // we also generate a new string if hash_passwords is set + } + } else { + if (r->fields[1]) { + password=r->fields[1]; + } else { + password=(char *)""; + } + } GloMyAuth->add( r->fields[0], // username - (r->fields[1]==NULL ? (char *)"" : r->fields[1]), //password + //(r->fields[1]==NULL ? (char *)"" : r->fields[1]), //password + password, // before #676, wewere always passing the password. Now it is possible that the password can be hashed usertype, // backend/frontend (strcmp(r->fields[2],"1")==0 ? true : false) , // use_ssl atoi(r->fields[3]), // default_hostgroup @@ -3765,6 +3825,9 @@ void ProxySQL_Admin::__add_active_users(enum cred_username_type usertype) { (strcmp(r->fields[7],"1")==0 ? true : false), // fast_forward ( atoi(r->fields[8])>0 ? atoi(r->fields[8]) : 0) // max_connections ); + if (variables.hash_passwords) { + free(password); // because we always generate a new string + } } } // if (error) free(error); From 1cbedc8057c7d3f8a9559a55eafedc7ef3bdde7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Mon, 12 Sep 2016 00:50:58 +0000 Subject: [PATCH 05/10] Documentation on passwords --- doc/passwords.md | 100 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 doc/passwords.md diff --git a/doc/passwords.md b/doc/passwords.md new file mode 100644 index 000000000..e870d142d --- /dev/null +++ b/doc/passwords.md @@ -0,0 +1,100 @@ +# MySQL Passwords in ProxySQL + +ProxySQL is a protocol aware proxy. +Because ProxySQL performs routing based on traffic, when a client connects it cannot yet identify a destination HG, therefore ProxySQL needs to authenticate the client. +For this reason, ProxySQL needs to have some information related to the password of the user: enough information to allow the authentication. + +ProxySQL also needs these information to later establish connections to backends, or issue `CHANGE_USER` within already established connections. + +The 3 layers configuration architecture applies also for users information. +ProxySQL stores users information in table `mysql_users`: +* an object `MySQL_Authentication()` is responsible to store these information at runtime; +* `main`.`mysql_users` is the in-memory database; +* `runtime`.`mysql_users` is the on-disk database. + +In `mysql_users` tables, both in-memory and on-disk, the credentials are stored in columns `username` and `password`. + +## Password formats + +Password can be stored in 2 formats in `mysql_users`.`password` , no matter if in-memory or on-disk: +* plain text +* hashed password + +Passwords in plain text are simple as that, very easy to read. If database and config file are kept in a safe location the security concern is limited, yet present. +Hashed passwords have the same format of the passwords in MySQL server, as stored into column `mysql`.`user`.`password`. + +ProxySQL considers a password starting with `*` has a hashed password. + +### Hashed passwords and authentication + +In MySQL and in ProxySQL, a hashed password is `SHA1(SHA1('clear_password'))` . +From a hashed password is not possible to derive a plain text password. +When a client connects to ProxySQL, this is able to authenticate it using the hashed password. +During the first client authentication, ProxySQL can derive a partially hashed password: `SHA1('clear_password')` . This information is internally stored at runtime and allows ProxySQL to connect to backends. + + +### How to input new passwords + +The Admin interface of ProxySQL does not have any `PASSWORD()` function. That means that: +* passwords are stored in the format they are inserted, either in plain text or hashed +* while inputting password in the Admin interface, it is not possible to derive an hashed password from a plain text password (yet you can run `SELECT PASSWORD('password')` in MySQL server and copy paste the result) + + +### Variable `admin-hash_passwords` + +To facilitate the support of hashed passwords, ProxySQL v1.2.3 introduced a new global boolean variable, `admin-hash_password`, enabled by default. +When `admin-hash_password=true` , password are automatically _at_RUNTIME_only_ hashed when running `LOAD MYSQL USERS TO RUNTIME` . +Passwords in `mysql_users` are yet *not* automatically hashed. +Nonetheless, it is easily possible to hash the password in `mysql_users` table, both in-memory and on-disk. It is enough to copy users from RUNTIME, for example running `SAVE MYSQL USERS FROM RUNTIME` after `LOAD MYSQL USERS TO RUNTIME`, and then `SAVE MYSQL USERS TO DISK` (recommended). + +Here an example: +```sql +Admin> SELECT * FROM mysql_users; +Empty set (0.00 sec) + +Admin> INSERT INTO mysql_users(username,password) VALUES ('user1','password1'), ('user2','password2'); +Query OK, 2 rows affected (0.00 sec) + +Admin> SELECT username,password FROM mysql_users; ++----------+-----------+ +| username | password | ++----------+-----------+ +| user1 | password1 | +| user2 | password2 | ++----------+-----------+ +2 rows in set (0.00 sec) + +Admin> LOAD MYSQL USERS TO RUNTIME; +Query OK, 0 rows affected (0.00 sec) + +Admin> SELECT username,password FROM mysql_users; ++----------+-----------+ +| username | password | ++----------+-----------+ +| user1 | password1 | +| user2 | password2 | ++----------+-----------+ +2 rows in set (0.00 sec) +``` + +At this stage, passwords are hashed at runtime, but still not hashed on `mysql_users`. To hash them also on `mysql_users` : + +```sql +Admin> SAVE MYSQL USERS FROM RUNTIME; +Query OK, 0 rows affected (0.00 sec) + +Admin> SELECT username,password FROM mysql_users; ++----------+-------------------------------------------+ +| username | password | ++----------+-------------------------------------------+ +| user1 | *668425423DB5193AF921380129F465A6425216D0 | +| user2 | *DC52755F3C09F5923046BD42AFA76BD1D80DF2E9 | ++----------+-------------------------------------------+ +2 rows in set (0.00 sec) +``` + +The hashed password can now be saved to disk running `SAVE MYSQL USERS TO DISK` . + + +**Note**: `admin-hash_passwords` is an `admin-` variable, not a `mysql-` variable. This because it affects the behaviour of Admin. +This details is important because to apply changes in `admin-hash_passwords` you need to run `LOAD ADMIN VARIABLES TO RUNTIME` and **not** `LOAD MYSQL VARIABLES TO RUNTIME` From ffce0fb0c0e7c02b6befe7f4dfd0aac64aedc674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Mon, 12 Sep 2016 01:20:21 -0700 Subject: [PATCH 06/10] Update passwords.md --- doc/passwords.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/passwords.md b/doc/passwords.md index e870d142d..9a543b4fa 100644 --- a/doc/passwords.md +++ b/doc/passwords.md @@ -2,7 +2,7 @@ ProxySQL is a protocol aware proxy. Because ProxySQL performs routing based on traffic, when a client connects it cannot yet identify a destination HG, therefore ProxySQL needs to authenticate the client. -For this reason, ProxySQL needs to have some information related to the password of the user: enough information to allow the authentication. +For this reason, it needs to have some information related to the password of the user: enough information to allow the authentication. ProxySQL also needs these information to later establish connections to backends, or issue `CHANGE_USER` within already established connections. @@ -35,7 +35,7 @@ During the first client authentication, ProxySQL can derive a partially hashed p ### How to input new passwords -The Admin interface of ProxySQL does not have any `PASSWORD()` function. That means that: +The Admin interface of ProxySQL does not have any `PASSWORD()` function. This means that: * passwords are stored in the format they are inserted, either in plain text or hashed * while inputting password in the Admin interface, it is not possible to derive an hashed password from a plain text password (yet you can run `SELECT PASSWORD('password')` in MySQL server and copy paste the result) @@ -43,9 +43,9 @@ The Admin interface of ProxySQL does not have any `PASSWORD()` function. That m ### Variable `admin-hash_passwords` To facilitate the support of hashed passwords, ProxySQL v1.2.3 introduced a new global boolean variable, `admin-hash_password`, enabled by default. -When `admin-hash_password=true` , password are automatically _at_RUNTIME_only_ hashed when running `LOAD MYSQL USERS TO RUNTIME` . -Passwords in `mysql_users` are yet *not* automatically hashed. -Nonetheless, it is easily possible to hash the password in `mysql_users` table, both in-memory and on-disk. It is enough to copy users from RUNTIME, for example running `SAVE MYSQL USERS FROM RUNTIME` after `LOAD MYSQL USERS TO RUNTIME`, and then `SAVE MYSQL USERS TO DISK` (recommended). +When `admin-hash_password=true` , password are automatically hashed _at RUNTIME only_ when running `LOAD MYSQL USERS TO RUNTIME` . +Passwords in `mysql_users` tables are yet *not* automatically hashed. +Nonetheless, it is easily possible to hash the passwords in `mysql_users` table, both in-memory and on-disk. It is enough to copy users _from RUNTIME_, for example running `SAVE MYSQL USERS FROM RUNTIME` after `LOAD MYSQL USERS TO RUNTIME`, and then `SAVE MYSQL USERS TO DISK` (recommended). Here an example: ```sql From c8d92b05d37f863ff894bcbb07dcd0155d2c61c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Mon, 12 Sep 2016 09:15:20 +0000 Subject: [PATCH 07/10] Preparation to compile on Ubuntu16 --- Makefile | 60 +++++++++---------- .../proxysql/centos67-build/proxysql.spec | 2 +- .../proxysql/centos7-build/proxysql.spec | 2 +- .../proxysql/debian-7.8-build/proxysql.ctl | 2 +- .../proxysql/debian-8.2-build/proxysql.ctl | 2 +- .../proxysql/ubuntu-12.04-build/proxysql.ctl | 2 +- .../proxysql/ubuntu-14.04-build/proxysql.ctl | 2 +- .../proxysql/ubuntu-15.10-build/proxysql.ctl | 2 +- .../proxysql/ubuntu-16.04-build/proxysql.ctl | 24 ++++++++ include/proxysql.h | 2 +- 10 files changed, 62 insertions(+), 38 deletions(-) create mode 100644 docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl diff --git a/Makefile b/Makefile index 3fd5d6ebf..4d83eae22 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ DEBUG=${ALL_DEBUG} #export DEBUG #export OPTZ #export EXTRALINK -CURVER=1.2.2 +CURVER=1.2.3 DISTRO := $(shell gawk -F= '/^NAME/{print $$2}' /etc/os-release) .PHONY: default @@ -48,7 +48,7 @@ clean: cd lib && ${MAKE} clean cd src && ${MAKE} clean -packages: centos6.7 centos7 centos6.7-dbg centos7-dbg ubuntu12 ubuntu14 debian7 debian8 ubuntu12-dbg ubuntu14-dbg debian7-dbg debian8-dbg +packages: centos6.7 centos7 centos6.7-dbg centos7-dbg ubuntu12 ubuntu14 debian7 debian8 ubuntu12-dbg ubuntu14-dbg debian7-dbg debian8-dbg ubuntu16 ubuntu16-dbg .PHONY: packages @@ -70,8 +70,8 @@ ubuntu12: binaries/proxysql_${CURVER}-ubuntu12_amd64.deb ubuntu14: binaries/proxysql_${CURVER}-ubuntu14_amd64.deb .PHONY: ubuntu14 -ubuntu15: binaries/proxysql_${CURVER}-ubuntu15_amd64.deb -.PHONY: ubuntu15 +ubuntu16: binaries/proxysql_${CURVER}-ubuntu16_amd64.deb +.PHONY: ubuntu16 debian7: binaries/proxysql_${CURVER}-debian7_amd64.deb .PHONY: debian7 @@ -85,8 +85,8 @@ ubuntu12-dbg: binaries/proxysql_${CURVER}-dbg-ubuntu12_amd64.deb ubuntu14-dbg: binaries/proxysql_${CURVER}-dbg-ubuntu14_amd64.deb .PHONY: ubuntu14-dbg -ubuntu15-dbg: binaries/proxysql_${CURVER}-dbg-ubuntu15_amd64.deb -.PHONY: ubuntu15-dbg +ubuntu16-dbg: binaries/proxysql_${CURVER}-dbg-ubuntu16_amd64.deb +.PHONY: ubuntu16-dbg debian7-dbg: binaries/proxysql_${CURVER}-dbg-debian7_amd64.deb .PHONY: debian7-dbg @@ -194,18 +194,18 @@ binaries/proxysql_${CURVER}-ubuntu14_amd64.deb: docker stop ubuntu14_build docker rm ubuntu14_build -binaries/proxysql_${CURVER}-ubuntu15_amd64.deb: - docker stop ubuntu15_build || true - docker rm ubuntu15_build || true - docker create --name ubuntu15_build renecannao/proxysql:build-ubuntu15 bash -c "while : ; do sleep 10 ; done" - docker start ubuntu15_build - docker exec ubuntu15_build bash -c "cd /opt; git clone -b v${CURVER} https://github.com/sysown/proxysql.git proxysql" - docker exec ubuntu15_build bash -c "cd /opt/proxysql; ${MAKE} clean && ${MAKE} -j 4 build_deps && ${MAKE} -j 4" - docker cp docker/images/proxysql/ubuntu-15.10-build/proxysql.ctl ubuntu15_build:/opt/proxysql/ - docker exec ubuntu15_build bash -c "cd /opt/proxysql; cp src/proxysql . ; equivs-build proxysql.ctl" - docker cp ubuntu15_build:/opt/proxysql/proxysql_${CURVER}_amd64.deb ./binaries/proxysql_${CURVER}-ubuntu15_amd64.deb - docker stop ubuntu15_build - docker rm ubuntu15_build +binaries/proxysql_${CURVER}-ubuntu16_amd64.deb: + docker stop ubuntu16_build || true + docker rm ubuntu16_build || true + docker create --name ubuntu16_build renecannao/proxysql:build-ubuntu16 bash -c "while : ; do sleep 10 ; done" + docker start ubuntu16_build + docker exec ubuntu16_build bash -c "cd /opt; git clone -b v${CURVER} https://github.com/sysown/proxysql.git proxysql" + docker exec ubuntu16_build bash -c "cd /opt/proxysql; ${MAKE} clean && ${MAKE} -j 4 build_deps && ${MAKE} -j 4" + docker cp docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl ubuntu16_build:/opt/proxysql/ + docker exec ubuntu16_build bash -c "cd /opt/proxysql; cp src/proxysql . ; equivs-build proxysql.ctl" + docker cp ubuntu16_build:/opt/proxysql/proxysql_${CURVER}_amd64.deb ./binaries/proxysql_${CURVER}-ubuntu16_amd64.deb + docker stop ubuntu16_build + docker rm ubuntu16_build binaries/proxysql_${CURVER}-debian7_amd64.deb: docker stop debian7_build || true @@ -260,18 +260,18 @@ binaries/proxysql_${CURVER}-dbg-ubuntu14_amd64.deb: docker stop ubuntu14_build docker rm ubuntu14_build -binaries/proxysql_${CURVER}-dbg-ubuntu15_amd64.deb: - docker stop ubuntu15_build || true - docker rm ubuntu15_build || true - docker create --name ubuntu15_build renecannao/proxysql:build-ubuntu15 bash -c "while : ; do sleep 10 ; done" - docker start ubuntu15_build - docker exec ubuntu15_build bash -c "cd /opt; git clone -b v${CURVER} https://github.com/sysown/proxysql.git proxysql" - docker exec ubuntu15_build bash -c "cd /opt/proxysql; ${MAKE} clean && ${MAKE} -j 4 build_deps && ${MAKE} debug" - docker cp docker/images/proxysql/ubuntu-15.10-build/proxysql.ctl ubuntu15_build:/opt/proxysql/ - docker exec ubuntu15_build bash -c "cd /opt/proxysql; cp src/proxysql . ; equivs-build proxysql.ctl" - docker cp ubuntu15_build:/opt/proxysql/proxysql_${CURVER}_amd64.deb ./binaries/proxysql_${CURVER}-ubuntu15_amd64.deb - docker stop ubuntu15_build - docker rm ubuntu15_build +binaries/proxysql_${CURVER}-dbg-ubuntu16_amd64.deb: + docker stop ubuntu16_build || true + docker rm ubuntu16_build || true + docker create --name ubuntu16_build renecannao/proxysql:build-ubuntu16 bash -c "while : ; do sleep 10 ; done" + docker start ubuntu16_build + docker exec ubuntu16_build bash -c "cd /opt; git clone -b v${CURVER} https://github.com/sysown/proxysql.git proxysql" + docker exec ubuntu16_build bash -c "cd /opt/proxysql; ${MAKE} clean && ${MAKE} -j 4 build_deps && ${MAKE} debug" + docker cp docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl ubuntu16_build:/opt/proxysql/ + docker exec ubuntu16_build bash -c "cd /opt/proxysql; cp src/proxysql . ; equivs-build proxysql.ctl" + docker cp ubuntu16_build:/opt/proxysql/proxysql_${CURVER}_amd64.deb ./binaries/proxysql_${CURVER}-ubuntu16_amd64.deb + docker stop ubuntu16_build + docker rm ubuntu16_build binaries/proxysql_${CURVER}-dbg-debian7_amd64.deb: docker stop debian7_build || true diff --git a/docker/images/proxysql/centos67-build/proxysql.spec b/docker/images/proxysql/centos67-build/proxysql.spec index 482c71de8..694210102 100644 --- a/docker/images/proxysql/centos67-build/proxysql.spec +++ b/docker/images/proxysql/centos67-build/proxysql.spec @@ -7,7 +7,7 @@ Summary: A high-performance MySQL proxy Name: proxysql -Version: 1.2.2 +Version: 1.2.3 Release: 1 License: GPL+ Group: Development/Tools diff --git a/docker/images/proxysql/centos7-build/proxysql.spec b/docker/images/proxysql/centos7-build/proxysql.spec index 482c71de8..694210102 100644 --- a/docker/images/proxysql/centos7-build/proxysql.spec +++ b/docker/images/proxysql/centos7-build/proxysql.spec @@ -7,7 +7,7 @@ Summary: A high-performance MySQL proxy Name: proxysql -Version: 1.2.2 +Version: 1.2.3 Release: 1 License: GPL+ Group: Development/Tools diff --git a/docker/images/proxysql/debian-7.8-build/proxysql.ctl b/docker/images/proxysql/debian-7.8-build/proxysql.ctl index 5c86af112..8475d93a5 100644 --- a/docker/images/proxysql/debian-7.8-build/proxysql.ctl +++ b/docker/images/proxysql/debian-7.8-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.2.2 +Version: 1.2.3 Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md diff --git a/docker/images/proxysql/debian-8.2-build/proxysql.ctl b/docker/images/proxysql/debian-8.2-build/proxysql.ctl index 5c86af112..8475d93a5 100644 --- a/docker/images/proxysql/debian-8.2-build/proxysql.ctl +++ b/docker/images/proxysql/debian-8.2-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.2.2 +Version: 1.2.3 Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md diff --git a/docker/images/proxysql/ubuntu-12.04-build/proxysql.ctl b/docker/images/proxysql/ubuntu-12.04-build/proxysql.ctl index 5c86af112..8475d93a5 100644 --- a/docker/images/proxysql/ubuntu-12.04-build/proxysql.ctl +++ b/docker/images/proxysql/ubuntu-12.04-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.2.2 +Version: 1.2.3 Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md diff --git a/docker/images/proxysql/ubuntu-14.04-build/proxysql.ctl b/docker/images/proxysql/ubuntu-14.04-build/proxysql.ctl index 5c86af112..8475d93a5 100644 --- a/docker/images/proxysql/ubuntu-14.04-build/proxysql.ctl +++ b/docker/images/proxysql/ubuntu-14.04-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.2.2 +Version: 1.2.3 Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md diff --git a/docker/images/proxysql/ubuntu-15.10-build/proxysql.ctl b/docker/images/proxysql/ubuntu-15.10-build/proxysql.ctl index 5c86af112..8475d93a5 100644 --- a/docker/images/proxysql/ubuntu-15.10-build/proxysql.ctl +++ b/docker/images/proxysql/ubuntu-15.10-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.2.2 +Version: 1.2.3 Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md diff --git a/docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl b/docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl new file mode 100644 index 000000000..8475d93a5 --- /dev/null +++ b/docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl @@ -0,0 +1,24 @@ +Section: misc +Priority: optional +Homepage: http://www.proxysql.com +Standards-Version: 3.9.2 + +Package: proxysql +Version: 1.2.3 +Maintainer: Rene Cannao +Architecture: amd64 +# Changelog: CHANGELOG.md +# Readme: README.md +Files: proxysql /usr/bin/ + etc/proxysql.cnf / + etc/init.d/proxysql / + tools/proxysql_galera_checker.sh /usr/share/proxysql/ + tools/proxysql_galera_writer.pl /usr/share/proxysql/ +Description: High performance MySQL proxy + ProxySQL is a fast, reliable MySQL proxy with advanced runtime configuration management (virtually no configuration change requires a restart). + . + It features query routing, query caching, query rewriting (for queries generated by ORMs, for example) and is most of the time a drop-in replacement for mysqld from the point of view of the application. It can be configured and remote controlled through an SQL-compatible admin interface. +File: postinst + #!/bin/sh -e + if [ ! -d /var/lib/proxysql ]; then mkdir /var/lib/proxysql ; fi + update-rc.d proxysql defaults diff --git a/include/proxysql.h b/include/proxysql.h index 1a79c3b8c..3397ef58d 100644 --- a/include/proxysql.h +++ b/include/proxysql.h @@ -99,7 +99,7 @@ #else #define DEB "" #endif /* DEBUG */ -#define PROXYSQL_VERSION "1.2.2" DEB +#define PROXYSQL_VERSION "1.2.3-RC" DEB #define PROXYSQL_CODENAME "Truls" #ifndef PROXYSQL_FUNC_DEFS From 816a443276869a96a75038c29e290aa23b4be6c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Mon, 12 Sep 2016 12:45:04 +0000 Subject: [PATCH 08/10] Preparation to build on Fedora24 #686 --- Makefile | 45 ++++++++++++- .../proxysql/fedora24-build/proxysql.spec | 66 +++++++++++++++++++ .../images/proxysql/fedora24-build/rpmmacros | 2 + 3 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 docker/images/proxysql/fedora24-build/proxysql.spec create mode 100644 docker/images/proxysql/fedora24-build/rpmmacros diff --git a/Makefile b/Makefile index 4d83eae22..fb0c3b2c1 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ clean: cd lib && ${MAKE} clean cd src && ${MAKE} clean -packages: centos6.7 centos7 centos6.7-dbg centos7-dbg ubuntu12 ubuntu14 debian7 debian8 ubuntu12-dbg ubuntu14-dbg debian7-dbg debian8-dbg ubuntu16 ubuntu16-dbg +packages: centos6.7 centos7 centos6.7-dbg centos7-dbg ubuntu12 ubuntu14 debian7 debian8 ubuntu12-dbg ubuntu14-dbg debian7-dbg debian8-dbg ubuntu16 ubuntu16-dbg fedora24 fedora24-dbg .PHONY: packages @@ -64,6 +64,12 @@ centos6.7-dbg: binaries/proxysql-${CURVER}-1-dbg-centos67.x86_64.rpm centos7-dbg: binaries/proxysql-${CURVER}-1-dbg-centos7.x86_64.rpm .PHONY: centos7-dbg +fedora24: binaries/proxysql-${CURVER}-1-fedora24.x86_64.rpm +.PHONY: fedora24 + +fedora24-dbg: binaries/proxysql-${CURVER}-1-dbg-fedora24.x86_64.rpm +.PHONY: fedora24-dbg + ubuntu12: binaries/proxysql_${CURVER}-ubuntu12_amd64.deb .PHONY: ubuntu12 @@ -168,6 +174,43 @@ binaries/proxysql-${CURVER}-1-dbg-centos7.x86_64.rpm: docker rm centos7_build +binaries/proxysql-${CURVER}-1-fedora24.x86_64.rpm: + docker stop fedora24_build || true + docker rm fedora24_build || true + docker create --name fedora24_build renecannao/proxysql:build-fedora24 bash -c "while : ; do sleep 10 ; done" + docker start fedora24_build + docker exec fedora24_build bash -c "cd /opt; git clone -b v${CURVER} https://github.com/sysown/proxysql.git proxysql" + docker exec fedora24_build bash -c "cd /opt/proxysql; ${MAKE} clean && ${MAKE} -j 4 build_deps && ${MAKE}" + docker exec -it fedora24_build bash -c "cd /opt/proxysql ; mkdir -p proxysql/usr/bin; mkdir -p proxysql/etc; cp src/proxysql proxysql/usr/bin/; cp -a etc proxysql ; mkdir -p proxysql/usr/share/proxysql/tools ; cp -a tools/proxysql_galera_checker.sh tools/proxysql_galera_writer.pl proxysql/usr/share/proxysql/tools ; mv proxysql proxysql-${CURVER} ; tar czvf proxysql-${CURVER}.tar.gz proxysql-${CURVER}" + docker exec -it fedora24_build bash -c "mkdir -p /root/rpmbuild/{RPMS,SRPMS,BUILD,SOURCES,SPECS,tmp}" + docker cp docker/images/proxysql/fedora24-build/rpmmacros fedora24_build:/root/.rpmmacros + docker cp docker/images/proxysql/fedora24-build/proxysql.spec fedora24_build:/root/rpmbuild/SPECS/proxysql.spec + docker exec -it fedora24_build bash -c "cp /opt/proxysql/proxysql-${CURVER}.tar.gz /root/rpmbuild/SOURCES" + docker exec -it fedora24_build bash -c "cd /root/rpmbuild; rpmbuild -ba SPECS/proxysql.spec" + docker exec -it fedora24_build bash -c "cp /root/rpmbuild/RPMS/x86_64/proxysql-${CURVER}-1.x86_64.rpm /root/rpm" + docker cp fedora24_build:/root/rpmbuild/RPMS/x86_64/proxysql-${CURVER}-1.x86_64.rpm ./binaries/proxysql-${CURVER}-1-fedora24.x86_64.rpm + docker stop fedora24_build + docker rm fedora24_build + +binaries/proxysql-${CURVER}-1-dbg-fedora24.x86_64.rpm: + docker stop fedora24_build || true + docker rm fedora24_build || true + docker create --name fedora24_build renecannao/proxysql:build-fedora24 bash -c "while : ; do sleep 10 ; done" + docker start fedora24_build + docker exec fedora24_build bash -c "cd /opt; git clone -b v${CURVER} https://github.com/sysown/proxysql.git proxysql" + docker exec fedora24_build bash -c "cd /opt/proxysql; ${MAKE} clean && ${MAKE} -j 4 build_deps && ${MAKE} debug" + docker exec -it fedora24_build bash -c "cd /opt/proxysql ; mkdir -p proxysql/usr/bin; mkdir -p proxysql/etc; cp src/proxysql proxysql/usr/bin/; cp -a etc proxysql ; mkdir -p proxysql/usr/share/proxysql/tools ; cp -a tools/proxysql_galera_checker.sh tools/proxysql_galera_writer.pl proxysql/usr/share/proxysql/tools ; mv proxysql proxysql-${CURVER} ; tar czvf proxysql-${CURVER}.tar.gz proxysql-${CURVER}" + docker exec -it fedora24_build bash -c "mkdir -p /root/rpmbuild/{RPMS,SRPMS,BUILD,SOURCES,SPECS,tmp}" + docker cp docker/images/proxysql/fedora24-build/rpmmacros fedora24_build:/root/.rpmmacros + docker cp docker/images/proxysql/fedora24-build/proxysql.spec fedora24_build:/root/rpmbuild/SPECS/proxysql.spec + docker exec -it fedora24_build bash -c "cp /opt/proxysql/proxysql-${CURVER}.tar.gz /root/rpmbuild/SOURCES" + docker exec -it fedora24_build bash -c "cd /root/rpmbuild; rpmbuild -ba SPECS/proxysql.spec" + docker exec -it fedora24_build bash -c "cp /root/rpmbuild/RPMS/x86_64/proxysql-${CURVER}-1.x86_64.rpm /root/rpm" + docker cp fedora24_build:/root/rpmbuild/RPMS/x86_64/proxysql-${CURVER}-1.x86_64.rpm ./binaries/proxysql-${CURVER}-1-dbg-fedora24.x86_64.rpm + docker stop fedora24_build + docker rm fedora24_build + + binaries/proxysql_${CURVER}-ubuntu12_amd64.deb: docker stop ubuntu12_build || true docker rm ubuntu12_build || true diff --git a/docker/images/proxysql/fedora24-build/proxysql.spec b/docker/images/proxysql/fedora24-build/proxysql.spec new file mode 100644 index 000000000..694210102 --- /dev/null +++ b/docker/images/proxysql/fedora24-build/proxysql.spec @@ -0,0 +1,66 @@ +# Don't try fancy stuff like debuginfo, which is useless on binary-only +# packages. Don't strip binary too +# Be sure buildpolicy set to do nothing +%define __spec_install_post %{nil} +%define debug_package %{nil} +%define __os_install_post %{_dbpath}/brp-compress + +Summary: A high-performance MySQL proxy +Name: proxysql +Version: 1.2.3 +Release: 1 +License: GPL+ +Group: Development/Tools +SOURCE0 : %{name}-%{version}.tar.gz +URL: http://www.proxysql.com/ + +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root + +%description +%{summary} + +%prep +%setup -q + +%build +# Empty section. + +%install +rm -rf %{buildroot} +mkdir -p %{buildroot} + +# in builddir +cp -a * %{buildroot} + +%clean +rm -rf %{buildroot} + +%post +mkdir /var/run/%{name} +chkconfig --add %{name} + +%postun +rm -rf /var/run/%{name} +chkconfig --del %{name} + +%files +%defattr(-,root,root,-) +%config(noreplace) %{_sysconfdir}/%{name}.cnf +%{_bindir}/* +%{_sysconfdir}/init.d/%{name} +/usr/share/proxysql/tools/proxysql_galera_checker.sh +/usr/share/proxysql/tools/proxysql_galera_writer.pl + +%changelog +* Fri Sep 2 2016 Rene Cannao 1.2.2 +- Second stable release of 1.2 +* Tue Aug 2 2016 Rene Cannao 1.2.1 +- First stable release of 1.2 +* Mon Mar 14 2016 Rene Cannao 1.2.0 +- First testing release of 1.2 +* Sat Mar 11 2016 Rene Cannao 1.1.2 +- Upgraded to release 1.1.2 +* Sat Oct 31 2015 Rene Cannao 1.0.1 +- Compiles 1.0.1 +* Wed Sep 9 2015 Andrei Ismail 0.2 +- Added support for automatic packaging on Ubuntu 14.04 and CentOS 7. diff --git a/docker/images/proxysql/fedora24-build/rpmmacros b/docker/images/proxysql/fedora24-build/rpmmacros new file mode 100644 index 000000000..a1d0f32f0 --- /dev/null +++ b/docker/images/proxysql/fedora24-build/rpmmacros @@ -0,0 +1,2 @@ +%_topdir %(echo $HOME)/rpmbuild +%_tmppath %{_topdir}/tmp \ No newline at end of file From 3380f6df482b4ec00179f15831189285d92021d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 13 Sep 2016 22:41:07 +0000 Subject: [PATCH 09/10] Adding parsing for several SQL commands ALTER VIEW CALL CHANGE MASTER CREATE DATABASE CREATE INDEX CREATE SCHEMA CREATE TABLE CREATE TEMPORARY CREATE TRIGGER CREATE USER CREATE VIEW DESCRIBE DROP TABLE DROP TRIGGER DROP USER DROP VIEW FLUSH GRANT KILL LOCK TABLE LOAD OPTIMIZE RENAME TABLE RESET MASTER RESET SLAVE REVOKE SAVEPOINT TRUNCATE TABLE UNLOCK --- include/query_processor.h | 5 + lib/Query_Processor.cpp | 191 +++++++++++++++++++++++++++++++++++++- 2 files changed, 191 insertions(+), 5 deletions(-) diff --git a/include/query_processor.h b/include/query_processor.h index 4abe7019c..cd6cdaee1 100644 --- a/include/query_processor.h +++ b/include/query_processor.h @@ -9,8 +9,10 @@ typedef std::unordered_map umap_query_digest; enum MYSQL_COM_QUERY_command { MYSQL_COM_QUERY_ALTER_TABLE, + MYSQL_COM_QUERY_ALTER_VIEW, MYSQL_COM_QUERY_ANALYZE_TABLE, MYSQL_COM_QUERY_BEGIN, + MYSQL_COM_QUERY_CALL, MYSQL_COM_QUERY_CHANGE_MASTER, MYSQL_COM_QUERY_COMMIT, MYSQL_COM_QUERY_CREATE_DATABASE, @@ -19,6 +21,7 @@ enum MYSQL_COM_QUERY_command { MYSQL_COM_QUERY_CREATE_TEMPORARY, MYSQL_COM_QUERY_CREATE_TRIGGER, MYSQL_COM_QUERY_CREATE_USER, + MYSQL_COM_QUERY_CREATE_VIEW, MYSQL_COM_QUERY_DEALLOCATE, MYSQL_COM_QUERY_DELETE, MYSQL_COM_QUERY_DESCRIBE, @@ -27,6 +30,7 @@ enum MYSQL_COM_QUERY_command { MYSQL_COM_QUERY_DROP_TABLE, MYSQL_COM_QUERY_DROP_TRIGGER, MYSQL_COM_QUERY_DROP_USER, + MYSQL_COM_QUERY_DROP_VIEW, MYSQL_COM_QUERY_GRANT, MYSQL_COM_QUERY_EXECUTE, MYSQL_COM_QUERY_EXPLAIN, @@ -50,6 +54,7 @@ enum MYSQL_COM_QUERY_command { MYSQL_COM_QUERY_SET, MYSQL_COM_QUERY_SHOW_TABLE_STATUS, MYSQL_COM_QUERY_START_TRANSACTION, + MYSQL_COM_QUERY_TRUNCATE_TABLE, MYSQL_COM_QUERY_UNLOCK_TABLES, MYSQL_COM_QUERY_UPDATE, MYSQL_COM_QUERY_USE, diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 9353876ef..19b4715ff 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -298,8 +298,10 @@ Query_Processor::Query_Processor() { for (int i=0; i Date: Wed, 14 Sep 2016 17:43:32 +0000 Subject: [PATCH 10/10] Return empty result for SHOW WARNINGS #696 The current version of ProxySQL filters SHOW WARNINGS returning an empty resultset. This implementation is logically incorrect, and in future version a better implementation will be developed --- lib/MySQL_Session.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index c91fab819..b55caf281 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -597,7 +597,18 @@ bool MySQL_Session::handler_special_queries(PtrSize_t *pkt) { free(unstripped); return true; } - + if ( (pkt->size == 18) && (strncasecmp((char *)"SHOW WARNINGS",(char *)pkt->ptr+5,13)==0) ) { + SQLite3_result * resultset=new SQLite3_result(3); + resultset->add_column_definition(SQLITE_TEXT,"Level"); + resultset->add_column_definition(SQLITE_TEXT,"Code"); + resultset->add_column_definition(SQLITE_TEXT,"Message"); + SQLite3_to_MySQL(resultset, NULL, 0, &client_myds->myprot); + delete resultset; + client_myds->DSS=STATE_SLEEP; + status=WAITING_CLIENT_DATA; + l_free(pkt->size,pkt->ptr); + return true; + } return false; }