aherman
/
proxysql
mirror of https://github.com/sysown/proxysql
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/v2.2.0' into v2.2.0-3317

Browse Source
pull/3451/head
Javier Jaramago Fernández 5 years ago
parent 32c60576b9 a2c82bd95a
commit c1b705bf2b
16 changed files with 1204 additions and 2201 deletions
Show Stats Download Patch File Download Diff File

28
include/MySQL_Thread.h
Unescape View File

@ -216,14 +216,21 @@ class MySQL_Thread
~MySQL_Thread();
MySQL_Session * create_new_session_and_client_data_stream(int _fd);
bool init();
void run();
void run___get_multiple_idle_connections(int& num_idles);
void run___cleanup_mirror_queue();
void ProcessAllMyDS_BeforePoll();
void ProcessAllMyDS_AfterPoll();
void run();
void poll_listener_add(int sock);
void poll_listener_del(int sock);
void register_session(MySQL_Session*, bool up_start=true);
void unregister_session(int);
struct pollfd * get_pollfd(unsigned int i);
bool process_data_on_data_stream(MySQL_Data_Stream *myds, unsigned int n);
void process_all_sessions();
void ProcessAllSessions_SortingSessions();
void ProcessAllSessions_CompletedMirrorSession(unsigned int& n, MySQL_Session *sess);
void ProcessAllSessions_MaintenanceLoop(MySQL_Session *sess, unsigned long long sess_time, unsigned int& total_active_transactions_);
void process_all_sessions();
void refresh_variables();
void register_session_connection_handler(MySQL_Session *_sess, bool _new=false);
void unregister_session_connection_handler(int idx, bool _new=false);
@ -361,6 +368,20 @@ class MySQL_Threads_Handler
pthread_rwlock_t rwlock;
PtrArray *bind_fds;
MySQL_Listeners_Manager *MLM;
// VariablesPointers_int stores:
// key: variable name
// tuple:
// variable address
// min value
// max value
// special variable : if true, min and max values are ignored, and further input validation is required
std::unordered_map<std::string, std::tuple<int *, int, int, bool>> VariablesPointers_int;
// VariablesPointers_bool stores:
// key: variable name
// tuple:
// variable address
// special variable : if true, further input validation is required
std::unordered_map<std::string, std::tuple<bool *, bool>> VariablesPointers_bool;
public:
struct {
int monitor_history;
@ -427,7 +448,7 @@ class MySQL_Threads_Handler
char *server_version;
char *keep_multiplexing_variables;
//unsigned int default_charset; // removed in 2.0.13 . Obsoleted previously using MySQL_Variables instead
unsigned int handle_unknown_charset;
int handle_unknown_charset;
bool servers_stats;
bool commands_stats;
bool query_digests;
@ -545,7 +566,6 @@ class MySQL_Threads_Handler
~MySQL_Threads_Handler();
char *get_variable_string(char *name);
unsigned int get_variable_uint(char *name);
uint16_t get_variable_uint16(char *name);
int get_variable_int(const char *name);
void print_version();

12
include/ProxySQL_Cluster.hpp
Unescape View File

@ -86,6 +86,7 @@ class ProxySQL_Node_Entry {
struct {
ProxySQL_Checksum_Value_2 admin_variables;
ProxySQL_Checksum_Value_2 mysql_variables;
ProxySQL_Checksum_Value_2 ldap_variables;
ProxySQL_Checksum_Value_2 mysql_query_rules;
ProxySQL_Checksum_Value_2 mysql_servers;
ProxySQL_Checksum_Value_2 mysql_users;
@ -116,6 +117,7 @@ class ProxySQL_Cluster_Nodes {
void get_peer_to_sync_mysql_users(char **host, uint16_t *port);
void get_peer_to_sync_mysql_variables(char **host, uint16_t *port);
void get_peer_to_sync_admin_variables(char **host, uint16_t* port);
void get_peer_to_sync_ldap_variables(char **host, uint16_t *port);
void get_peer_to_sync_proxysql_servers(char **host, uint16_t *port);
};
@ -149,12 +151,19 @@ struct p_cluster_counter {
pulled_admin_variables_success,
pulled_admin_variables_failure,
pulled_ldap_variables_success,
pulled_ldap_variables_failure,
pulled_mysql_ldap_mapping_success,
pulled_mysql_ldap_mapping_failure,
sync_conflict_mysql_query_rules_share_epoch,
sync_conflict_mysql_servers_share_epoch,
sync_conflict_proxysql_servers_share_epoch,
sync_conflict_mysql_users_share_epoch,
sync_conflict_mysql_variables_share_epoch,
sync_conflict_admin_variables_share_epoch,
sync_conflict_ldap_variables_share_epoch,
sync_delayed_mysql_query_rules_version_one,
sync_delayed_mysql_servers_version_one,
@ -162,6 +171,7 @@ struct p_cluster_counter {
sync_delayed_proxysql_servers_version_one,
sync_delayed_mysql_variables_version_one,
sync_delayed_admin_variables_version_one,
sync_delayed_ldap_variables_version_one,
__size
};
@ -225,12 +235,14 @@ class ProxySQL_Cluster {
int cluster_mysql_users_diffs_before_sync;
int cluster_proxysql_servers_diffs_before_sync;
int cluster_mysql_variables_diffs_before_sync;
int cluster_ldap_variables_diffs_before_sync;
int cluster_admin_variables_diffs_before_sync;
bool cluster_mysql_query_rules_save_to_disk;
bool cluster_mysql_servers_save_to_disk;
bool cluster_mysql_users_save_to_disk;
bool cluster_proxysql_servers_save_to_disk;
bool cluster_mysql_variables_save_to_disk;
bool cluster_ldap_variables_save_to_disk;
bool cluster_admin_variables_save_to_disk;
ProxySQL_Cluster();
~ProxySQL_Cluster();

4
include/proxysql_admin.h
Unescape View File

@ -152,12 +152,14 @@ class ProxySQL_Admin {
int cluster_proxysql_servers_diffs_before_sync;
int cluster_mysql_variables_diffs_before_sync;
int cluster_admin_variables_diffs_before_sync;
int cluster_ldap_variables_diffs_before_sync;
bool cluster_mysql_query_rules_save_to_disk;
bool cluster_mysql_servers_save_to_disk;
bool cluster_mysql_users_save_to_disk;
bool cluster_proxysql_servers_save_to_disk;
bool cluster_mysql_variables_save_to_disk;
bool cluster_admin_variables_save_to_disk;
bool cluster_ldap_variables_save_to_disk;
int stats_mysql_connection_pool;
int stats_mysql_connections;
int stats_mysql_query_cache;
@ -262,6 +264,7 @@ class ProxySQL_Admin {
bool checksum_mysql_users;
bool checksum_mysql_variables;
bool checksum_admin_variables;
bool checksum_ldap_variables;
} checksum_variables;
void public_add_active_users(enum cred_username_type usertype, char *user=NULL) {
__add_active_users(usertype, user);
@ -311,6 +314,7 @@ class ProxySQL_Admin {
void flush_mysql_variables__from_memory_to_disk();
void flush_admin_variables__from_disk_to_memory();
void flush_admin_variables__from_memory_to_disk();
void flush_ldap_variables__from_memory_to_disk();
void load_mysql_servers_to_runtime();
void save_mysql_servers_from_runtime();

1
include/proxysql_glovars.hpp
Unescape View File

@ -111,6 +111,7 @@ class ProxySQL_GlobalVariables {
ProxySQL_Checksum_Value mysql_servers;
ProxySQL_Checksum_Value mysql_users;
ProxySQL_Checksum_Value mysql_variables;
ProxySQL_Checksum_Value ldap_variables;
ProxySQL_Checksum_Value proxysql_servers;
uint64_t global_checksum;
unsigned long long updates_cnt;

4
include/proxysql_structs.h
Unescape View File

@ -758,7 +758,7 @@ __thread int mysql_thread___set_query_lock_on_hostgroup;
__thread int mysql_thread___reset_connection_algorithm;
__thread uint32_t mysql_thread___server_capabilities;
__thread int mysql_thread___auto_increment_delay_multiplex;
__thread unsigned int mysql_thread___handle_unknown_charset;
__thread int mysql_thread___handle_unknown_charset;
__thread int mysql_thread___poll_timeout;
__thread int mysql_thread___poll_timeout_on_failure;
__thread bool mysql_thread___connection_warming;
@ -909,7 +909,7 @@ extern __thread int mysql_thread___set_query_lock_on_hostgroup;
extern __thread int mysql_thread___reset_connection_algorithm;
extern __thread uint32_t mysql_thread___server_capabilities;
extern __thread int mysql_thread___auto_increment_delay_multiplex;
extern __thread unsigned int mysql_thread___handle_unknown_charset;
extern __thread int mysql_thread___handle_unknown_charset;
extern __thread int mysql_thread___poll_timeout;
extern __thread int mysql_thread___poll_timeout_on_failure;
extern __thread bool mysql_thread___connection_warming;

2903
lib/MySQL_Thread.cpp
View File

File diff suppressed because it is too large Load Diff

2
lib/MySQL_Variables.cpp
Unescape View File

@ -24,6 +24,8 @@ MySQL_Variables::MySQL_Variables() {
ignore_vars.push_back("interactive_timeout");
ignore_vars.push_back("wait_timeout");
ignore_vars.push_back("net_read_timeout");
// NOTE: This variable has been temporarily ignored. Check issues #3442 and #3441.
ignore_vars.push_back("session_track_schema");
variables_regexp = "";
for (auto i = 0; i < SQL_NAME_LAST; i++) {
if (i == SQL_CHARACTER_SET || i == SQL_CHARACTER_ACTION || i == SQL_SET_NAMES) {

96
lib/ProxySQL_Admin.cpp
Unescape View File

@ -535,17 +535,20 @@ static char * admin_variables_names[]= {
(char *)"cluster_proxysql_servers_diffs_before_sync",
(char *)"cluster_mysql_variables_diffs_before_sync",
(char *)"cluster_admin_variables_diffs_before_sync",
(char *)"cluster_ldap_variables_diffs_before_sync",
(char *)"cluster_mysql_query_rules_save_to_disk",
(char *)"cluster_mysql_servers_save_to_disk",
(char *)"cluster_mysql_users_save_to_disk",
(char *)"cluster_proxysql_servers_save_to_disk",
(char *)"cluster_mysql_variables_save_to_disk",
(char *)"cluster_admin_variables_save_to_disk",
(char *)"cluster_ldap_variables_save_to_disk",
(char *)"checksum_mysql_query_rules",
(char *)"checksum_mysql_servers",
(char *)"checksum_mysql_users",
(char *)"checksum_mysql_variables",
(char *)"checksum_admin_variables",
(char *)"checksum_ldap_variables",
(char *)"restapi_enabled",
(char *)"restapi_port",
(char *)"web_enabled",
@ -5265,17 +5268,20 @@ ProxySQL_Admin::ProxySQL_Admin() :
variables.cluster_proxysql_servers_diffs_before_sync = 3;
variables.cluster_mysql_variables_diffs_before_sync = 3;
variables.cluster_admin_variables_diffs_before_sync = 3;
variables.cluster_ldap_variables_diffs_before_sync = 3;
checksum_variables.checksum_mysql_query_rules = true;
checksum_variables.checksum_mysql_servers = true;
checksum_variables.checksum_mysql_users = true;
checksum_variables.checksum_mysql_variables = true;
checksum_variables.checksum_admin_variables = true;
checksum_variables.checksum_ldap_variables = true;
variables.cluster_mysql_query_rules_save_to_disk = true;
variables.cluster_mysql_servers_save_to_disk = true;
variables.cluster_mysql_users_save_to_disk = true;
variables.cluster_proxysql_servers_save_to_disk = true;
variables.cluster_mysql_variables_save_to_disk = true;
variables.cluster_admin_variables_save_to_disk = true;
variables.cluster_ldap_variables_save_to_disk = true;
variables.stats_mysql_connection_pool = 60;
variables.stats_mysql_connections = 60;
variables.stats_mysql_query_cache = 60;
@ -6935,6 +6941,33 @@ void ProxySQL_Admin::flush_ldap_variables___database_to_runtime(SQLite3DB *db, b
}
}
GloMyLdapAuth->wrunlock();
// update variables checksum
if (checksum_variables.checksum_ldap_variables) {
pthread_mutex_lock(&GloVars.checksum_mutex);
// generate checksum for cluster
flush_ldap_variables___runtime_to_database(admindb, false, false, false, true);
char *error=NULL;
int cols=0;
int affected_rows=0;
SQLite3_result *resultset=NULL;
char *q=(char *)"SELECT variable_name, variable_value FROM runtime_global_variables WHERE variable_name LIKE 'ldap-\%' ORDER BY variable_name";
admindb->execute_statement(q, &error , &cols , &affected_rows , &resultset);
uint64_t hash1 = resultset->raw_checksum();
uint32_t d32[2];
char buf[20];
memcpy(&d32, &hash1, sizeof(hash1));
sprintf(buf,"0x%0X%0X", d32[0], d32[1]);
GloVars.checksums_values.ldap_variables.set_checksum(buf);
GloVars.checksums_values.ldap_variables.version++;
time_t t = time(NULL);
GloVars.checksums_values.ldap_variables.epoch = t;
GloVars.epoch_version = t;
GloVars.generate_global_checksum();
GloVars.checksums_values.updates_cnt++;
pthread_mutex_unlock(&GloVars.checksum_mutex);
delete resultset;
}
}
if (resultset) delete resultset;
}
@ -7099,6 +7132,10 @@ char * ProxySQL_Admin::get_variable(char *name) {
sprintf(intbuf,"%d",variables.cluster_admin_variables_diffs_before_sync);
return strdup(intbuf);
}
if (!strcasecmp(name,"cluster_ldap_variables_diffs_before_sync")) {
sprintf(intbuf,"%d",variables.cluster_ldap_variables_diffs_before_sync);
return strdup(intbuf);
}
if (!strcasecmp(name,"cluster_mysql_query_rules_save_to_disk")) {
return strdup((variables.cluster_mysql_query_rules_save_to_disk ? "true" : "false"));
}
@ -7117,6 +7154,9 @@ char * ProxySQL_Admin::get_variable(char *name) {
if (!strcasecmp(name,"cluster_admin_variables_save_to_disk")) {
return strdup((variables.cluster_admin_variables_save_to_disk ? "true" : "false"));
}
if (!strcasecmp(name,"cluster_ldap_variables_save_to_disk")) {
return strdup((variables.cluster_ldap_variables_save_to_disk ? "true" : "false"));
}
if (!strcasecmp(name,"refresh_interval")) {
sprintf(intbuf,"%d",variables.refresh_interval);
return strdup(intbuf);
@ -7145,6 +7185,9 @@ char * ProxySQL_Admin::get_variable(char *name) {
if (!strcasecmp(name,"checksum_admin_variables")) {
return strdup((checksum_variables.checksum_admin_variables ? "true" : "false"));
}
if (!strcasecmp(name,"checksum_ldap_variables")) {
return strdup((checksum_variables.checksum_ldap_variables ? "true" : "false"));
}
if (!strcasecmp(name,"restapi_enabled")) {
return strdup((variables.restapi_enabled ? "true" : "false"));
}
@ -7609,6 +7652,16 @@ bool ProxySQL_Admin::set_variable(char *name, char *value) { // this is the pub
return false;
}
}
if (!strcasecmp(name,"cluster_ldap_variables_diffs_before_sync")) {
int intv=atoi(value);
if (intv >= 0 && intv <= 1000) {
variables.cluster_ldap_variables_diffs_before_sync=intv;
__sync_lock_test_and_set(&GloProxyCluster->cluster_ldap_variables_diffs_before_sync, intv);
return true;
} else {
return false;
}
}
if (!strcasecmp(name,"version")) {
if (strcasecmp(value,(char *)PROXYSQL_VERSION)==0) {
return true;
@ -7775,6 +7828,20 @@ bool ProxySQL_Admin::set_variable(char *name, char *value) { // this is the pub
}
return rt;
}
if (!strcasecmp(name,"cluster_ldap_variables_save_to_disk")) {
bool rt = false;
if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) {
variables.cluster_ldap_variables_save_to_disk=true;
rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_ldap_variables_save_to_disk, true);
return true;
}
if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) {
variables.cluster_ldap_variables_save_to_disk=false;
rt = __sync_lock_test_and_set(&GloProxyCluster->cluster_ldap_variables_save_to_disk, false);
return true;
}
return rt;
}
if (!strcasecmp(name,"checksum_mysql_query_rules")) {
if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) {
checksum_variables.checksum_mysql_query_rules=true;
@ -7830,6 +7897,17 @@ bool ProxySQL_Admin::set_variable(char *name, char *value) { // this is the pub
}
return false;
}
if (!strcasecmp(name,"checksum_ldap_variables")) {
if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) {
checksum_variables.checksum_ldap_variables=true;
return true;
}
if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) {
checksum_variables.checksum_ldap_variables=false;
return true;
}
return false;
}
if (!strcasecmp(name,"read_only")) {
if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) {
variables.admin_read_only=true;
@ -9774,6 +9852,14 @@ void ProxySQL_Admin::flush_admin_variables__from_memory_to_disk() {
admindb->wrunlock();
}
void ProxySQL_Admin::flush_ldap_variables__from_memory_to_disk() {
admindb->wrlock();
admindb->execute("PRAGMA foreign_keys = OFF");
admindb->execute("INSERT OR REPLACE INTO disk.global_variables SELECT * FROM main.global_variables WHERE variable_name LIKE 'ldap-%'");
admindb->execute("PRAGMA foreign_keys = ON");
admindb->wrunlock();
}
void ProxySQL_Admin::__attach_db(SQLite3DB *db1, SQLite3DB *db2, char *alias) {
const char *a="ATTACH DATABASE '%s' AS %s";
int l=strlen(a)+strlen(db2->get_url())+strlen(alias)+5;
@ -10271,6 +10357,16 @@ void ProxySQL_Admin::dump_checksums_values_table() {
rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb);
rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb);
if (GloMyLdapAuth) {
rc=(*proxy_sqlite3_bind_text)(statement1, 1, "ldap_variables", -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb);
rc=(*proxy_sqlite3_bind_int64)(statement1, 2, GloVars.checksums_values.ldap_variables.version); ASSERT_SQLITE_OK(rc, admindb);
rc=(*proxy_sqlite3_bind_int64)(statement1, 3, GloVars.checksums_values.ldap_variables.epoch); ASSERT_SQLITE_OK(rc, admindb);
rc=(*proxy_sqlite3_bind_text)(statement1, 4, GloVars.checksums_values.ldap_variables.checksum, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb);
SAFE_SQLITE3_STEP2(statement1);
rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, admindb);
rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, admindb);
}
admindb->execute((char *)"COMMIT");
pthread_mutex_unlock(&GloVars.checksum_mutex);
(*proxy_sqlite3_finalize)(statement1);

238
lib/ProxySQL_Cluster.cpp
Unescape View File

@ -5,6 +5,7 @@
#include "prometheus_helpers.h"
#include "ProxySQL_Cluster.hpp"
#include "MySQL_LDAP_Authentication.hpp"
#ifdef DEBUG
#define DEB "_DEBUG"
@ -37,8 +38,8 @@
static char *NODE_COMPUTE_DELIMITER=(char *)"-gtyw23a-"; // a random string used for hashing
extern ProxySQL_Cluster * GloProxyCluster;
extern ProxySQL_Admin *GloAdmin;
extern MySQL_LDAP_Authentication* GloMyLdapAuth;
typedef struct _proxy_node_address_t {
pthread_t thrid;
@ -495,6 +496,26 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) {
}
continue;
}
if (GloMyLdapAuth && strcmp(row[0],"ldap_variables")==0) {
checksums_values.ldap_variables.version = atoll(row[1]);
checksums_values.ldap_variables.epoch = atoll(row[2]);
checksums_values.ldap_variables.last_updated = now;
if (strcmp(checksums_values.ldap_variables.checksum, row[3])) {
strcpy(checksums_values.ldap_variables.checksum, row[3]);
checksums_values.ldap_variables.last_changed = now;
checksums_values.ldap_variables.diff_check = 1;
proxy_info("Cluster: detected a new checksum for ldap_variables from peer %s:%d, version %llu, epoch %llu, checksum %s . Not syncing yet ...\n", hostname, port, checksums_values.ldap_variables.version, checksums_values.ldap_variables.epoch, checksums_values.ldap_variables.checksum);
if (strcmp(checksums_values.ldap_variables.checksum, GloVars.checksums_values.ldap_variables.checksum) == 0) {
proxy_info("Cluster: checksum for ldap_variables from peer %s:%d matches with local checksum %s , we won't sync.\n", hostname, port, GloVars.checksums_values.ldap_variables.checksum);
}
} else {
checksums_values.ldap_variables.diff_check++;
}
if (strcmp(checksums_values.ldap_variables.checksum, GloVars.checksums_values.ldap_variables.checksum) == 0) {
checksums_values.ldap_variables.diff_check = 0;
}
continue;
}
}
if (_r == NULL) {
ProxySQL_Checksum_Value_2 *v = NULL;
@ -540,6 +561,13 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) {
}
if (v->diff_check)
v->diff_check++;
v = &checksums_values.ldap_variables;
v->last_updated = now;
if (strcmp(v->checksum, GloVars.checksums_values.ldap_variables.checksum) == 0) {
v->diff_check = 0;
}
if (v->diff_check)
v->diff_check++;
}
pthread_mutex_unlock(&GloVars.checksum_mutex);
// we now do a series of checks, and we take action
@ -550,6 +578,7 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) {
unsigned int diff_mu = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_users_diffs_before_sync,0);
unsigned int diff_ps = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_proxysql_servers_diffs_before_sync,0);
unsigned int diff_mv = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_mysql_variables_diffs_before_sync,0);
unsigned int diff_lv = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_ldap_variables_diffs_before_sync,0);
unsigned int diff_av = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_admin_variables_diffs_before_sync,0);
ProxySQL_Checksum_Value_2 *v = NULL;
if (diff_mqr) {
@ -716,6 +745,34 @@ void ProxySQL_Node_Entry::set_checksums(MYSQL_RES *_r) {
}
}
}
if (GloMyLdapAuth && diff_lv) {
v = &checksums_values.ldap_variables;
unsigned long long own_version = __sync_fetch_and_add(&GloVars.checksums_values.ldap_variables.version, 0);
unsigned long long own_epoch = __sync_fetch_and_add(&GloVars.checksums_values.ldap_variables.epoch, 0);
char* own_checksum = __sync_fetch_and_add(&GloVars.checksums_values.ldap_variables.checksum, 0);
if (v->version > 1) {
if (
(own_version == 1) // we just booted
||
(v->epoch > own_epoch) // epoch is newer
) {
if (v->diff_check >= diff_lv) {
proxy_info("Cluster: detected a peer %s:%d with ldap_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. Proceeding with remote sync\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch);
GloProxyCluster->pull_global_variables_from_peer("ldap");
}
}
if ((v->epoch == own_epoch) && v->diff_check && ((v->diff_check % (diff_lv*10)) == 0)) {
proxy_error("Cluster: detected a peer %s:%d with ldap_variables version %llu, epoch %llu, diff_check %u, checksum %s. Own version: %llu, epoch: %llu, checksum %s. Sync conflict, epoch times are EQUAL, can't determine which server holds the latest config, we won't sync. This message will be repeated every %llu checks until LOAD LDAP VARIABLES is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, v->checksum, own_version, own_epoch, own_checksum, (diff_lv*10));
GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_conflict_ldap_variables_share_epoch]->Increment();
}
} else {
if (v->diff_check && (v->diff_check % (diff_lv*10)) == 0) {
proxy_warning("Cluster: detected a peer %s:%d with ldap_variables version %llu, epoch %llu, diff_check %u. Own version: %llu, epoch: %llu. diff_check is increasing, but version 1 doesn't allow sync. This message will be repeated every %llu checks until LOAD LDAP VARIABLES TO RUNTIME is executed on candidate master.\n", hostname, port, v->version, v->epoch, v->diff_check, own_version, own_epoch, (diff_lv*10));
GloProxyCluster->metrics.p_counter_array[p_cluster_counter::sync_delayed_ldap_variables_version_one]->Increment();
}
}
}
}
void ProxySQL_Cluster::pull_mysql_query_rules_from_peer() {
@ -944,7 +1001,7 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer() {
proxy_info("Cluster: Fetching MySQL Users from peer %s:%d completed\n", hostname, port);
proxy_info("Cluster: Loading to runtime MySQL Users from peer %s:%d\n", hostname, port);
GloAdmin->init_users();
if (GloProxyCluster->cluster_mysql_query_rules_save_to_disk == true) {
if (GloProxyCluster->cluster_mysql_users_save_to_disk == true) {
proxy_info("Cluster: Saving to disk MySQL Users from peer %s:%d\n", hostname, port);
GloAdmin->flush_mysql_users__from_memory_to_disk();
} else {
@ -955,9 +1012,57 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer() {
proxy_info("Cluster: Fetching MySQL Users from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn));
metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_failure]->Increment();
}
if (GloMyLdapAuth) {
rc_query = mysql_query(
conn,
"SELECT priority, frontend_entity, backend_entity, comment FROM runtime_mysql_ldap_mapping"
);
if (rc_query == 0) {
MYSQL_RES *result = mysql_store_result(conn);
GloAdmin->admindb->execute("DELETE FROM mysql_ldap_mapping");
MYSQL_ROW row;
char* q = const_cast<char*>(
"INSERT INTO mysql_ldap_mapping (priority, frontend_entity, backend_entity, comment)"
" VALUES (?1 , ?2 , ?3 , ?4)"
);
sqlite3_stmt *statement1 = NULL;
rc = GloAdmin->admindb->prepare_v2(q, &statement1);
ASSERT_SQLITE_OK(rc, GloAdmin->admindb);
while ((row = mysql_fetch_row(result))) {
rc=(*proxy_sqlite3_bind_int64)(statement1, 1, atoll(row[0])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // priority
rc=(*proxy_sqlite3_bind_text)(statement1, 2, row[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // frontend_entity
rc=(*proxy_sqlite3_bind_text)(statement1, 3, row[2], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // backend_entity
rc=(*proxy_sqlite3_bind_text)(statement1, 4, row[3], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // comment
SAFE_SQLITE3_STEP2(statement1);
rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb);
rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb);
}
mysql_free_result(result);
proxy_info("Cluster: Fetching LDAP Mappings from peer %s:%d completed\n", hostname, port);
proxy_info("Cluster: Loading to runtime LDAP Mappings from peer %s:%d\n", hostname, port);
GloAdmin->init_users();
if (GloProxyCluster->cluster_mysql_users_save_to_disk == true) {
proxy_info("Cluster: Saving to disk LDAP Mappings from peer %s:%d\n", hostname, port);
GloAdmin->flush_mysql_users__from_memory_to_disk();
} else {
proxy_info("Cluster: Saving to disk LDAP Mappings Rules from peer %s:%d\n", hostname, port);
}
metrics.p_counter_array[p_cluster_counter::pulled_mysql_ldap_mapping_success]->Increment();
} else {
proxy_info("Cluster: Fetching LDAP Mappings from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn));
metrics.p_counter_array[p_cluster_counter::pulled_mysql_ldap_mapping_failure]->Increment();
}
}
} else {
proxy_info("Cluster: Fetching MySQL Users from peer %s:%d failed: %s\n", hostname, port, mysql_error(conn));
metrics.p_counter_array[p_cluster_counter::pulled_mysql_users_failure]->Increment();
metrics.p_counter_array[p_cluster_counter::pulled_mysql_ldap_mapping_failure]->Increment();
}
}
__exit_pull_mysql_users_from_peer:
@ -1377,16 +1482,25 @@ void ProxySQL_Cluster::pull_global_variables_from_peer(const std::string& var_ty
vars_type_str = const_cast<char*>("Admin");
success_metric = p_cluster_counter::pulled_admin_variables_success;
failure_metric = p_cluster_counter::pulled_admin_variables_failure;
} else if (var_type == "ldap") {
vars_type_str = const_cast<char*>("LDAP");
success_metric = p_cluster_counter::pulled_ldap_variables_success;
failure_metric = p_cluster_counter::pulled_ldap_variables_failure;
} else {
proxy_error("Invalid parameter supplied to 'pull_global_variables_from_peer': var_type=%s", var_type.c_str());
proxy_error("Invalid parameter supplied to 'pull_global_variables_from_peer': var_type=%s\n", var_type.c_str());
assert(0);
}
pthread_mutex_lock(&GloProxyCluster->update_mysql_variables_mutex);
if (var_type == "mysql") {
nodes.get_peer_to_sync_mysql_variables(&hostname, &port);
} else {
} else if (var_type == "admin") {
nodes.get_peer_to_sync_admin_variables(&hostname, &port);
} else if (var_type == "ldap"){
nodes.get_peer_to_sync_ldap_variables(&hostname, &port);
} else {
proxy_error("Invalid parameter supplied to 'pull_global_variables_from_peer': var_type=%s\n", var_type.c_str());
assert(0);
}
if (hostname) {
@ -1450,13 +1564,23 @@ void ProxySQL_Cluster::pull_global_variables_from_peer(const std::string& var_ty
proxy_info("Cluster: Saving to disk MySQL Variables from peer %s:%d\n", hostname, port);
GloAdmin->flush_mysql_variables__from_memory_to_disk();
}
} else {
} else if (var_type == "admin") {
GloAdmin->load_admin_variables_to_runtime();
if (GloProxyCluster->cluster_admin_variables_save_to_disk == true) {
proxy_info("Cluster: Saving to disk Admin Variables from peer %s:%d\n", hostname, port);
GloAdmin->flush_admin_variables__from_memory_to_disk();
}
} else if (var_type == "ldap") {
GloAdmin->load_ldap_variables_to_runtime();
if (GloProxyCluster->cluster_ldap_variables_save_to_disk == true) {
proxy_info("Cluster: Saving to disk LDAP Variables from peer %s:%d\n", hostname, port);
GloAdmin->flush_ldap_variables__from_memory_to_disk();
}
} else {
proxy_error("Invalid parameter supplied to 'pull_global_variables_from_peer': var_type=%s\n", var_type.c_str());
assert(0);
}
metrics.p_counter_array[success_metric]->Increment();
} else {
@ -1973,6 +2097,48 @@ void ProxySQL_Cluster_Nodes::get_peer_to_sync_admin_variables(char **host, uint1
}
}
void ProxySQL_Cluster_Nodes::get_peer_to_sync_ldap_variables(char **host, uint16_t *port) {
unsigned long long version = 0;
unsigned long long epoch = 0;
unsigned long long max_epoch = 0;
char *hostname = NULL;
uint16_t p = 0;
unsigned int diff_mu = (unsigned int)__sync_fetch_and_add(&GloProxyCluster->cluster_ldap_variables_diffs_before_sync,0);
for (std::unordered_map<uint64_t, ProxySQL_Node_Entry *>::iterator it = umap_proxy_nodes.begin(); it != umap_proxy_nodes.end();) {
ProxySQL_Node_Entry * node = it->second;
ProxySQL_Checksum_Value_2 * v = &node->checksums_values.ldap_variables;
if (v->version > 1) {
if ( v->epoch > epoch ) {
max_epoch = v->epoch;
if (v->diff_check > diff_mu) {
epoch = v->epoch;
version = v->version;
if (hostname) {
free(hostname);
}
hostname=strdup(node->get_hostname());
p = node->get_port();
}
}
}
it++;
}
if (epoch) {
if (max_epoch > epoch) {
proxy_warning("Cluster: detected a peer with ldap_variables epoch %llu, but not enough diff_check. We won't sync from epoch %llu: temporarily skipping sync\n", max_epoch, epoch);
if (hostname) {
free(hostname);
hostname = NULL;
}
}
}
if (hostname) {
*host = hostname;
*port = p;
proxy_info("Cluster: detected peer %s:%d with ldap_variables version %llu, epoch %llu\n", hostname, p, version, epoch);
}
}
void ProxySQL_Cluster_Nodes::get_peer_to_sync_proxysql_servers(char **host, uint16_t *port) {
unsigned long long version = 0;
unsigned long long epoch = 0;
@ -2454,6 +2620,46 @@ cluster_metrics_map = std::make_tuple(
}
),
// ldap_variables_*
std::make_tuple (
p_cluster_counter::pulled_ldap_variables_success,
"proxysql_cluster_pulled_total",
"Number of times a 'module' have been pulled from a peer.",
metric_tags {
{ "module_name", "ldap_variables" },
{ "status", "success" }
}
),
std::make_tuple (
p_cluster_counter::pulled_ldap_variables_failure,
"proxysql_cluster_pulled_total",
"Number of times a 'module' have been pulled from a peer.",
metric_tags {
{ "module_name", "ldap_variables" },
{ "status", "failure" }
}
),
// mysql_ldap_mappings_*
std::make_tuple (
p_cluster_counter::pulled_mysql_ldap_mapping_success,
"proxysql_cluster_pulled_total",
"Number of times a 'module' have been pulled from a peer.",
metric_tags {
{ "module_name", "mysql_ldap_mapping" },
{ "status", "success" }
}
),
std::make_tuple (
p_cluster_counter::pulled_mysql_ldap_mapping_failure,
"proxysql_cluster_pulled_total",
"Number of times a 'module' have been pulled from a peer.",
metric_tags {
{ "module_name", "mysql_ldap_mapping" },
{ "status", "failure" }
}
),
// sync_conflict same epoch
// ====================================================================
std::make_tuple (
@ -2510,6 +2716,15 @@ cluster_metrics_map = std::make_tuple(
{ "reason", "servers_share_epoch" }
}
),
std::make_tuple (
p_cluster_counter::sync_conflict_ldap_variables_share_epoch,
"proxysql_cluster_syn_conflict_total",
"Number of times a 'module' has not been able to be synced.",
metric_tags {
{ "module_name", "ldap_variables" },
{ "reason", "servers_share_epoch" }
}
),
// ====================================================================
// sync_delayed due to version one
@ -2567,7 +2782,16 @@ cluster_metrics_map = std::make_tuple(
{ "module_name", "admin_variables" },
{ "reason", "version_one" }
}
)
),
std::make_tuple (
p_cluster_counter::sync_delayed_ldap_variables_version_one,
"proxysql_cluster_syn_conflict_total",
"Number of times a 'module' has not been able to be synced.",
metric_tags {
{ "module_name", "ldap_variables" },
{ "reason", "version_one" }
}
),
// ====================================================================
},
cluster_gauge_vector {}

11
lib/ProxySQL_GloVars.cpp
Unescape View File

@ -7,6 +7,10 @@
#include "SpookyV2.h"
#include <cxxabi.h>
#include "MySQL_LDAP_Authentication.hpp"
extern MySQL_LDAP_Authentication* GloMyLdapAuth;
static void term_handler(int sig) {
proxy_warning("Received TERM signal: shutdown in progress...\n");
#ifdef DEBUG
@ -367,6 +371,13 @@ uint64_t ProxySQL_GlobalVariables::generate_global_checksum() {
myhash.Update(v->checksum,strlen(v->checksum));
myhash.Update(&v->version,sizeof(v->version));
}
if (GloMyLdapAuth) {
v = &checksums_values.ldap_variables;
if (v->version) {
myhash.Update(v->checksum,strlen(v->checksum));
myhash.Update(&v->version,sizeof(v->version));
}
}
uint64_t h1, h2;
myhash.Final(&h1, &h2);
h1 = h1/2; // ugly way to make it signed within LLONG_MAX

46
test/tap/tests/set_testing-multi-t.cpp
Unescape View File

@ -12,7 +12,10 @@
#include <iostream>
#include <fstream>
#include <mutex>
#include "json.hpp"
#include "re2/re2.h"
#include "re2/regexp.h"
#include "tap.h"
#include "utils.h"
@ -571,9 +574,48 @@ void * my_conn_thread(void *arg) {
fprintf(stderr, "Variable %s->%s in proxysql resultset was not found.\nmysql data : %s\nproxysql data: %s\ncsv data %s\n",
el.value().dump().c_str(), el.key().c_str(), mysql_vars.dump().c_str(), proxysql_vars.dump().c_str(), vars.dump().c_str());
bool verified_special_sqlmode = false;
bool special_sqlmode = false;
if (el.key() == "sql_mode") {
if (!el.value().is_string()) {
diag("Invalid value for 'sql_mode' found. Provided value should be of 'string' type");
exit(EXIT_FAILURE);
}
std::string str_val { el.value() };
re2::RE2::Options options(RE2::Quiet);
options.set_case_sensitive(false);
options.set_longest_match(false);
re2::RE2 concat_re("^CONCAT\\((|@@|@@session\\.)SQL_MODE,\"(.*)\"\\)", options);
re2::StringPiece sp_input(str_val);
std::string f_match {};
std::string s_match {};
re2::RE2::Consume(&sp_input, concat_re, &f_match, &s_match);
if (!s_match.empty()) {
special_sqlmode = true;
// remove the initial 'comma' if exists
if (s_match[0] == ',') {
s_match = s_match.substr(1, std::string::npos);
}
std::string k_str_val { k.value() };
verified_special_sqlmode =
strcasestr(k_str_val.c_str(), s_match.c_str()) != NULL;
}
}
if (
(el.key() != "session_track_gtids" && (k.value() != el.value() || s.value() != el.value())) ||
(el.key() == "session_track_gtids" && !check_session_track_gtids(el.value(), s.value(), k.value()))
(special_sqlmode == true && verified_special_sqlmode == false) ||
(special_sqlmode == false &&
(el.key() != "session_track_gtids" && (k.value() != el.value() || s.value() != el.value())) ||
(el.key() == "session_track_gtids" && !check_session_track_gtids(el.value(), s.value(), k.value()))
)
) {
__sync_fetch_and_add(&g_failed, 1);
testPassed = false;

46
test/tap/tests/set_testing-t.cpp
Unescape View File

@ -22,7 +22,10 @@
#include <iostream>
#include <fstream>
#include <mutex>
#include "json.hpp"
#include "re2/re2.h"
#include "re2/regexp.h"
#include "tap.h"
#include "utils.h"
@ -565,9 +568,48 @@ void * my_conn_thread(void *arg) {
fprintf(stderr, "Variable %s->%s in proxysql resultset was not found.\nmysql data : %s\nproxysql data: %s\ncsv data %s\n",
el.value().dump().c_str(), el.key().c_str(), mysql_vars.dump().c_str(), proxysql_vars.dump().c_str(), vars.dump().c_str());
bool verified_special_sqlmode = false;
bool special_sqlmode = false;
if (el.key() == "sql_mode") {
if (!el.value().is_string()) {
diag("Invalid value for 'sql_mode' found. Provided value should be of 'string' type");
exit(EXIT_FAILURE);
}
std::string str_val { el.value() };
re2::RE2::Options options(RE2::Quiet);
options.set_case_sensitive(false);
options.set_longest_match(false);
re2::RE2 concat_re("^CONCAT\\((|@@|@@session\\.)SQL_MODE,\"(.*)\"\\)", options);
re2::StringPiece sp_input(str_val);
std::string f_match {};
std::string s_match {};
re2::RE2::Consume(&sp_input, concat_re, &f_match, &s_match);
if (!s_match.empty()) {
special_sqlmode = true;
// remove the initial 'comma' if exists
if (s_match[0] == ',') {
s_match = s_match.substr(1, std::string::npos);
}
std::string k_str_val { k.value() };
verified_special_sqlmode =
strcasestr(k_str_val.c_str(), s_match.c_str()) != NULL;
}
}
if (
(el.key() != "session_track_gtids" && (k.value() != el.value() || s.value() != el.value())) ||
(el.key() == "session_track_gtids" && !check_session_track_gtids(el.value(), s.value(), k.value()))
(special_sqlmode == true && verified_special_sqlmode == false) ||
(special_sqlmode == false &&
(el.key() != "session_track_gtids" && (k.value() != el.value() || s.value() != el.value())) ||
(el.key() == "session_track_gtids" && !check_session_track_gtids(el.value(), s.value(), k.value()))
)
) {
__sync_fetch_and_add(&g_failed, 1);
testPassed = false;

1
test/tap/tests/set_testing-t.csv
Unescape View File

@ -91,3 +91,4 @@
"SET time_zone='+04:00', sql_mode='NO_ENGINE_SUBSTITUTION', max_join_size=10000; SET CHARACTER SET 'latin1'", "{'time_zone':'+04:00', 'sql_mode':'NO_ENGINE_SUBSTITUTION', 'max_join_size':'10000', 'character_set_results':'latin1', 'character_set_client':'latin1'}", "['character_set_connection', 'collation_connection']"
"SET time_zone='+04:00', sql_mode='NO_ENGINE_SUBSTITUTION', max_join_size=10000; SET CHARSET 'latin1'", "{'time_zone':'+04:00', 'sql_mode':'NO_ENGINE_SUBSTITUTION', 'max_join_size':'10000', 'character_set_results':'latin1', 'character_set_client':'latin1'}", "['character_set_connection', 'collation_connection']"
"SET session_track_gtids=ALL_GTIDS", "{'session_track_gtids':'ALL_GTIDS'}"
"SET sql_safe_updates=1, session_track_schema=1, sql_mode = concat(@@sql_mode,',STRICT_TRANS_TABLES')", "{'sql_safe_updates':'ON', 'sql_mode':'concat(@@sql_mode,\',STRICT_TRANS_TABLES\')'}"

Can't render this file because it has a wrong number of fields in line 6.

5
test/tap/tests/sqlite3-t.cpp
Unescape View File

@ -6,9 +6,12 @@
#include <cstring>
#include <memory>
#include <openssl/ssl.h>
#include "proxysql_structs.h"
#include "proxysql_glovars.hpp"
#include "sqlite3db.h"
#include "MySQL_LDAP_Authentication.hpp"
MySQL_LDAP_Authentication* GloMyLdapAuth = nullptr;
int main() {
SQLite3DB::LoadPlugin(NULL);

6
test/tap/tests/test_cluster_sync-t.cpp
Unescape View File

@ -975,9 +975,9 @@ int main(int, char**) {
std::make_tuple("mysql-monitor_writer_is_also_reader" , "true" ),
std::make_tuple("mysql-max_allowed_packet" , "67108864" ),
std::make_tuple("mysql-tcp_keepalive_time" , "0" ),
std::make_tuple("mysql-use_tcp_keepalive" , "0" ),
std::make_tuple("mysql-automatic_detect_sqli" , "0" ),
std::make_tuple("mysql-firewall_whitelist_enabled" , "0" ),
std::make_tuple("mysql-use_tcp_keepalive" , "false" ),
std::make_tuple("mysql-automatic_detect_sqli" , "false" ),
std::make_tuple("mysql-firewall_whitelist_enabled" , "false" ),
std::make_tuple("mysql-firewall_whitelist_errormsg" , "Firewall blocked this query"),
std::make_tuple("mysql-throttle_connections_per_sec_to_hostgroup" , "1000001" ),
std::make_tuple("mysql-max_transaction_time" , "14400001" ),

2
test/tap/tests/test_filtered_set_statements-t.cpp
Unescape View File

@ -46,6 +46,8 @@ std::vector<std::pair<std::string, std::string>> filtered_set_queries {
{ "session_track_gtids", "OWN_GTID" },
{ "interactive_timeout", "28801" },
{ "net_read_timeout", "28801" },
// NOTE: This variable has been temporarily ignored. Check issues #3442 and #3441.
{ "session_track_schema", "1" },
};
std::vector<std::string> get_valid_set_query_set(const std::string& set_query, const std::string param) {

Write Preview
Loading…
Cancel
Save