Cluster: Speedup of processing of MySQL Users and MySQL Servers

- Cluster now syncs the server tables via 'incoming_*' tables generated
  during 'load_mysql_servers_to_runtime'.
- Cluster now syncs 'mysql_users' table via a resultset generated via
  '__refresh_users'.
pull/3921/head
Javier Jaramago Fernández 4 years ago
parent f12839acc1
commit 099c2a5f06

@ -55,6 +55,11 @@ typedef struct _creds_group_t {
class MySQL_Authentication {
private:
/**
* @brief Holds the current value for 'runtime_mysql_users' used by 'ProxySQL_Admin' to reply to
* 'CLUSTER_QUERY_MYSQL_USERS'.
*/
std::unique_ptr<SQLite3_result> mysql_users_resultset { nullptr };
creds_group_t creds_backends;
creds_group_t creds_frontends;
bool _reset(enum cred_username_type usertype);
@ -84,9 +89,21 @@ class MySQL_Authentication {
* schema_locked, transaction_persistent, fast_forward, backend, frontend, max_connections,
* attributes, comment FROM runtime_mysql_users"'
* The order isn't relevant in the query itself because ordering is performed while processing.
* @param mysql_users A 'unique_ptr' to be filled with the 'frontend' and 'backend' users found in the
* provided resulset.
* @return The computed hash for the provided resultset.
*/
uint64_t get_runtime_checksum(MYSQL_RES* resultset);
uint64_t get_runtime_checksum(MYSQL_RES* resultset, unique_ptr<SQLite3_result>& mysql_users);
/**
* @brief Takes ownership of the supplied resultset and stores it in 'mysql_users_resultset' field.
* @param users Holds the current value for 'runtime_mysql_users'.
*/
void save_mysql_users(std::unique_ptr<SQLite3_result>&& users);
/**
* @brief Return a pointer to internally managed 'mysql_users_resultset' field. DO NOT FREE.
* @return A pointer to the internally managed 'mysql_users_resultset'.
*/
SQLite3_result* get_current_mysql_users();
};
#endif /* __CLASS_MYSQL_AUTHENTICATION_H */

@ -375,6 +375,20 @@ class MySQL_HostGroups_Manager {
void generate_mysql_replication_hostgroups_table();
Galera_Info *get_galera_node_info(int hostgroup);
/**
* @brief These resultset holds the latest values for 'incoming_*' tables used promoted servers to runtime.
* @details All these resultsets are used by 'Cluster' to fetch and promote the same configuration used in the
* node across the whole cluster. For these, the queries:
* - 'CLUSTER_QUERY_MYSQL_SERVERS'
* - 'CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS'
* - 'CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS'
* - 'CLUSTER_QUERY_MYSQL_GALERA'
* - 'CLUSTER_QUERY_MYSQL_AWS_AURORA'
* Issued by 'Cluster' are intercepted by 'ProxySQL_Admin' and return the content of these resultsets. This is
* possible because 'incoming_*' tables represent the actual applied configuration to the node servers before
* reconfiguration ('commit') is triggered, making this process convergent on the supplied config.
*/
SQLite3_result* incoming_mysql_servers;
SQLite3_result *incoming_replication_hostgroups;
void generate_mysql_group_replication_hostgroups_table();
@ -540,10 +554,23 @@ class MySQL_HostGroups_Manager {
int servers_add(SQLite3_result *resultset);
bool commit(const std::string& checksum = "", const time_t epoch = 0);
/**
* @brief These setters/getter functions store and retrieve the currently hold resultset for the 'incoming_*'
* table set that have been loaded to runtime. Whose hold the config to be propagated by Cluster.
* @param The resulset to be stored replacing the current one.
*/
void set_incoming_mysql_servers(SQLite3_result *);
void set_incoming_replication_hostgroups(SQLite3_result *);
void set_incoming_group_replication_hostgroups(SQLite3_result *);
void set_incoming_galera_hostgroups(SQLite3_result *);
void set_incoming_aws_aurora_hostgroups(SQLite3_result *);
SQLite3_result* get_current_mysql_servers_inner();
SQLite3_result* get_current_mysql_replication_hostgroups_inner();
SQLite3_result* get_current_mysql_group_replication_hostgroups_inner();
SQLite3_result* get_current_mysql_galera_hostgroups();
SQLite3_result* get_current_mysql_aws_aurora_hostgroups();
SQLite3_result * execute_query(char *query, char **error);
SQLite3_result *dump_table_mysql_servers();
SQLite3_result *dump_table_mysql_replication_hostgroups();

@ -12,24 +12,45 @@
#define PROXYSQL_NODE_METRICS_LEN 5
/**
* @brief This query is intercepted by 'ProxySQL_Admin' and doesn't represent the actual resultset being received when
* issuing it. Instead it represent the 'intended' resultset that should be received when fetching a cluster peer
* `MYSQL_SERVERS` table. The caller issuing the query is responsable of the extra-processing.
* @details Received resultset corresponds to 'MySQL_HostGroups_Manager::dump_table_proxysql_servers' resultset.
* CLUSTER QUERIES DEFINITION
* ==========================
*
* The following queries are used by 'ProxySQL_Cluster' and intercepted by 'ProxySQL_Admin'. These queries should match
* the queries issued for generating the checksum for each of the target modules, for simpler reasoning, they should
* also represent the actual resultset being received when issuing them, since this resultset is used for computing the
* 'expected checksum' for the fetched config before loading it to runtime. This is done for the following modules:
* - 'runtime_mysql_servers': tables 'mysql_servers', 'mysql_replication_hostgroups', 'mysql_group_replication_hostroups',
* 'mysql_galera_hostgroups', 'mysql_aws_aurora_hostgroups'.
* - 'runtime_mysql_users'.
* - 'runtime_mysql_query_rules'.
*
* IMPORTANT: For further clarify this means that it's important that the actual resultset produced by the intercepted
* query preserve the filtering and ordering expressed in this queries.
*/
#define CLUSTER_QUERY_MYSQL_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM runtime_mysql_servers WHERE status<>'OFFLINE_HARD' ORDER BY hostgroup_id, hostname, port"
/**
* @brief The result of this query is later used for the computation of the checksum of the received 'mysql_servers'.
* Since it differs from the query being issued by 'MySQL_HostGroups_Manager::commit' for the checksum computation,
* it should be revisit whenever the table 'mysql_replication_hostgroups' change.
*/
#define CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS "SELECT writer_hostgroup, reader_hostgroup, comment FROM runtime_mysql_replication_hostgroups ORDER BY writer_hostgroup"
// the following two queries are the same used in ProxySQL_Admin::load_mysql_query_rules_to_runtime() , but on runtime_ tables.
// It is important to note that the queries in ProxySQL_Admin::load_mysql_query_rules_to_runtime() are used to compute the checksum,
// and are the same resultset saved. Therefore it Cluster can retrieve the same resultset saved it can easily compute the checksum before loading.
#define CLUSTER_QUERY_MYSQL_QUERY_RULES "SELECT rule_id, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, cache_timeout, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, ok_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, attributes, comment FROM runtime_mysql_query_rules ORDER BY rule_id"
#define CLUSTER_QUERY_MYSQL_QUERY_RULES_FAST_ROUTING "SELECT username, schemaname, flagIN, destination_hostgroup, comment FROM runtime_mysql_query_rules_fast_routing ORDER BY username, schemaname, flagIN"
/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_servers'. See top comment for details. */
#define CLUSTER_QUERY_MYSQL_SERVERS "PROXY_SELECT hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM runtime_mysql_servers WHERE status<>'OFFLINE_HARD' ORDER BY hostgroup_id, hostname, port"
/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_replication_hostgroups'. See top comment for details. */
#define CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS "PROXY_SELECT writer_hostgroup, reader_hostgroup, comment FROM runtime_mysql_replication_hostgroups ORDER BY writer_hostgroup"
/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_group_replication_hostgroups'. See top comment for details. */
#define CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS "PROXY_SELECT writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup, active, max_writers, writer_is_also_reader, max_transactions_behind, comment FROM runtime_mysql_group_replication_hostgroups ORDER BY writer_hostgroup"
/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_aws_aurora_hostgroups'. See top comment for details. */
#define CLUSTER_QUERY_MYSQL_AWS_AURORA "PROXY_SELECT writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment FROM runtime_mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup"
/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_galera_hostgroups'. See top comment for details. */
#define CLUSTER_QUERY_MYSQL_GALERA "PROXY_SELECT writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup, active, max_writers, writer_is_also_reader, max_transactions_behind, comment FROM runtime_mysql_galera_hostgroups ORDER BY writer_hostgroup"
/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_users'. See top comment for details. */
#define CLUSTER_QUERY_MYSQL_USERS "PROXY_SELECT username, password, use_ssl, default_hostgroup, default_schema, schema_locked, transaction_persistent, fast_forward, backend, frontend, max_connections, attributes, comment FROM runtime_mysql_users"
/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_users'. See top comment for details. */
#define CLUSTER_QUERY_MYSQL_QUERY_RULES "PROXY_SELECT rule_id, username, schemaname, flagIN, client_addr, proxy_addr, proxy_port, digest, match_digest, match_pattern, negate_match_pattern, re_modifiers, flagOUT, replace_pattern, destination_hostgroup, cache_ttl, cache_empty_result, cache_timeout, reconnect, timeout, retries, delay, next_query_flagIN, mirror_flagOUT, mirror_hostgroup, error_msg, ok_msg, sticky_conn, multiplex, gtid_from_hostgroup, log, apply, attributes, comment FROM runtime_mysql_query_rules ORDER BY rule_id"
/* @brief Query to be intercepted by 'ProxySQL_Admin' for 'runtime_mysql_users'. See top comment for details. */
#define CLUSTER_QUERY_MYSQL_QUERY_RULES_FAST_ROUTING "PROXY_SELECT username, schemaname, flagIN, destination_hostgroup, comment FROM runtime_mysql_query_rules_fast_routing ORDER BY username, schemaname, flagIN"
class ProxySQL_Checksum_Value_2: public ProxySQL_Checksum_Value {
public:

@ -116,6 +116,14 @@ struct admin_metrics_map_idx {
// ProxySQL_Admin shared variables
extern int admin__web_verbosity;
struct incoming_servers_t {
SQLite3_result* incoming_mysql_servers = NULL;
SQLite3_result* incoming_replication_hostgroups = NULL;
SQLite3_result* incoming_group_replication_hostgroups = NULL;
SQLite3_result* incoming_galera_hostgroups = NULL;
SQLite3_result* incoming_aurora_hostgroups = NULL;
};
class ProxySQL_Admin {
private:
volatile int main_shutdown;
@ -232,10 +240,31 @@ class ProxySQL_Admin {
void __insert_or_replace_disktable_select_maintable();
void __attach_db(SQLite3DB *db1, SQLite3DB *db2, char *alias);
void __add_active_users(enum cred_username_type usertype, char *user=NULL, uint64_t *hash1 = NULL);
/**
* @brief Loads to runtime either supplied users via params or users in 'mysql_users' table.
* @details If the 'usertype' and 'user' parameters are supplied, it loads the target user to runtime. If
* 'user' parameter is not supplied, and 'resulset' param is, it loads the users contained in this
* resultset. If both these params are 'nullptr' current contents of 'mysql_users' table are load to
* runtime. Param 'usertype' is ignored when 'resultset' param is supplied. It always return a
* 'SQLite3_result*' with the users that have been loaded to runtime.
*
* NOTE: The returned resultset doesn't contains duplicated rows for the 'frontend'/'backend' users,
* instead, contains a single row for representing both. This is by design, and the checksum computation
* in the received end should take this into account.
*
* @param usertype The target usertype supplied in param 'user' to 'load to runtime'.
* @param user The username of the user to LOAD TO RUNTIME.
* @param resultset If supplied, must contain all the users to be 'load to runtime'. Typically the
* parameter supplied here is the resultset of query 'CLUSTER_QUERY_MYSQL_USERS'.
*
* @return A 'SQLite3_result*' containing all the users that have been 'loaded to runtime'. When
* param 'resultset' is supplied, it will match it's value, otherwise it will be a locally created
* 'SQLite3_result*' that should be freed.
*/
SQLite3_result* __add_active_users(enum cred_username_type usertype, char *user=NULL, SQLite3_result* resultset = nullptr);
void __delete_inactive_users(enum cred_username_type usertype);
void add_admin_users();
void __refresh_users(const std::string& checksum = "", const time_t epoch = 0);
void __refresh_users(std::unique_ptr<SQLite3_result>&& all_users = nullptr, const std::string& checksum = "", const time_t epoch = 0);
void __add_active_users_ldap();
void flush_mysql_variables___runtime_to_database(SQLite3DB *db, bool replace, bool del, bool onlyifempty, bool runtime=false, bool use_lock=true);
@ -306,7 +335,7 @@ class ProxySQL_Admin {
bool get_read_only() { return variables.admin_read_only; }
bool set_read_only(bool ro) { variables.admin_read_only=ro; return variables.admin_read_only; }
bool has_variable(const char *name);
void init_users(const std::string& checksum = "", const time_t epoch = 0);
void init_users(std::unique_ptr<SQLite3_result>&& mysql_users_resultset = nullptr, const std::string& checksum = "", const time_t epoch = 0);
void init_mysql_servers();
void init_mysql_query_rules();
void init_mysql_firewall();
@ -339,7 +368,7 @@ class ProxySQL_Admin {
// void flush_admin_variables__from_disk_to_memory(); // commented in 2.3 because unused
void flush_admin_variables__from_memory_to_disk();
void flush_ldap_variables__from_memory_to_disk();
void load_mysql_servers_to_runtime(const std::string& checksum = "", const time_t epoch = 0);
void load_mysql_servers_to_runtime(const incoming_servers_t& incoming_servers = {}, const std::string& checksum = "", const time_t epoch = 0);
void save_mysql_servers_from_runtime();
/**
* @brief Performs the load to runtime of the current configuration in 'main' for 'mysql_query_rules' and

@ -39,7 +39,7 @@ enum log_event_type {
PROXYSQL_COM_STMT_PREPARE
};
enum cred_username_type { USERNAME_BACKEND, USERNAME_FRONTEND };
enum cred_username_type { USERNAME_BACKEND, USERNAME_FRONTEND, USERNAME_NONE };
#define PROXYSQL_USE_RESULT

@ -641,7 +641,7 @@ uint64_t MySQL_Authentication::get_runtime_checksum() {
return hashB+hashF;
}
pair<umap_auth, umap_auth> extract_accounts_details(MYSQL_RES* resultset) {
pair<umap_auth, umap_auth> extract_accounts_details(MYSQL_RES* resultset, unique_ptr<SQLite3_result>& all_users) {
if (resultset == nullptr) { return { umap_auth {}, umap_auth {} }; }
// The following order is assumed for the resulset received fields:
@ -650,48 +650,76 @@ pair<umap_auth, umap_auth> extract_accounts_details(MYSQL_RES* resultset) {
umap_auth f_accs_map {};
umap_auth b_accs_map {};
while (MYSQL_ROW row = mysql_fetch_row(resultset)) {
// Create the SQLite3 resultsets for 'frontend' and 'backend' users
uint32_t num_fields = mysql_num_fields(resultset);
MYSQL_FIELD* fields = mysql_fetch_fields(resultset);
SQLite3_result* _all_users { new SQLite3_result(num_fields) };
for (uint32_t i = 0; i < num_fields; i++) {
_all_users->add_column_definition(SQLITE_TEXT, fields[i].name);
}
const auto create_account_details = [] (MYSQL_ROW row) -> account_details_t* {
account_details_t* acc_details { new account_details_t {} };
acc_details->username = row[0];
acc_details->password = row[1] ? row[1] : const_cast<char*>("");
acc_details->__active = strcmp(row[2], "1") == 0 ? true : false;
acc_details->use_ssl = strcmp(row[3], "1") == 0 ? true : false;
acc_details->default_hostgroup = atoi(row[4]);
acc_details->default_schema = row[5] ? row[5] : const_cast<char*>("");
acc_details->schema_locked = strcmp(row[6], "1") == 0 ? true : false;
acc_details->transaction_persistent = strcmp(row[7], "1") == 0 ? true : false;
acc_details->fast_forward = strcmp(row[8], "1") == 0 ? true : false;
acc_details->__backend = strcmp(row[9], "1") == 0 ? true : false;
acc_details->__frontend = strcmp(row[10], "1") == 0 ? true : false;
acc_details->max_connections = atoi(row[11]);
acc_details->attributes = row[12] ? row[12] : const_cast<char*>("");
acc_details->comment = row[13] ? row[13] : const_cast<char*>("");
acc_details->__active = true;
acc_details->use_ssl = strcmp(row[2], "1") == 0 ? true : false;
acc_details->default_hostgroup = atoi(row[3]);
acc_details->default_schema = row[4] ? row[4] : const_cast<char*>("");
acc_details->schema_locked = strcmp(row[5], "1") == 0 ? true : false;
acc_details->transaction_persistent = strcmp(row[6], "1") == 0 ? true : false;
acc_details->fast_forward = strcmp(row[7], "1") == 0 ? true : false;
acc_details->__backend = strcmp(row[8], "1") == 0 ? true : false;
acc_details->__frontend = strcmp(row[9], "1") == 0 ? true : false;
acc_details->max_connections = atoi(row[10]);
acc_details->attributes = row[11] ? row[11] : const_cast<char*>("");
acc_details->comment = row[12] ? row[12] : const_cast<char*>("");
return acc_details;
};
vector<char*> pta(static_cast<size_t>(num_fields));
while (MYSQL_ROW row = mysql_fetch_row(resultset)) {
// compute the 'username' hash for the map
uint64_t u_hash = 0, _u_hash2 = 0;
SpookyHash myhash {};
myhash.Init(1,2);
myhash.Update(acc_details->username, strlen(acc_details->username));
myhash.Update(row[0], strlen(row[0]));
myhash.Final(&u_hash, &_u_hash2);
if (acc_details->__backend) {
// is backend
if (strcmp(row[8], "1") == 0) {
account_details_t* acc_details = create_account_details(row);
b_accs_map.insert({u_hash, acc_details});
} else {
}
// is frontend
if (strcmp(row[9], "1") == 0) {
account_details_t* acc_details = create_account_details(row);
f_accs_map.insert({u_hash, acc_details});
}
// Update the contents of the row for the SQLite3 resultset
for (uint32_t i = 0; i < num_fields; i++) {
pta[i] = row[i];
}
_all_users->add_row(&pta[0]);
}
mysql_data_seek(resultset, 0);
// Update the supplied 'unique_ptr' with the target resultsets
all_users.reset(_all_users);
return { b_accs_map, f_accs_map };
}
uint64_t MySQL_Authentication::get_runtime_checksum(MYSQL_RES* resultset) {
uint64_t MySQL_Authentication::get_runtime_checksum(MYSQL_RES* resultset, unique_ptr<SQLite3_result>& all_users) {
if (resultset == NULL) { return 0; }
pair<umap_auth, umap_auth> acc_maps { extract_accounts_details(resultset) };
pair<umap_auth, umap_auth> acc_maps { extract_accounts_details(resultset, all_users) };
uint64_t b_acc_hash = compute_accounts_hash(acc_maps.first);
uint64_t f_acc_hash = compute_accounts_hash(acc_maps.second);
@ -705,3 +733,11 @@ uint64_t MySQL_Authentication::get_runtime_checksum(MYSQL_RES* resultset) {
return b_acc_hash + f_acc_hash;
}
void MySQL_Authentication::save_mysql_users(unique_ptr<SQLite3_result>&& users) {
this->mysql_users_resultset = std::move(users);
}
SQLite3_result* MySQL_Authentication::get_current_mysql_users() {
return this->mysql_users_resultset.get();
}

@ -1410,6 +1410,7 @@ MySQL_HostGroups_Manager::MySQL_HostGroups_Manager() {
mydb->execute(MYHGM_MYSQL_AWS_AURORA_HOSTGROUPS);
mydb->execute("CREATE INDEX IF NOT EXISTS idx_mysql_servers_hostname_port ON mysql_servers (hostname,port)");
MyHostGroups=new PtrArray();
incoming_mysql_servers=NULL;
incoming_replication_hostgroups=NULL;
incoming_group_replication_hostgroups=NULL;
incoming_galera_hostgroups=NULL;
@ -3788,7 +3789,19 @@ __exit_get_multiple_idle_connections:
return num_conn_current;
}
void MySQL_HostGroups_Manager::set_incoming_mysql_servers(SQLite3_result *s) {
if (incoming_mysql_servers) {
delete incoming_mysql_servers;
incoming_mysql_servers = nullptr;
}
incoming_mysql_servers=s;
}
void MySQL_HostGroups_Manager::set_incoming_replication_hostgroups(SQLite3_result *s) {
if (incoming_replication_hostgroups) {
delete incoming_replication_hostgroups;
incoming_replication_hostgroups = nullptr;
}
incoming_replication_hostgroups=s;
}
@ -3816,6 +3829,26 @@ void MySQL_HostGroups_Manager::set_incoming_aws_aurora_hostgroups(SQLite3_result
incoming_aws_aurora_hostgroups=s;
}
SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_servers_inner() {
return this->incoming_mysql_servers;
}
SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_replication_hostgroups_inner() {
return this->incoming_replication_hostgroups;
}
SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_group_replication_hostgroups_inner() {
return this->incoming_group_replication_hostgroups;
}
SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_galera_hostgroups() {
return this->incoming_galera_hostgroups;
}
SQLite3_result* MySQL_HostGroups_Manager::get_current_mysql_aws_aurora_hostgroups() {
return this->incoming_aws_aurora_hostgroups;
}
SQLite3_result * MySQL_HostGroups_Manager::SQL3_Free_Connections() {
const int colnum=13;
proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 4, "Dumping Free Connections in Pool\n");

@ -52,6 +52,9 @@
#include <uuid/uuid.h>
using std::string;
using std::unique_ptr;
#ifdef WITHGCOV
extern "C" void __gcov_dump();
extern "C" void __gcov_reset();
@ -3698,24 +3701,118 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) {
// handle special queries from Cluster
// for bug #1188 , ProxySQL Admin needs to know the exact query
if (!strncasecmp(CLUSTER_QUERY_MYSQL_SERVERS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_SERVERS))) {
//ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa;
if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats
resultset=MyHGM->dump_table_mysql_servers();
if (resultset) {
GloAdmin->mysql_servers_wrlock();
resultset = MyHGM->get_current_mysql_servers_inner();
GloAdmin->mysql_servers_wrunlock();
if (resultset == nullptr) {
resultset=MyHGM->dump_table_mysql_servers();
if (resultset) {
sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot);
delete resultset;
run_query=false;
goto __run_query;
}
} else {
sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot);
delete resultset;
run_query=false;
goto __run_query;
}
}
}
if (!strncasecmp(CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_REPLICATION_HOSTGROUPS))) {
//ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa;
if (sess->session_type == PROXYSQL_SESSION_ADMIN) { // no stats
resultset=MyHGM->dump_table_mysql_replication_hostgroups();
if (resultset) {
if (sess->session_type == PROXYSQL_SESSION_ADMIN) {
GloAdmin->mysql_servers_wrlock();
resultset = MyHGM->get_current_mysql_replication_hostgroups_inner();
GloAdmin->mysql_servers_wrunlock();
if (resultset == nullptr) {
resultset=MyHGM->dump_table_mysql_replication_hostgroups();
if (resultset) {
sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot);
delete resultset;
run_query=false;
goto __run_query;
}
} else {
sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot);
run_query=false;
goto __run_query;
}
}
}
if (!strncasecmp(CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS))) {
if (sess->session_type == PROXYSQL_SESSION_ADMIN) {
GloAdmin->mysql_servers_wrlock();
resultset = MyHGM->get_current_mysql_group_replication_hostgroups_inner();
GloAdmin->mysql_servers_wrunlock();
if (resultset == nullptr) {
resultset=MyHGM->dump_table_mysql_group_replication_hostgroups();
if (resultset) {
sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot);
delete resultset;
run_query=false;
goto __run_query;
}
} else {
sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot);
run_query=false;
goto __run_query;
}
}
}
if (!strncasecmp(CLUSTER_QUERY_MYSQL_GALERA, query_no_space, strlen(CLUSTER_QUERY_MYSQL_GALERA))) {
if (sess->session_type == PROXYSQL_SESSION_ADMIN) {
GloAdmin->mysql_servers_wrlock();
resultset = MyHGM->get_current_mysql_galera_hostgroups();
GloAdmin->mysql_servers_wrunlock();
if (resultset == nullptr) {
resultset=MyHGM->dump_table_mysql_galera_hostgroups();
if (resultset) {
sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot);
delete resultset;
run_query=false;
goto __run_query;
}
} else {
sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot);
run_query=false;
goto __run_query;
}
}
}
if (!strncasecmp(CLUSTER_QUERY_MYSQL_AWS_AURORA, query_no_space, strlen(CLUSTER_QUERY_MYSQL_AWS_AURORA))) {
if (sess->session_type == PROXYSQL_SESSION_ADMIN) {
GloAdmin->mysql_servers_wrlock();
resultset = MyHGM->get_current_mysql_aws_aurora_hostgroups();
GloAdmin->mysql_servers_wrunlock();
if (resultset == nullptr) {
resultset=MyHGM->dump_table_mysql_aws_aurora_hostgroups();
if (resultset) {
sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot);
delete resultset;
run_query=false;
goto __run_query;
}
} else {
sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot);
run_query=false;
goto __run_query;
}
}
}
if (!strncasecmp(CLUSTER_QUERY_MYSQL_USERS, query_no_space, strlen(CLUSTER_QUERY_MYSQL_USERS))) {
if (sess->session_type == PROXYSQL_SESSION_ADMIN) {
pthread_mutex_lock(&users_mutex);
resultset = GloMyAuth->get_current_mysql_users();
pthread_mutex_unlock(&users_mutex);
if (resultset != nullptr) {
sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot);
delete resultset;
run_query=false;
goto __run_query;
}
@ -10481,9 +10578,11 @@ void ProxySQL_Admin::__attach_db(SQLite3DB *db1, SQLite3DB *db2, char *alias) {
}
void ProxySQL_Admin::init_users(const std::string& checksum, const time_t epoch) {
void ProxySQL_Admin::init_users(
unique_ptr<SQLite3_result>&& mysql_users_resultset, const std::string& checksum, const time_t epoch
) {
pthread_mutex_lock(&users_mutex);
__refresh_users(checksum, epoch);
__refresh_users(std::move(mysql_users_resultset), checksum, epoch);
pthread_mutex_unlock(&users_mutex);
}
@ -10523,55 +10622,73 @@ void ProxySQL_Admin::add_admin_users() {
#endif /* DEBUG */
}
void ProxySQL_Admin::__refresh_users(const std::string& checksum, const time_t epoch) {
void ProxySQL_Admin::__refresh_users(
unique_ptr<SQLite3_result>&& mysql_users_resultset, const string& checksum, const time_t epoch
) {
bool calculate_checksum = false;
bool no_resultset_supplied = mysql_users_resultset == nullptr;
if (checksum_variables.checksum_mysql_users) {
calculate_checksum = true;
}
if (calculate_checksum)
pthread_mutex_lock(&GloVars.checksum_mutex);
__delete_inactive_users(USERNAME_BACKEND);
__delete_inactive_users(USERNAME_FRONTEND);
GloMyAuth->set_all_inactive(USERNAME_BACKEND);
GloMyAuth->set_all_inactive(USERNAME_FRONTEND);
add_admin_users();
// uint64_t hashB, hashF;
// if (calculate_checksum) {
// __add_active_users(USERNAME_BACKEND, NULL, &hashB);
// __add_active_users(USERNAME_FRONTEND, NULL, &hashF);
// } else {
__add_active_users(USERNAME_BACKEND);
__add_active_users(USERNAME_FRONTEND);
// }
SQLite3_result* added_users { __add_active_users(USERNAME_NONE, NULL, mysql_users_resultset.get()) };
if (mysql_users_resultset == nullptr && added_users != nullptr) {
mysql_users_resultset.reset(added_users);
}
if (GloMyLdapAuth) {
__add_active_users_ldap();
}
GloMyAuth->remove_inactives(USERNAME_BACKEND);
GloMyAuth->remove_inactives(USERNAME_FRONTEND);
uint64_t hash1 = 0;
set_variable((char *)"admin_credentials",(char *)"");
if (calculate_checksum) {
hash1 = GloMyAuth->get_runtime_checksum();
//uint64_t hash1 = hashB + hashF; // overflow allowed
if (GloMyLdapAuth) {
hash1 += GloMyLdapAuth->get_ldap_mapping_runtime_checksum();
char* buff = nullptr;
char buf[20] = { 0 };
if (no_resultset_supplied) {
uint64_t hash1 = GloMyAuth->get_runtime_checksum();
if (GloMyLdapAuth) {
hash1 += GloMyLdapAuth->get_ldap_mapping_runtime_checksum();
}
uint32_t d32[2];
memcpy(&d32, &hash1, sizeof(hash1));
sprintf(buf,"0x%0X%0X", d32[0], d32[1]);
buff = buf;
} else {
buff = const_cast<char*>(checksum.c_str());
}
uint32_t d32[2];
char buf[20];
memcpy(&d32, &hash1, sizeof(hash1));
sprintf(buf,"0x%0X%0X", d32[0], d32[1]);
GloVars.checksums_values.mysql_users.set_checksum(buf);
GloVars.checksums_values.mysql_users.set_checksum(buff);
GloVars.checksums_values.mysql_users.version++;
time_t t = time(NULL);
if (epoch != 0 && checksum != "" && GloVars.checksums_values.mysql_users.checksum == checksum) {
const bool same_checksum = no_resultset_supplied == false;
const bool matching_checksums = same_checksum || (GloVars.checksums_values.mysql_users.checksum == checksum);
if (epoch != 0 && checksum != "" && matching_checksums) {
GloVars.checksums_values.mysql_users.epoch = epoch;
} else {
GloVars.checksums_values.mysql_users.epoch = t;
}
GloVars.epoch_version = t;
GloVars.generate_global_checksum();
GloVars.checksums_values.updates_cnt++;
// store the new 'added_users' resultset after generating the new checksum
GloMyAuth->save_mysql_users(std::move(mysql_users_resultset));
pthread_mutex_unlock(&GloVars.checksum_mutex);
}
proxy_info(
@ -10680,64 +10797,49 @@ void ProxySQL_Admin::__add_active_users_ldap() {
#define ADDUSER_STMT_RAW
void ProxySQL_Admin::__add_active_users(enum cred_username_type usertype, char *__user, uint64_t *hash1) {
SQLite3_result* ProxySQL_Admin::__add_active_users(
enum cred_username_type usertype, char *__user, SQLite3_result* mysql_users_resultset
) {
char *error=NULL;
int cols=0;
int affected_rows=0;
bool empty = true;
SpookyHash myhash;
if (hash1) {
myhash.Init(19,3);
}
#ifdef ADDUSER_STMT_RAW
sqlite3_stmt *statement=NULL;
#else
SQLite3_result *resultset=NULL;
#endif
char *str=NULL;
char *query=NULL;
if (__user==NULL) {
if (hash1) {
str=(char *)"SELECT username,password,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,max_connections,attributes,comment FROM main.mysql_users WHERE %s=1 AND active=1 AND default_hostgroup>=0 ORDER BY username";
if (mysql_users_resultset == nullptr) {
str = (char*)"SELECT username,password,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,backend,frontend,max_connections,attributes,comment FROM main.mysql_users WHERE active=1 AND default_hostgroup>=0";
admindb->execute_statement(str, &error, &cols, &affected_rows, &resultset);
} else {
str=(char *)"SELECT username,password,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,max_connections,attributes,comment FROM main.mysql_users WHERE %s=1 AND active=1 AND default_hostgroup>=0";
resultset = mysql_users_resultset;
}
query=(char *)malloc(strlen(str)+15);
sprintf(query,str,(usertype==USERNAME_BACKEND ? "backend" : "frontend"));
} else {
str=(char *)"SELECT username,password,use_ssl,default_hostgroup,default_schema,schema_locked,transaction_persistent,fast_forward,max_connections,attributes,comment FROM main.mysql_users WHERE %s=1 AND active=1 AND default_hostgroup>=0 AND username='%s'";
query=(char *)malloc(strlen(str)+strlen(__user)+15);
sprintf(query,str,(usertype==USERNAME_BACKEND ? "backend" : "frontend"),__user);
admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset);
}
#ifdef ADDUSER_STMT_RAW
admindb->execute_statement_raw(query, &error , &cols , &affected_rows , &statement);
#else
admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset);
#endif
SQLite3_result* added_users { nullptr };
if (error) {
proxy_error("Error on %s : %s\n", query, error);
} else {
#ifdef ADDUSER_STMT_RAW
int rc;
while ((rc=(*proxy_sqlite3_step)(statement))==SQLITE_ROW) {
SQLite3_row *r=new SQLite3_row(cols);
r->add_fields(statement);
if (hash1) {
empty = false;
for (int i=0; i<cols;i++) {
if (r->fields[i]) {
myhash.Update(r->fields[i],r->sizes[i]);
} else {
myhash.Update("",0);
}
}
SQLite3_result* sqlite_result { nullptr };
if (mysql_users_resultset == nullptr) {
sqlite_result = new SQLite3_result(resultset->columns);
for (SQLite3_column* c : resultset->column_definition) {
sqlite_result->add_column_definition(c->datatype, c->name);
}
}
#else
for (std::vector<SQLite3_row *>::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) {
SQLite3_row *r=*it;
#endif
char *password=NULL;
if (variables.hash_passwords) { // We must use hashed password. See issue #676
// Admin needs to hash the password
@ -10766,44 +10868,86 @@ void ProxySQL_Admin::__add_active_users(enum cred_username_type usertype, char *
password=(char *)"";
}
}
GloMyAuth->add(
r->fields[0], // username
password, // before #676, wewere always passing the password. Now it is possible that the password can be hashed
usertype, // backend/frontend
(strcmp(r->fields[2],"1")==0 ? true : false) , // use_ssl
atoi(r->fields[3]), // default_hostgroup
(r->fields[4]==NULL ? (char *)"" : r->fields[4]), //default_schema
(strcmp(r->fields[5],"1")==0 ? true : false) , // schema_locked
(strcmp(r->fields[6],"1")==0 ? true : false) , // transaction_persistent
(strcmp(r->fields[7],"1")==0 ? true : false), // fast_forward
( atoi(r->fields[8])>0 ? atoi(r->fields[8]) : 0), // max_connections
(r->fields[9] == NULL ? (char *)"" : r->fields[9]), // attributes
(r->fields[10]==NULL ? (char *)"" : r->fields[10]) //comment
);
std::vector<enum cred_username_type> usertypes {};
char* max_connections = nullptr;
char* attributes = nullptr;
char* comment = nullptr;
if (__user != nullptr) {
usertypes.push_back(usertype);
max_connections = r->fields[8];
attributes = r->fields[9];
comment = r->fields[10];
} else {
if (strcasecmp(r->fields[8], "1") == 0) {
usertypes.push_back(USERNAME_BACKEND);
}
if (strcasecmp(r->fields[9], "1") == 0) {
usertypes.push_back(USERNAME_FRONTEND);
}
max_connections = r->fields[10];
attributes = r->fields[11];
comment = r->fields[12];
}
for (const enum cred_username_type usertype : usertypes) {
GloMyAuth->add(
r->fields[0], // username
password, // before #676, wewere always passing the password. Now it is possible that the password can be hashed
usertype, // backend/frontend
(strcmp(r->fields[2],"1")==0 ? true : false) , // use_ssl
atoi(r->fields[3]), // default_hostgroup
(r->fields[4]==NULL ? (char *)"" : r->fields[4]), //default_schema
(strcmp(r->fields[5],"1")==0 ? true : false) , // schema_locked
(strcmp(r->fields[6],"1")==0 ? true : false) , // transaction_persistent
(strcmp(r->fields[7],"1")==0 ? true : false), // fast_forward
( atoi(max_connections)>0 ? atoi(max_connections) : 0), // max_connections
(attributes == NULL ? (char *)"" : attributes), // attributes
(comment ==NULL ? (char *)"" : comment) //comment
);
}
if (sqlite_result != nullptr) {
vector<char*> pta(static_cast<size_t>(resultset->columns));
for (uint32_t i = 0; i < resultset->columns; i++) {
if (i == 1) {
pta[i] = password;
} else {
if (r->fields[i] != nullptr) {
pta[i] = r->fields[i];
} else {
pta[i] = const_cast<char*>("");
}
}
}
sqlite_result->add_row(&pta[0]);
}
if (variables.hash_passwords) {
free(password); // because we always generate a new string
}
#ifdef ADDUSER_STMT_RAW
delete r;
#endif
}
}
#ifdef ADDUSER_STMT_RAW
if (statement) {
(*proxy_sqlite3_finalize)(statement);
}
if (hash1) {
uint64_t h1, h2;
myhash.Final(&h1, &h2);
*hash1 = h1;
if (empty) {
*hash1 = 0;
if (__user == nullptr) {
if (mysql_users_resultset == nullptr) {
added_users = sqlite_result;
} else {
added_users = mysql_users_resultset;
}
}
// resulset has been locally allocated and must be deleted
if (resultset != mysql_users_resultset) {
delete resultset;
}
}
#else
if (resultset) delete resultset;
#endif
free(query);
return added_users;
}
#ifdef PROXYSQLCLICKHOUSE
@ -11716,26 +11860,56 @@ void ProxySQL_Admin::load_scheduler_to_runtime() {
resultset=NULL;
}
void ProxySQL_Admin::load_mysql_servers_to_runtime(const std::string& checksum, const time_t epoch) {
void ProxySQL_Admin::load_mysql_servers_to_runtime(
const incoming_servers_t& incoming_servers, const std::string& checksum, const time_t epoch
) {
// make sure that the caller has called mysql_servers_wrlock()
char *error=NULL;
int cols=0;
int affected_rows=0;
SQLite3_result *resultset=NULL;
SQLite3_result *resultset_servers=NULL;
SQLite3_result *resultset_replication=NULL;
SQLite3_result *resultset_group_replication=NULL;
SQLite3_result *resultset_galera=NULL;
SQLite3_result *resultset_aws_aurora=NULL;
char *query=(char *)"SELECT hostgroup_id,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM main.mysql_servers";
SQLite3_result* incoming_mysql_servers = incoming_servers.incoming_mysql_servers;
SQLite3_result* incoming_replication_hostgroups = incoming_servers.incoming_replication_hostgroups;
SQLite3_result* incoming_group_replication_hostgroups = incoming_servers.incoming_group_replication_hostgroups;
SQLite3_result* incoming_galera_hostgroups = incoming_servers.incoming_galera_hostgroups;
SQLite3_result* incoming_aurora_hostgroups = incoming_servers.incoming_aurora_hostgroups;
// TODO: Fix ordering here and place mixed ordering in TEST
char *query=(char *)"SELECT hostgroup_id,hostname,port,gtid_port,status,weight,compression,max_connections,max_replication_lag,use_ssl,max_latency_ms,comment FROM main.mysql_servers ORDER BY hostgroup_id, hostname, port";
proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query);
admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset);
if (incoming_mysql_servers == nullptr) {
admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_servers);
} else {
resultset_servers = incoming_mysql_servers;
}
//MyHGH->wrlock();
if (error) {
proxy_error("Error on %s : %s\n", query, error);
} else {
MyHGM->servers_add(resultset);
MyHGM->servers_add(resultset_servers);
size_t init_row_count = resultset_servers->rows_count;
size_t rm_rows_count = 0;
const auto is_offline_server = [&rm_rows_count] (SQLite3_row* row) {
if (strcasecmp(row->fields[4], "OFFLINE_HARD") == 0) {
rm_rows_count += 1;
return true;
} else {
return false;
}
};
resultset_servers->rows.erase(
std::remove_if(resultset_servers->rows.begin(), resultset_servers->rows.end(), is_offline_server),
resultset_servers->rows.end()
);
resultset_servers->rows_count = init_row_count - rm_rows_count;
MyHGM->set_incoming_mysql_servers(resultset_servers);
}
if (resultset) delete resultset;
resultset=NULL;
query=(char *)"SELECT a.* FROM mysql_replication_hostgroups a JOIN mysql_replication_hostgroups b ON a.writer_hostgroup=b.reader_hostgroup WHERE b.reader_hostgroup";
@ -11752,10 +11926,13 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const std::string& checksum,
if (resultset) delete resultset;
resultset=NULL;
query=(char *)"SELECT a.* FROM mysql_replication_hostgroups a LEFT JOIN mysql_replication_hostgroups b ON a.writer_hostgroup=b.reader_hostgroup WHERE b.reader_hostgroup IS NULL";
query=(char *)"SELECT a.* FROM mysql_replication_hostgroups a LEFT JOIN mysql_replication_hostgroups b ON a.writer_hostgroup=b.reader_hostgroup WHERE b.reader_hostgroup IS NULL ORDER BY writer_hostgroup";
proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query);
admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_replication);
if (incoming_replication_hostgroups == nullptr) {
admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_replication);
} else {
resultset_replication = incoming_replication_hostgroups;
}
//MyHGH->wrlock();
if (error) {
proxy_error("Error on %s : %s\n", query, error);
@ -11783,9 +11960,14 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const std::string& checksum,
if (resultset) delete resultset;
resultset=NULL;
query=(char *)"SELECT a.* FROM mysql_group_replication_hostgroups a LEFT JOIN mysql_group_replication_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup OR a.writer_hostgroup=b.backup_writer_hostgroup OR a.writer_hostgroup=b.offline_hostgroup) WHERE b.reader_hostgroup IS NULL AND b.backup_writer_hostgroup IS NULL AND b.offline_hostgroup IS NULL";
query=(char *)"SELECT a.* FROM mysql_group_replication_hostgroups a LEFT JOIN mysql_group_replication_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup OR a.writer_hostgroup=b.backup_writer_hostgroup OR a.writer_hostgroup=b.offline_hostgroup) WHERE b.reader_hostgroup IS NULL AND b.backup_writer_hostgroup IS NULL AND b.offline_hostgroup IS NULL ORDER BY writer_hostgroup";
proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query);
admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_group_replication);
if (incoming_group_replication_hostgroups == nullptr) {
admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_group_replication);
} else {
resultset_group_replication = incoming_group_replication_hostgroups;
}
if (error) {
proxy_error("Error on %s : %s\n", query, error);
} else {
@ -11810,9 +11992,13 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const std::string& checksum,
if (resultset) delete resultset;
resultset=NULL;
query=(char *)"SELECT a.* FROM mysql_galera_hostgroups a LEFT JOIN mysql_galera_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup OR a.writer_hostgroup=b.backup_writer_hostgroup OR a.writer_hostgroup=b.offline_hostgroup) WHERE b.reader_hostgroup IS NULL AND b.backup_writer_hostgroup IS NULL AND b.offline_hostgroup IS NULL";
query=(char *)"SELECT a.* FROM mysql_galera_hostgroups a LEFT JOIN mysql_galera_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup OR a.writer_hostgroup=b.backup_writer_hostgroup OR a.writer_hostgroup=b.offline_hostgroup) WHERE b.reader_hostgroup IS NULL AND b.backup_writer_hostgroup IS NULL AND b.offline_hostgroup IS NULL ORDER BY writer_hostgroup";
proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query);
admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_galera);
if (incoming_galera_hostgroups == nullptr) {
admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_galera);
} else {
resultset_galera = incoming_galera_hostgroups;
}
if (error) {
proxy_error("Error on %s : %s\n", query, error);
} else {
@ -11838,12 +12024,16 @@ void ProxySQL_Admin::load_mysql_servers_to_runtime(const std::string& checksum,
resultset=NULL;
//#ifdef TEST_AURORA // temporary enabled only for testing purpose
query=(char *)"SELECT a.* FROM mysql_aws_aurora_hostgroups a LEFT JOIN mysql_aws_aurora_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup) WHERE b.reader_hostgroup IS NULL";
query=(char *)"SELECT a.* FROM mysql_aws_aurora_hostgroups a LEFT JOIN mysql_aws_aurora_hostgroups b ON (a.writer_hostgroup=b.reader_hostgroup) WHERE b.reader_hostgroup IS NULL ORDER BY writer_hostgroup";
//#else
// query=(char *)"SELECT a.* FROM mysql_aws_aurora_hostgroups a WHERE 1=0";
//#endif
proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query);
admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_aws_aurora);
if (incoming_aurora_hostgroups == nullptr) {
admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset_aws_aurora);
} else {
resultset_aws_aurora = incoming_aurora_hostgroups;
}
if (error) {
proxy_error("Error on %s : %s\n", query, error);
} else {

@ -894,8 +894,8 @@ void ProxySQL_Cluster::pull_mysql_query_rules_from_peer(const string& expected_c
result2 = mysql_store_result(conn);
proxy_info("Cluster: Fetching MySQL Query Rules from peer %s:%d completed\n", hostname, port);
std::unique_ptr<SQLite3_result> SQLite3_query_rules_resultset = get_SQLite3_resulset(result1);
std::unique_ptr<SQLite3_result> SQLite3_query_rules_fast_routing_resultset = get_SQLite3_resulset(result2);
std::unique_ptr<SQLite3_result> SQLite3_query_rules_resultset { get_SQLite3_resulset(result1) };
std::unique_ptr<SQLite3_result> SQLite3_query_rules_fast_routing_resultset { get_SQLite3_resulset(result2) };
const uint64_t query_rules_hash =
SQLite3_query_rules_resultset->raw_checksum() + SQLite3_query_rules_fast_routing_resultset->raw_checksum();
@ -1054,58 +1054,10 @@ __exit_pull_mysql_query_rules_from_peer:
pthread_mutex_unlock(&GloProxyCluster->update_mysql_query_rules_mutex);
}
int fetch_mysql_users_checksum(MYSQL* conn, char* hostname, uint16_t port, string& checksum) {
const char* CLUSTER_QUERY_RUNTIME_CHECKS = "SELECT checksum FROM runtime_checksums_values WHERE name='mysql_users' LIMIT 1";
proxy_info("Cluster: Fetching checksum for MySQL Users from peer %s:%d before processing\n", hostname, port);
int res = -1;
int my_rc = mysql_query(conn, CLUSTER_QUERY_RUNTIME_CHECKS);
if (!my_rc) {
MYSQL_RES* my_res = mysql_store_result(conn);
if (my_res != NULL && mysql_num_rows(my_res) == 1) {
MYSQL_ROW row = mysql_fetch_row(my_res);
if (row[0] != nullptr) {
checksum = row[0];
res = 0;
} else {
proxy_error(
"Cluster: Received empty checksum checksum for MySQL Users from peer %s:%d. Please report a bug\n",
hostname, port
);
}
} else {
if (mysql_errno(conn)) {
proxy_info(
"Cluster: Fetching checksum for MySQL Users from peer %s:%d failed: '%s'\n",
hostname, port, mysql_error(conn)
);
} else {
uint64_t num_rows = mysql_num_rows(my_res);
if (num_rows != 1) {
proxy_error(
"Cluster: Empty resulset fetching checksum for MySQL Users from peer %s:%d. Please report a bug\n",
hostname, port
);
} else {
proxy_error(
"Cluster: Invalid row num '%d' fetching checksum for MySQL Users from peer %s:%d. Please report a bug\n",
num_rows, hostname, port
);
}
}
}
} else {
proxy_info(
"Cluster: Fetching checksum for MySQL Users from peer %s:%d failed: '%s'\n", hostname, port, mysql_error(conn)
);
}
return res;
}
uint64_t get_mysql_users_checksum(MYSQL_RES* resultset, MYSQL_RES* ldap_resultset) {
uint64_t raw_users_checksum = GloMyAuth->get_runtime_checksum(resultset);
uint64_t get_mysql_users_checksum(
MYSQL_RES* resultset, MYSQL_RES* ldap_resultset, unique_ptr<SQLite3_result>& all_users
) {
uint64_t raw_users_checksum = GloMyAuth->get_runtime_checksum(resultset, all_users);
if (GloMyLdapAuth) {
raw_users_checksum += mysql_raw_checksum(ldap_resultset);
@ -1127,18 +1079,18 @@ void update_mysql_users(MYSQL_RES* result) {
while (MYSQL_ROW row = mysql_fetch_row(result)) {
rc=(*proxy_sqlite3_bind_text)(statement1, 1, row[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // username
rc=(*proxy_sqlite3_bind_text)(statement1, 2, row[1], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // password
rc=(*proxy_sqlite3_bind_int64)(statement1, 3, atoll(row[2])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // active
rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atoll(row[3])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // use_ssl
rc=(*proxy_sqlite3_bind_int64)(statement1, 5, atoll(row[4])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // default_hostgroup
rc=(*proxy_sqlite3_bind_text)(statement1, 6, row[5], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // default_schema
rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoll(row[6])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // schema_locked
rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoll(row[7])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // transaction_persistent
rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoll(row[8])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // fast_forward
rc=(*proxy_sqlite3_bind_int64)(statement1, 10, atoll(row[9])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // backend
rc=(*proxy_sqlite3_bind_int64)(statement1, 11, atoll(row[10])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // frontend
rc=(*proxy_sqlite3_bind_int64)(statement1, 12, atoll(row[11])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // max_connection
rc=(*proxy_sqlite3_bind_text)(statement1, 13, row[12], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // attributes
rc=(*proxy_sqlite3_bind_text)(statement1, 14, row[13], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // comment
rc=(*proxy_sqlite3_bind_int64)(statement1, 3, 1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // active
rc=(*proxy_sqlite3_bind_int64)(statement1, 4, atoll(row[2])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // use_ssl
rc=(*proxy_sqlite3_bind_int64)(statement1, 5, atoll(row[3])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // default_hostgroup
rc=(*proxy_sqlite3_bind_text)(statement1, 6, row[4], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // default_schema
rc=(*proxy_sqlite3_bind_int64)(statement1, 7, atoll(row[5])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // schema_locked
rc=(*proxy_sqlite3_bind_int64)(statement1, 8, atoll(row[6])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // transaction_persistent
rc=(*proxy_sqlite3_bind_int64)(statement1, 9, atoll(row[7])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // fast_forward
rc=(*proxy_sqlite3_bind_int64)(statement1, 10, atoll(row[8])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // backend
rc=(*proxy_sqlite3_bind_int64)(statement1, 11, atoll(row[9])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // frontend
rc=(*proxy_sqlite3_bind_int64)(statement1, 12, atoll(row[10])); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // max_connection
rc=(*proxy_sqlite3_bind_text)(statement1, 13, row[11], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // attributes
rc=(*proxy_sqlite3_bind_text)(statement1, 14, row[12], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, GloAdmin->admindb); // comment
SAFE_SQLITE3_STEP2(statement1);
rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, GloAdmin->admindb);
@ -1207,11 +1159,7 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksu
goto __exit_pull_mysql_users_from_peer;
}
rc_query = mysql_query(
conn,
"SELECT username, password, active, use_ssl, default_hostgroup, default_schema, schema_locked,"
" transaction_persistent, fast_forward, backend, frontend, max_connections, attributes, comment FROM runtime_mysql_users"
);
rc_query = mysql_query(conn, CLUSTER_QUERY_MYSQL_USERS);
if (rc_query == 0) {
MYSQL_RES* mysql_users_result = mysql_store_result(conn);
MYSQL_RES* ldap_mapping_result = nullptr;
@ -1234,7 +1182,9 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksu
}
}
const uint64_t users_raw_checksum = get_mysql_users_checksum(mysql_users_result, ldap_mapping_result);
unique_ptr<SQLite3_result> mysql_users_resultset { nullptr };
const uint64_t users_raw_checksum =
get_mysql_users_checksum(mysql_users_result, ldap_mapping_result, mysql_users_resultset);
const string computed_checksum { get_checksum_from_hash(users_raw_checksum) };
if (expected_checksum == computed_checksum) {
@ -1251,7 +1201,7 @@ void ProxySQL_Cluster::pull_mysql_users_from_peer(const string& expected_checksu
proxy_info("Cluster: Loading to runtime LDAP Mappings from peer %s:%d\n", hostname, port);
}
GloAdmin->init_users(expected_checksum, epoch);
GloAdmin->init_users(std::move(mysql_users_resultset), expected_checksum, epoch);
if (GloProxyCluster->cluster_mysql_users_save_to_disk == true) {
proxy_info("Cluster: Saving to disk MySQL Users from peer %s:%d\n", hostname, port);
if (GloMyLdapAuth) {
@ -1464,6 +1414,20 @@ uint64_t compute_servers_tables_raw_checksum(const vector<MYSQL_RES*>& results)
return servers_hash;
}
incoming_servers_t convert_servers_resultsets(const std::vector<MYSQL_RES*>& results) {
if (results.size() != sizeof(incoming_servers_t) / sizeof(void*)) {
return {};
} else {
return {
get_SQLite3_resulset(results[0]).release(),
get_SQLite3_resulset(results[1]).release(),
get_SQLite3_resulset(results[2]).release(),
get_SQLite3_resulset(results[3]).release(),
get_SQLite3_resulset(results[4]).release()
};
}
}
void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum, const time_t epoch) {
char * hostname = NULL;
uint16_t port = 0;
@ -1493,34 +1457,25 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum,
if (rc_conn) {
std::vector<MYSQL_RES*> results {};
// Server query messages
// servers messages
std::string fetch_servers_done = "";
string_format("Cluster: Fetching MySQL Servers from peer %s:%d completed\n", fetch_servers_done, hostname, port);
std::string fetch_servers_err = "";
string_format("Cluster: Fetching MySQL Servers from peer %s:%d failed: \n", fetch_servers_err, hostname, port);
// group_replication_hostgroups query and messages
const char* CLUSTER_QUERY_MYSQL_GROUP_REPLICATION_HOSTGROUPS =
"SELECT writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup, active, "
"max_writers, writer_is_also_reader, max_transactions_behind, comment FROM runtime_mysql_group_replication_hostgroups ORDER BY writer_hostgroup";
// group_replication_hostgroups messages
std::string fetch_group_replication_hostgroups = "";
string_format("Cluster: Fetching 'MySQL Group Replication Hostgroups' from peer %s:%d\n", fetch_group_replication_hostgroups, hostname, port);
std::string fetch_group_replication_hostgroups_err = "";
string_format("Cluster: Fetching 'MySQL Group Replication Hostgroups' from peer %s:%d failed: \n", fetch_group_replication_hostgroups_err, hostname, port);
// AWS Aurora query and messages
const char* CLUSTER_QUERY_MYSQL_AWS_AURORA =
"SELECT writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, "
"check_timeout_ms, writer_is_also_reader, new_reader_weight, add_lag_ms, min_lag_ms, lag_num_checks, comment FROM runtime_mysql_aws_aurora_hostgroups ORDER BY writer_hostgroup";
// AWS Aurora messages
std::string fetch_aws_aurora_start = "";
string_format("Cluster: Fetching 'MySQL Aurora Hostgroups' from peer %s:%d\n", fetch_aws_aurora_start, hostname, port);
std::string fetch_aws_aurora_err = "";
string_format("Cluster: Fetching 'MySQL Aurora Hostgroups' from peer %s:%d failed: \n", fetch_aws_aurora_err, hostname, port);
// Galera query and messages
const char* CLUSTER_QUERY_MYSQL_GALERA =
"SELECT writer_hostgroup, backup_writer_hostgroup, reader_hostgroup, offline_hostgroup, active, "
"max_writers, writer_is_also_reader, max_transactions_behind, comment FROM runtime_mysql_galera_hostgroups ORDER BY writer_hostgroup";
// Galera messages
std::string fetch_galera_start = "";
string_format("Cluster: Fetching 'MySQL Galera Hostgroups' from peer %s:%d\n", fetch_galera_start, hostname, port);
std::string fetch_galera_err = "";
@ -1583,6 +1538,8 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum,
const string computed_checksum { get_checksum_from_hash(servers_hash) };
if (computed_checksum == peer_checksum) {
// No need to perform the conversion if checksums don't match
const incoming_servers_t incoming_servers { convert_servers_resultsets(results) };
// we are OK to sync!
proxy_info("Cluster: Fetching checksum for 'MySQL Servers' from peer %s:%d successful. Checksum: %s\n", hostname, port, computed_checksum.c_str());
// sync mysql_servers
@ -1756,7 +1713,7 @@ void ProxySQL_Cluster::pull_mysql_servers_from_peer(const std::string& checksum,
delete resultset;
proxy_info("Cluster: Loading to runtime MySQL Servers from peer %s:%d\n", hostname, port);
GloAdmin->load_mysql_servers_to_runtime(checksum, epoch);
GloAdmin->load_mysql_servers_to_runtime(incoming_servers, checksum, epoch);
if (GloProxyCluster->cluster_mysql_servers_save_to_disk == true) {
proxy_info("Cluster: Saving to disk MySQL Servers from peer %s:%d\n", hostname, port);
GloAdmin->flush_mysql_servers__from_memory_to_disk();

Loading…
Cancel
Save