aherman
/
proxysql
mirror of https://github.com/sysown/proxysql
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4288 from sysown/v2.x_read_only_actions_offline_hard_servers

Browse Source 
Fixed potential crash when read_only_actions creates a server that is already present in hgm container in OFFLINE HARD state
pull/4172/merge v2.5.4
René Cannaò 3 years ago committed by GitHub
parent ef2f4f7b80 792db85b20
commit d15b40ab43
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 738 additions and 11 deletions
Show Stats Download Patch File Download Diff File

56
lib/MySQL_HostGroups_Manager.cpp
Unescape View File

@ -7805,7 +7805,7 @@ void MySQL_HostGroups_Manager::HostGroup_Server_Mapping::copy_if_not_exists(Type
for (const auto& src_node : src_nodes) {
for (auto& dest_node : dest_nodes) {
for (const auto& dest_node : dest_nodes) {
if (src_node.reader_hostgroup_id == dest_node.reader_hostgroup_id &&
src_node.writer_hostgroup_id == dest_node.writer_hostgroup_id) {
@ -7825,7 +7825,6 @@ void MySQL_HostGroups_Manager::HostGroup_Server_Mapping::copy_if_not_exists(Type
if (dest_nodes.capacity() < (dest_nodes.size() + append.size()))
dest_nodes.reserve(dest_nodes.size() + append.size());
//dest_nodes.insert(dest_nodes.end(), append.begin(), append.end());
for (auto& node : append) {
@ -7883,23 +7882,58 @@ unsigned int MySQL_HostGroups_Manager::HostGroup_Server_Mapping::get_hostgroup_i
MySrvC* MySQL_HostGroups_Manager::HostGroup_Server_Mapping::insert_HGM(unsigned int hostgroup_id, const MySrvC* srv) {
MyHGC* hostgroup_container = myHGM->MyHGC_lookup(hostgroup_id);
MyHGC* myhgc = myHGM->MyHGC_lookup(hostgroup_id);
if (!hostgroup_container)
if (!myhgc)
return NULL;
if (GloMTH->variables.hostgroup_manager_verbose) {
proxy_info("Creating new server in HG %d : %s:%d , gtid_port=%d, weight=%d, status=%d\n", hostgroup_id, srv->address, srv->port, srv->gtid_port, srv->weight, srv->status);
MySrvC* ret_srv = NULL;
for (uint32_t j = 0; j < myhgc->mysrvs->cnt(); j++) {
MySrvC* mysrvc = static_cast<MySrvC*>(myhgc->mysrvs->servers->index(j));
if (strcmp(mysrvc->address, srv->address) == 0 && mysrvc->port == srv->port) {
if (mysrvc->status == MYSQL_SERVER_STATUS_OFFLINE_HARD) {
mysrvc->gtid_port = srv->gtid_port;
mysrvc->weight = srv->weight;
mysrvc->compression = srv->compression;
mysrvc->max_connections = srv->max_connections;
mysrvc->max_replication_lag = srv->max_replication_lag;
mysrvc->use_ssl = srv->use_ssl;
mysrvc->max_latency_us = srv->max_latency_us;
mysrvc->comment = strdup(srv->comment);
mysrvc->status = MYSQL_SERVER_STATUS_ONLINE;
if (GloMTH->variables.hostgroup_manager_verbose) {
proxy_info(
"Found server node in Host Group Container %s:%d as 'OFFLINE_HARD', setting back as 'ONLINE' with:"
" hostgroup_id=%d, gtid_port=%d, weight=%ld, compression=%d, max_connections=%ld, use_ssl=%d,"
" max_replication_lag=%ld, max_latency_ms=%ld, comment=%s\n",
mysrvc->address, mysrvc->port, hostgroup_id, mysrvc->gtid_port, mysrvc->weight, mysrvc->compression,
mysrvc->max_connections, mysrvc->use_ssl, mysrvc->max_replication_lag, (mysrvc->max_latency_us / 1000),
mysrvc->comment
);
}
ret_srv = mysrvc;
break;
}
}
}
if (!ret_srv) {
if (GloMTH->variables.hostgroup_manager_verbose) {
proxy_info("Creating new server in HG %d : %s:%d , gtid_port=%d, weight=%d, status=%d\n", hostgroup_id, srv->address, srv->port, srv->gtid_port, srv->weight, srv->status);
}
proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Adding new server %s:%d , weight=%d, status=%d, mem_ptr=%p into hostgroup=%d\n", srv->address, srv->port, srv->weight, srv->status, srv, hostgroup_id);
proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Adding new server %s:%d , weight=%d, status=%d, mem_ptr=%p into hostgroup=%d\n", srv->address, srv->port, srv->weight, srv->status, srv, hostgroup_id);
MySrvC* new_srv = new MySrvC(srv->address, srv->port, srv->gtid_port, srv->weight, srv->status, srv->compression,
srv->max_connections, srv->max_replication_lag, srv->use_ssl, (srv->max_latency_us/1000), srv->comment);
ret_srv = new MySrvC(srv->address, srv->port, srv->gtid_port, srv->weight, srv->status, srv->compression,
srv->max_connections, srv->max_replication_lag, srv->use_ssl, (srv->max_latency_us / 1000), srv->comment);
hostgroup_container->mysrvs->add(new_srv);
myhgc->mysrvs->add(ret_srv);
}
return new_srv;
return ret_srv;
}
void MySQL_HostGroups_Manager::HostGroup_Server_Mapping::remove_HGM(MySrvC* srv) {

693
test/tap/tests/test_read_only_actions_offline_hard_servers-t.cpp
Unescape View File

@ -0,0 +1,693 @@
#include <unistd.h>
#include <atomic>
#include <vector>
#include <string>
#include <mysql.h>
#include "tap.h"
#include "command_line.h"
#include "utils.h"
#include "proxysql_utils.h"
#define BACKEND_SERVER_HOST "127.0.0.1"
#define BACKEND_SERVER_PORT 13306
#define BACKEND_SERVER_USER "root"
#define BACKEND_SERVER_PASS "root"
#define MYSQL_QUERY__(mysql, query) \
do { \
if (mysql_query(mysql, query)) { \
fprintf(stderr, "File %s, line %d, Error: %s\n", \
__FILE__, __LINE__, mysql_error(mysql)); \
goto cleanup; \
} \
} while(0)
const uint32_t SYNC_TIMEOUT = 10;
using mysql_server_tuple = std::tuple<int,std::string,int,int,std::string,int,int,int,int,int,int,std::string>;
using replication_hostgroups_tuple = std::tuple<int,int,std::string>;
MYSQL* create_new_connection(const char* host, const char* username, const char* password, int port) {
MYSQL* mysql = mysql_init(NULL);
if (!mysql) {
fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(mysql));
goto __exit;
}
if (!mysql_real_connect(mysql, host, username, password, NULL, port, NULL, 0)) {
fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(mysql));
mysql_close(mysql);
mysql = NULL;
goto __exit;
}
__exit:
return mysql;
}
/**
* @brief Helper function to verify that the sync of a table (or variable) have been performed.
*
* @param r_proxy_admin An already opened connection to ProxySQL.
* @param queries Queries to be executed that should return a **non-zero** value after the sync has taken place.
* @param sync_timeout Timeout for the sync to happen.
*
* @return EXIT_SUCCESS in case of success, otherwise:
* - '-1' if a query against Admin fails to be performed (failure is logged).
* - '-2' if timeout expired without sync being completed.
*/
int sync_checker(MYSQL* r_proxy_admin, const std::vector<std::string>& queries, uint32_t sync_timeout) {
bool not_synced_query = false;
uint waited = 0;
while (waited < sync_timeout) {
not_synced_query = false;
// Check that all the entries have been synced
for (const auto& query : queries) {
int q_res = mysql_query(r_proxy_admin, query.c_str());
if (q_res != EXIT_SUCCESS) {
fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(r_proxy_admin));
return -1;
}
MYSQL_RES* proxysql_servers_res = mysql_store_result(r_proxy_admin);
MYSQL_ROW row = mysql_fetch_row(proxysql_servers_res);
int row_value = atoi(row[0]);
mysql_free_result(proxysql_servers_res);
if (row_value == 0) {
not_synced_query = true;
break;
}
}
if (not_synced_query) {
waited += 1;
sleep(1);
} else {
break;
}
}
if (not_synced_query) {
return -2;
} else {
return EXIT_SUCCESS;
}
}
int check_nodes_sync(
const CommandLine& cl, const std::vector<mysql_res_row>& core_nodes, const std::string& check_query, uint32_t sync_timeout
) {
int ret_status = EXIT_FAILURE;
for (const auto& node : core_nodes) {
const std::string host { node[0] };
const int port = std::stol(node[1]);
MYSQL* c_node_admin = mysql_init(NULL);
if (!mysql_real_connect(c_node_admin, host.c_str(), cl.admin_username, cl.admin_password, NULL, port, NULL, 0)) {
fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(c_node_admin));
goto __exit;
}
int not_synced = sync_checker(c_node_admin, { check_query }, sync_timeout);
if (not_synced != EXIT_SUCCESS) {
const std::string err_msg { "Node '" + host + ":" + std::to_string(port) + "' sync timed out" };
fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, err_msg.c_str());
goto __exit;
}
}
ret_status = EXIT_SUCCESS;
__exit:
return ret_status;
}
int insert_mysql_servers_records(MYSQL* proxy_admin, const std::vector<mysql_server_tuple>& insert_mysql_servers_values,
const std::vector<replication_hostgroups_tuple>& insert_replication_hostgroups_values) {
MYSQL_QUERY(proxy_admin, "DELETE FROM mysql_servers");
MYSQL_QUERY(proxy_admin, "DELETE FROM mysql_replication_hostgroups");
// Configure 'mysql_servers' and check sync with NULL comments
const char* t_insert_mysql_servers =
"INSERT INTO mysql_servers ("
" hostgroup_id, hostname, port, gtid_port, status, weight, compression, max_connections,"
" max_replication_lag, use_ssl, max_latency_ms, comment"
") VALUES (%d, '%s', %d, %d, '%s', %d, %d, %d, %d, %d, %d, '%s')";
const char* t_mysql_replication_hostgroups =
"INSERT INTO mysql_replication_hostgroups (writer_hostgroup, reader_hostgroup, check_type) VALUES (%d,%d,'%s')";
for (auto const& values : insert_mysql_servers_values) {
std::string insert_mysql_servers_hostgroup_query;
string_format(
t_insert_mysql_servers,
insert_mysql_servers_hostgroup_query,
std::get<0>(values),
std::get<1>(values).c_str(),
std::get<2>(values),
std::get<3>(values),
std::get<4>(values).c_str(),
std::get<5>(values),
std::get<6>(values),
std::get<7>(values),
std::get<8>(values),
std::get<9>(values),
std::get<10>(values),
std::get<11>(values).c_str()
);
// Insert the new mysql_servers hostgroups values
MYSQL_QUERY(proxy_admin, insert_mysql_servers_hostgroup_query.c_str());
}
for (auto const& values : insert_replication_hostgroups_values) {
std::string insert_mysql_replication_hostgroups_query;
string_format(
t_mysql_replication_hostgroups,
insert_mysql_replication_hostgroups_query,
std::get<0>(values),
std::get<1>(values),
std::get<2>(values).c_str()
);
// Insert the new mysql_replication_hostgroups values
MYSQL_QUERY(proxy_admin, insert_mysql_replication_hostgroups_query.c_str());
}
MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME");
return EXIT_SUCCESS;
}
/**
* @brief Assumes that 'proxysql_servers' holds at least the one entry required for this test.
* @details It's assumed that primary ProxySQL is part of a Cluster.
*/
int update_proxysql_servers(const CommandLine& cl, MYSQL* admin) {
const char update_proxysql_servers_t[] {
"UPDATE proxysql_servers SET comment='%s' WHERE hostname='%s' and port=%d"
};
cfmt_t update_servers {
cstr_format(update_proxysql_servers_t, std::to_string(time(NULL)).c_str(), cl.host, cl.admin_port)
};
MYSQL_QUERY_T(admin, update_servers.str.c_str());
MYSQL_QUERY_T(admin, "LOAD PROXYSQL SERVERS TO RUNTIME");
return EXIT_SUCCESS;
}
int get_read_only_value(const std::string& host, uint16_t port, const std::string& username, const std::string& password,
int* read_only_val) {
MYSQL* mysqldb = create_new_connection(host.c_str(), username.c_str(), password.c_str(), port);
if (!mysqldb) {
fprintf(stderr, "File %s, line %d, Error: create_new_connection() failed\n", __FILE__, __LINE__);
return EXIT_FAILURE;
}
const int rc_query = mysql_query(mysqldb,"SELECT @@global.read_only read_only");
if (rc_query == 0) {
MYSQL_RES *result = mysql_store_result(mysqldb);
MYSQL_ROW row;
while ((row = mysql_fetch_row(result))) {
if (row[0]) {
*read_only_val = static_cast<uint16_t>(std::strtoul(row[0], NULL, 10));
}
}
mysql_free_result(result);
}
mysql_close(mysqldb);
return EXIT_SUCCESS;
}
int set_read_only_value(const std::string& host, uint16_t port, const std::string& username, const std::string& password,
int read_only_val) {
int rc_query = -1;
int ret_status = EXIT_FAILURE;
MYSQL* mysqldb = create_new_connection(host.c_str(), username.c_str(), password.c_str(), port);
if (!mysqldb) {
fprintf(stderr, "File %s, line %d, Error: create_new_connection() failed\n", __FILE__, __LINE__);
goto __cleanup;
}
char query[256];
sprintf(query, "SET @@global.read_only=%d", read_only_val);
rc_query = mysql_query(mysqldb,query);
if (rc_query != 0) {
fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(mysqldb));
goto __cleanup;
}
ret_status = EXIT_SUCCESS;
__cleanup:
if (mysqldb) mysql_close(mysqldb);
return ret_status;
}
int test_scenario_1(MYSQL* proxy_admin, const CommandLine& cl) {
diag("Running test_scenario_1 ...");
int ret_status = EXIT_FAILURE;
int read_only_val = -1;
MYSQL* dummy_mysqldb = NULL;
const std::vector<mysql_server_tuple> insert_mysql_servers_values {
std::make_tuple(0, BACKEND_SERVER_HOST, BACKEND_SERVER_PORT, 12, "ONLINE", 1, 1, 1000, 300, 1, 200, "") // this server has read_only value 0 (writer)
};
const std::vector<replication_hostgroups_tuple> insert_replication_hostgroups_values {
std::make_tuple(0, 1, "read_only")
};
// cleaning old records
MYSQL_QUERY__(proxy_admin, "DELETE FROM mysql_servers");
MYSQL_QUERY__(proxy_admin, "DELETE FROM mysql_replication_hostgroups");
MYSQL_QUERY__(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME");
MYSQL_QUERY__(proxy_admin, "SET mysql-monitor_read_only_interval=200"); // setting read_only variables
MYSQL_QUERY__(proxy_admin, "SET mysql-monitor_read_only_timeout=100");
MYSQL_QUERY__(proxy_admin, "SET mysql-monitor_enabled='true'"); // enabling monitor
MYSQL_QUERY__(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME");
{
int result = get_read_only_value(BACKEND_SERVER_HOST, BACKEND_SERVER_PORT, BACKEND_SERVER_USER, BACKEND_SERVER_PASS, &read_only_val);
if (result != EXIT_SUCCESS) {
fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching read_only value from mysql server failed.");
goto cleanup;
}
ok(read_only_val == 0, "MySQL Server '%s:%d' should function as a writer", BACKEND_SERVER_HOST, BACKEND_SERVER_PORT);
// Inserting new records into 'mysql_servers' and 'mysql_replication_hostgroups'.
result = insert_mysql_servers_records(proxy_admin, insert_mysql_servers_values, insert_replication_hostgroups_values);
if (result != EXIT_SUCCESS) {
fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Failed to insert records in mysql_servers table.");
goto cleanup;
}
std::string variable_val;
// get read_only interval variable value
result = get_variable_value(proxy_admin, "mysql-monitor_read_only_interval", variable_val);
if (result) { goto cleanup; }
const long monitor_read_only_interval = std::stol(variable_val);
// get read_only timeout variable value
result = get_variable_value(proxy_admin, "mysql-monitor_read_only_timeout", variable_val);
if (result) { goto cleanup; }
const long monitor_read_only_timeout = std::stol(variable_val);
// Wait till read_only actions have been performed
const uint64_t wait = monitor_read_only_interval + monitor_read_only_timeout;
usleep((wait * 1000) * 2);
dummy_mysqldb = create_new_connection(cl.host, cl.username, cl.password, cl.port);
ok(dummy_mysqldb != NULL, "Connection created successfully");
if (!dummy_mysqldb) {
fprintf(stderr, "File %s, line %d, Error: create_new_connection() failed\n", __FILE__, __LINE__);
goto cleanup;
}
diag("Starting transaction");
MYSQL_QUERY__(dummy_mysqldb, "BEGIN");
MYSQL_QUERY__(dummy_mysqldb, "DO 1");
result = set_read_only_value(BACKEND_SERVER_HOST, BACKEND_SERVER_PORT, BACKEND_SERVER_USER, BACKEND_SERVER_PASS, 1);
if (result != EXIT_SUCCESS) {
fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching read_only value from mysql server failed.");
goto cleanup;
}
result = get_read_only_value(BACKEND_SERVER_HOST, BACKEND_SERVER_PORT, BACKEND_SERVER_USER, BACKEND_SERVER_PASS, &read_only_val);
if (result != EXIT_SUCCESS) {
fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching read_only value from mysql server failed.");
goto cleanup;
}
ok(read_only_val == 1, "MySQL Server '%s:%d' should function as a reader", BACKEND_SERVER_HOST, BACKEND_SERVER_PORT);
// Wait till read_only actions have been performed
usleep((wait * 1000) * 2);
// checking if proxysql instance is still alive?
result = mysql_query(proxy_admin, "SELECT 1");
ok(result == 0, "ProxySQL instance is alive");
if (result) {
fprintf(stderr, "File %s, line %d, Error: %s\n", \
__FILE__, __LINE__, mysql_error(proxy_admin)); \
goto cleanup;
}
mysql_free_result(mysql_store_result(proxy_admin));
result = set_read_only_value(BACKEND_SERVER_HOST, BACKEND_SERVER_PORT, BACKEND_SERVER_USER, BACKEND_SERVER_PASS, 0);
if (result != EXIT_SUCCESS) {
fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching read_only value from mysql server failed.");
goto cleanup;
}
result = get_read_only_value(BACKEND_SERVER_HOST, BACKEND_SERVER_PORT, BACKEND_SERVER_USER, BACKEND_SERVER_PASS, &read_only_val);
if (result != EXIT_SUCCESS) {
fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching read_only value from mysql server failed.");
goto cleanup;
}
ok(read_only_val == 0, "MySQL Server '%s:%d' should function as a writer", BACKEND_SERVER_HOST, BACKEND_SERVER_PORT);
// Wait till read_only actions have been performed
usleep((wait * 1000) * 2);
// checking if proxysql instance is still alive?
result = mysql_query(proxy_admin, "SELECT 1");
ok(result == 0, "ProxySQL instance is alive");
if (result) {
fprintf(stderr, "File %s, line %d, Error: %s\n", \
__FILE__, __LINE__, mysql_error(proxy_admin)); \
goto cleanup;
}
mysql_free_result(mysql_store_result(proxy_admin));
ret_status = EXIT_SUCCESS;
}
cleanup:
if (dummy_mysqldb) {
mysql_query(dummy_mysqldb, "ROLLBACK");
mysql_close(dummy_mysqldb);
}
// Restoring MySQL Server read_only value
if (read_only_val != -1) {
diag("Restoring MySQL Server %s:%d 'read_only' value to '0'", BACKEND_SERVER_HOST, BACKEND_SERVER_PORT);
if (set_read_only_value(BACKEND_SERVER_HOST, BACKEND_SERVER_PORT, BACKEND_SERVER_USER, BACKEND_SERVER_PASS, 0) != EXIT_SUCCESS) {
fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Restoring read_only value failed.");
}
}
diag("test_scenario_1 execution completed\n");
return ret_status;
}
int test_scenario_2(MYSQL* proxy_admin, const CommandLine& cl) {
diag("Running test_scenario_2 ...");
int ret_status = EXIT_FAILURE;
int read_only_val = -1;
MYSQL* dummy_mysqldb = NULL;
const std::vector<mysql_server_tuple> insert_mysql_servers_values {
std::make_tuple(1, BACKEND_SERVER_HOST, BACKEND_SERVER_PORT, 12, "ONLINE", 1, 1, 1000, 300, 1, 200, "") // this server has read_only value 0 (writer)
};
const std::vector<replication_hostgroups_tuple> insert_replication_hostgroups_values {
std::make_tuple(0, 1, "read_only")
};
// cleaning old records
MYSQL_QUERY__(proxy_admin, "DELETE FROM mysql_servers");
MYSQL_QUERY__(proxy_admin, "DELETE FROM mysql_replication_hostgroups");
MYSQL_QUERY__(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME");
MYSQL_QUERY__(proxy_admin, "SET mysql-monitor_read_only_interval=200"); // setting read_only variables
MYSQL_QUERY__(proxy_admin, "SET mysql-monitor_read_only_timeout=100");
MYSQL_QUERY__(proxy_admin, "SET mysql-monitor_writer_is_also_reader='false'");
MYSQL_QUERY__(proxy_admin, "SET mysql-monitor_enabled='true'"); // enabling monitor
MYSQL_QUERY__(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME");
{
int result = get_read_only_value(BACKEND_SERVER_HOST, BACKEND_SERVER_PORT, BACKEND_SERVER_USER, BACKEND_SERVER_PASS, &read_only_val);
if (result != EXIT_SUCCESS) {
fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Fetching read_only value from mysql server failed.");
goto cleanup;
}
ok(read_only_val == 0, "MySQL Server '%s:%d' should function as a writer", BACKEND_SERVER_HOST, BACKEND_SERVER_PORT);
// Inserting new records into 'mysql_servers' and 'mysql_replication_hostgroups'.
result = insert_mysql_servers_records(proxy_admin, insert_mysql_servers_values, insert_replication_hostgroups_values);
if (result != EXIT_SUCCESS) {
fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Failed to insert records in mysql_servers table.");
goto cleanup;
}
std::string variable_val;
// get read_only interval variable value
result = get_variable_value(proxy_admin, "mysql-monitor_read_only_interval", variable_val);
if (result) { goto cleanup; }
const long monitor_read_only_interval = std::stol(variable_val);
// get read_only timeout variable value
result = get_variable_value(proxy_admin, "mysql-monitor_read_only_timeout", variable_val);
if (result) { goto cleanup; }
const long monitor_read_only_timeout = std::stol(variable_val);
// Wait till read_only actions have been performed
const uint64_t wait = monitor_read_only_interval + monitor_read_only_timeout;
usleep((wait * 1000) * 2);
dummy_mysqldb = create_new_connection(cl.host, cl.username, cl.password, cl.port);
ok(dummy_mysqldb != NULL, "Connection created successfully");
if (!dummy_mysqldb) {
fprintf(stderr, "File %s, line %d, Error: create_new_connection() failed\n", __FILE__, __LINE__);
goto cleanup;
}
diag("Starting transaction");
MYSQL_QUERY__(dummy_mysqldb, "BEGIN");
MYSQL_QUERY__(dummy_mysqldb, "DO 1");
// this will remove server
MYSQL_QUERY__(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME");
// Wait till read_only actions have been performed
usleep((wait * 1000) * 2);
// checking if proxysql instance is still alive?
result = mysql_query(proxy_admin, "SELECT 1");
ok(result == 0, "ProxySQL instance is alive");
if (result) {
fprintf(stderr, "File %s, line %d, Error: %s\n", \
__FILE__, __LINE__, mysql_error(proxy_admin)); \
goto cleanup;
}
mysql_free_result(mysql_store_result(proxy_admin));
ret_status = EXIT_SUCCESS;
}
cleanup:
if (dummy_mysqldb) {
mysql_query(dummy_mysqldb, "ROLLBACK");
mysql_close(dummy_mysqldb);
}
// Restoring MySQL Server read_only value
if (read_only_val != -1) {
diag("Restoring MySQL Server %s:%d 'read_only' value to '0'", BACKEND_SERVER_HOST, BACKEND_SERVER_PORT);
if (set_read_only_value(BACKEND_SERVER_HOST, BACKEND_SERVER_PORT, BACKEND_SERVER_USER, BACKEND_SERVER_PASS, 0) != EXIT_SUCCESS) {
fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, "Restoring read_only value failed.");
}
}
diag("test_scenario_2 execution completed\n");
return ret_status;
}
int test_read_only_offline_hard_servers(MYSQL* proxy_admin, const CommandLine& cl, bool isolate_primary_node) {
std::vector<mysql_res_row> core_nodes;
std::string check_no_primary_query;
if (isolate_primary_node) {
const std::string t_update_proxysql_servers{
"INSERT INTO proxysql_servers (hostname, port, weight, comment) VALUES ('%s', %d, 0, 'proxysql')"
};
std::string update_proxysql_servers;
string_format(t_update_proxysql_servers, update_proxysql_servers, cl.host, cl.admin_port);
// 1. Backup the Core nodes from current cluster configuration
MYSQL_QUERY__(proxy_admin, "DROP TABLE IF EXISTS proxysql_servers_sync_test_backup_2687");
MYSQL_QUERY__(proxy_admin, "CREATE TABLE proxysql_servers_sync_test_backup_2687 AS SELECT * FROM proxysql_servers");
// 2. Remove primary from Core nodes
MYSQL_QUERY__(proxy_admin, "DELETE FROM proxysql_servers WHERE hostname=='127.0.0.1' AND PORT==6032");
MYSQL_QUERY__(proxy_admin, "LOAD PROXYSQL SERVERS TO RUNTIME");
MYSQL_QUERY__(proxy_admin, "SELECT hostname,port FROM proxysql_servers");
MYSQL_RES* my_res = mysql_store_result(proxy_admin);
core_nodes = { extract_mysql_rows(my_res) };
mysql_free_result(my_res);
// 3. Wait for all Core nodes to sync (confirm primary out of core nodes)
string_format(
"SELECT CASE COUNT(*) WHEN 0 THEN 1 ELSE 0 END FROM proxysql_servers WHERE hostname=='%s' AND port==%d",
check_no_primary_query, cl.host, cl.admin_port
);
int check_res = check_nodes_sync(cl, core_nodes, check_no_primary_query, SYNC_TIMEOUT);
if (check_res != EXIT_SUCCESS) {
goto cleanup;
}
// 4. Remove all current servers from primary instance
MYSQL_QUERY__(proxy_admin, "DELETE FROM proxysql_servers");
MYSQL_QUERY__(proxy_admin, update_proxysql_servers.c_str());
MYSQL_QUERY__(proxy_admin, "LOAD PROXYSQL SERVERS TO RUNTIME");
}
if (test_scenario_1(proxy_admin, cl) != EXIT_SUCCESS) {
goto cleanup;
}
if (test_scenario_2(proxy_admin, cl) != EXIT_SUCCESS) {
goto cleanup;
}
cleanup:
if (isolate_primary_node) {
// Recover primary ProxySQL MySQL and ProxySQL servers
diag("RESTORING: Recovering primary configuration...");
{
// Recover previous MySQL servers and generate a newer checksum for primary
MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS FROM DISK");
MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME");
// Insert primary into another Core node config and wait for replication
diag("RESTORING: Inserting primary back into Core nodes");
std::string insert_query{};
string_format(
"INSERT INTO proxysql_servers (hostname,port,weight,comment) VALUES ('%s',%d,0,'proxysql')",
insert_query, cl.host, cl.admin_port
);
for (const auto& row : core_nodes) {
const std::string host{ row[0] };
const int port = std::stol(row[1]);
MYSQL* c_node_admin = mysql_init(NULL);
diag("RESTORING: Inserting into node '%s:%d'", host.c_str(), port);
if (!mysql_real_connect(c_node_admin, host.c_str(), cl.admin_username, cl.admin_password, NULL, port, NULL, 0)) {
const std::string err_msg{
"Connection to core node failed with '" + std::string { mysql_error(c_node_admin) } + "'. Retrying..."
};
fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, err_msg.c_str());
mysql_close(c_node_admin);
continue;
}
int my_rc = mysql_query(c_node_admin, insert_query.c_str());
if (my_rc == EXIT_SUCCESS) {
mysql_query(c_node_admin, "LOAD PROXYSQL SERVERS TO RUNTIME");
break;
} else {
const std::string err_msg{
"Insert primary into node failed with: '" + std::string { mysql_error(c_node_admin) } + "'"
};
fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, err_msg.c_str());
}
}
// Wait for sync after primary insertion into Core node
std::string check_for_primary{};
string_format(
"SELECT COUNT(*) FROM proxysql_servers WHERE hostname=='%s' AND port==%d", check_no_primary_query,
cl.host, cl.admin_port
);
// Wait for the other nodes to sync ProxySQL servers to include Primary
int check_res = check_nodes_sync(cl, core_nodes, check_no_primary_query, SYNC_TIMEOUT);
if (check_res != EXIT_SUCCESS) { return EXIT_FAILURE; }
// Recover the old ProxySQL servers from backup in primary
MYSQL_QUERY(proxy_admin, "DELETE FROM proxysql_servers");
MYSQL_QUERY(proxy_admin, "INSERT INTO proxysql_servers SELECT * FROM proxysql_servers_sync_test_backup_2687");
MYSQL_QUERY(proxy_admin, "DROP TABLE proxysql_servers_sync_test_backup_2687");
MYSQL_QUERY(proxy_admin, "LOAD PROXYSQL SERVERS TO RUNTIME");
}
}
return (tests_failed() == 0) ? EXIT_SUCCESS : EXIT_FAILURE;
}
int main(int, char**) {
CommandLine cl;
if (cl.getEnv()) {
diag("Failed to get the required environmental variables.");
return EXIT_FAILURE;
}
plan(9+9);
MYSQL* proxy_admin = mysql_init(NULL);
// Initialize connections
if (!proxy_admin) {
fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxy_admin));
return EXIT_FAILURE;
}
// Connnect to local proxysql
if (!mysql_real_connect(proxy_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) {
fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxy_admin));
return EXIT_FAILURE;
}
diag(">> test_read_only_offline_hard_servers() >> Primary node included in cluster\n");
if (test_read_only_offline_hard_servers(proxy_admin, cl, false) != EXIT_SUCCESS) {
goto cleanup;
}
diag(">> test_read_only_offline_hard_servers() >> Primary node isolated from cluster\n");
if (test_read_only_offline_hard_servers(proxy_admin, cl, true) != EXIT_SUCCESS) {
goto cleanup;
}
cleanup:
mysql_close(proxy_admin);
return exit_status();
}
Write Preview
Loading…
Cancel
Save