diff --git a/test/tap/tap/utils.cpp b/test/tap/tap/utils.cpp index 510a9cae6..b36732226 100644 --- a/test/tap/tap/utils.cpp +++ b/test/tap/tap/utils.cpp @@ -641,3 +641,27 @@ int get_proxysql_cpu_usage(const CommandLine& cl, uint32_t intv, uint32_t& cpu_u return 0; } + +MYSQL* wait_for_proxysql(const conn_opts_t& opts, int timeout) { + uint con_waited = 0; + MYSQL* admin = mysql_init(NULL); + + const char* user = opts.user.c_str(); + const char* pass = opts.pass.c_str(); + const char* host = opts.host.c_str(); + const int port = opts.port; + + while (!mysql_real_connect(admin, host, user, pass, NULL, port, NULL, 0) && con_waited < timeout) { + mysql_close(admin); + admin = mysql_init(NULL); + + con_waited += 1; + sleep(1); + } + + if (con_waited >= timeout) { + return nullptr; + } else { + return admin; + } +} diff --git a/test/tap/tap/utils.h b/test/tap/tap/utils.h index fc67664dd..cb5c2243a 100644 --- a/test/tap/tap/utils.h +++ b/test/tap/tap/utils.h @@ -216,4 +216,26 @@ std::string tap_curtime(); */ int get_proxysql_cpu_usage(const CommandLine& cl, uint32_t intv, uint32_t& cpu_usage); +/** + * @brief Helper struct holding connection options for helper functions creating MySQL connections. + */ +struct conn_opts_t { + std::string host; + std::string user; + std::string pass; + int port; + uint64_t client_flags; +}; + +/** + * @brief Helper function for waiting until ProxySQL is replying to client connections. + * + * @param opts The options to be used for performing the MySQL connection. + * @param timeout Timeout for the wait. + * + * @return An opened MySQL* connection in case a connection could be created before timeout expired, + * 'nullptr' otherwise. + */ +MYSQL* wait_for_proxysql(const conn_opts_t& opts, int timeout); + #endif // #define UTILS_H diff --git a/test/tap/tests/reg_test_3765_ssl_pollout-t.cpp b/test/tap/tests/reg_test_3765_ssl_pollout-t.cpp index ed409b742..539fd4ea8 100644 --- a/test/tap/tests/reg_test_3765_ssl_pollout-t.cpp +++ b/test/tap/tests/reg_test_3765_ssl_pollout-t.cpp @@ -26,14 +26,6 @@ using std::string; using std::vector; -struct conn_opts_t { - string host; - string user; - string pass; - int port; - uint64_t client_flags; -}; - /** * @brief TODO: Refactor this into utils, also used in another PR. */ diff --git a/test/tap/tests/reg_test_3847_admin_lock-t.cpp b/test/tap/tests/reg_test_3847_admin_lock-t.cpp new file mode 100644 index 000000000..de92226bb --- /dev/null +++ b/test/tap/tests/reg_test_3847_admin_lock-t.cpp @@ -0,0 +1,225 @@ +/** + * @file reg_test_3847_admin_lock-t.cpp + * @brief This is a regression test for the deadlock described in issue #3847. + */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "proxysql_utils.h" +#include "tap.h" +#include "command_line.h" +#include "utils.h" + +using std::string; +using std::vector; + +int main(int argc, char** argv) { + CommandLine cl; + + const char* WORKSPACE = getenv("WORKSPACE"); + + if (cl.getEnv() || WORKSPACE == nullptr) { + diag("Failed to get the required environmental variables."); + return EXIT_FAILURE; + } + + bool stop = false; + int q_load_res = -1; + int q_globals_res = -1; + + MYSQL* p_proxy_admin = mysql_init(NULL); + + if (!mysql_real_connect(p_proxy_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(p_proxy_admin)); + return EXIT_FAILURE; + } + + const string sec_cfg_file = string { cl.workdir } + "reg_test_3847_node_datadir/proxysql_sec.cfg"; + const string sec_log_file = string { cl.workdir } + "reg_test_3847_node_datadir/proxysql_sec.log"; + + const string sec_proxy_cmd { + string { "ASAN_OPTIONS=abort_on_error=0:halt_on_error=0:fast_unwind_on_fatal=1:detect_leaks=0 " } + + string { WORKSPACE } + "/src/proxysql -M -c \"" + sec_cfg_file + "\" > " + sec_log_file + " 2>&1" + }; + + int launch_res = -1; + + std::thread launch_sec_proxy = std::thread([&WORKSPACE,&cl] (int& err_code) -> void { + to_opts wexecvp_opts {}; + wexecvp_opts.select_to_us = 100*1000; + wexecvp_opts.it_delay_us = 500*1000; + // Stop launched process after 20s + wexecvp_opts.timeout_us = 20000 * 1000; + // Send sigkill 3s after timeout + wexecvp_opts.sigkill_timeout_us = 3000 * 1000; + + const string sec_cfg_file = string { cl.workdir } + "reg_test_3847_node_datadir/proxysql_sec.cfg"; + const string sec_log_file = string { cl.workdir } + "reg_test_3847_node_datadir/proxysql_sec.log"; + const string proxysql_path { string { WORKSPACE } + "/src/proxysql" }; + + const vector proxy_args { "-f", "-M", "-c", sec_cfg_file.c_str() }; + + string s_stdout {}; + string s_stderr {}; + + int w_res = wexecvp(proxysql_path, proxy_args, &wexecvp_opts, s_stdout, s_stderr); + if (w_res != EXIT_SUCCESS) { + diag("'wexecvp' failed with error: %d", w_res); + } + + err_code = w_res; + + // Write process output to log file + try { + std::ofstream os_logfile { sec_log_file, std::ios::out }; + os_logfile << s_stderr; + } catch (const std::exception& ex) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, ex.what()); + } + }, std::ref(launch_res)); + + // Check that the second ProxySQL is up and responsive + conn_opts_t conn_opts { "127.0.0.1", "radmin", "radmin", 26081 }; + MYSQL* s_proxy_admin = wait_for_proxysql(conn_opts, 5); + + if (s_proxy_admin == nullptr) { + fprintf(stderr, "Error: %s\n", "Waiting for ProxySQL replica timedout"); + return EXIT_FAILURE; + } + + // Configure Cluster access for primary ProxySQL + MYSQL_QUERY(p_proxy_admin, "SET admin-admin_credentials='admin:admin;radmin:radmin;cluster1:secret1pass'"); + MYSQL_QUERY(p_proxy_admin, "SET admin-cluster_username='cluster1'"); + MYSQL_QUERY(p_proxy_admin, "SET admin-cluster_password='secret1pass'"); + MYSQL_QUERY(p_proxy_admin, "LOAD ADMIN VARIABLES TO RUNTIME"); + + // Configure secondary node + MYSQL_QUERY(s_proxy_admin, "SET admin-cluster_check_interval_ms=10"); + MYSQL_QUERY(s_proxy_admin, "SET admin-cluster_mysql_variables_diffs_before_sync=1"); + MYSQL_QUERY(s_proxy_admin, "LOAD ADMIN VARIABLES TO RUNTIME"); + + std::thread th_load_mysql_vars([&cl] (bool& stop, int& load_res) -> void { + MYSQL* admin = mysql_init(NULL); + + if (!mysql_real_connect(admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(admin)); + load_res = mysql_errno(admin); + } + + int default_query_timeout = 36000000; + + while (stop == false) { + string set_query { "SET mysql-default_query_timeout=" + std::to_string(default_query_timeout) }; + mysql_query(admin, set_query.c_str()); + int my_res = mysql_query(admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + if (my_res) { + load_res = mysql_errno(admin); + break; + } else { + usleep(1000 * 10 * 2); + } + + if (default_query_timeout > 36000000) { + default_query_timeout = 36000000; + } else { + default_query_timeout += 1; + } + } + + load_res = 0; + + mysql_close(admin); + }, std::ref(stop), std::ref(q_load_res)); + + std::thread th_query_globals([&cl] (bool& stop, int& save_res) -> void { + MYSQL* admin = mysql_init(NULL); + + if (!mysql_real_connect(admin, cl.host, cl.admin_username, cl.admin_password, NULL, 26081, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(admin)); + save_res = mysql_errno(admin); + } + + while (stop == false) { + int my_res = mysql_query(admin, "SELECT COUNT(*) FROM runtime_global_variables"); + if (my_res) { + save_res = mysql_errno(admin); + break; + } else { + mysql_free_result(mysql_store_result(admin)); + } + } + + save_res = 0; + + mysql_close(admin); + }, std::ref(stop), std::ref(q_globals_res)); + + uint32_t timeout = 10; + uint32_t wait = 0; + + while (wait < timeout) { + if (q_globals_res != -1) { + fprintf(stderr, "'th_admin_save' failed with error: %d\n", q_globals_res); + break; + } + if (q_load_res != -1) { + fprintf(stderr, "'th_admin_load' failed with error: %d\n", q_load_res); + break; + } + + sleep(1); + wait += 1; + } + + { + diag("Shutting down worker threads"); + int timeout = 2; + int waited = 0; + + stop = true; + + if ((q_globals_res == -1 || q_load_res == -1) && waited < timeout) { + sleep(1); + waited += 1; + } + } + + ok(q_load_res == 0, "'th_load_mysql_vars' thread didn't deadlock: %d", q_load_res); + ok(q_globals_res == 0, "'th_query_globals' thread didn't deadlock: %d", q_globals_res); + + th_load_mysql_vars.detach(); + th_query_globals.detach(); + + { + // NOTE: Can lock for a max of 20s (child process timeout) + diag("Shutting down ProxySQL replica"); + mysql_query(s_proxy_admin, "PROXYSQL SHUTDOWN SLOW"); + + int timeout = 3; + int waited = 0; + + // Wait for shutdown + if (launch_res == -1 && waited < timeout) { + sleep(1); + waited += 1; + } + + ok(launch_res == 0, "Replica was properly shutdown and no deadlock took place"); + } + + launch_sec_proxy.join(); + + mysql_close(p_proxy_admin); + mysql_close(s_proxy_admin); + + return exit_status(); +} diff --git a/test/tap/tests/reg_test_3847_node_datadir/proxysql_sec.cfg b/test/tap/tests/reg_test_3847_node_datadir/proxysql_sec.cfg new file mode 100644 index 000000000..45f866d04 --- /dev/null +++ b/test/tap/tests/reg_test_3847_node_datadir/proxysql_sec.cfg @@ -0,0 +1,31 @@ +cluster_sync_interfaces=false +restart_on_missing_heartbeats=3 + +admin_variables= +{ + admin_credentials="admin:admin;cluster1:secret1pass" + mysql_ifaces="0.0.0.0:26081" + cluster_username="cluster1" + cluster_password="secret1pass" +} + +mysql_variables= +{ + interfaces="0.0.0.0:36081" +} + +proxysql_servers = +( + { + hostname="127.0.0.1" + port=26081 + weight=0 + comment="replica" + }, + { + hostname="127.0.0.1" + port=6032 + weight=0 + comment="primary" + } +)