diff --git a/deps/Makefile b/deps/Makefile index 0e5dc8f5c..1b618fab6 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -115,6 +115,7 @@ ev: libev/libev/.libs/libev.a coredumper/coredumper/src/libcoredumper.a: cd coredumper && rm -rf coredumper-*/ || true cd coredumper && tar -zxf coredumper-*.tar.gz + cd coredumper/coredumper && patch -p1 < ../includes.patch cd coredumper/coredumper && cmake . -DBUILD_TESTING=OFF -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE=Debug cd coredumper/coredumper && CC=${CC} CXX=${CXX} ${MAKE} coredumper: coredumper/coredumper/src/libcoredumper.a diff --git a/deps/coredumper/includes.patch b/deps/coredumper/includes.patch new file mode 100644 index 000000000..c4c1ec09a --- /dev/null +++ b/deps/coredumper/includes.patch @@ -0,0 +1,11 @@ +--- coredumper/src/thread_lister.c 2022-12-07 14:57:26.000000000 +0000 ++++ coredumper.patched/src/thread_lister.c 2024-07-09 10:58:42.500458663 +0000 +@@ -35,6 +35,8 @@ + + #include /* needed for NULL on some powerpc platforms (?!) */ + #include ++#include ++#include + + #include "linuxthreads.h" + /* Include other thread listers here that define THREADS macro diff --git a/deps/mariadb-client-library/x509cache.patch b/deps/mariadb-client-library/x509cache.patch index 442730f80..612cfec25 100644 --- a/deps/mariadb-client-library/x509cache.patch +++ b/deps/mariadb-client-library/x509cache.patch @@ -35,9 +35,9 @@ index 916024a8..79564a10 100644 int STDCALL mysql_set_server_option(MYSQL *mysql, diff --git libmariadb/secure/openssl.c libmariadb/secure/openssl.c -+index 916024a8..79564a10 100644 -+--- libmariadb/secure/openssl.c -++++ libmariadb/secure/openssl.c +index 916024a8..79564a10 100644 +--- libmariadb/secure/openssl.c ++++ libmariadb/secure/openssl.c @@ -30,6 +30,11 @@ #include #include diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 32765f49c..08901f561 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -448,6 +448,7 @@ class MySQL_Monitor { static void trigger_dns_cache_update(); void process_discovered_topology(const std::string& originating_server_hostname, const vector& discovered_servers, int reader_hostgroup); + bool is_aws_rds_multi_az_db_cluster_topology(const std::vector& discovered_servers); private: std::vector *tables_defs_monitor; diff --git a/include/MySQL_Variables.h b/include/MySQL_Variables.h index 678d594a0..4d0aa5c8a 100644 --- a/include/MySQL_Variables.h +++ b/include/MySQL_Variables.h @@ -20,6 +20,7 @@ bool update_server_variable(MySQL_Session* session, int idx, int &_rc); bool verify_server_variable(MySQL_Session* session, int idx, uint32_t client_hash, uint32_t server_hash); bool verify_set_names(MySQL_Session* session); bool logbin_update_server_variable(MySQL_Session* session, int idx, int &_rc); +bool is_perm_track_err(int err, const char* varname); class MySQL_Variables { static verify_var verifiers[SQL_NAME_LAST_HIGH_WM]; diff --git a/include/proxysql.h b/include/proxysql.h index d10e0f3cf..1d674bc85 100644 --- a/include/proxysql.h +++ b/include/proxysql.h @@ -108,6 +108,10 @@ void proxy_info_(const char* msg, ...); #ifdef DEBUG void init_debug_struct(); void init_debug_struct_from_cmdline(); +/** + * @brief Add a debug entry in the error log. To be used through 'proxy_debug' macro. + * @details This function saves/restores the previous 'errno' value. + */ __attribute__((__format__ (__printf__, 7, 8))) void proxy_debug_func(enum debug_module, int, int, const char *, int, const char *, const char *, ...); void proxy_debug_get_filters(std::set&); diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index d021a4ea3..61f52c2b2 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -285,6 +285,11 @@ typedef struct { char * default_value; // default value bool is_global_variable; // is it a global variable? } mysql_variable_st; + +typedef struct { + int err; + const char* name; +} var_track_err_st; #endif enum mysql_data_stream_status { @@ -1409,10 +1414,9 @@ mysql_variable_st mysql_tracked_variables[] { session_track_system_variables session_track_transaction_info */ - - }; #else extern mysql_variable_st mysql_tracked_variables[]; +extern var_track_err_st perm_track_errs[]; #endif // PROXYSQL_EXTERN #endif // MYSQL_TRACKED_VARIABLES diff --git a/include/set_parser.h b/include/set_parser.h index 542186834..bd89bbab2 100644 --- a/include/set_parser.h +++ b/include/set_parser.h @@ -42,6 +42,12 @@ class SetParser { // First implemenation of the parser for TRANSACTION ISOLATION LEVEL and TRANSACTION READ/WRITE std::map> parse2(); std::string parse_character_set(); + std::string parse_USE_query(); + std::string remove_comments(const std::string& q); +#ifdef DEBUG + // built-in testing + void test_parse_USE_query(); +#endif // DEBUG ~SetParser(); }; diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 4f7925959..31659be7b 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -2467,6 +2467,15 @@ MySQL_Connection * MySQL_HostGroups_Manager::get_MyConn_from_pool(unsigned int _ } void MySQL_HostGroups_Manager::destroy_MyConn_from_pool(MySQL_Connection *c, bool _lock) { + // 'libmariadbclient' only performs a cleanup of SSL error queue during connect when making use of + // 'auth_caching_sha2_client|auth_sha256_client' during connect. If any SSL errors took place during the + // previous operation, we must cleanup the queue to avoid polluting other backend conns. + int myerr=mysql_errno(c->mysql); + if (myerr >= 2000 && myerr < 3000 && c->mysql->options.use_ssl) { + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Client error %d detected on SSL connection, cleaning SSL error queue\n", myerr); + ERR_clear_error(); + } + bool to_del=true; // the default, legacy behavior MySrvC *mysrvc=(MySrvC *)c->parent; if (mysrvc->get_status() == MYSQL_SERVER_STATUS_ONLINE && c->send_quit && queue.size() < __sync_fetch_and_add(&GloMTH->variables.connpoll_reset_queue_length, 0)) { diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 09c3e1d08..c6e996b30 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -3367,6 +3367,35 @@ void MySQL_Monitor::process_discovered_topology(const std::string& originating_s } } +/** +* @brief Check if a list of servers is matching the description of an AWS RDS Multi-AZ DB Cluster. +* @details This method takes a vector of discovered servers and checks that there are exactly three which are named "instance-[1|2|3]" respectively, as expected on an AWS RDS Multi-AZ DB Cluster. +* @param discovered_servers A vector of servers discovered when querying the cluster's topology. +* @return Returns 'true' if all conditions are met and 'false' otherwise. +*/ +bool MySQL_Monitor::is_aws_rds_multi_az_db_cluster_topology(const std::vector& discovered_servers) { + if (discovered_servers.size() != 3) { + return false; + } + + const std::vector instance_names = {"-instance-1", "-instance-2", "-instance-3"}; + int identified_hosts = 0; + for (const std::string& instance_str : instance_names) { + for (MYSQL_ROW server : discovered_servers) { + if (server[2] == NULL || (server[2][0] == '\0')) { + continue; + } + + std::string current_discovered_hostname = server[2]; + if (current_discovered_hostname.find(instance_str) != std::string::npos) { + ++identified_hosts; + break; + } + } + } + return (identified_hosts == 3); +} + void * MySQL_Monitor::monitor_read_only() { mysql_close(mysql_init(NULL)); // initialize the MySQL Thread (note: this is not a real thread, just the structures associated with it) @@ -3381,9 +3410,9 @@ void * MySQL_Monitor::monitor_read_only() { unsigned long long t2; unsigned long long next_loop_at=0; int topology_loop = 0; - int topology_loop_max = mysql_thread___monitor_aws_rds_topology_discovery_interval; while (GloMyMon->shutdown==false && mysql_thread___monitor_enabled==true) { + int topology_loop_max = mysql_thread___monitor_aws_rds_topology_discovery_interval; bool do_discovery_check = false; unsigned int glover; @@ -3418,11 +3447,13 @@ void * MySQL_Monitor::monitor_read_only() { goto __end_monitor_read_only_loop; } - if (topology_loop >= topology_loop_max) { - do_discovery_check = true; - topology_loop = 0; + if (topology_loop_max > 0) { // if the discovery interval is set to zero, do not query for the topology + if (topology_loop >= topology_loop_max) { + do_discovery_check = true; + topology_loop = 0; + } + topology_loop += 1; } - topology_loop += 1; // resultset must be initialized before calling monitor_read_only_async monitor_read_only_async(resultset, do_discovery_check); @@ -7400,8 +7431,8 @@ VALGRIND_ENABLE_ERROR_REPORTING; discovered_servers.push_back(row); } - // Process the discovered servers and add them to 'runtime_mysql_servers' - if (!discovered_servers.empty()) { + // Process the discovered servers and add them to 'runtime_mysql_servers' (process only for AWS RDS Multi-AZ DB Clusters) + if (!discovered_servers.empty() && is_aws_rds_multi_az_db_cluster_topology(discovered_servers)) { process_discovered_topology(originating_server_hostname, discovered_servers, mmsd->reader_hostgroup); } } else { diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index 41d116620..814dc3a6f 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -2566,6 +2566,8 @@ bool MySQL_Session::handler_again___status_SETTING_GENERIC_VARIABLE(int *_rc, co (myerr == 1193) // variable is not found || (myerr == 1651) // Query cache is disabled + || + (is_perm_track_err(myerr, var_name)) // Special permitted tracking errors (~= '1193') ) { int idx = SQL_NAME_LAST_HIGH_WM; for (int i=0; istatus.frontend_use_db, 1); string nq=string((char *)pkt->ptr+sizeof(mysql_hdr)+1,pkt->size-sizeof(mysql_hdr)-1); - RE2::GlobalReplace(&nq,(char *)"(?U)/\\*.*\\*/",(char *)" "); - char *sn_tmp = (char *)nq.c_str(); - while (sn_tmp < ( nq.c_str() + nq.length() - 4 ) && *sn_tmp == ' ') - sn_tmp++; - //char *schemaname=strdup(nq.c_str()+4); - char *schemaname=strdup(sn_tmp+3); - char *schemanameptr=trim_spaces_and_quotes_in_place(schemaname); - // handle cases like "USE `schemaname` - if(schemanameptr[0]=='`' && schemanameptr[strlen(schemanameptr)-1]=='`') { - schemanameptr[strlen(schemanameptr)-1]='\0'; - schemanameptr++; - } - client_myds->myconn->userinfo->set_schemaname(schemanameptr,strlen(schemanameptr)); - free(schemaname); - if (mirror==false) { + SetParser parser(nq); + string schemaname = parser.parse_USE_query(); + if (schemaname != "") { + client_myds->myconn->userinfo->set_schemaname((char *)schemaname.c_str(),schemaname.length()); + if (mirror==false) { + RequestEnd(NULL); + } + l_free(pkt->size,pkt->ptr); + client_myds->setDSS_STATE_QUERY_SENT_NET(); + unsigned int nTrx=NumActiveTransactions(); + uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0 ); + if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; + client_myds->myprot.generate_pkt_OK(true,NULL,NULL,1,0,0,setStatus,0,NULL); + GloMyLogger->log_audit_entry(PROXYSQL_MYSQL_INITDB, this, NULL); + } else { + l_free(pkt->size,pkt->ptr); + client_myds->setDSS_STATE_QUERY_SENT_NET(); + std::string msg = "Unable to parse: " + nq; + client_myds->myprot.generate_pkt_ERR(true,NULL,NULL,client_myds->pkt_sid+1,1148,(char *)"42000", msg.c_str()); RequestEnd(NULL); } - l_free(pkt->size,pkt->ptr); - client_myds->setDSS_STATE_QUERY_SENT_NET(); - unsigned int nTrx=NumActiveTransactions(); - uint16_t setStatus = (nTrx ? SERVER_STATUS_IN_TRANS : 0 ); - if (autocommit) setStatus |= SERVER_STATUS_AUTOCOMMIT; - client_myds->myprot.generate_pkt_OK(true,NULL,NULL,1,0,0,setStatus,0,NULL); - GloMyLogger->log_audit_entry(PROXYSQL_MYSQL_INITDB, this, NULL); client_myds->DSS=STATE_SLEEP; } else { l_free(pkt->size,pkt->ptr); diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index 350503ab4..a76453516 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -989,7 +989,7 @@ MySQL_Threads_Handler::MySQL_Threads_Handler() { variables.monitor_ping_interval=8000; variables.monitor_ping_max_failures=3; variables.monitor_ping_timeout=1000; - variables.monitor_aws_rds_topology_discovery_interval=1000; + variables.monitor_aws_rds_topology_discovery_interval=0; variables.monitor_read_only_interval=1000; variables.monitor_read_only_timeout=800; variables.monitor_read_only_max_timeout_count=3; @@ -2155,7 +2155,7 @@ char ** MySQL_Threads_Handler::get_variables_list() { VariablesPointers_int["monitor_ping_timeout"] = make_tuple(&variables.monitor_ping_timeout, 100, 600*1000, false); VariablesPointers_int["monitor_ping_max_failures"] = make_tuple(&variables.monitor_ping_max_failures, 1, 1000*1000, false); - VariablesPointers_int["monitor_aws_rds_topology_discovery_interval"] = make_tuple(&variables.monitor_aws_rds_topology_discovery_interval, 1, 100000, false); + VariablesPointers_int["monitor_aws_rds_topology_discovery_interval"] = make_tuple(&variables.monitor_aws_rds_topology_discovery_interval, 0, 100000, false); VariablesPointers_int["monitor_read_only_interval"] = make_tuple(&variables.monitor_read_only_interval, 100, 7*24*3600*1000, false); VariablesPointers_int["monitor_read_only_timeout"] = make_tuple(&variables.monitor_read_only_timeout, 100, 600*1000, false); VariablesPointers_int["monitor_read_only_max_timeout_count"] = make_tuple(&variables.monitor_read_only_max_timeout_count, 1, 1000*1000, false); diff --git a/lib/MySQL_Variables.cpp b/lib/MySQL_Variables.cpp index 6f5c1f048..c0df46782 100644 --- a/lib/MySQL_Variables.cpp +++ b/lib/MySQL_Variables.cpp @@ -9,6 +9,7 @@ #endif #include +#include "mysqld_error.h" static inline char is_digit(char c) { @@ -17,6 +18,23 @@ static inline char is_digit(char c) { return 0; } +var_track_err_st perm_track_errs[] { + // ERROR 1210 (HY000): Variable not supported in combination with Galera: + // - Changed by MySQL, previously 'ER_UNKNOWN_SYSTEM_VARIABLE' + { ER_WRONG_ARGUMENTS, "sql_generate_invisible_primary_key" } +}; + +bool is_perm_track_err(int err, const char* varname) { + const size_t count = sizeof(perm_track_errs) / sizeof(var_track_err_st); + + for (size_t i = 0; i < count; i++) { + if (perm_track_errs[i].err == err && (strcasecmp(varname, perm_track_errs[i].name) == 0)) { + return true; + } + } + return false; +} + #include "proxysql_find_charset.h" verify_var MySQL_Variables::verifiers[SQL_NAME_LAST_HIGH_WM]; diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index d1088fc12..9e576fa8f 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -193,7 +193,8 @@ static void BQE1(SQLite3DB *db, const vector& tbs, const string& p1, con } -static int round_intv_to_time_interval(int& intv) { +static int round_intv_to_time_interval(const char* name, int _intv) { + int intv = _intv; if (intv > 300) { intv = 600; } else { @@ -221,6 +222,9 @@ static int round_intv_to_time_interval(int& intv) { } } } + if (intv != _intv) { + proxy_warning("Variable '%s' rounded to interval '%d'\n", name, intv); + } return intv; } @@ -3437,7 +3441,7 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this if (!strcasecmp(name,"stats_mysql_connection_pool")) { int intv=atoi(value); if (intv >= 0 && intv <= 300) { - intv = round_intv_to_time_interval(intv); + intv = round_intv_to_time_interval(name, intv); variables.stats_mysql_connection_pool=intv; GloProxyStats->variables.stats_mysql_connection_pool=intv; return true; @@ -3448,7 +3452,7 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this if (!strcasecmp(name,"stats_mysql_connections")) { int intv=atoi(value); if (intv >= 0 && intv <= 300) { - intv = round_intv_to_time_interval(intv); + intv = round_intv_to_time_interval(name, intv); variables.stats_mysql_connections=intv; GloProxyStats->variables.stats_mysql_connections=intv; return true; @@ -3459,7 +3463,7 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this if (!strcasecmp(name,"stats_mysql_query_cache")) { int intv=atoi(value); if (intv >= 0 && intv <= 300) { - intv = round_intv_to_time_interval(intv); + intv = round_intv_to_time_interval(name, intv); variables.stats_mysql_query_cache=intv; GloProxyStats->variables.stats_mysql_query_cache=intv; return true; @@ -3480,7 +3484,7 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this if (!strcasecmp(name,"stats_system_cpu")) { int intv=atoi(value); if (intv >= 0 && intv <= 600) { - intv = round_intv_to_time_interval(intv); + intv = round_intv_to_time_interval(name, intv); variables.stats_system_cpu=intv; GloProxyStats->variables.stats_system_cpu=intv; return true; @@ -3492,7 +3496,7 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this if (!strcasecmp(name,"stats_system_memory")) { int intv=atoi(value); if (intv >= 0 && intv <= 600) { - intv = round_intv_to_time_interval(intv); + intv = round_intv_to_time_interval(name, intv); variables.stats_system_memory=intv; GloProxyStats->variables.stats_system_memory=intv; return true; diff --git a/lib/debug.cpp b/lib/debug.cpp index e4eda648f..4f3755c9c 100644 --- a/lib/debug.cpp +++ b/lib/debug.cpp @@ -131,15 +131,22 @@ void proxy_debug_load_filters(std::set& f) { //pthread_mutex_unlock(&debug_mutex); } +// REMINDER: This function should always save/restore 'errno', otherwise it could influence error handling. void proxy_debug_func(enum debug_module module, int verbosity, int thr, const char *__file, int __line, const char *__func, const char *fmt, ...) { + int saved_errno = errno; assert(moduleuse_ssl) { { // mariadb client library disables NONBLOCK for SSL connections ... re-enable it! + // CONTEXT: This shouldn't be confused with the previous incompatibility that 'mariadbclient' + // had between the NONBLOCK flags and SSL connections, and that was reported and solved via + // CONC-320, our patch for this incompatibility was dropped on connector upgrade for v2.0.9. + // Setting the socket back to NONBLOCK is SAFE. This is because a connection can be used for + // BOTH blocking and not blocking calls, as described here: + // - https://mariadb.com/kb/en/using-the-non-blocking-library/#mixing-blocking-and-non-blocking-operation + // In case of SSL connections, it's still required to set the socket back to NONBLOCK prior to + // non-blockig calls. mysql_options(mysql, MYSQL_OPT_NONBLOCK, 0); int f=fcntl(mysql->net.fd, F_GETFL); #ifdef FD_CLOEXEC diff --git a/lib/mysql_data_stream.cpp b/lib/mysql_data_stream.cpp index f996cc2be..76fe1e889 100644 --- a/lib/mysql_data_stream.cpp +++ b/lib/mysql_data_stream.cpp @@ -574,9 +574,7 @@ int MySQL_Data_Stream::read_from_net() { int r=0; int s=queue_available(queueIN); - if (encrypted) { - // proxy_info("Queue available of %d bytes\n", s); - } + if (encrypted == false) { if (pkts_recv) { r = recv(fd, queue_w_ptr(queueIN), s, 0); @@ -596,41 +594,24 @@ int MySQL_Data_Stream::read_from_net() { } } } else { // encrypted == true -/* - if (!SSL_is_init_finished(ssl)) { - int ret = SSL_do_handshake(ssl); - int ret2; - if (ret != 1) { - //ERR_print_errors_fp(stderr); - ret2 = SSL_get_error(ssl, ret); - fprintf(stderr,"%d\n",ret2); - } - return 0; - } else { - r = SSL_read (ssl, queue_w_ptr(queueIN), s); - } -*/ PROXY_TRACE(); if (s < MY_SSL_BUFFER) { return 0; // no enough space for reads } char buf[MY_SSL_BUFFER]; - //ssize_t n = read(fd, buf, sizeof(buf)); - int n = recv(fd, buf, sizeof(buf), 0); - //proxy_info("SSL recv of %d bytes\n", n); - proxy_debug(PROXY_DEBUG_NET, 7, "Session=%p: recv() read %d bytes. num_write: %lu , num_read: %lu\n", sess, n, rbio_ssl->num_write , rbio_ssl->num_read); - if (n > 0 || rbio_ssl->num_write > rbio_ssl->num_read) { - //on_read_cb(buf, (size_t)n); + int ssl_recv_bytes = recv(fd, buf, sizeof(buf), 0); + proxy_debug(PROXY_DEBUG_NET, 7, "Session=%p: recv() read %d bytes. num_write: %lu , num_read: %lu\n", sess, ssl_recv_bytes, rbio_ssl->num_write , rbio_ssl->num_read); + if (ssl_recv_bytes > 0 || rbio_ssl->num_write > rbio_ssl->num_read) { char buf2[MY_SSL_BUFFER]; int n2; enum sslstatus status; char *src = buf; - int len = n; + int len = ssl_recv_bytes; while (len > 0) { n2 = BIO_write(rbio_ssl, src, len); proxy_debug(PROXY_DEBUG_NET, 5, "Session=%p: write %d bytes into BIO %p, len=%d\n", sess, n2, rbio_ssl, len); - //proxy_info("BIO_write with len = %d and %d bytes\n", len , n2); + if (n2 <= 0) { shut_soft(); return -1; @@ -638,40 +619,29 @@ int MySQL_Data_Stream::read_from_net() { src += n2; len -= n2; if (!SSL_is_init_finished(ssl)) { - //proxy_info("SSL_is_init_finished NOT completed\n"); + proxy_debug(PROXY_DEBUG_NET, 5, "SSL handshake not finished yet session=%p bytes=%d BIO=%p len=%d\n", sess, n2, rbio_ssl, len); if (do_ssl_handshake() == SSLSTATUS_FAIL) { - //proxy_info("SSL_is_init_finished failed!!\n"); + proxy_debug(PROXY_DEBUG_NET, 5, "SSL handshake failed session=%p bytes=%d BIO=%p len=%d\n", sess, n2, rbio_ssl, len); shut_soft(); return -1; } if (!SSL_is_init_finished(ssl)) { - //proxy_info("SSL_is_init_finished yet NOT completed\n"); + proxy_debug(PROXY_DEBUG_NET, 5, "SSL handshake not finished yet session=%p bytes=%d BIO=%p len=%d\n", sess, n2, rbio_ssl, len); return 0; } } else { - //proxy_info("SSL_is_init_finished completed\n"); + proxy_debug(PROXY_DEBUG_NET, 5, "SSL handshake finished session=%p bytes=%d BIO=%p len=%d\n", sess, n2, rbio_ssl, len); } } n2 = SSL_read (ssl, queue_w_ptr(queueIN), s); proxy_debug(PROXY_DEBUG_NET, 5, "Session=%p: read %d bytes from BIO %p into a buffer with %d bytes free\n", sess, n2, rbio_ssl, s); r = n2; - //proxy_info("Read %d bytes from SSL\n", r); - if (n2 > 0) { - } -/* - do { - n2 = SSL_read(ssl, buf2, sizeof(buf2)); - if (n2 > 0) { - - } - } while (n > 0); -*/ status = get_sslstatus(ssl, n2); - //proxy_info("SSL status = %d\n", status); + if (status == SSLSTATUS_WANT_IO) { do { n2 = BIO_read(wbio_ssl, buf2, sizeof(buf2)); - //proxy_info("BIO_read with %d bytes\n", n2); + if (n2 > 0) { queue_encrypted_bytes(buf2, n2); } else if (!BIO_should_retry(wbio_ssl)) { @@ -685,14 +655,18 @@ int MySQL_Data_Stream::read_from_net() { return -1; } } else { - r = n; - //r += SSL_read (ssl, queue_w_ptr(queueIN), s); - //proxy_info("Read %d bytes from SSL\n", r); + // Shutdown if we either received the EOF, or operation failed with non-retryable error. + if (ssl_recv_bytes==0 || (ssl_recv_bytes==-1 && errno != EINTR && errno != EAGAIN)) { + proxy_debug(PROXY_DEBUG_NET, 5, "Received EOF, shutting down soft socket -- Session=%p, Datastream=%p\n", sess, this); + shut_soft(); + return -1; + } + r = ssl_recv_bytes; } } -//__exit_read_from_next: + proxy_debug(PROXY_DEBUG_NET, 5, "Session=%p: read %d bytes from fd %d into a buffer of %d bytes free\n", sess, r, fd, s); - //proxy_error("read %d bytes from fd %d into a buffer of %d bytes free\n", r, fd, s); + if (r < 1) { if (encrypted==false) { int myds_errno=errno; diff --git a/lib/set_parser.cpp b/lib/set_parser.cpp index 219f12723..f083df2de 100644 --- a/lib/set_parser.cpp +++ b/lib/set_parser.cpp @@ -3,9 +3,11 @@ #include #include #include -#ifdef PARSERDEBUG +#include +#include // for std::pair +//#ifdef PARSERDEBUG #include -#endif +//#endif #ifdef DEBUG //#define VALGRIND_ENABLE_ERROR_REPORTING @@ -515,3 +517,148 @@ std::string SetParser::parse_character_set() { return value4; } +std::string SetParser::parse_USE_query() { +#ifdef DEBUG + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Parsing query %s\n", query.c_str()); +#endif // DEBUG + + re2::RE2::Options opt2(RE2::Quiet); + opt2.set_case_sensitive(false); + opt2.set_longest_match(false); + + std::string dbname = remove_comments(query); + re2::RE2 re0("^\\s*", opt2); + re2::RE2::Replace(&dbname, re0, ""); + if (dbname.size() >= 4) { + if ( + strncasecmp(dbname.c_str(), "USE ",4) == 0 + || + strncasecmp(dbname.c_str(), "USE`",4) == 0 + ) { + re2::RE2 re1("^USE\\s*", opt2); + re2::RE2::Replace(&dbname, re1, ""); + re2::RE2 re2("\\s*$", opt2); + re2::RE2::Replace(&dbname, re2, ""); + if (dbname[0] == '`') { + if (dbname.length() > 2) { + if (dbname[dbname.length()-1] == '`') { + // Remove the first character + dbname.erase(0, 1); + // Remove the last character + dbname.erase(dbname.length() - 1); + return dbname; + } + } + } else { + return dbname; + } + } + } else { + return ""; + } + + return ""; +} + + +std::string SetParser::remove_comments(const std::string& q) { + std::string result = ""; + bool in_multiline_comment = false; + + for (size_t i = 0; i < query.size(); ++i) { + char current_char = query[i]; + + // Check for multiline comment start + if (current_char == '/' && i + 1 < query.size() && query[i + 1] == '*') { + in_multiline_comment = true; + i++; // Skip the '*' + continue; + } + + // Check for multiline comment end + if (in_multiline_comment && current_char == '*' && i + 1 < query.size() && query[i + 1] == '/') { + in_multiline_comment = false; + i++; // Skip the '/' + continue; + } + + // Skip characters inside multiline comment + if (in_multiline_comment) { + continue; + } + + // Check for single-line comments + if (current_char == '#' || (current_char == '-' && i + 1 < query.size() && query[i + 1] == '-')) { + // Skip until the end of the line + while (i < query.size() && query[i] != '\n') { + i++; + } + continue; + } + + // Append the character to the result if it's not a comment + result += current_char; + } + + return result; +} + + +#ifdef DEBUG +void SetParser::test_parse_USE_query() { + + // Define vector of pairs (query, expected dbname) + std::vector> testCases = { + {"USE my_database", "my_database"}, // Basic Case + {"USE my_database", "my_database"}, // Basic Case + {"USE my_database ", "my_database"}, // Basic Case + {"/* comment */USE dbname /* comment */", "dbname"}, // With Comments + {"/* comment */ USE dbname", "dbname"}, // With Comments + {"USE dbname /* comment */", "dbname"}, // With Comments + {"/* comment */USE `dbname` /* comment */", "dbname"}, // With backtick + {"/* comment */USE `dbname`/* comment */", "dbname"}, // With backtick + {"/* comment */USE`dbname` /* comment */", "dbname"}, // With backtick + {"/* comment */USE `dbname`/* comment */", "dbname"}, // With backtick + {"/* comment\nmultiline comment */USE dbname /* comment */", "dbname"}, // Multiline Comment + {"/* comment */USE dbname # comment", "dbname"}, // Hash Comment + {"/* comment */USE dbname -- comment", "dbname"}, // Double Dash Comment + {"/* comment */USE dbname # comment", "dbname"}, // Hash Comment + {"/* comment */USE dbname -- comment", "dbname"}, // Double Dash Comment + {"USE dbname # comment", "dbname"}, // Hash Comment + {"USE dbname -- comment", "dbname"}, // Double Dash Comment + {"SELECT * FROM my_table", ""}, // No match + {"/*+ placeholder_comment */ USE test_use_comment", "test_use_comment"}, + + {"USE /*+ placeholder_comment */ `test_use_comment-a1`", "test_use_comment-a1"}, + {"USE /*+ placeholder_comment */ `test_use_comment_1`", "test_use_comment_1"}, + {"USE/*+ placeholder_comment */ `test_use_comment_2`", "test_use_comment_2"}, + {"USE /*+ placeholder_comment */`test_use_comment_3`", "test_use_comment_3"}, + {"USE /*+ placeholder_comment */ test_use_comment_4", "test_use_comment_4"}, + {"USE/*+ placeholder_comment */ test_use_comment_5", "test_use_comment_5"}, + {"USE /*+ placeholder_comment */test_use_comment_6", "test_use_comment_6"}, + {"USE /*+ placeholder_comment */ `test_use_comment-1`", "test_use_comment-1"}, + {"use my_database", "my_database"}, + {"/* comment */ use dbname -- comment", "dbname"}, + {"/* comment\nmultiline comment */USE dbname /* comment\nmultiline comment */", "dbname"}, // Multiline Comment + + {"USE/*+ placeholder_comment */ `test_use_comment-2`", "test_use_comment-2"}, + {"USE /*+ placeholder_comment */`test_use_comment-3`", "test_use_comment-3"}, + {"/*+ placeholder_comment */USE `test_use_comment-4`", "test_use_comment-4"}, + {"USE/*+ placeholder_comment */`test_use_comment-5`", "test_use_comment-5"}, + {"/* comment */USE`test_use_comment-6`", "test_use_comment-6"}, + {"USE`test_use_comment-7`", "test_use_comment-7"}, + }; + + // Run tests for each pair + for (const auto& p : testCases) { + set_query(p.first); + std::string dbname = parse_USE_query(); + if (dbname != p.second) { + // we call parse_USE_query() again just to make it easier to create a breakpoint + std::string s = parse_USE_query(); + assert(s == p.second); + } + } + +} +#endif // DEBUG diff --git a/src/main.cpp b/src/main.cpp index f38ebee6f..41578db81 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2156,6 +2156,13 @@ int main(int argc, const char * argv[]) { // std::cerr << "Main init phase0 completed in "; #endif } +#ifdef DEBUG + { + // Automated testing + SetParser parser(""); + parser.test_parse_USE_query(); + } +#endif // DEBUG { cpu_timer t; ProxySQL_Main_process_global_variables(argc, argv); diff --git a/test/tap/tap/tap.cpp b/test/tap/tap/tap.cpp index b067432a2..5270516f8 100644 --- a/test/tap/tap/tap.cpp +++ b/test/tap/tap/tap.cpp @@ -20,7 +20,6 @@ #include "tap.h" -//#include "my_global.h" typedef char my_bool; #include @@ -28,8 +27,13 @@ typedef char my_bool; #include #include #include +#include #include +#include + +using std::size_t; + static ulong start_timer(void); static void end_timer(ulong start_time,char *buff); static void nice_time(double sec,char *buff,my_bool part_second); @@ -68,6 +72,34 @@ static TEST_DATA g_test = { NO_PLAN, 0, 0, "" }; */ #define tapout stdout +/** + @brief Writes current time ("%Y-%m-%d %H:%M:%S") in the supplied buffer. + + @param tm_buf Buffer to be written. + @param us True for enabling microseconds in log output. + @param len Buffer len. + */ +size_t get_fmt_time(char* tm_buf, size_t len, bool us=false) { + time_t __timer; + time(&__timer); + struct tm __tm_info {}; + localtime_r(&__timer, &__tm_info); + size_t rc = strftime(tm_buf, len, "%Y-%m-%d %H:%M:%S", &__tm_info); + + if (rc == 0) { + return rc; + } else if (us) { + struct timeval tv; + struct tm *tm_info; + + gettimeofday(&tv, NULL); + tm_info = localtime(&tv.tv_sec); + rc = snprintf(tm_buf + rc, len - rc, ".%06ld", tv.tv_usec); + } + + return rc; +} + /** Emit the beginning of a test line, that is: "(not) ok", test number, and description. @@ -85,9 +117,12 @@ static TEST_DATA g_test = { NO_PLAN, 0, 0, "" }; static void vemit_tap(int pass, char const *fmt, va_list ap) { - fprintf(tapout, "%sok %d%s", + char tm_buf[28] = { 0 }; + get_fmt_time(tm_buf, sizeof(tm_buf), __sync_add_and_fetch(&tap_log_us, 0)); + fprintf(tapout, "%sok %d - %s%s", pass ? "" : "not ", __sync_add_and_fetch(&g_test.last, 1), + tm_buf, (fmt && *fmt) ? " - " : ""); if (fmt && *fmt) vfprintf(tapout, fmt, ap); @@ -156,7 +191,9 @@ diag(char const *fmt, ...) { va_list ap; va_start(ap, fmt); - fprintf(tapout, "# "); + char tm_buf[28] = { 0 }; + get_fmt_time(tm_buf, sizeof(tm_buf), __sync_add_and_fetch(&tap_log_us, 0)); + fprintf(tapout, "# %s ", tm_buf); vfprintf(tapout, fmt, ap); emit_endl(); va_end(ap); @@ -192,6 +229,7 @@ static signal_entry install_signal[]= { }; int skip_big_tests= 1; +volatile int tap_log_us = 1; ulong start_time= 0; void diff --git a/test/tap/tap/tap.h b/test/tap/tap/tap.h index 7678c1a39..f40dad0e9 100644 --- a/test/tap/tap/tap.h +++ b/test/tap/tap/tap.h @@ -78,6 +78,10 @@ extern "C" { @see SKIP_BIG_TESTS */ extern int skip_big_tests; +/** + * @brief Specifies if time logging should include microseconds; either 1 or 0. + */ +extern volatile int tap_log_us; /** @defgroup MyTAP_API MyTAP API diff --git a/test/tap/tap/utils.cpp b/test/tap/tap/utils.cpp index d55e3521c..b656e03aa 100644 --- a/test/tap/tap/utils.cpp +++ b/test/tap/tap/utils.cpp @@ -187,6 +187,45 @@ my_bool mysql_stmt_close_override(MYSQL_STMT* stmt, const char* file, int line) #endif +pair> disable_core_nodes_scheduler(CommandLine& cl, MYSQL* admin) { + vector nodes_conns {}; + + pair> nodes_fetch { fetch_cluster_nodes(admin, true) }; + if (nodes_fetch.first) { + diag("Failed to fetch cluster nodes. Aborting further testing"); + return { EXIT_FAILURE, {} }; + } + + // Ensure a more idle status for ProxySQL + for (const srv_addr_t& node : nodes_fetch.second) { + const char* user { cl.admin_username }; + const char* pass { cl.admin_password }; + + MYSQL* myconn = mysql_init(NULL); + + if (!mysql_real_connect(myconn, node.host.c_str(), user, pass, NULL, node.port, NULL, 0)) { + diag( + "Failed to connect to Cluster node. Abort further testing" + " host=%s port=%d errno=%d error='%s'", + node.host.c_str(), node.port, mysql_errno(myconn), mysql_error(myconn) + ); + return { EXIT_FAILURE, {} }; + } + + const vector queries { "DELETE FROM scheduler", "LOAD SCHEDULER TO RUNTIME" }; + for (const char* q : queries) { + if (mysql_query_t(myconn, q)) { + diag("Failed to execute query query=%s error='%s'", q, mysql_error(myconn)); + return { EXIT_FAILURE, {} }; + } + } + + nodes_conns.push_back(myconn); + } + + return { EXIT_SUCCESS, nodes_conns }; +} + std::size_t count_matches(const string& str, const string& substr) { std::size_t result = 0; std::size_t pos = 0; @@ -199,8 +238,8 @@ std::size_t count_matches(const string& str, const string& substr) { return result; } -int mysql_query_t(MYSQL* mysql, const char* query) { - diag("%s: Issuing query '%s' to ('%s':%d)", get_formatted_time().c_str(), query, mysql->host, mysql->port); +int mysql_query_t__(MYSQL* mysql, const char* query, const char* f, int ln, const char* fn) { + diag("%s:%d:%s(): Issuing query '%s' to ('%s':%d)", f, ln, fn, query, mysql->host, mysql->port); return mysql_query(mysql, query); } @@ -946,20 +985,71 @@ string tap_curtime() { } int get_proxysql_cpu_usage(const CommandLine& cl, uint32_t intv, double& cpu_usage) { - // check if proxysql process is consuming higher cpu than it should + // Create Admin connection MYSQL* proxysql_admin = mysql_init(NULL); if (!mysql_real_connect(proxysql_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql_admin)); return EXIT_FAILURE; } - // recover admin variables - string set_stats_query { "SET admin-stats_system_cpu=" + std::to_string(intv) }; + // Set new interval + const string set_stats_query { "SET admin-stats_system_cpu=" + std::to_string(intv) }; MYSQL_QUERY(proxysql_admin, set_stats_query.c_str()); MYSQL_QUERY(proxysql_admin, "LOAD ADMIN VARIABLES TO RUNTIME"); + // Wait until 'system_cpu' is filled with newer entries + time_t curtime = time(NULL); + uint32_t entry_count { 0 }; + + const char runtime_stats[] { + "SELECT variable_value FROM runtime_global_variables WHERE variable_name='admin-stats_system_cpu'" + }; + ext_val_t ext_rintv { mysql_query_ext_val(proxysql_admin, runtime_stats, 10) }; + if (ext_rintv.err != EXIT_SUCCESS) { + const string err { get_ext_val_err(proxysql_admin, ext_rintv) }; + diag("Failed query query:`%s`, err:`%s`", runtime_stats, err.c_str()); + return EXIT_FAILURE; + } + + if (ext_rintv.val != intv) { + diag( + "WARNING: Supplied interval not available, using rounded value intv=%d rintv=%d", + intv, ext_rintv.val + ); + } + // sleep during the required interval + safe threshold - sleep(2 * intv + 2); + const uint32_t init_wait = 2 * ext_rintv.val + 2; + diag("Waiting for %d secs for new 'system_cpu' entries... curtime=%ld", init_wait, curtime); + sleep(init_wait); + + const string count_query { + "SELECT COUNT(*) FROM system_cpu WHERE timestamp > " + std::to_string(curtime) + }; + ext_val_t ext_stats_count { mysql_query_ext_val(proxysql_admin, count_query, 10) }; + + if (ext_stats_count.err != EXIT_SUCCESS) { + const string err { get_ext_val_err(proxysql_admin, ext_stats_count) }; + diag("Failed query query:`%s`, err:`%s`", count_query.c_str(), err.c_str()); + return EXIT_FAILURE; + } + + entry_count = ext_stats_count.val; + diag("Finished initial wait for 'system_cpu' entry_count=%d", entry_count); + + while (entry_count < 2) { + diag("Waiting for more 'system_cpu' entries... entry_count=%d", entry_count); + ext_val_t ext_stats_count { mysql_query_ext_val(proxysql_admin, count_query, 10) }; + + if (ext_stats_count.err != EXIT_SUCCESS) { + const string err { get_ext_val_err(proxysql_admin, ext_stats_count) }; + diag("Failed query query:`%s`, err:`%s`", count_query.c_str(), err.c_str()); + return EXIT_FAILURE; + } + + entry_count = ext_stats_count.val; + sleep(1); + } MYSQL_QUERY(proxysql_admin, "SELECT * FROM system_cpu ORDER BY timestamp DESC LIMIT 2"); MYSQL_RES* admin_res = mysql_store_result(proxysql_admin); @@ -977,7 +1067,7 @@ int get_proxysql_cpu_usage(const CommandLine& cl, uint32_t intv, double& cpu_usa int init_stime_s = atoi(row[2]) * s_clk; int init_t_s = init_utime_s + init_stime_s; - cpu_usage = 100.0 * ((final_t_s - init_t_s) / (static_cast(intv) * 1000)); + cpu_usage = 100.0 * ((final_t_s - init_t_s) / (static_cast(ext_rintv.val) * 1000)); // free the result mysql_free_result(admin_res); @@ -1496,10 +1586,16 @@ cleanup: return res; } -json fetch_internal_session(MYSQL* proxy) { - int rc = mysql_query_t(proxy, "PROXYSQL INTERNAL SESSION"); +json fetch_internal_session(MYSQL* proxy, bool verbose) { + int rc = 0; - if (rc ) { + if (verbose) { + rc = mysql_query_t(proxy, "PROXYSQL INTERNAL SESSION"); + } else { + rc = mysql_query(proxy, "PROXYSQL INTERNAL SESSION"); + } + + if (rc) { return json {}; } else { MYSQL_RES* myres = mysql_store_result(proxy); @@ -1511,6 +1607,29 @@ json fetch_internal_session(MYSQL* proxy) { } } +map parse_prometheus_metrics(const string& s) { + vector lines { split(s, '\n') }; + map metrics_map {}; + + for (const string ln : lines) { + const vector line_values { split(ln, ' ') }; + + if (ln.empty() == false && ln[0] != '#') { + if (line_values.size() > 2) { + size_t delim_pos_st = ln.rfind("} "); + string metric_key = ln.substr(0, delim_pos_st); + string metric_val = ln.substr(delim_pos_st + 2); + + metrics_map.insert({metric_key, stod(metric_val)}); + } else { + metrics_map.insert({line_values.front(), stod(line_values.back())}); + } + } + } + + return metrics_map; +} + struct cols_table_info_t { vector names; vector widths; @@ -1690,42 +1809,49 @@ pair fetch_conn_stats(MYSQL* admin, const vector hgs } } -int wait_for_cond(MYSQL* mysql, const std::string& query, uint32_t timeout) { - int result = EXIT_FAILURE; +int check_cond(MYSQL* mysql, const string& q) { + diag("Checking condition '%s' in ('%s':%d)", q.c_str(), mysql->host, mysql->port); - auto start = std::chrono::system_clock::now(); - std::chrono::duration elapsed {}; + int rc = mysql_query(mysql, q.c_str()); + int res = 1; - while (elapsed.count() < timeout && result == EXIT_FAILURE) { - int rc = mysql_query(mysql, query.c_str()); - fprintf( - stderr, "# %s: Waiting for condition '%s' in ('%s':%d)\n", - get_formatted_time().c_str(), query.c_str(), mysql->host, mysql->port - ); + if (rc == 0) { + MYSQL_RES* myres = mysql_store_result(mysql); - if (rc == EXIT_SUCCESS) { - MYSQL_RES* myres = mysql_store_result(mysql); - if (myres) { - uint32_t field_num = mysql_num_fields(myres); - uint32_t row_num = mysql_num_rows(myres); + if (myres) { + uint32_t field_num = mysql_num_fields(myres); + uint32_t row_num = mysql_num_rows(myres); - if (field_num == 1 && row_num == 1) { - MYSQL_ROW row = mysql_fetch_row(myres); + if (field_num == 1 && row_num == 1) { + MYSQL_ROW myrow = mysql_fetch_row(myres); - if (row && strcasecmp("TRUE", row[0]) == 0) { - result = EXIT_SUCCESS; - } + if (myrow && strcasecmp("TRUE", myrow[0]) == 0) { + res = 0; + } else if (myrow && atoi(myrow[0]) >= 1) { + res = 0; } + } + } + } else { + diag("Check failed with error '%s'", mysql_error(mysql)); + res = -1; + } + + return res; +} - mysql_free_result(myres); +int wait_for_cond(MYSQL* mysql, const string& q, uint32_t to) { + diag("Waiting for condition '%s' in ('%s':%d)", q.c_str(), mysql->host, mysql->port); - if (result == EXIT_SUCCESS) { - break; - } - } - } else { - diag("Condition query failed with error: ('%d','%s')", mysql_errno(mysql), mysql_error(mysql)); - result = EXIT_FAILURE; + int result = 1; + std::chrono::duration elapsed {}; + + auto start = std::chrono::system_clock::now(); + + while (elapsed.count() < to && result == EXIT_FAILURE) { + result = check_cond(mysql, q); + + if (result == 0 || result == -1) { break; } @@ -1738,6 +1864,74 @@ int wait_for_cond(MYSQL* mysql, const std::string& query, uint32_t timeout) { return result; } +vector wait_for_conds(MYSQL* mysql, const vector& qs, uint32_t to) { + diag("Waiting multiple conditions in ('%s':%d):", mysql->host, mysql->port); + for (const string& q : qs) { + diag(" - cond: '%s'", q.c_str()); + } + + std::chrono::duration elapsed {}; + + vector res {}; + std::transform(qs.begin(), qs.end(), std::back_inserter(res), + [] (const string& q) { + return check_res_t { 1, q }; + } + ); + auto start = std::chrono::system_clock::now(); + + while (elapsed.count() < to) { + int chk_res = 0; + + for (std::size_t i = 0; i < qs.size(); i++) { + chk_res = check_cond(mysql, qs[i]); + + if (chk_res == -1) { + diag("Error during query. Aborting further checks"); + res[i].first = -1; + break; + } else if (chk_res == 0) { + res[i].first = 0; + } + } + + int acc = std::accumulate(res.begin(), res.end(), size_t(0), + [] (size_t acc, const check_res_t& p) -> size_t { + if (p.first == 0) { + return acc + 1; + } else { + return acc; + } + }); + + if (acc == qs.size() || chk_res == -1) { + break; + } else { + usleep(500 * 1000); + auto it_end = std::chrono::system_clock::now(); + elapsed = it_end - start; + } + } + + return res; +} + +int proc_wait_checks(const vector& chks) { + int res = 0; + + for (const check_res_t& r : chks) { + if (r.first == -1) { + res = -1; + diag("Waiting check FAILED to execute '%s'", r.second.c_str()); + } else if (r.first == 1) { + res = res == 0 ? 1 : res; + diag("Waiting check TIMEOUT '%s'", r.second.c_str()); + } + } + + return res; +} + void check_conn_count(MYSQL* admin, const string& conn_type, uint32_t conn_num, int32_t hg) { const string hg_s { to_string(hg) }; const string conn_num_s { to_string(conn_num) }; @@ -1807,8 +2001,67 @@ void check_query_count(MYSQL* admin, vector queries, uint32_t hg) { } }; -const char* get_env_str(const char* envname, const char* envdefault) { +pair> fetch_cluster_nodes(MYSQL* admin, bool dump_fetch) { + int rc = mysql_query_t(admin, "SELECT hostname,port FROM proxysql_servers"); + if (rc) { return { static_cast(mysql_errno(admin)), {} }; } + + MYSQL_RES* myres = mysql_store_result(admin); + if (myres == NULL) { + diag("Storing resultset failed error='%s'", mysql_error(admin)); + return { static_cast(mysql_errno(admin)), {} }; + } + + if (dump_fetch) { + const string res_table { dump_as_table(myres) }; + diag("Dumping fetched cluster nodes:"); + + printf("%s", res_table.c_str()); + } + + vector nodes_rows { extract_mysql_rows(myres) }; + mysql_free_result(myres); + + vector nodes {}; + std::transform(nodes_rows.begin(), nodes_rows.end(), std::back_inserter(nodes), + [] (const mysql_res_row& row) { + return srv_addr_t { row[0], std::atoi(row[1].c_str()) }; + } + ); + + return { 0, nodes }; +} +int check_nodes_sync( + const CommandLine& cl, const vector& nodes, const string& check, uint32_t to +) { + for (const auto& node : nodes) { + MYSQL* admin = mysql_init(NULL); + + if ( + !mysql_real_connect( + admin, node.host.c_str(), cl.admin_username, cl.admin_password, NULL, node.port, NULL, 0 + ) + ) { + diag("File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(admin)); + return EXIT_FAILURE; + } + + const vector wres { wait_for_conds(admin, { check }, to) }; + int node_sync = proc_wait_checks(wres); + + if (node_sync != EXIT_SUCCESS) { + const string err { "Node '" + node.host + ":" + std::to_string(node.port) + "' sync timed out" }; + diag("File %s, line %d, Error: %s\n", __FILE__, __LINE__, err.c_str()); + return EXIT_FAILURE; + } + + mysql_close(admin); + } + + return EXIT_SUCCESS; +} + +const char* get_env_str(const char* envname, const char* envdefault) { const char* envval = std::getenv(envname); if (envval != NULL) @@ -1818,13 +2071,11 @@ const char* get_env_str(const char* envname, const char* envdefault) { }; int get_env_int(const char* envname, int envdefault) { - const char* envval = std::getenv(envname); int res = envdefault; if (envval != NULL) res = strtol(envval, NULL, 0); -// diag("%s: %s='%s' >>> %d", __FUNCTION__, envname, envval, res); return res; }; @@ -1852,7 +2103,5 @@ bool get_env_bool(const char* envname, bool envdefault) { } } -// diag("%s: %s='%s' >>> %d", __FUNCTION__, envname, envval, res); - return (bool) res; }; diff --git a/test/tap/tap/utils.h b/test/tap/tap/utils.h index 19b1c32e3..e2b71a4a9 100644 --- a/test/tap/tap/utils.h +++ b/test/tap/tap/utils.h @@ -56,6 +56,18 @@ my_bool mysql_stmt_close_override(MYSQL_STMT* stmt, const char* file, int line); #endif +/** + * @brief Helper function to disable Core nodes scheduler from ProxySQL Cluster nodes. + * @details In the CI environment, 'Scheduler' is used to induce extra load via Admin interface on + * all the cluster nodes. Disabling this allows for more accurate measurements on the primary. + * @param cl CommandLine arguments supplied to the test. + * @param admin Already opened Admin conn to the primary instance. + * @return On success, the opened Admin connections to the Core nodes used to change their config, + * these conns should later be used to restore their original config. On failure, a pair of shape + * '{ EXIT_FAILURE, {} }'. + */ +std::pair> disable_core_nodes_scheduler(CommandLine& cl, MYSQL* admin); + inline std::string get_formatted_time() { time_t __timer; char __buffer[30]; @@ -68,7 +80,18 @@ inline std::string get_formatted_time() { return std::string(__buffer); } -int mysql_query_t(MYSQL* mysql, const char* query); +/** + * @brief Wrapper for 'mysql_query' with logging for convenience. + * @details Should be used through 'mysql_query_t' macro. + * @return Result of calling 'mysql_query'. + */ +int mysql_query_t__(MYSQL* mysql, const char* query, const char* f, int ln, const char* fn); + +/** + * @brief Convenience macro with query logging. + */ +#define mysql_query_t(mysql, query)\ + mysql_query_t__(mysql, query, __FILE__, __LINE__, __func__) #define MYSQL_QUERY(mysql, query) \ do { \ @@ -89,8 +112,7 @@ int mysql_query_t(MYSQL* mysql, const char* query); #define MYSQL_QUERY_T(mysql, query) \ do { \ - const std::string time { get_formatted_time() }; \ - fprintf(stderr, "# %s: Issuing query '%s' to ('%s':%d)\n", time.c_str(), query, mysql->host, mysql->port); \ + diag("Issuing query '%s' to ('%s':%d)", query, mysql->host, mysql->port); \ if (mysql_query(mysql, query)) { \ fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(mysql)); \ return EXIT_FAILURE; \ @@ -220,7 +242,7 @@ std::string get_ext_val_err(MYSQL* mysql, const ext_val_t& ext_val) { } else if (ext_val.err == -2) { return "Failed to parse response value '" + ext_val.str + "'"; } else { - return "Query failed with error '" + std::string { mysql_error(mysql) } + "'"; + return std::string { mysql_error(mysql) }; } } @@ -660,8 +682,18 @@ int64_t get_elem_idx(const T& e, const std::vector& v) { /** * @brief Returns a 'JSON' object holding 'PROXYSQL INTERNAL SESSION' contents. * @param proxy And already openned connection to ProxySQL. + * @param verbose Wether to log or not the issued queries. + */ +nlohmann::json fetch_internal_session(MYSQL* proxy, bool verbose=true); + +/** + * @brief Extract the metrics values from the output of: + * - The admin command 'SHOW PROMETHEUS METRICS'. + * - Querying the RESTAPI metrics endpoint. + * @param s ProxySQL prometheus exporter output. + * @return A map of metrics identifiers and values. */ -nlohmann::json fetch_internal_session(MYSQL* proxy); +std::map parse_prometheus_metrics(const std::string& s); /** * @brief Returns a string table representation of the supplied resultset. @@ -720,11 +752,56 @@ std::pair fetch_conn_stats(MYSQL* admin, const std::vector; + +/** + * @brief Waits for multiple conditions to take place before returning. + * @param mysql Already oppened connection in which to execute the queries. + * @param qs Conditions represented as queries; must pass 'check_cond' requirements. + * @param to Timeout in which all the conditions should be accomplished. + * @return Vector of pairs of shape '{err, check}'. + */ +std::vector wait_for_conds(MYSQL* mysql, const std::vector& qs, uint32_t to); + +/** + * @brief Reduces a vector of 'check_res_t' to either success or failure. + * @param chks Vector to be fold into single value. + * @return -1 in case a check failed to execute, 1 if any check timedout, 0 for success. + */ +int proc_wait_checks(const std::vector& chks); + +/** + * @brief Encapsulates a server address. + */ +struct srv_addr_t { + const std::string host; + const int port; +}; + // Helpers using 'wait_for_cond' on 'stats_mysql_connection' void check_conn_count(MYSQL* admin, const std::string& conn_type, uint32_t conn_num, int32_t hg=-1); void check_query_count(MYSQL* admin, uint32_t queries, uint32_t hg); void check_query_count(MYSQL* admin, std::vector queries, uint32_t hg); +/** + * @brief Fetches the ProxySQL nodes configured in the supplied instance. + * @param cl Parameters for performing the connection to the instance. + * @return Pair of shape '{err, {srv_addr}}'. + */ +std::pair> fetch_cluster_nodes(MYSQL* admin, bool dump_fetch=false); + +/** + * @brief Helper function that waits for a check in all the supplied nodes. + * @param cl Used for credentials to open conns to the nodes. + * @param nodes The nodes addresses in which to perform the checks. + * @param check The check itself to be performed in all the nodes. Must pass 'check_cond' requirements. + * @param to Timeout for synchronization to take place. + * @return 0 in case of success, 1 in case of timeout, and -1 in case of check failure. + */ +int check_nodes_sync( + const CommandLine& cl, const std::vector& nodes, const std::string& check, uint32_t to +); + /** * @brief fetches and converts env var value to str/int/bool if possible otherwise uses default * @details helper function for fetching str/int/bool from env diff --git a/test/tap/tests/max_connections_ff-t.cpp b/test/tap/tests/max_connections_ff-t.cpp index f85a66f33..211c939c5 100644 --- a/test/tap/tests/max_connections_ff-t.cpp +++ b/test/tap/tests/max_connections_ff-t.cpp @@ -18,14 +18,12 @@ #include #include -#include #include #include #include #include #include "mysql.h" -#include "mysqld_error.h" #include "json.hpp" @@ -66,10 +64,10 @@ int set_max_conns(MYSQL* proxy_admin, int max_conns, int hg_id) { string max_conn_query {}; string_format("UPDATE mysql_servers SET max_connections=%d WHERE hostgroup_id=%d", max_conn_query, max_conns, hg_id); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), max_conn_query.c_str()); + diag("Executing query `%s`...", max_conn_query.c_str()); MYSQL_QUERY(proxy_admin, max_conn_query.c_str()); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "LOAD MYSQL SERVERS TO RUNTIME"); + diag("Executing query `%s`...", "LOAD MYSQL SERVERS TO RUNTIME"); MYSQL_QUERY(proxy_admin, "LOAD MYSQL SERVERS TO RUNTIME"); return EXIT_SUCCESS; @@ -79,10 +77,10 @@ int set_srv_conn_to(MYSQL* proxy_admin, int connect_to) { string srv_conn_to_query {}; string_format("SET mysql-connect_timeout_server_max=%d", srv_conn_to_query, connect_to); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), srv_conn_to_query.c_str()); + diag("Executing query `%s`...", srv_conn_to_query.c_str()); MYSQL_QUERY(proxy_admin, srv_conn_to_query.c_str()); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "LOAD MYSQL VARIABLES TO RUNTIME"); + diag("Executing query `%s`...", "LOAD MYSQL VARIABLES TO RUNTIME"); MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); return EXIT_SUCCESS; @@ -93,10 +91,10 @@ int set_ff_for_user(MYSQL* proxy_admin, const string& user, bool ff) { string upd_ff_query {}; string_format("UPDATE mysql_users SET fast_forward=%d WHERE username='%s'", upd_ff_query, ff, user.c_str()); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), upd_ff_query.c_str()); + diag("Executing query `%s`...", upd_ff_query.c_str()); MYSQL_QUERY(proxy_admin, upd_ff_query.c_str()); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "LOAD MYSQL VARIABLES TO RUNTIME"); + diag("Executing query `%s`...", "LOAD MYSQL VARIABLES TO RUNTIME"); MYSQL_QUERY(proxy_admin, "LOAD MYSQL USERS TO RUNTIME"); return EXIT_SUCCESS; @@ -264,9 +262,9 @@ cleanup: string reset_conn_to_srv {}; string_format("SET mysql-connect_timeout_server_max=%s", reset_conn_to_srv, str_connect_timeout_server_max.c_str()); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), reset_conn_to_srv.c_str()); + diag("Executing query `%s`...", reset_conn_to_srv.c_str()); MYSQL_QUERY(proxy_admin, reset_conn_to_srv.c_str()); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "LOAD MYSQL VARIABLES TO RUNTIME"); + diag("Executing query `%s`...", "LOAD MYSQL VARIABLES TO RUNTIME"); MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); return EXIT_SUCCESS; @@ -313,7 +311,7 @@ int test_ff_only_one_free_conn(const CommandLine& cl, MYSQL* proxy_admin, int ma // Reset all the current stats for 'stats_mysql_connection_pool' my_err = mysql_query(proxy_admin, reset_connpool_stats); - diag("%s: Executing query `%s` in new 'fast_forward' conn...", tap_curtime().c_str(), reset_connpool_stats); + diag("Executing query `%s` in new 'fast_forward' conn...", reset_connpool_stats); if (my_err) { diag("Query '%s' failed", reset_connpool_stats); res = EXIT_FAILURE; @@ -333,7 +331,7 @@ int test_ff_only_one_free_conn(const CommandLine& cl, MYSQL* proxy_admin, int ma MYSQL* trx_conn = trx_conns.back(); diag("Freeing ONE connection by committing the transaction..."); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "COMMIT"); + diag("Executing query `%s`...", "COMMIT"); my_err = mysql_query(trx_conn, "COMMIT"); if (my_err) { diag( @@ -375,7 +373,7 @@ int test_ff_only_one_free_conn(const CommandLine& cl, MYSQL* proxy_admin, int ma } // 3.1 Issue a simple query into the new 'fast_forward' connection - diag("%s: Executing query `%s` in new 'fast_forward' conn...", tap_curtime().c_str(), "DO 1"); + diag("Executing query `%s` in new 'fast_forward' conn...", "DO 1"); int q_my_err = mysql_query(proxy_ff, "DO 1"); if (q_my_err) { diag( diff --git a/test/tap/tests/reg_test_3223-restapi_return_codes-t.cpp b/test/tap/tests/reg_test_3223-restapi_return_codes-t.cpp index bd6fa98ba..86c3336d7 100644 --- a/test/tap/tests/reg_test_3223-restapi_return_codes-t.cpp +++ b/test/tap/tests/reg_test_3223-restapi_return_codes-t.cpp @@ -172,7 +172,7 @@ int main(int argc, char** argv) { for (const string& params : req.params) { const string ept { join_path(base_address, req.ept_info.name) }; diag( - "%s: Checking valid '%s' request - ept: '%s', params: '%s'", tap_curtime().c_str(), + "Checking valid '%s' request - ept: '%s', params: '%s'", req.ept_info.method.c_str(), ept.c_str(), params.c_str() ); std::chrono::nanoseconds duration; @@ -308,7 +308,7 @@ int main(int argc, char** argv) { const string ept { join_path(base_address, req.ept_info.name) }; diag( - "%s: Checking valid '%s' request - ept: '%s', params: '%s'", tap_curtime().c_str(), + "Checking valid '%s' request - ept: '%s', params: '%s'", req.ept_info.method.c_str(), ept.c_str(), ept_pl.params.c_str() ); diff --git a/test/tap/tests/reg_test_3765_ssl_pollout-t.cpp b/test/tap/tests/reg_test_3765_ssl_pollout-t.cpp index 898c56f4e..cdc48d14f 100644 --- a/test/tap/tests/reg_test_3765_ssl_pollout-t.cpp +++ b/test/tap/tests/reg_test_3765_ssl_pollout-t.cpp @@ -13,16 +13,17 @@ #include #include #include +#include #include #include #include "mysql.h" -#include #include "tap.h" #include "command_line.h" #include "utils.h" +using std::pair; using std::string; using std::vector; @@ -59,13 +60,15 @@ int create_connections(const conn_opts_t& conn_opts, uint32_t cons_num, std::vec const uint32_t ADMIN_CONN_NUM = 100; const uint32_t MYSQL_CONN_NUM = 100; const uint32_t REPORT_INTV_SEC = 5; -double MAX_ALLOWED_CPU_USAGE = (double) get_env_int("TAP_MAX_ALLOWED_CPU_USAGE", 13); -int get_idle_conns_cpu_usage(CommandLine& cl, uint64_t mode, double& idle_cpu_ms, double& final_cpu_ms) { +double MAX_IDLE_CPU_USAGE = (double) get_env_int("MAX_IDLE_CPU_USAGE", 10); +double MAX_INCREASE_CPU_USAGE = (double) get_env_int("TAP_MAX_INCREASE_CPU_USAGE", 2); + +int get_idle_conns_cpu_usage(CommandLine& cl, uint64_t mode, double& no_conns_cpu, double& idle_conns_cpu) { // get ProxySQL idle cpu usage - int idle_err = get_proxysql_cpu_usage(cl, REPORT_INTV_SEC, idle_cpu_ms); + int idle_err = get_proxysql_cpu_usage(cl, REPORT_INTV_SEC, no_conns_cpu); if (idle_err) { - diag("File %s, line %d, Error: '%s'", __FILE__, __LINE__, "Unable to get 'idle_cpu' usage."); + diag("Unable to get 'no_conns_cpu' usage."); return idle_err; } @@ -87,9 +90,9 @@ int get_idle_conns_cpu_usage(CommandLine& cl, uint64_t mode, double& idle_cpu_ms return EXIT_FAILURE; } - int final_err = get_proxysql_cpu_usage(cl, REPORT_INTV_SEC, final_cpu_ms); + int final_err = get_proxysql_cpu_usage(cl, REPORT_INTV_SEC, idle_conns_cpu); if (final_err) { - diag("File %s, line %d, Error: '%s'", __FILE__, __LINE__, "Unable to get 'idle_cpu' usage."); + diag("Unable to get 'idle_conns_cpu' usage."); return idle_err; } @@ -111,11 +114,11 @@ int main(int argc, char** argv) { // For ASAN builds we don't care about correctness in this measurement. if (get_env_int("WITHASAN", 0)) { - MAX_ALLOWED_CPU_USAGE = 80; + MAX_IDLE_CPU_USAGE = 80; } - double idle_cpu_ms = 0; - double final_cpu_ms = 0; + double no_conns_cpu = 0; + double idle_conns_cpu = 0; MYSQL* proxysql_admin = mysql_init(NULL); if (!mysql_real_connect(proxysql_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { @@ -123,57 +126,71 @@ int main(int argc, char** argv) { return EXIT_FAILURE; } + pair> p_err_nodes_conns { disable_core_nodes_scheduler(cl, proxysql_admin) }; + if (p_err_nodes_conns.first) { return EXIT_FAILURE; } + vector& nodes_conns { p_err_nodes_conns.second }; + MYSQL_QUERY(proxysql_admin, "SET mysql-have_ssl=1"); MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); mysql_close(proxysql_admin); diag("Testing regular connections..."); - int ret_cpu_usage = get_idle_conns_cpu_usage(cl, 0, idle_cpu_ms, final_cpu_ms); + int ret_cpu_usage = get_idle_conns_cpu_usage(cl, 0, no_conns_cpu, idle_conns_cpu); if (ret_cpu_usage != EXIT_SUCCESS) { return EXIT_FAILURE; } ok( - idle_cpu_ms < MAX_ALLOWED_CPU_USAGE, - "ProxySQL 'no clients' CPU usage should be below expected: (Exp: %%%lf, Act: %%%lf)", - MAX_ALLOWED_CPU_USAGE, idle_cpu_ms + no_conns_cpu < MAX_IDLE_CPU_USAGE, + "ProxySQL 'no clients' CPU usage should be below expected: (MAX_IDLE_CPU_USAGE: %%%lf, Act: %%%lf)", + MAX_IDLE_CPU_USAGE, no_conns_cpu ); ok( - final_cpu_ms < MAX_ALLOWED_CPU_USAGE, - "ProxySQL 'clients' CPU usage should be below expected: (Exp: %%%lf, Act: %%%lf)", - MAX_ALLOWED_CPU_USAGE, final_cpu_ms + idle_conns_cpu < MAX_IDLE_CPU_USAGE + MAX_INCREASE_CPU_USAGE, + "ProxySQL 'with clients' CPU usage should be below expected:" + " (MAX_IDLE_CPU_USAGE + MAX_INCREASE_CPU_USAGE: %%%lf, Act: %%%lf)", + MAX_IDLE_CPU_USAGE + MAX_INCREASE_CPU_USAGE, idle_conns_cpu ); diag("Testing SSL connections..."); - ret_cpu_usage = get_idle_conns_cpu_usage(cl, CLIENT_SSL, idle_cpu_ms, final_cpu_ms); + ret_cpu_usage = get_idle_conns_cpu_usage(cl, CLIENT_SSL, no_conns_cpu, idle_conns_cpu); if (ret_cpu_usage != EXIT_SUCCESS) { return EXIT_FAILURE; } ok( - idle_cpu_ms < MAX_ALLOWED_CPU_USAGE, + no_conns_cpu < MAX_IDLE_CPU_USAGE, "ProxySQL 'no clients' CPU usage should be below expected: (Exp: %%%lf, Act: %%%lf)", - MAX_ALLOWED_CPU_USAGE, idle_cpu_ms + MAX_IDLE_CPU_USAGE, no_conns_cpu ); ok( - final_cpu_ms < MAX_ALLOWED_CPU_USAGE, - "ProxySQL 'SSL clients' CPU usage should be below expected: (Exp: %%%lf, Act: %%%lf)", - MAX_ALLOWED_CPU_USAGE, final_cpu_ms + idle_conns_cpu < MAX_IDLE_CPU_USAGE + MAX_INCREASE_CPU_USAGE, + "ProxySQL 'with SSL clients' CPU usage should be below expected:" + " (MAX_IDLE_CPU_USAGE + MAX_INCREASE_CPU_USAGE: %%%lf, Act: %%%lf)", + MAX_IDLE_CPU_USAGE + MAX_INCREASE_CPU_USAGE, idle_conns_cpu ); diag("Testing SSL and compressed connections..."); - ret_cpu_usage = get_idle_conns_cpu_usage(cl, CLIENT_SSL|CLIENT_COMPRESS, idle_cpu_ms, final_cpu_ms); + ret_cpu_usage = get_idle_conns_cpu_usage(cl, CLIENT_SSL|CLIENT_COMPRESS, no_conns_cpu, idle_conns_cpu); if (ret_cpu_usage != EXIT_SUCCESS) { return EXIT_FAILURE; } ok( - idle_cpu_ms < MAX_ALLOWED_CPU_USAGE, + no_conns_cpu < MAX_IDLE_CPU_USAGE, "ProxySQL 'no clients' CPU usage should be below expected: (Exp: %%%lf, Act: %%%lf)", - MAX_ALLOWED_CPU_USAGE, idle_cpu_ms + MAX_IDLE_CPU_USAGE, no_conns_cpu ); ok( - final_cpu_ms < MAX_ALLOWED_CPU_USAGE, - "ProxySQL 'SSL|COMPRESS clients' CPU usage should be below expected: (Exp: %%%lf, Act: %%%lf)", - MAX_ALLOWED_CPU_USAGE, final_cpu_ms + idle_conns_cpu < MAX_IDLE_CPU_USAGE + MAX_INCREASE_CPU_USAGE, + "ProxySQL 'with SSL|COMPRESS clients' CPU usage should be below expected: (Exp: %%%lf, Act: %%%lf)", + MAX_IDLE_CPU_USAGE + MAX_INCREASE_CPU_USAGE, idle_conns_cpu ); + // Recover cluster scheduler + for (MYSQL* myconn : nodes_conns) { + MYSQL_QUERY_T(myconn, "LOAD SCHEDULER FROM DISK"); + MYSQL_QUERY_T(myconn, "LOAD SCHEDULER TO RUNTIME"); + + mysql_close(myconn); + } + return exit_status(); } diff --git a/test/tap/tests/reg_test_4402-mysql_fields-t.cpp b/test/tap/tests/reg_test_4402-mysql_fields-t.cpp index bf1c0a1c1..ff0d3e586 100644 --- a/test/tap/tests/reg_test_4402-mysql_fields-t.cpp +++ b/test/tap/tests/reg_test_4402-mysql_fields-t.cpp @@ -83,7 +83,9 @@ int main(int argc, char** argv) { // to check column alias issue: { - const std::string& query = "SELECT testdb.echo_int(1) AS " + generate_random_string(length); + // NOTE: The randomly generated string should be escaped \`\`, otherwise could collide + // with SQL reserved words, causing an invalid test failure. + const std::string& query = "SELECT testdb.echo_int(1) AS `" + generate_random_string(length) + "`"; MYSQL_QUERY__(proxysql, query.c_str()); MYSQL_RES* res = mysql_use_result(proxysql); diff --git a/test/tap/tests/reg_test_4556-ssl_error_queue-t.cpp b/test/tap/tests/reg_test_4556-ssl_error_queue-t.cpp new file mode 100644 index 000000000..87d334c16 --- /dev/null +++ b/test/tap/tests/reg_test_4556-ssl_error_queue-t.cpp @@ -0,0 +1,845 @@ +/** + * @file reg_test_4556-ssl_error_queue-t.cpp + * @brief Regression test for SSL error queue cleanup in frontend/backend conns. + * @details Two different kind of coherence checks are performed: + * 1. SSL errors on fronted conns don't propagate or pollute other frontend/backend conns: + * 1.1 Config ProxySQL for either perform conn retention or not (session_idle_ms). This ensure we + * test threads conn-sharing via 'idle-threads'. + * 1.2 Warm-up the conn-pool with multiple conns per-thread, test even conn distribution. + * 1.3 Force different kinds of SSL failures on frontend conns. + * 1.4 Check that exercising all the other client conns doesn't result in errors. This ensures + * the thread that received the error, is not polluting other conns while processing the queries. + * 2. SSL errors on backend conns don't propagate or pollute other frontend/backend conns: + * 2.1 Config ProxySQL with the desired PING intv, will be used to check backend conns, and '100' + * as 'free_connections_pct' to prevent early cleanups. + * 2.1 Warm-up the conn-pool creating backend conns for multiple threads. + * 2.2 Connect to MySQL, and kill a random conn count from conn-pool. + * 2.3 Let ProxySQL exercise backend conns via PING checks, ensure only killed conns died. + * 2.4 Exercise all backend conns via trxs, exhausting the conn-pool, no errors should take place. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "mysql.h" +#include "openssl/ssl.h" +#include "json.hpp" + +#include "tap.h" +#include "command_line.h" +#include "utils.h" + +using std::map; +using std::pair; +using std::string; +using std::vector; + +using nlohmann::json; + +#define TAP_NAME "TAP_SSL_ERROR_QUEUE___" + +// Check ENV for partially disable test sections +const char* TEST_FRONTEND = getenv(TAP_NAME"TEST_FRONTEND_CONNS"); +const char* TEST_CONN_DIST = getenv(TAP_NAME"TEST_CONN_DIST"); +const char* TEST_BACKEND = getenv(TAP_NAME"TEST_BACKEND_CONNS"); +const int HG_ID = get_env_int(TAP_NAME"MYSQL_SERVER_HOSTGROUP_ID", 0); +const int PER_THREAD_CONN_COUNT = get_env_int(TAP_NAME"PER_THREAD_CONN_COUNT", 20); + +/* Helper function to do the waiting for events on the socket. */ +static int wait_for_mysql(MYSQL *mysql, int status) { + struct pollfd pfd; + int timeout, res; + + pfd.fd = mysql_get_socket(mysql); + pfd.events = + (status & MYSQL_WAIT_READ ? POLLIN : 0) | + (status & MYSQL_WAIT_WRITE ? POLLOUT : 0) | + (status & MYSQL_WAIT_EXCEPT ? POLLPRI : 0); + if (status & MYSQL_WAIT_TIMEOUT) + timeout = 1000*mysql_get_timeout_value(mysql); + else + timeout = -1; + res = poll(&pfd, 1, timeout); + if (res == 0) + return MYSQL_WAIT_TIMEOUT; + else if (res < 0) + return MYSQL_WAIT_TIMEOUT; + else { + int status = 0; + if (pfd.revents & POLLIN) status |= MYSQL_WAIT_READ; + if (pfd.revents & POLLOUT) status |= MYSQL_WAIT_WRITE; + if (pfd.revents & POLLPRI) status |= MYSQL_WAIT_EXCEPT; + return status; + } +} + +// Thread Input +struct th_args__in_t { + CommandLine& cl; +}; + +// Thread Output +struct th_args__out_t { + std::string thread_addr {}; +}; + +struct th_args_t { + th_args__in_t in_args; + th_args__out_t out_args {}; +}; + +void* create_ssl_conn_and_close_socket(void* arg) { + th_args_t* th_args = static_cast(arg); + CommandLine& cl = th_args->in_args.cl; + + MYSQL* myconn = mysql_init(NULL); + mysql_ssl_set(myconn, NULL, NULL, NULL, NULL, NULL); + + if (!mysql_real_connect(myconn, cl.host, cl.username, cl.password, NULL, cl.port, NULL, CLIENT_SSL)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(myconn)); + return NULL; + } + + json j_session = fetch_internal_session(myconn, false); + string thread_addr { j_session["thread"] }; + + th_args->out_args.thread_addr = thread_addr; + + diag("Early closing socket of first conn thread_addr=%s", thread_addr.c_str()); + close(myconn->net.fd); + + return NULL; +} + +void* create_ssl_conn_and_close_socket_async(void* arg) { + th_args_t* th_args = static_cast(arg); + CommandLine& cl = th_args->in_args.cl; + + MYSQL* myconn = mysql_init(NULL); + mysql_options(myconn, MYSQL_OPT_NONBLOCK, 0); + mysql_ssl_set(myconn, NULL, NULL, NULL, NULL, NULL); + + MYSQL* ret = nullptr; + diag("Starting async 'MySQL' connection thread=%ld", pthread_self()); + int status = mysql_real_connect_start( + &ret, myconn, cl.host, cl.username, cl.password, NULL, cl.port, NULL, CLIENT_SSL + ); + + diag("Early closing socket of non-complete async conn ret=%p status=%d", ret, status); + close(myconn->net.fd); + + return NULL; +} + +void* create_ssl_conn_inv_cert(void* arg) { + th_args_t* th_args = static_cast(arg); + CommandLine& cl = th_args->in_args.cl; + + MYSQL* myconn = mysql_init(NULL); + mysql_options(myconn, MYSQL_OPT_NONBLOCK, 0); + + char* inv_cert_path = tempnam(nullptr, "tap"); + diag("Setting invalid CERT for conn with tmp file path=%s", inv_cert_path); + mysql_ssl_set(myconn, NULL, NULL, inv_cert_path, NULL, NULL); + + MYSQL* ret = nullptr; + diag("Starting 'MySQL' connection with invalid CERT thread=%ld", pthread_self()); + + if (!mysql_real_connect(myconn, cl.host, cl.username, cl.password, NULL, cl.port, NULL, CLIENT_SSL)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(myconn)); + return NULL; + } + + return NULL; +} + +/** + * @brief Invalid cert data to place in temporary file. + */ +const char inv_cert[] { + "-----BEGIN CERTIFICATE-----\n" + "kDeWG8U5N5v61p9QRwUutjUnRgmtYQOoe52Ib8k6KTVhSk/BsxsKNBQ2CdbjhuDl\n" + "5QIDc+5Z9FBIHFEL+jfivaA2X4jVRTZ52RPDDxqMK5Y3mTEyJZGE\n" + "-----END CERTIFICATE-----" +}; + +void* create_ssl_conn_missing_cert(void* arg) { + th_args_t* th_args = static_cast(arg); + CommandLine& cl = th_args->in_args.cl; + + MYSQL* myconn = mysql_init(NULL); + mysql_options(myconn, MYSQL_OPT_NONBLOCK, 0); + + char* inv_cert_path = tempnam(nullptr, "tap"); + FILE *tmp_file = fopen(inv_cert_path, "w"); + fprintf(tmp_file, inv_cert); + fflush(tmp_file); + + diag("Setting invalid CERT for conn with tmp file path=%s", inv_cert_path); + mysql_ssl_set(myconn, NULL, NULL, inv_cert_path, NULL, NULL); + + MYSQL* ret = nullptr; + diag("Starting 'MySQL' connection with invalid CERT thread=%ld", pthread_self()); + + if (!mysql_real_connect(myconn, cl.host, cl.username, cl.password, NULL, cl.port, NULL, CLIENT_SSL)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(myconn)); + return NULL; + } + + fclose(tmp_file); + + return NULL; +} + +void check_threads_conns(const map>& m_th_conns, const string& th_addr) { + if (th_addr.empty()) { + diag("Checking ALL threads conns"); + } else { + diag("Checking conns filtered by thread thread_addr=%s", th_addr.c_str()); + } + + for (const pair>& p_th_conns : m_th_conns) { + if (!th_addr.empty() && p_th_conns.first != th_addr) { continue; } + + diag("Checking thread conns thread_addr=%s", p_th_conns.first.c_str()); + const vector& th_conns { p_th_conns.second }; + + for (size_t i = 0; i < th_conns.size(); i++) { + int rc = mysql_query(th_conns[i], "SELECT 1"); + + ok( + rc == 0, + "Query should execute without error rc=%d mysql_error='%s'", + rc, mysql_error(th_conns[i]) + ); + + MYSQL_RES* myres = mysql_store_result(th_conns[i]); + + ok( + myres != nullptr && mysql_errno(th_conns[i]) == 0, + "Resultset should be properly retreived myres=%p mysql_error='%s'", + myres, mysql_error(th_conns[i]) + ); + + mysql_free_result(myres); + } + } +} + +int create_conn(const CommandLine& cl) { + struct sockaddr_in server_addr; + + int sock = socket(AF_INET, SOCK_STREAM, 0); + if (sock < 0) { + perror("Unable to create socket"); + return -1; + } + + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(uint32_t(cl.port)); + + if (inet_pton(AF_INET, cl.host, &server_addr.sin_addr) <= 0) { + perror("'inet_pton' failed"); + close(sock); + return -1; + } + + if (connect(sock, (struct sockaddr*)&server_addr, sizeof(server_addr)) < 0) { + perror("Unable to connect"); + close(sock); + return -1; + } + + return sock; +} + +char net_buf[4096] { 0 }; + +struct _mysql_hdr { + u_int pkt_length:24, pkt_id:8; +}; + +int read_srv_handshake(int fd) { + char* buf_pos = net_buf; + + int r = read(fd, buf_pos, sizeof(net_buf)); + if (r == -1) { + perror("'read' failed"); + return r; + } + + buf_pos += r; + + while (r > 0 && r < NET_HEADER_SIZE) { + r = read(fd, buf_pos + r, sizeof(buf_pos)); + buf_pos += r; + + if (r == -1) { + perror("'read' failed"); + return r; + } + } + + _mysql_hdr myhdr; + memcpy(&myhdr, net_buf, sizeof(_mysql_hdr)); + + while (r > 0 && r < myhdr.pkt_length) { + r = read(fd, buf_pos + r, sizeof(buf_pos)); + buf_pos += r; + + if (r == -1) { + perror("'read' failed"); + return r; + } + } + + return 0; +} + +/** + * @brief Hardcoded SSL_Request packet. + */ +unsigned char SSL_REQUEST_PKT[] = { + 0x20, 0x00, 0x00, 0x01, 0x85, 0xae, 0xff, 0x19, + 0x00, 0x00, 0x00, 0x01, 0xe0, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 +}; + +void* force_ssl_pre_handshake_failure(void* arg) { + th_args_t* th_args = static_cast(arg); + CommandLine& cl = th_args->in_args.cl; + + diag("Creating TCP connection to ProxySQL"); + int sock = create_conn(cl); + + diag("Reading ServerHandshake packet"); + int rc = read_srv_handshake(sock); + if (rc == -1) { + diag("Failed to read ProxySQL init hanshake"); + return NULL; + } + + diag("Sending harcoded 'SSLRequest'"); + rc = send(sock, SSL_REQUEST_PKT, sizeof(SSL_REQUEST_PKT), 0); + if (rc == -1) { + perror("'send' failed"); + return NULL; + } + + diag("Closing socket just after 'SSLRequest'"); + close(sock); + + return NULL; +} + +MYSQL* create_server_conn(CommandLine& cl) { + MYSQL* server = mysql_init(NULL); + + if ( + !mysql_real_connect( + server, + cl.mysql_host, + cl.mysql_username, + cl.mysql_password, + NULL, + cl.mysql_port, + NULL, + 0 + ) + ) { + diag( + "Failed to create conn to MySQL error=%s port=%d", + mysql_error(server), cl.mysql_port + ); + return NULL; + } + + return server; +} + +int create_test_database(CommandLine& cl, const string& name) { + MYSQL* server = create_server_conn(cl); + if (!server) { return EXIT_FAILURE; } + + const string q { "CREATE DATABASE IF NOT EXISTS " + name }; + if (mysql_query_t(server, q.c_str())) { + diag("Query failed to execute query=%s err=%s", q.c_str(), mysql_error(server)); + return EXIT_FAILURE; + } + + mysql_close(server); + return EXIT_SUCCESS; +} + +const char CONNPOOL_DB[] { "reg_test_4556" }; + +pair> warmup_conn_pool(CommandLine& cl, uint32_t CONNS_TOTAL) { + // Create database to use to flag conn-pool connections + diag("Creating testing database for connpool warming database=%s", CONNPOOL_DB); + if (create_test_database(cl, CONNPOOL_DB)) { + diag("Failed to create testing db database=%s", CONNPOOL_DB); + return { EXIT_FAILURE, {} }; + } + + diag("Elevating process limits for conns creation"); + struct rlimit limits { 0, 0 }; + getrlimit(RLIMIT_NOFILE, &limits); + diag("Old process limits rlim_cur=%ld rlim_max=%ld", limits.rlim_cur, limits.rlim_max); + if (limits.rlim_cur < CONNS_TOTAL * 2) { + diag("Updating process max FD limit"); + limits.rlim_cur = CONNS_TOTAL * 2; + setrlimit(RLIMIT_NOFILE, &limits); + } + diag("New process limits rlim_cur=%ld rlim_max=%ld", limits.rlim_cur, limits.rlim_max); + + vector conns {}; + + for (int i = 0; i < CONNS_TOTAL; i++) { + MYSQL* myconn = mysql_init(NULL); + + if (!mysql_real_connect(myconn, cl.host, cl.username, cl.password, NULL, cl.port, NULL, 0)) { + diag( + "Failed to connect addr=%s port=%d user=%s pass=%s err=%s", + cl.host, cl.port, cl.username, cl.password, mysql_error(myconn) + ); + return { EXIT_FAILURE, {} }; + } + + const vector CONN_CREATE_QUERIES { + "USE reg_test_4556", + "/* create_new_connection=1 */ DO 1" + }; + + // Make sure to fill the connection pool + for (const char* q : CONN_CREATE_QUERIES) { + if (mysql_query_t(myconn, q)) { + diag("Query failed to execute query=%s err=%s", q, mysql_error(myconn)); + return { mysql_errno(myconn), {} }; + } + } + + conns.push_back(myconn); + } + + return { EXIT_SUCCESS, conns }; +} + +map> create_conn_thread_map(vector& conns) { + map> m_thread_conns {}; + + for (MYSQL* myconn : conns) { + json j_session = fetch_internal_session(myconn, false); + string thread_addr { j_session["thread"] }; + + m_thread_conns[thread_addr].push_back(myconn); + } + + return m_thread_conns; +} + +void check_thread_conn_dist(const map>& m_thread_conns) { + if (TEST_CONN_DIST && strcasecmp(TEST_CONN_DIST, "0") == 0) { return; } + + size_t lo_count = 0; + size_t hg_count = 0; + + diag("Dumping per-thread conn count:"); + for (const pair>& thread_conns : m_thread_conns) { + if (lo_count == 0 || thread_conns.second.size() < lo_count) { + lo_count = thread_conns.second.size(); + } + if (hg_count == 0 || thread_conns.second.size() > hg_count) { + hg_count = thread_conns.second.size(); + } + fprintf(stderr, "Map entry thread=%s count=%ld\n", thread_conns.first.c_str(), thread_conns.second.size()); + } + + ok(lo_count != 0, "No thread should be left without connections lo_count=%ld", lo_count); +} + +int clean_conn_pool(MYSQL* admin) { + // 0. Ensure connection pool cleanup + MYSQL_QUERY_T(admin, "UPDATE mysql_servers SET max_connections=0"); + MYSQL_QUERY_T(admin, "LOAD MYSQL SERVERS TO RUNTIME"); + + { + MYSQL_QUERY(admin, "SELECT * FROM stats_mysql_connection_pool"); + MYSQL_RES* myres = mysql_store_result(admin); + diag("stats_mysql_connection_pool:\n%s", dump_as_table(myres).c_str()); + } + + const string COND_CONN_CLEANUP { + "SELECT IIF((SELECT SUM(ConnUsed + ConnFree) FROM stats.stats_mysql_connection_pool" + " WHERE hostgroup=" + std::to_string(HG_ID) + ")=0, 'TRUE', 'FALSE')" + }; + int w_res = wait_for_cond(admin, COND_CONN_CLEANUP, 10); + if (w_res) { + { + MYSQL_QUERY(admin, "SELECT * FROM stats_mysql_connection_pool"); + MYSQL_RES* myres = mysql_store_result(admin); + diag("stats_mysql_connection_pool:\n%s", dump_as_table(myres).c_str()); + } + + diag("Waiting for backend connections failed res:'%d'", w_res); + return EXIT_FAILURE; + } + + MYSQL_QUERY_T(admin, "UPDATE mysql_servers SET max_connections=500"); + MYSQL_QUERY_T(admin, "LOAD MYSQL SERVERS TO RUNTIME"); + + { + MYSQL_QUERY(admin, "SELECT * FROM stats_mysql_connection_pool"); + MYSQL_RES* myres = mysql_store_result(admin); + diag("stats_mysql_connection_pool:\n%s", dump_as_table(myres).c_str()); + } + + return EXIT_SUCCESS; +} + +int check_frontend_ssl_errs( + CommandLine& cl, MYSQL* admin, int64_t thread_count, int64_t idle_sess_ms, void*(*ssl_err_cb)(void*) +) { + // 0. Ensure connection pool cleanup + int clean_rc = clean_conn_pool(admin); + if (clean_rc) { + diag("Conn-pool cleanup failed; aborting futher testing"); + return EXIT_FAILURE; + } + + // 1. Configure ProxySQL forcing threads to retains conns + MYSQL_QUERY_T(admin, ("SET mysql-session_idle_ms=" + std::to_string(idle_sess_ms)).c_str()); + MYSQL_QUERY_T(admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + // 2. Create connections based on number of threads + pair> p_rc_conns {}; + { + uint32_t CONNS_TOTAL = thread_count * PER_THREAD_CONN_COUNT; + diag( + "Creating connections threads=%ld per_thread_conns=%d total=%d", + thread_count, PER_THREAD_CONN_COUNT, CONNS_TOTAL + ); + + p_rc_conns = warmup_conn_pool(cl, CONNS_TOTAL); + if (p_rc_conns.first) { return EXIT_FAILURE; } + } + + // 3. Check connection distribution on ProxySQL + map> m_thread_conns { create_conn_thread_map(p_rc_conns.second) }; + check_thread_conn_dist(m_thread_conns); + + // 4. Force a failure in a new connection; different kinds: + // - SSL failure due to pure SSL error, inv cert. + // - SSL failure due to closed socket; during query, or premature close. + { + pthread_t unexp_socket_close; + void* th_ret = nullptr; + std::unique_ptr th_args { + new th_args_t { th_args__in_t { cl }, th_args__out_t {} } + }; + + diag("Force early SSL failure in thread"); + pthread_create(&unexp_socket_close, NULL, ssl_err_cb, th_args.get()); + pthread_join(unexp_socket_close, &th_ret); + } + + // 5. Check all the others client conns in same thread are not broken (SSL err queue). + { + diag("Checking connections handled by ALL threads"); + check_threads_conns(m_thread_conns, string {}); + } + + for (MYSQL* myconn : p_rc_conns.second) { + mysql_close(myconn); + } + + return EXIT_SUCCESS; +} + +const uint32_t PING_INTV_MS { 1000 }; + +pair fetch_metric_val(CommandLine& cl, const string& metric_id) { + uint64_t curl_res_code = 0; + string curl_res_data {}; + const char URL[] { "http://localhost:6070/metrics/" }; + + diag("Fetching metric values via RESTAPI URL=%s", URL); + CURLcode code = perform_simple_get(URL, curl_res_code, curl_res_data); + + if (code != CURLE_OK || curl_res_code != 200) { + diag("Failed to fetch current metrics error=%s", curl_res_data.c_str()); + return { EXIT_FAILURE, 0 }; + } + + const map m_id_val { parse_prometheus_metrics(curl_res_data) }; + diag( + "Searching in fetched metrics metric_id=%s map_size=%ld", + metric_id.c_str(), m_id_val.size() + ); + + double error_count = 0; + + auto m_it = m_id_val.find(metric_id); + if (m_it != m_id_val.end()) { + error_count = m_it->second; + } + + return { EXIT_SUCCESS, error_count }; +} + +/** + * @brief Perform coherence checks on the backend conns. + * @details The checks are performed in the following way: + * 1. Warmup the conn-pool creating backend conns for multiple threads. + * 2. Connect to MySQL, and kill random conns from conn-pool. + * 3. Exercise all backend conns via PING checks, ensure only killed conns died. + * 4. Exercise all backend conns via trxs, exhausting the conn-pool, no errors should take place. + * @param cl Env config for conn creation. + * @param admin Already open Admin conn for ProxySQL config. + * @param thread_count Number of threads used by ProxySQL. + * @return EXIT_SUCCESS if checks could be performed, EXIT_FAILURE otherwise. + */ +int check_backend_ssl_errs(CommandLine& cl, MYSQL* admin, int64_t thread_count) { + // Ensure connection pool cleanup + int clean_rc = clean_conn_pool(admin); + if (clean_rc) { + diag("Conn-pool cleanup failed; aborting futher testing"); + return EXIT_FAILURE; + } + + // Configure ProxySQL to exercise backend connections via PING + const string ping_intv_str { std::to_string(PING_INTV_MS) }; + MYSQL_QUERY(admin, ("SET mysql-ping_interval_server_msec=" + ping_intv_str).c_str()); + // Prevent early closing of backend conns; will interfere with error counting + MYSQL_QUERY(admin, "SET mysql-free_connections_pct=100"); + MYSQL_QUERY(admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + // Create connections based on number of threads + uint32_t CONNS_TOTAL = thread_count * PER_THREAD_CONN_COUNT; + pair> p_rc_conns {}; + + diag( + "Creating connections threads=%ld per_thread_conns=%d total=%d", + thread_count, PER_THREAD_CONN_COUNT, CONNS_TOTAL + ); + + p_rc_conns = warmup_conn_pool(cl, CONNS_TOTAL); + if (p_rc_conns.first) { return EXIT_FAILURE; } + + // Create connection map for backend connetions; assuming high idle-sessions. + map> m_thread_conns { create_conn_thread_map(p_rc_conns.second) }; + + // Kill several backend connnections; check ProxySQL only destroys the relevant ones + MYSQL* server = create_server_conn(cl); + if (!server) { return EXIT_FAILURE; } + + uint32_t TO_KILL = (thread_count * 2) + rand() % 10; + uint32_t CONN_PCT = TO_KILL * 100 / CONNS_TOTAL; + diag("Random conn kill count kills=%d conn_pct=%d", TO_KILL, CONN_PCT); + + const string proc_list_q { + "SELECT ID FROM information_schema.processlist" + " WHERE DB='" + string { CONNPOOL_DB } + "'" + " ORDER BY RAND() LIMIT " + std::to_string(TO_KILL) + }; + diag("Fetching conns to be killed query=%s", proc_list_q.c_str()); + const pair> p_conns_ids { + mysql_query_ext_rows(server, proc_list_q) + }; + if (p_conns_ids.first) { + diag("Failed to fetch conns ids from processlist error=%s", mysql_error(server)); + return EXIT_FAILURE; + } + + // Fetch current connections errors to target server + const string myport { std::to_string(cl.mysql_port) }; + const string m_srv_id { + "mysql_error_total{" + "address=\"" + string {cl.mysql_host} + "\",code=\"2013\"," + "hostgroup=\"" + std::to_string(HG_ID) + "\",port=\"" + myport + "\"" + "}" + }; + + pair p_pre_count { fetch_metric_val(cl, m_srv_id) }; + if (p_pre_count.first) { return EXIT_FAILURE; } + + for (const mysql_res_row& conn_id_row : p_conns_ids.second) { + MYSQL_QUERY_T(server, string {"KILL " + conn_id_row[0]}.c_str()); + } + + // Give time for ProxySQL to detect broken connections + sleep((PING_INTV_MS / 1000 ) * 3); + + pair p_post_count { fetch_metric_val(cl, m_srv_id) }; + + ok( + p_pre_count.second + TO_KILL == p_post_count.second, + "Errors should be increased **ONLY** by killed conns pre=%lf post=%lf to_kill=%d", + p_pre_count.second, p_post_count.second, TO_KILL + ); + + // Check all conns remains viable using trxs to exhaust the conn-pool + diag("Starting trxs count=%ld", p_rc_conns.second.size()); + vector trxs_rcs {}; + for (MYSQL* mysql : p_rc_conns.second) { + int q_rc = mysql_query_t(mysql, "BEGIN"); + if (q_rc) { + diag("Trx start failed error=%s", mysql_error(mysql)); + } + trxs_rcs.push_back(q_rc); + } + + size_t failed_trxs = std::accumulate(trxs_rcs.begin(), trxs_rcs.end(), 0, + [] (size_t acc, int n) { + if (n != 0) { return acc + 1; } + else { return acc; } + } + ); + + ok(failed_trxs == 0, "No trxs should fail to start failed_trxs=%ld", failed_trxs); + + for (MYSQL* mysql : p_rc_conns.second) { + mysql_close(mysql); + } + + return EXIT_SUCCESS; +} + +const vector idle_sess_ms { + 10000 /* No session sharing on threads */, + 1 /* Session sharing between; via 'idle-threads' */ +}; + +const vector> ssl_failure_rts { + { "force_ssl_pre_handshake_failure", force_ssl_pre_handshake_failure }, + { "create_ssl_conn_inv_cert", create_ssl_conn_inv_cert }, + { "create_ssl_conn_missing_cert", create_ssl_conn_missing_cert } +}; + +int main(int argc, char** argv) { + CommandLine cl; + + if (cl.getEnv()) { + diag("Failed to get the required environmental variables."); + return EXIT_FAILURE; + } + + diag("Init rand seed with current time"); + srand(time(NULL)); + + MYSQL* admin = mysql_init(NULL); + + if (!mysql_real_connect(admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(admin)); + return EXIT_FAILURE; + } + + // Disable query retry; required for further tests + MYSQL_QUERY_T(admin, "SET mysql-query_retries_on_failure=0"); + MYSQL_QUERY_T(admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + // Ensure RESTAPI is enabled for backend conn errors fetching + MYSQL_QUERY_T(admin, "SET admin-restapi_enabled=1"); + MYSQL_QUERY_T(admin, "LOAD ADMIN VARIABLES TO RUNTIME"); + + // Update default hostgroup for user with target hostgroup + MYSQL_QUERY_T(admin, + ("UPDATE mysql_users SET default_hostgroup=" + std::to_string(HG_ID) + + " WHERE username='" + cl.username + "'").c_str() + ); + MYSQL_QUERY_T(admin, "LOAD MYSQL USERS TO RUNTIME"); + + // Disable all queries rules if present; not required + MYSQL_QUERY_T(admin, "UPDATE mysql_query_rules SET active=0"); + MYSQL_QUERY_T(admin, "LOAD MYSQL QUERY RULES TO RUNTIME"); + + // Update MySQL servers config + MYSQL_QUERY_T(admin, + ("UPDATE mysql_servers SET use_ssl=1 WHERE hostgroup_id=" + std::to_string(HG_ID)).c_str() + ); + MYSQL_QUERY_T(admin, "LOAD MYSQL SERVERS TO RUNTIME"); + + const char q_thread_count[] { + "SELECT variable_value FROM global_variables WHERE variable_name='mysql-threads'" + }; + ext_val_t ext_thread_count { mysql_query_ext_val(admin, q_thread_count, int64_t(0)) }; + + if (ext_thread_count.err != EXIT_SUCCESS) { + const string err { get_ext_val_err(admin, ext_thread_count) }; + diag("Failed getting 'mysql-threads' query:`%s`, err:`%s`", q_thread_count, err.c_str()); + return EXIT_FAILURE; + } + + const long conn_queries { PER_THREAD_CONN_COUNT * ext_thread_count.val * 2 }; + const size_t end_to_end_conns_checks { conn_queries * idle_sess_ms.size() * ssl_failure_rts.size() }; + const size_t thread_conn_dist_checks { ssl_failure_rts.size() * idle_sess_ms.size() }; + + size_t frontend_conns_checks { 0 }; + size_t conns_dist_checks { 0 }; + size_t backend_conns_checks { 0 }; + + if (!TEST_FRONTEND || (TEST_FRONTEND && strcasecmp(TEST_FRONTEND, "0") != 0)) { + frontend_conns_checks = end_to_end_conns_checks; + } + if (!TEST_CONN_DIST || (TEST_CONN_DIST && strcasecmp(TEST_CONN_DIST, "0") != 0)) { + conns_dist_checks = thread_conn_dist_checks; + } + if (!TEST_BACKEND || (TEST_BACKEND && strcasecmp(TEST_BACKEND, "0") != 0)) { + backend_conns_checks = 2; + } + + plan(frontend_conns_checks + conns_dist_checks + backend_conns_checks); + + if (!frontend_conns_checks) { + goto backend_checks; + } + +frontend_checks: + diag("START: Regression testing of #4556 for frontend conns"); + for (const pair p_name_rt : ssl_failure_rts) { + const char* rt_name = p_name_rt.first.c_str(); + void*(*ssl_fail_rt)(void*) = p_name_rt.second; + + for (size_t ms_idle : idle_sess_ms) { + diag("Forcing SSL failure on fronted connection routine=%s", p_name_rt.first.c_str()); + int rc = check_frontend_ssl_errs(cl, admin, ext_thread_count.val, ms_idle, ssl_fail_rt); + if (rc) { + diag("Unable to perform check, operation failed routine=%s", p_name_rt.first.c_str()); + return EXIT_FAILURE; + } + } + } + + if (!backend_conns_checks) { + goto cleanup; + } + +backend_checks: + diag("START: Regression testing for SSL errors on backend conns"); + { + int rc = check_backend_ssl_errs(cl, admin, ext_thread_count.val); + if (rc) { + diag("Unable to perform check, operation failed"); + return EXIT_FAILURE; + } + } + +cleanup: + mysql_close(admin); + + return exit_status(); +} diff --git a/test/tap/tests/reg_test_4556-ssl_error_queue-t.env b/test/tap/tests/reg_test_4556-ssl_error_queue-t.env new file mode 100644 index 000000000..0700ee438 --- /dev/null +++ b/test/tap/tests/reg_test_4556-ssl_error_queue-t.env @@ -0,0 +1,3 @@ +TAP_MYSQLUSERNAME=root +TAP_MYSQLPASSWORD=root +TAP_MYSQLPORT=13306 diff --git a/test/tap/tests/reg_test__ssl_client_busy_wait-t.cpp b/test/tap/tests/reg_test__ssl_client_busy_wait-t.cpp new file mode 100644 index 000000000..098689881 --- /dev/null +++ b/test/tap/tests/reg_test__ssl_client_busy_wait-t.cpp @@ -0,0 +1,352 @@ +/** + * @file reg_test_3273_ssl_con-t.cpp + * @brief Regression test for SSL busy/infinite loops for frontend connections. + * @details When client disconnects unexpectedly closing the socket on a SSL connection, depending on the + * timing conditions, either an infinite loop or a busy loop could take place. These scenarios are: + * 1. Closed socket while query running on backend (before data arrives), leads to busy loop. + * 2. Closed socket after all the data has been written into the socket, since no more writing would take + * place in the socket an infinite loop would take place. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + + +#include "mysql.h" + +#include "tap.h" +#include "command_line.h" +#include "utils.h" + +using std::string; +using std::pair; +using std::vector; + +/* Helper function to do the waiting for events on the socket. */ +static int wait_for_mysql(MYSQL *mysql, int status) { + struct pollfd pfd; + int timeout, res; + + pfd.fd = mysql_get_socket(mysql); + pfd.events = + (status & MYSQL_WAIT_READ ? POLLIN : 0) | + (status & MYSQL_WAIT_WRITE ? POLLOUT : 0) | + (status & MYSQL_WAIT_EXCEPT ? POLLPRI : 0); + if (status & MYSQL_WAIT_TIMEOUT) + timeout = 1000*mysql_get_timeout_value(mysql); + else + timeout = -1; + res = poll(&pfd, 1, timeout); + if (res == 0) + return MYSQL_WAIT_TIMEOUT; + else if (res < 0) + return MYSQL_WAIT_TIMEOUT; + else { + int status = 0; + if (pfd.revents & POLLIN) status |= MYSQL_WAIT_READ; + if (pfd.revents & POLLOUT) status |= MYSQL_WAIT_WRITE; + if (pfd.revents & POLLPRI) status |= MYSQL_WAIT_EXCEPT; + return status; + } +} + +enum BUSY_LOOP_T { + BUSY_LOOP=0, + INF_LOOP=1 +}; + +struct th_args__in_t { + // Input + int argc { 0 }; + char** argv { nullptr }; + int secs { 0 }; + int busy_loop_type = BUSY_LOOP_T::BUSY_LOOP; + CommandLine& cl; +}; + +struct th_args__out_t { + // Output + volatile int* query_started { nullptr }; + volatile int* routine_rc { nullptr }; +}; + +struct th_args_t { + th_args__in_t in_args; + th_args__out_t out_args {}; +}; + +void* perform_async_query(void* arg) { + th_args_t* th_args = static_cast(arg); + + MYSQL* mysql = nullptr; + + { + CommandLine& cl = th_args->in_args.cl; + mysql = mysql_init(NULL); + MYSQL* ret = NULL; + int query_ret = 0; + + mysql_options(mysql, MYSQL_OPT_NONBLOCK, 0); + mysql_ssl_set(mysql, NULL, NULL, NULL, NULL, NULL); + + int status = 0; + + if (th_args->in_args.argc == 2 && (strcmp(th_args->in_args.argv[1], "admin") == 0)) { + status = mysql_real_connect_start( + &ret, mysql, cl.host, "radmin", "radmin", NULL, 6032, NULL, CLIENT_SSL + ); + diag("Creating 'Admin' connection thread=%ld ret=%p status=%d", pthread_self(), ret, status); + } else { + status = mysql_real_connect_start( + &ret, mysql, cl.host, cl.username, cl.password, NULL, cl.port, NULL, CLIENT_SSL + ); + + diag("Creating 'MySQL' connection thread=%ld ret=%p status=%d", pthread_self(), ret, status); + } + if (status == 0) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(mysql)); + __sync_fetch_and_add(th_args->out_args.routine_rc, 1); + return NULL; + } + + diag("Continue connection establishment thread=%ld ret=%p status=%d", pthread_self(), ret, status); + while (status) { + status = wait_for_mysql(mysql, status); + status = mysql_real_connect_cont(&ret, mysql, status); + diag("'mysql_real_connect_cont' thread=%ld ret=%p status=%d", pthread_self(), ret, status); + } + + // NOTE: mariadbclient has an incompatibility between SSL and NONBLOCK flags. Flag needs to be reset + // after 'mysql_real_connect_cont', otherwise API would become blocking. + mysql_options(mysql, MYSQL_OPT_NONBLOCK, 0); + int f=fcntl(mysql->net.fd, F_GETFL); + fcntl(mysql->net.fd, F_SETFL, f|O_NONBLOCK); + + const string sleep_query { "SELECT SLEEP(" + std::to_string(th_args->in_args.secs) + ")" }; + diag("mysql_query_start thread=%ld query=%s", pthread_self(), sleep_query.c_str()); + + status = mysql_real_query_start(&query_ret, mysql, sleep_query.c_str(), sleep_query.size()); + + if (th_args->in_args.busy_loop_type == BUSY_LOOP_T::INF_LOOP) { + // NOTE: When signaling after 'mysql_query_start' has finished, ProxySQL wont attempt to write more + // to the closed pipe, this corresponds to 'busy_loop_type=2' and infinite loop. + while (status) { + status = wait_for_mysql(mysql, status); + status = mysql_real_query_cont(&query_ret, mysql, status); + diag("'mysql_real_connect_cont' thread=%ld ret=%p status=%d", pthread_self(), ret, status); + } + } + + diag("Signaling query start thread=%ld status=%d query_ret=%d", pthread_self(), status, query_ret); + __sync_fetch_and_add(th_args->out_args.query_started, 1); + + // NOTE: Required for triggering the issue, thread exit isn't enough, either 'process exit' or + // 'close()'. They should be immediate to the previous action, otherwise timing could be invalid. + close(mysql->net.fd); + + while (true) { + diag( + "Sleeping after query started... thread=%ld status=%d query_ret=%d", + pthread_self(), status, query_ret + ); + sleep(1); + } + } + + return NULL; +} + +struct pthread_data_t { + pthread_t id { 0 }; + int query_started { 0 }; + int routine_rc { EXIT_SUCCESS }; +}; + +const int BUSY_THREADS = get_env_int("TAP_SSL_BUSY_WAIT__BUSY_THREADS", 4); +const int MAX_IDLE_CPU = get_env_int("TAP_SSL_BUSY_WAIT__MAX_IDLE_CPU", 20); +const int MAX_BUSY_CPU = get_env_int("TAP_SSL_BUSY_WAIT__MAX_BUSY_CPU", 25); + +// NOTE: '10' is a nice value due to it's relationship with the 'system_cpu' interval +const int BUSY_WAIT_SECS = get_env_int("TAP_SSL_BUSY_WAIT__BUSY_WAIT_SECS", 10); +// NOTE: '5' is the min value due to time interval rounding 'round_intv_to_time_interval' +const int SAMPLE_INTV_SECS = get_env_int("TAP_SSL_BUSY_WAIT__SAMPLE_INTV_SEC", BUSY_WAIT_SECS / 2); + +void create_busy_loops(int argc, char** argv, CommandLine& cl, BUSY_LOOP_T loop_type) { + vector ths_data {}; + vector> ths_args {}; + + ths_data.resize(BUSY_THREADS); + + for (size_t i = 0; i < BUSY_THREADS; i++) { + pthread_data_t& th_data = ths_data[i]; + std::unique_ptr th_args { + new th_args_t { + th_args__in_t { + argc, argv, BUSY_WAIT_SECS, loop_type, cl + }, + th_args__out_t { + &th_data.query_started, + &th_data.routine_rc + } + } + }; + + pthread_create(&th_data.id, NULL, perform_async_query, th_args.get()); + ths_args.push_back(std::move(th_args)); + + diag("Thread created thread=%ld", th_data.id); + } + + bool missing_query = true; + bool query_failed = false; + + while (missing_query && !query_failed) { + bool all_query_started = true; + + for (pthread_data_t& th_data : ths_data) { + bool query_started = __sync_fetch_and_add(&th_data.query_started, 0); + diag( + "Thread data thread=%ld routine_rc=%d query_started=%d", + th_data.id, th_data.routine_rc, query_started + ); + + if (th_data.id == 0 && query_started == 1) { + diag( + "Thread alreay cancelled thread=%ld routine_rc=%d query_started=%d", + th_data.id, th_data.routine_rc, query_started + ); + continue; + } + + query_failed = __sync_fetch_and_add(&th_data.routine_rc, 0); + + if (query_failed) { + diag( + "Async query failed; aborting test thread=%ld routine_rc=%d query_started=%d", + th_data.id, th_data.routine_rc, query_started + ); + break; + } + + all_query_started &= query_started; + + if (query_started) { + diag( + "Async query started, killing thread thread=%ld routine_rc=%d query_started=%d", + th_data.id, th_data.routine_rc, query_started + ); + pthread_cancel(th_data.id); + th_data.id = 0; + } else { + diag( + "Waiting for async query to start... thread=%ld routine_rc=%d query_started=%d", + th_data.id, th_data.routine_rc, query_started + ); + } + } + + missing_query = !all_query_started; + usleep(500 * 1000); + } +} + +int main(int argc, char** argv) { + CommandLine cl; + + if (cl.getEnv()) { + diag("Failed to get the required environmental variables."); + return -1; + } + + plan(4); + + MYSQL* admin = mysql_init(NULL); + if (!mysql_real_connect(admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(admin)); + return EXIT_FAILURE; + } + + pair> p_err_nodes_conns { disable_core_nodes_scheduler(cl, admin) }; + if (p_err_nodes_conns.first) { return EXIT_FAILURE; } + vector& nodes_conns { p_err_nodes_conns.second }; + + diag("Checking ProxySQL idle CPU usage"); + double idle_cpu = 0; + int ret_i_cpu = get_proxysql_cpu_usage(cl, SAMPLE_INTV_SECS, idle_cpu); + if (ret_i_cpu) { + diag("Getting initial CPU usage failed with error - %d", ret_i_cpu); + diag("Aborting further testing"); + + return EXIT_FAILURE; + } + + ok( + idle_cpu < MAX_IDLE_CPU, "Idle CPU usage should be below expected - Exp:%d%%, Act: %lf%%", + MAX_IDLE_CPU, idle_cpu + ); + + diag("Trigger BUSY_LOOP regression BUSY_THREADS=%d BUSY_WAIT_SECS=%d", BUSY_THREADS, BUSY_WAIT_SECS); + create_busy_loops(argc, argv, cl, BUSY_LOOP_T::BUSY_LOOP); + + diag("Checking ProxySQL final CPU usage for 'BUSY_LOOP'"); + double final_cpu_usage = 0; + int ret_f_cpu = get_proxysql_cpu_usage(cl, SAMPLE_INTV_SECS, final_cpu_usage); + + ok( + final_cpu_usage < MAX_BUSY_CPU, + "ProxySQL CPU usage should be below expected - Exp: %d%%, Act: %lf%%", + MAX_BUSY_CPU, final_cpu_usage + ); + + // Extra wait to ensure cleanup of faulty client conns. See 'BUSY_WAIT_SECS' NOTE in def. + int BUSY_WAIT_CLEANUP = BUSY_WAIT_SECS < 5 ? 5 : BUSY_WAIT_SECS / 2; + diag("Sleeping for %d secs for BUSY_LOOP client cleanup", BUSY_WAIT_CLEANUP); + sleep(BUSY_WAIT_CLEANUP); + + diag("Checking ProxySQL idle CPU usage"); + ret_i_cpu = get_proxysql_cpu_usage(cl, SAMPLE_INTV_SECS, idle_cpu); + if (ret_i_cpu) { + diag("Getting initial CPU usage failed with error - %d", ret_i_cpu); + diag("Aborting further testing"); + + return EXIT_FAILURE; + } + + ok( + idle_cpu < MAX_IDLE_CPU, "Idle CPU usage should be below expected - Exp:%d%%, Act: %lf%%", + MAX_IDLE_CPU, idle_cpu + ); + + diag("Trigger INF_LOOP regression BUSY_THREADS=%d BUSY_WAIT_SECS=%d", BUSY_THREADS, BUSY_WAIT_SECS); + create_busy_loops(argc, argv, cl, BUSY_LOOP_T::INF_LOOP); + + diag("Checking ProxySQL final CPU usage for 'BUSY_LOOP'"); + final_cpu_usage = 0; + ret_f_cpu = get_proxysql_cpu_usage(cl, SAMPLE_INTV_SECS, final_cpu_usage); + + ok( + final_cpu_usage < MAX_BUSY_CPU, + "ProxySQL CPU usage should be below expected - Exp: %d%%, Act: %lf%%", + MAX_BUSY_CPU, final_cpu_usage + ); + + // Recover cluster scheduler + for (MYSQL* myconn : nodes_conns) { + MYSQL_QUERY_T(myconn, "LOAD SCHEDULER FROM DISK"); + MYSQL_QUERY_T(myconn, "LOAD SCHEDULER TO RUNTIME"); + + mysql_close(myconn); + } + + mysql_close(admin); + + return exit_status(); +} diff --git a/test/tap/tests/reg_test_sql_calc_found_rows-t.cpp b/test/tap/tests/reg_test_sql_calc_found_rows-t.cpp index 674a1f9b7..0a7200ef1 100644 --- a/test/tap/tests/reg_test_sql_calc_found_rows-t.cpp +++ b/test/tap/tests/reg_test_sql_calc_found_rows-t.cpp @@ -13,7 +13,6 @@ #include #include "mysql.h" -#include "mysqld_error.h" #include "json.hpp" @@ -74,7 +73,7 @@ int main(int argc, char** argv) { // 1. Prepare the 'SQL_CALC_FOUND_ROWS' stmt in a connection MYSQL* proxy_mysql = mysql_init(NULL); - diag("%s: Openning initial connection...", tap_curtime().c_str()); + diag("Openning initial connection..."); if (!mysql_real_connect(proxy_mysql, cl.host, cl.username, cl.password, NULL, cl.port, NULL, 0)) { fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxy_mysql)); return EXIT_FAILURE; @@ -84,7 +83,7 @@ int main(int argc, char** argv) { MYSQL_STMT* stmt_2 = mysql_stmt_init(proxy_mysql); MYSQL_STMT* stmt_3 = nullptr; - diag("%s: Issuing the prepare for `%s` in init conn", tap_curtime().c_str(), Q_CALC_FOUND_ROWS_1); + diag("Issuing the prepare for `%s` in init conn", Q_CALC_FOUND_ROWS_1); int my_err = mysql_stmt_prepare(stmt_1, Q_CALC_FOUND_ROWS_1, strlen(Q_CALC_FOUND_ROWS_1)); if (my_err) { diag( @@ -94,7 +93,7 @@ int main(int argc, char** argv) { goto cleanup; } - diag("%s: Issuing the prepare for `%s` in init conn", tap_curtime().c_str(), Q_CALC_FOUND_ROWS_2); + diag("Issuing the prepare for `%s` in init conn", Q_CALC_FOUND_ROWS_2); my_err = mysql_stmt_prepare(stmt_2, Q_CALC_FOUND_ROWS_2, strlen(Q_CALC_FOUND_ROWS_2)); if (my_err) { diag( @@ -107,13 +106,13 @@ int main(int argc, char** argv) { mysql_stmt_close(stmt_1); mysql_stmt_close(stmt_2); - diag("%s: Closing initial connection...", tap_curtime().c_str()); + diag("Closing initial connection..."); mysql_close(proxy_mysql); // 2. Open a new connection and prepare the stmts it again in a new connection proxy_mysql = mysql_init(NULL); - diag("%s: Openning new connection for testing 'Multiplex' disabling", tap_curtime().c_str()); + diag("Openning new connection for testing 'Multiplex' disabling"); if (!mysql_real_connect(proxy_mysql, cl.host, cl.username, cl.password, NULL, cl.port, NULL, 0)) { fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxy_mysql)); return EXIT_FAILURE; @@ -123,7 +122,7 @@ int main(int argc, char** argv) { stmt_2 = mysql_stmt_init(proxy_mysql); stmt_3 = mysql_stmt_init(proxy_mysql); - diag("%s: Issuing the prepare for `%s` in new conn", tap_curtime().c_str(), Q_CALC_FOUND_ROWS_1); + diag("Issuing the prepare for `%s` in new conn", Q_CALC_FOUND_ROWS_1); my_err = mysql_stmt_prepare(stmt_1, Q_CALC_FOUND_ROWS_1, strlen(Q_CALC_FOUND_ROWS_1)); if (my_err) { diag( @@ -134,7 +133,7 @@ int main(int argc, char** argv) { } { - diag("%s: Issuing execute for `%s` in new conn", tap_curtime().c_str(), Q_CALC_FOUND_ROWS_1); + diag("Issuing execute for `%s` in new conn", Q_CALC_FOUND_ROWS_1); my_err = mysql_stmt_execute(stmt_1); if (my_err) { diag("'mysql_stmt_execute' at line %d failed: %s", __LINE__, mysql_stmt_error(stmt_1)); @@ -149,7 +148,7 @@ int main(int argc, char** argv) { } } { - diag("%s: Issuing the prepare for `%s` in new conn", tap_curtime().c_str(), Q_CALC_FOUND_ROWS_2); + diag("Issuing the prepare for `%s` in new conn", Q_CALC_FOUND_ROWS_2); my_err = mysql_stmt_prepare(stmt_2, Q_CALC_FOUND_ROWS_2, strlen(Q_CALC_FOUND_ROWS_2)); if (my_err) { diag( @@ -160,7 +159,7 @@ int main(int argc, char** argv) { } { - diag("%s: Issuing execute for `%s` in new conn", tap_curtime().c_str(), Q_CALC_FOUND_ROWS_2); + diag("Issuing execute for `%s` in new conn", Q_CALC_FOUND_ROWS_2); my_err = mysql_stmt_execute(stmt_2); if (my_err) { diag("'mysql_stmt_execute' at line %d failed: %s", __LINE__, mysql_stmt_error(stmt_2)); @@ -175,7 +174,7 @@ int main(int argc, char** argv) { } } - diag("%s: Issuing the prepare for `%s` in new conn", tap_curtime().c_str(), Q_FOUND_ROWS); + diag("Issuing the prepare for `%s` in new conn", Q_FOUND_ROWS); my_err = mysql_stmt_prepare(stmt_3, Q_FOUND_ROWS, strlen(Q_FOUND_ROWS)); if (my_err) { diag( diff --git a/test/tap/tests/test_auto_increment_delay_multiplex-t.cpp b/test/tap/tests/test_auto_increment_delay_multiplex-t.cpp index ee50da721..64f531da0 100644 --- a/test/tap/tests/test_auto_increment_delay_multiplex-t.cpp +++ b/test/tap/tests/test_auto_increment_delay_multiplex-t.cpp @@ -25,11 +25,9 @@ #include #include #include -#include #include #include "mysql.h" -#include "mysqld_error.h" #include "json.hpp" @@ -142,14 +140,14 @@ int check_auto_increment_timeout( const string set_auto_inc_to_query { "SET mysql-auto_increment_delay_multiplex_timeout_ms=" + std::to_string(auto_inc_delay_to) }; - diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_auto_inc_to_query.c_str()); + diag("Executing query `%s`...", set_auto_inc_to_query.c_str()); MYSQL_QUERY(proxy_admin, set_auto_inc_to_query.c_str()); MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "LOAD MYSQL VARIABLES TO RUNTIME"); + diag("Executing query `%s`...", "LOAD MYSQL VARIABLES TO RUNTIME"); MYSQL_QUERY(proxy_mysql, INSERT_QUERY); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), INSERT_QUERY); + diag("Executing query `%s`...", INSERT_QUERY); // Wait at least '500' milliseconds over the poll period usleep((poll_to + 500) * 1000); @@ -197,7 +195,7 @@ int check_auto_increment_timeout( } MYSQL_QUERY(proxy_mysql, "DO 1"); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "DO 1"); + diag("Executing query `%s`...", "DO 1"); uint32_t old_auto_inc_delay_mult = cur_auto_inc_delay_mult; g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_auto_inc_delay_mult); @@ -244,9 +242,9 @@ int check_variables_config(MYSQL* proxy_mysql, MYSQL* proxy_admin) { int check_auto_increment_delay_multiplex(MYSQL* proxy_mysql, MYSQL* proxy_admin) { // Disable the 'timeout' for the this check since it can be fixated now - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "SET mysql-auto_increment_delay_multiplex_timeout_ms=0"); + diag("Executing query `%s`...", "SET mysql-auto_increment_delay_multiplex_timeout_ms=0"); MYSQL_QUERY(proxy_admin, "SET mysql-auto_increment_delay_multiplex_timeout_ms=0"); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "SET mysql-connection_delay_multiplex_ms=0"); + diag("Executing query `%s`...", "SET mysql-connection_delay_multiplex_ms=0"); MYSQL_QUERY(proxy_admin, "SET mysql-connection_delay_multiplex_ms=0"); int cur_auto_inc_delay_mult = 0; @@ -310,13 +308,13 @@ int check_auto_increment_delay_multiplex_timeout(MYSQL* proxy_mysql, MYSQL* prox uint64_t poll_timeout = 0; int g_res = 0; - diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_auto_inc_query.c_str()); + diag("Executing query `%s`...", set_auto_inc_query.c_str()); MYSQL_QUERY(proxy_admin, set_auto_inc_query.c_str()); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "SET mysql-connection_delay_multiplex_ms=0"); + diag("Executing query `%s`...", "SET mysql-connection_delay_multiplex_ms=0"); MYSQL_QUERY(proxy_admin, "SET mysql-connection_delay_multiplex_ms=0"); const string q_poll_timeout { "SELECT variable_value FROM global_variables WHERE variable_name='mysql-poll_timeout'" }; - diag("%s: Executing query `%s`...", tap_curtime().c_str(), q_poll_timeout.c_str()); + diag("Executing query `%s`...", q_poll_timeout.c_str()); g_res = get_query_result(proxy_admin, q_poll_timeout.c_str(), poll_timeout); if (g_res != EXIT_SUCCESS) { return EXIT_FAILURE; } @@ -357,14 +355,14 @@ int check_auto_increment_delay_multiplex_timeout(MYSQL* proxy_mysql, MYSQL* prox if (g_res != EXIT_SUCCESS) { return EXIT_FAILURE; } MYSQL_QUERY(proxy_admin, delay_query.c_str()); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), delay_query.c_str()); + diag("Executing query `%s`...", delay_query.c_str()); MYSQL_QUERY(proxy_admin, timeout_query.c_str()); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), timeout_query.c_str()); + diag("Executing query `%s`...", timeout_query.c_str()); MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); { // Insert disabling multiplexing for the connection - diag("%s: Executing query `%s`...", tap_curtime().c_str(), INSERT_QUERY); + diag("Executing query `%s`...", INSERT_QUERY); MYSQL_QUERY(proxy_mysql, INSERT_QUERY); // Perform queries in the same connection @@ -373,7 +371,7 @@ int check_auto_increment_delay_multiplex_timeout(MYSQL* proxy_mysql, MYSQL* prox while (waited < timeout_ms * 3) { sleep(1); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "/* hostgroup=0 */ DO 1"); + diag("Executing query `%s`...", "/* hostgroup=0 */ DO 1"); MYSQL_QUERY(proxy_mysql, "/* hostgroup=0 */ DO 1"); waited += 1000; } @@ -397,7 +395,7 @@ int check_auto_increment_delay_multiplex_timeout(MYSQL* proxy_mysql, MYSQL* prox while (waited < timeout_ms * 3) { sleep(1); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "SELECT 1"); + diag("Executing query `%s`...", "SELECT 1"); MYSQL_QUERY(proxy_mysql, "SELECT 1"); mysql_free_result(mysql_store_result(proxy_mysql)); @@ -423,16 +421,16 @@ int check_auto_increment_delay_multiplex_timeout(MYSQL* proxy_mysql, MYSQL* prox // Transactions connections should be preserved by 'auto_increment_delay_multiplex_timeout_ms' { - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "BEGIN"); + diag("Executing query `%s`...", "BEGIN"); MYSQL_QUERY(proxy_mysql, "BEGIN"); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), INSERT_QUERY); + diag("Executing query `%s`...", INSERT_QUERY); MYSQL_QUERY(proxy_mysql, INSERT_QUERY); // Wait for the timeout and check the value - diag("%s: Waiting for timeout to expire...", tap_curtime().c_str()); + diag("Waiting for timeout to expire..."); usleep(timeout_ms * 1000 + poll_timeout * 1000 + 500 * 1000 * 2); - diag("%s: Extracting current auto inc delay...", tap_curtime().c_str()); + diag("Extracting current auto inc delay..."); int cur_delay = 0; int g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_delay); if (g_res != EXIT_SUCCESS) { @@ -445,13 +443,13 @@ int check_auto_increment_delay_multiplex_timeout(MYSQL* proxy_mysql, MYSQL* prox delay, cur_delay ); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "COMMIT"); + diag("Executing query `%s`...", "COMMIT"); MYSQL_QUERY(proxy_mysql, "COMMIT"); - diag("%s: Waiting for timeout to expire...", tap_curtime().c_str()); + diag("Waiting for timeout to expire..."); usleep(timeout_ms * 1000 + poll_timeout * 1000 + 500 * 1000 * 2); - diag("%s: Extracting current auto inc delay...", tap_curtime().c_str()); + diag("Extracting current auto inc delay..."); cur_delay = 0; g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_delay); if (g_res != EXIT_SUCCESS) { @@ -468,16 +466,16 @@ int check_auto_increment_delay_multiplex_timeout(MYSQL* proxy_mysql, MYSQL* prox // Multiplex disabled by any action should take precedence over 'auto_increment_delay_multiplex_timeout_ms' { const char* set_query { "SET @local_var='foo'" }; - diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_query); + diag("Executing query `%s`...", set_query); MYSQL_QUERY(proxy_mysql, set_query); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), INSERT_QUERY); + diag("Executing query `%s`...", INSERT_QUERY); MYSQL_QUERY(proxy_mysql, INSERT_QUERY); // Wait for the timeout and check the value - diag("%s: Waiting for timeout to expire...", tap_curtime().c_str()); + diag("Waiting for timeout to expire..."); usleep(timeout_ms * 1000 + poll_timeout * 1000 + 500 * 1000 * 2); - diag("%s: Extracting current auto inc delay...", tap_curtime().c_str()); + diag("Extracting current auto inc delay..."); int cur_delay = 0; int g_res = get_conn_auto_inc_delay_token(proxy_mysql, cur_delay); if (g_res != EXIT_SUCCESS) { @@ -533,9 +531,9 @@ void check_connection_retained(MYSQL* proxy_mysql, uint32_t exp_conns) { int check_transactions_and_multiplex_disable( MYSQL* proxy_mysql, const char* query, const uint32_t timeout, uint64_t poll_timeout=2 ) { - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "BEGIN"); + diag("Executing query `%s`...", "BEGIN"); MYSQL_QUERY(proxy_mysql, "BEGIN"); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), query); + diag("Executing query `%s`...", query); MYSQL_QUERY(proxy_mysql, query); diag("Checking connection present before timeout..."); @@ -547,7 +545,7 @@ int check_transactions_and_multiplex_disable( diag("Checking connection is still present after timeout due to transaction..."); check_connection_retained(proxy_mysql, 1); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "COMMIT"); + diag("Executing query `%s`...", "COMMIT"); MYSQL_QUERY(proxy_mysql, "COMMIT"); diag("Sleeping for '%lf' seconds", timeout / 2.0); @@ -565,7 +563,7 @@ int check_transactions_and_multiplex_disable( diag("Checking multiplex disabled by any action take precedence over 'connection_delay_multiplex_ms'..."); const char* set_query { "SET @local_var='foo'" }; - diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_query); + diag("Executing query `%s`...", set_query); MYSQL_QUERY(proxy_mysql, set_query); diag("Sleeping for '%ld' seconds", timeout + poll_timeout); @@ -587,13 +585,13 @@ int check_connection_delay_multiplex_ms(MYSQL* proxy_mysql, MYSQL* proxy_admin) string_format("SET mysql-connection_delay_multiplex_ms=%d", set_delay_multiplex, timeout * 1000); const char* set_auto_inc_delay { "SET mysql-auto_increment_delay_multiplex_timeout_ms=0" }; - diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_delay_multiplex.c_str()); + diag("Executing query `%s`...", set_delay_multiplex.c_str()); MYSQL_QUERY(proxy_admin, set_delay_multiplex.c_str()); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_auto_inc_delay); + diag("Executing query `%s`...", set_auto_inc_delay); MYSQL_QUERY(proxy_admin, set_auto_inc_delay); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "LOAD MYSQL VARIABLES TO RUNTIME"); + diag("Executing query `%s`...", "LOAD MYSQL VARIABLES TO RUNTIME"); MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); MYSQL_QUERY(proxy_mysql, "SELECT 1"); @@ -632,13 +630,13 @@ int check_multiplex_disabled_connection_delay_multiplex_ms(MYSQL* proxy_mysql, M string_format("SET mysql-connection_delay_multiplex_ms=%d", set_delay_multiplex, timeout * 1000); const char* set_auto_inc_delay { "SET mysql-auto_increment_delay_multiplex_timeout_ms=0" }; - diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_delay_multiplex.c_str()); + diag("Executing query `%s`...", set_delay_multiplex.c_str()); MYSQL_QUERY(proxy_admin, set_delay_multiplex.c_str()); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_auto_inc_delay); + diag("Executing query `%s`...", set_auto_inc_delay); MYSQL_QUERY(proxy_admin, set_auto_inc_delay); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "LOAD MYSQL VARIABLES TO RUNTIME"); + diag("Executing query `%s`...", "LOAD MYSQL VARIABLES TO RUNTIME"); MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); // Check transactions behavior and multiplex disabling actions @@ -652,11 +650,11 @@ int check_traffic_connection_delay_multiplex_ms(MYSQL* proxy_mysql, MYSQL* proxy const char* set_delay_multiplex_query { "SET mysql-connection_delay_multiplex_ms=2000" }; const char* set_auto_inc_timeout_query { "SET mysql-auto_increment_delay_multiplex_timeout_ms=0" }; - diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_delay_multiplex_query); + diag("Executing query `%s`...", set_delay_multiplex_query); MYSQL_QUERY(proxy_admin, set_delay_multiplex_query); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_auto_inc_timeout_query); + diag("Executing query `%s`...", set_auto_inc_timeout_query); MYSQL_QUERY(proxy_admin, set_auto_inc_timeout_query); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "LOAD MYSQL VARIABLES TO RUNTIME"); + diag("Executing query `%s`...", "LOAD MYSQL VARIABLES TO RUNTIME"); MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); // Retain connection in 'hg=0' @@ -664,7 +662,7 @@ int check_traffic_connection_delay_multiplex_ms(MYSQL* proxy_mysql, MYSQL* proxy uint32_t waited = 0; while (waited < 2*timeout) { - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "DO 1"); + diag("Executing query `%s`...", "DO 1"); MYSQL_QUERY(proxy_mysql, "DO 1"); sleep(1); @@ -679,9 +677,9 @@ int check_traffic_connection_delay_multiplex_ms(MYSQL* proxy_mysql, MYSQL* proxy diag("Check connection expiring when traffic issued to different hostgroup..."); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "DO 1"); + diag("Executing query `%s`...", "DO 1"); MYSQL_QUERY(proxy_mysql, "DO 1"); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "SELECT 1"); + diag("Executing query `%s`...", "SELECT 1"); MYSQL_QUERY(proxy_mysql, "SELECT 1"); mysql_free_result(mysql_store_result(proxy_mysql)); @@ -719,7 +717,7 @@ int check_traffic_connection_delay_multiplex_ms(MYSQL* proxy_mysql, MYSQL* proxy // Check for connections retained in 'hg 0' waited = 0; while (waited < timeout * 2) { - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "SELECT 1"); + diag("Executing query `%s`...", "SELECT 1"); MYSQL_QUERY(proxy_mysql, "SELECT 1"); mysql_free_result(mysql_store_result(proxy_mysql)); @@ -768,18 +766,18 @@ int check_auto_inc_delay_and_conn_delay_multiplex(MYSQL* proxy_mysql, MYSQL* pro const char* set_delay_multiplex_query { "SET mysql-connection_delay_multiplex_ms=2000" }; const char* set_auto_inc_timeout_query { "SET mysql-auto_increment_delay_multiplex_timeout_ms=0" }; - diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_delay_multiplex_query); + diag("Executing query `%s`...", set_delay_multiplex_query); MYSQL_QUERY(proxy_admin, set_delay_multiplex_query); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_auto_inc_timeout_query); + diag("Executing query `%s`...", set_auto_inc_timeout_query); MYSQL_QUERY(proxy_admin, set_auto_inc_timeout_query); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "LOAD MYSQL VARIABLES TO RUNTIME"); + diag("Executing query `%s`...", "LOAD MYSQL VARIABLES TO RUNTIME"); MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); // Retain connection in 'hg=0' diag("Checking connection not expiring due to 'auto_increment_delay_multiplex'."); uint32_t waited = 0; - diag("%s: Executing query `%s`...", tap_curtime().c_str(), INSERT_QUERY); + diag("Executing query `%s`...", INSERT_QUERY); MYSQL_QUERY(proxy_mysql, INSERT_QUERY); diag("* Check connection retained after executing the query"); @@ -797,13 +795,13 @@ int check_auto_inc_delay_and_conn_delay_multiplex(MYSQL* proxy_mysql, MYSQL* pro ); string_format("SET mysql-auto_increment_delay_multiplex_timeout_ms=%d", auto_inc_timeout_query, timeout*2*1000); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), auto_inc_timeout_query.c_str()); + diag("Executing query `%s`...", auto_inc_timeout_query.c_str()); MYSQL_QUERY(proxy_admin, auto_inc_timeout_query.c_str()); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "LOAD MYSQL VARIABLES TO RUNTIME"); + diag("Executing query `%s`...", "LOAD MYSQL VARIABLES TO RUNTIME"); MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), INSERT_QUERY); + diag("Executing query `%s`...", INSERT_QUERY); MYSQL_QUERY(proxy_mysql, INSERT_QUERY); diag("Sleeping for '%d' seconds", timeout + 1); @@ -834,17 +832,17 @@ int check_auto_inc_delay_and_conn_delay_multiplex(MYSQL* proxy_mysql, MYSQL* pro ); string_format("SET mysql-auto_increment_delay_multiplex_timeout_ms=%d", auto_inc_timeout_query, timeout*1000); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), auto_inc_timeout_query.c_str()); + diag("Executing query `%s`...", auto_inc_timeout_query.c_str()); MYSQL_QUERY(proxy_admin, auto_inc_timeout_query.c_str()); const char* set_delay_multiplex_query_2 { "SET mysql-connection_delay_multiplex_ms=4000" }; - diag("%s: Executing query `%s`...", tap_curtime().c_str(), set_delay_multiplex_query_2); + diag("Executing query `%s`...", set_delay_multiplex_query_2); MYSQL_QUERY(proxy_admin, set_delay_multiplex_query_2); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), "LOAD MYSQL VARIABLES TO RUNTIME"); + diag("Executing query `%s`...", "LOAD MYSQL VARIABLES TO RUNTIME"); MYSQL_QUERY(proxy_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); - diag("%s: Executing query `%s`...", tap_curtime().c_str(), INSERT_QUERY); + diag("Executing query `%s`...", INSERT_QUERY); MYSQL_QUERY(proxy_mysql, INSERT_QUERY); diag("Sleeping for '%d' seconds", timeout + 1); diff --git a/test/tap/tests/test_binlog_fast_forward-t.cpp b/test/tap/tests/test_binlog_fast_forward-t.cpp index 6f9c98bef..c6569c7c4 100644 --- a/test/tap/tests/test_binlog_fast_forward-t.cpp +++ b/test/tap/tests/test_binlog_fast_forward-t.cpp @@ -84,7 +84,7 @@ int pull_replication(MYSQL *mysql, int server_id) { } } if (print_diag == true) - diag("%s: server_id %d , event: %d , received events: %d , received heartbeats: %d", tap_curtime().c_str(), server_id, event->event_type, num_events, num_heartbeats); + diag("server_id %d , event: %d , received events: %d , received heartbeats: %d", server_id, event->event_type, num_events, num_heartbeats); } // we expects NHB heartbeats ok(num_heartbeats == NHB , "For server_id %d received %d heartbeats", server_id, num_heartbeats); diff --git a/test/tap/tests/test_cluster_sync-t.cpp b/test/tap/tests/test_cluster_sync-t.cpp index d7f47989c..7d1e71501 100644 --- a/test/tap/tests/test_cluster_sync-t.cpp +++ b/test/tap/tests/test_cluster_sync-t.cpp @@ -29,8 +29,11 @@ * - Check that checksum is detected and fetched by the peer node (only checksum itself). * - Check that once checksum is detected and fetched, it takes '%_diffs_before_sync' before the actual sync * is performed, error log is used to verify this. - * - Finally check the config sync, the new checksum should match the previously detected. + * - Check the config sync, the new checksum should match the previously detected. * + Module 'admin_variables' may be the exception, since 'LOAD TO RUNTIME' generates a new checksum. + * - To avoid race conditions, and make the next check always start from a known state, we finally check that + * the primary has updated the monitoring checksums. This way we ensure that in the next check, a change in + * the checksum means the new computed checksum not a previous, yet not synced one. * * Each sync DISABLE check consists in: * @@ -59,11 +62,9 @@ */ #include -#include #include #include #include -#include #include #include @@ -72,7 +73,6 @@ #include #include #include -#include #include #include "libconfig.h" @@ -100,58 +100,6 @@ using std::tuple; using std::fstream; using std::function; -/** - * @brief Helper function to verify that the sync of a table (or variable) have been performed. - * - * @param r_proxy_admin An already opened connection to ProxySQL. - * @param queries Queries to be executed that should return a **non-zero** value after the sync has taken place. - * @param sync_timeout Timeout for the sync to happen. - * - * @return EXIT_SUCCESS in case of success, otherwise: - * - '-1' if a query against Admin fails to be performed (failure is logged). - * - '-2' if timeout expired without sync being completed. - */ -int sync_checker(MYSQL* r_proxy_admin, const vector& queries, uint32_t sync_timeout) { - bool not_synced_query = false; - uint waited = 0; - - while (waited < sync_timeout) { - not_synced_query = false; - - // Check that all the entries have been synced - for (const auto& query : queries) { - int q_res = mysql_query(r_proxy_admin, query.c_str()); - if (q_res != EXIT_SUCCESS) { - fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(r_proxy_admin)); - return -1; - } - - MYSQL_RES* proxysql_servers_res = mysql_store_result(r_proxy_admin); - MYSQL_ROW row = mysql_fetch_row(proxysql_servers_res); - int row_value = atoi(row[0]); - mysql_free_result(proxysql_servers_res); - - if (row_value == 0) { - not_synced_query = true; - break; - } - } - - if (not_synced_query) { - waited += 1; - sleep(1); - } else { - break; - } - } - - if (not_synced_query) { - return -2; - } else { - return EXIT_SUCCESS; - } -} - // GLOBAL TEST PARAMETERS const uint32_t SYNC_TIMEOUT = 10; const uint32_t CONNECT_TIMEOUT = 10; @@ -244,30 +192,6 @@ int setup_config_file(const CommandLine& cl) { return 0; } -int check_nodes_sync( - const CommandLine& cl, const vector& core_nodes, const string& check_query, uint32_t sync_timeout -) { - for (const auto& node : core_nodes) { - const string host { node[0] }; - const int port = std::stol(node[1]); - - MYSQL* c_node_admin = mysql_init(NULL); - if (!mysql_real_connect(c_node_admin, host.c_str(), cl.admin_username, cl.admin_password, NULL, port, NULL, 0)) { - fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(c_node_admin)); - return EXIT_FAILURE; - } - - int not_synced = sync_checker(c_node_admin, { check_query }, sync_timeout); - if (not_synced != EXIT_SUCCESS) { - const string err_msg { "Node '" + host + ":" + std::to_string(port) + "' sync timed out" }; - fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, err_msg.c_str()); - return EXIT_FAILURE; - } - } - - return EXIT_SUCCESS; -} - const std::string t_debug_query = "mysql -u%s -p%s -h %s -P%d -C -e \"%s\""; using mysql_server_tuple = tuple; @@ -942,6 +866,53 @@ int check_module_checksums_sync( } } + // Check that the primary has updated monitored checksums: + // - It's own checksum (monitoring itself). + // - The new checksum from replica after its sync. + // This is important to avoid race conditions. If this sync is not performed, the primary may detect the + // new checksum in the replica confusing this with the previous check. + { + const char prim_repl_sync_t[] { + "SELECT count(*) FROM stats_proxysql_servers_checksums WHERE " + "hostname='%s' AND port='%d' AND name='%s' AND checksum='%s'" + }; + cfmt_t wait_remote_chksm_syn { + cstr_format( prim_repl_sync_t, + conn_opts.host.c_str(), + conn_opts.port, + module.c_str(), + ext_checksum.str.c_str() + ) + }; + const char prim_own_sync_t[] { + "SELECT count(*) FROM stats_proxysql_servers_checksums WHERE " + "hostname='%s' AND port='%d' AND name='%s' AND checksum='%s'" + }; + cfmt_t wait_own_chksm_sync { + cstr_format( + prim_repl_sync_t, + conn_opts.host.c_str(), + conn_opts.port, + module.c_str(), + ext_checksum.str.c_str() + ) + }; + + int sync_res = wait_for_node_sync(admin, { wait_remote_chksm_syn.str }, CHECKSUM_SYNC_TIMEOUT); + ok( + sync_res == 0, + "Primary(%s:%d) has detected the new checksum '%s' in the replica(%s:%d)", + admin->host, admin->port, ext_checksum.str.c_str(), r_admin->host, r_admin->port + ); + + sync_res = wait_for_node_sync(admin, { wait_remote_chksm_syn.str }, CHECKSUM_SYNC_TIMEOUT); + ok( + sync_res == 0, + "Primary(%s:%d) has detected its own new checksum '%s'", + admin->host, admin->port, ext_checksum.str.c_str() + ); + } + return EXIT_SUCCESS; } @@ -1139,17 +1110,31 @@ int main(int, char**) { return EXIT_FAILURE; } - const size_t num_pls = module_sync_payloads.size(); + const size_t dis_mod_checks = 7; + const size_t ena_mod_checks = 5; + const size_t sync_pls = module_sync_payloads.size(); - const size_t all_mod_sync_checks = ((5+(3*(num_pls-1)))*(num_pls-1))*2 + (5+(3*(num_pls-1))); - const size_t mod_sync_checks = ((3*4*(num_pls-1)) + 3*2); - const size_t init_mod_sync_checks = (3*2*num_pls); + // check_all_modules_sync: 'ENABLED' mods sync and 'DISABLED' doesn't - REMOTE / MAIN + const size_t check_all_modules_sync__tests = dis_mod_checks + (ena_mod_checks * (sync_pls-1)); + + // check_modules_checksums_sync: All modules checksums tests + const size_t check_modules_checksums_sync__tests = + // 1: All 'ENABLED' modules sync - REMOTE / MAIN + sync_pls * ena_mod_checks * 2 + + // 2: Check all mods sync but disabled one + check_all_modules_sync__tests * sync_pls + + // 3: Re-enable module and check sync both ways + ena_mod_checks * 2 * sync_pls + + // 4: Disable module via checksums and check again + check_all_modules_sync__tests * (sync_pls - 1) + + // 5: Re-enable module and check sync both ways + ena_mod_checks * 2 * (sync_pls - 1); plan( // Sync tests by values 16 + - // Module with disabled sync checksum tests - init_mod_sync_checks + all_mod_sync_checks + mod_sync_checks + // Module checkums tests; enabled and disabled checksums + check_modules_checksums_sync__tests ); const std::string fmt_config_file = std::string(cl.workdir) + "test_cluster_sync_config/test_cluster_sync.cnf"; @@ -1188,10 +1173,9 @@ int main(int, char**) { MYSQL_QUERY(proxy_admin, "DELETE FROM proxysql_servers WHERE hostname=='127.0.0.1' AND PORT==6032"); MYSQL_QUERY(proxy_admin, "DELETE FROM proxysql_servers WHERE hostname=='127.0.0.1' AND PORT==16062"); MYSQL_QUERY(proxy_admin, "LOAD PROXYSQL SERVERS TO RUNTIME"); - MYSQL_QUERY(proxy_admin, "SELECT hostname,port FROM proxysql_servers"); - MYSQL_RES* my_res = mysql_store_result(proxy_admin); - vector core_nodes { extract_mysql_rows(my_res) }; - mysql_free_result(my_res); + + pair> nodes_fetch { fetch_cluster_nodes(proxy_admin) }; + if (nodes_fetch.first) { return EXIT_FAILURE; } // 3. Wait for all Core nodes to sync (confirm primary out of core nodes) string check_no_primary_query {}; @@ -1200,7 +1184,7 @@ int main(int, char**) { check_no_primary_query, cl.host, cl.admin_port ); - int check_res = check_nodes_sync(cl, core_nodes, check_no_primary_query, SYNC_TIMEOUT); + int check_res = check_nodes_sync(cl, nodes_fetch.second, check_no_primary_query, SYNC_TIMEOUT); if (check_res != EXIT_SUCCESS) { return EXIT_FAILURE; } // 4. Remove all current servers from primary instance (only secondary sync matters) @@ -1967,12 +1951,14 @@ int main(int, char**) { std::cout << "MASTER TABLE BEFORE SYNC:" << std::endl; system(print_master_proxysql_servers.c_str()); - int check_res = sync_checker(r_proxy_admin, select_proxysql_servers_queries, SYNC_TIMEOUT); + int wait_res = proc_wait_checks( + wait_for_conds(r_proxy_admin, select_proxysql_servers_queries, SYNC_TIMEOUT) + ); std::cout << "REPLICA TABLE AFTER SYNC:" << std::endl; system(print_replica_proxysql_servers.c_str()); - ok(check_res == EXIT_SUCCESS, "'proxysql_servers' with should be synced: '%d'", check_res); + ok(wait_res == EXIT_SUCCESS, "'proxysql_servers' with should be synced: '%d'", wait_res); // TEARDOWN CONFIG MYSQL_QUERY__(proxy_admin, "DELETE FROM proxysql_servers"); @@ -2009,10 +1995,13 @@ int main(int, char**) { system(print_master_proxysql_servers.c_str()); // 3. Check that the servers have been synced in the replica - int check_res = - sync_checker( - r_proxy_admin, { "SELECT CASE count(*) WHEN 0 THEN 1 ELSE 0 END from proxysql_servers" }, SYNC_TIMEOUT - ); + int wait_res = proc_wait_checks( + wait_for_conds( + r_proxy_admin, + { "SELECT CASE count(*) WHEN 0 THEN 1 ELSE 0 END from proxysql_servers" }, + SYNC_TIMEOUT + ) + ); std::cout << "REPLICA TABLE AFTER SYNC:" << std::endl; system(print_replica_proxysql_servers.c_str()); @@ -2028,7 +2017,7 @@ int main(int, char**) { MYSQL_QUERY__(r_proxy_admin, "DROP TABLE IF EXISTS proxysql_servers_backup"); MYSQL_QUERY__(r_proxy_admin, "LOAD PROXYSQL SERVERS TO RUNTIME"); - ok(check_res == EXIT_SUCCESS, "Empty 'proxysql_servers' table ('0x00' checksum) should be synced: '%d'", check_res); + ok(wait_res == EXIT_SUCCESS, "Empty 'proxysql_servers' table ('0x00' checksum) should be synced: '%d'", wait_res); } sleep(2); @@ -2525,7 +2514,7 @@ int main(int, char**) { // std::make_tuple("admin-cluster_username" , "" ), Known issue, can't clear // std::make_tuple("admin-cluster_password" , "" ), Known issue, can't clear // std::make_tuple("admin-debug" , "false" ), Should not be synced -// std::make_tuple("admin-hash_passwords" , "true" ), // deprecated variable + // std::make_tuple("admin-hash_passwords" , "true" ), // deprecated variable // std::make_tuple("admin-mysql_ifaces" , "0.0.0.0:6032" ), // disabled because of cluster_sync_interfaces=false std::make_tuple("admin-prometheus_memory_metrics_interval" , "61" ), std::make_tuple("admin-read_only" , "false" ), @@ -2715,14 +2704,16 @@ cleanup: insert_query, cl.host, cl.admin_port ); - for (const auto& row : core_nodes) { - const string host { row[0] }; - const int port = std::stol(row[1]); + for (const auto& node : nodes_fetch.second) { MYSQL* c_node_admin = mysql_init(NULL); - diag("RESTORING: Inserting into node '%s:%d'", host.c_str(), port); + diag("RESTORING: Inserting into node '%s:%d'", node.host.c_str(), node.port); - if (!mysql_real_connect(c_node_admin, host.c_str(), cl.admin_username, cl.admin_password, NULL, port, NULL, 0)) { + if ( + !mysql_real_connect( + c_node_admin, node.host.c_str(), cl.admin_username, cl.admin_password, NULL, node.port, NULL, 0 + ) + ) { const string err_msg { "Connection to core node failed with '" + string { mysql_error(c_node_admin) } + "'. Retrying..." }; @@ -2751,7 +2742,7 @@ cleanup: ); // Wait for the other nodes to sync ProxySQL servers to include Primary - int check_res = check_nodes_sync(cl, core_nodes, check_no_primary_query, SYNC_TIMEOUT); + int check_res = check_nodes_sync(cl, nodes_fetch.second, check_no_primary_query, SYNC_TIMEOUT); if (check_res != EXIT_SUCCESS) { return EXIT_FAILURE; } // Recover the old ProxySQL servers from backup in primary diff --git a/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp b/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp index 4a84270c2..6b4c8f4f9 100644 --- a/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp +++ b/test/tap/tests/test_cluster_sync_mysql_servers-t.cpp @@ -24,20 +24,16 @@ */ #include -#include #include #include #include -#include #include #include #include #include #include -#include #include -#include #include #include "libconfig.h" @@ -53,7 +49,9 @@ #include "command_line.h" #include "utils.h" +using std::pair; using std::string; +using std::vector; #define MYSQL_QUERY__(mysql, query) \ do { \ @@ -132,83 +130,6 @@ uint64_t mysql_servers_raw_checksum(MYSQL_RES* resultset) { return res_hash; } - -/** - * @brief Helper function to verify that the sync of a table (or variable) have been performed. - * - * @param r_proxy_admin An already opened connection to ProxySQL. - * @param queries Queries to be executed that should return a **non-zero** value after the sync has taken place. - * @param sync_timeout Timeout for the sync to happen. - * - * @return EXIT_SUCCESS in case of success, otherwise: - * - '-1' if a query against Admin fails to be performed (failure is logged). - * - '-2' if timeout expired without sync being completed. - */ -int sync_checker(MYSQL* r_proxy_admin, const std::vector& queries, uint32_t sync_timeout) { - bool not_synced_query = false; - uint waited = 0; - - while (waited < sync_timeout) { - not_synced_query = false; - - // Check that all the entries have been synced - for (const auto& query : queries) { - int q_res = mysql_query(r_proxy_admin, query.c_str()); - if (q_res != EXIT_SUCCESS) { - fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(r_proxy_admin)); - return -1; - } - - MYSQL_RES* proxysql_servers_res = mysql_store_result(r_proxy_admin); - MYSQL_ROW row = mysql_fetch_row(proxysql_servers_res); - int row_value = atoi(row[0]); - mysql_free_result(proxysql_servers_res); - - if (row_value == 0) { - not_synced_query = true; - break; - } - } - - if (not_synced_query) { - waited += 1; - sleep(1); - } else { - break; - } - } - - if (not_synced_query) { - return -2; - } else { - return EXIT_SUCCESS; - } -} - -int check_nodes_sync( - const CommandLine& cl, const std::vector& core_nodes, const std::string& check_query, uint32_t sync_timeout -) { - for (const auto& node : core_nodes) { - const std::string host { node[0] }; - const int port = std::stol(node[1]); - - MYSQL* c_node_admin = mysql_init(NULL); - if (!mysql_real_connect(c_node_admin, host.c_str(), cl.admin_username, cl.admin_password, NULL, port, NULL, 0)) { - fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(c_node_admin)); - return EXIT_FAILURE; - } - - int not_synced = sync_checker(c_node_admin, { check_query }, sync_timeout); - if (not_synced != EXIT_SUCCESS) { - const std::string err_msg { "Node '" + host + ":" + std::to_string(port) + "' sync timed out" }; - fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, err_msg.c_str()); - return EXIT_FAILURE; - } - } - - return EXIT_SUCCESS; -} - int insert_mysql_servers_records(MYSQL* proxy_admin, const std::vector& insert_mysql_servers_values, const std::vector& insert_replication_hostgroups_values) { @@ -860,15 +781,15 @@ int main(int, char**) { // 2. Remove primary from Core nodes MYSQL_QUERY(proxy_admin, "DELETE FROM proxysql_servers WHERE hostname=='127.0.0.1' AND PORT==6032"); MYSQL_QUERY(proxy_admin, "LOAD PROXYSQL SERVERS TO RUNTIME"); - MYSQL_QUERY(proxy_admin, "SELECT hostname,port FROM proxysql_servers"); - MYSQL_RES* my_res = mysql_store_result(proxy_admin); - std::vector core_nodes { extract_mysql_rows(my_res) }; - mysql_free_result(my_res); + + pair> core_nodes { fetch_cluster_nodes(proxy_admin) }; + if (core_nodes.first) { return EXIT_FAILURE; } // 2.1 If core nodes are not reachable, assume no cluster is running; make test gracefully exit - if (core_nodes.size()) { - const string host { core_nodes[0][0] }; - const int port = std::stol(core_nodes[0][1]); + if (core_nodes.second.size()) { + const string host { core_nodes.second[0].host }; + const int port { core_nodes.second[0].port }; + MYSQL* c_node_admin = mysql_init(NULL); if (!mysql_real_connect(c_node_admin, host.c_str(), cl.admin_username, cl.admin_password, NULL, port, NULL, 0)) { @@ -894,7 +815,7 @@ int main(int, char**) { check_no_primary_query, cl.host, cl.admin_port ); - int check_res = check_nodes_sync(cl, core_nodes, check_no_primary_query, SYNC_TIMEOUT); + int check_res = check_nodes_sync(cl, core_nodes.second, check_no_primary_query, SYNC_TIMEOUT); if (check_res != EXIT_SUCCESS) { return EXIT_FAILURE; } // 4. Remove all current servers from primary instance (only secondary sync matters) @@ -1048,9 +969,9 @@ cleanup: insert_query, cl.host, cl.admin_port ); - for (const auto& row : core_nodes) { - const std::string host { row[0] }; - const int port = std::stol(row[1]); + for (const auto& node : core_nodes.second) { + const std::string host { node.host }; + const int port = node.port; MYSQL* c_node_admin = mysql_init(NULL); diag("RESTORING: Inserting into node '%s:%d'", host.c_str(), port); @@ -1084,7 +1005,7 @@ cleanup: ); // Wait for the other nodes to sync ProxySQL servers to include Primary - int check_res = check_nodes_sync(cl, core_nodes, check_no_primary_query, SYNC_TIMEOUT); + int check_res = check_nodes_sync(cl, core_nodes.second, check_no_primary_query, SYNC_TIMEOUT); if (check_res != EXIT_SUCCESS) { return EXIT_FAILURE; } // Recover the old ProxySQL servers from backup in primary diff --git a/test/tap/tests/test_prometheus_metrics-t.cpp b/test/tap/tests/test_prometheus_metrics-t.cpp index a9bc27df6..82545cf6a 100644 --- a/test/tap/tests/test_prometheus_metrics-t.cpp +++ b/test/tap/tests/test_prometheus_metrics-t.cpp @@ -39,35 +39,6 @@ int mysql_query_d(MYSQL* mysql, const char* query) { return mysql_query(mysql, query); } -/** - * @brief Extract the metrics values from the output of the admin command - * 'SHOW PROMETHEUS METRICS'. - * @param metrics_output The output of the command 'SHOW PROMETHEUS METRICS'. - * @return A map holding the metrics identifier and its current value. - */ -std::map get_metric_values(std::string metrics_output) { - std::vector output_lines { split(metrics_output, '\n') }; - std::map metrics_map {}; - - for (const std::string line : output_lines) { - const std::vector line_values { split(line, ' ') }; - - if (line.empty() == false && line[0] != '#') { - if (line_values.size() > 2) { - size_t delim_pos_st = line.rfind("} "); - string metric_key = line.substr(0, delim_pos_st); - string metric_val = line.substr(delim_pos_st + 2); - - metrics_map.insert({metric_key, std::stod(metric_val)}); - } else { - metrics_map.insert({line_values.front(), std::stod(line_values.back())}); - } - } - } - - return metrics_map; -} - int get_cur_metrics(MYSQL* admin, map& metrics_vals) { MYSQL_QUERY(admin, "SHOW PROMETHEUS METRICS\\G"); MYSQL_RES* p_resulset = mysql_store_result(admin); @@ -81,7 +52,7 @@ int get_cur_metrics(MYSQL* admin, map& metrics_vals) { } mysql_free_result(p_resulset); - metrics_vals = get_metric_values(row_value); + metrics_vals = parse_prometheus_metrics(row_value); return EXIT_SUCCESS; } diff --git a/test/tap/tests/test_read_only_actions_offline_hard_servers-t.cpp b/test/tap/tests/test_read_only_actions_offline_hard_servers-t.cpp index 90380e20f..7ab4af208 100644 --- a/test/tap/tests/test_read_only_actions_offline_hard_servers-t.cpp +++ b/test/tap/tests/test_read_only_actions_offline_hard_servers-t.cpp @@ -1,8 +1,8 @@ - #include -#include #include #include +#include +#include #include "mysql.h" #include "tap.h" @@ -15,6 +15,9 @@ //#define BACKEND_SERVER_USER "root" //#define BACKEND_SERVER_PASS "root" +using std::vector; +using std::pair; + #define MYSQL_QUERY__(mysql, query) \ do { \ if (mysql_query(mysql, query)) { \ @@ -49,87 +52,6 @@ __exit: return mysql; } -/** - * @brief Helper function to verify that the sync of a table (or variable) have been performed. - * - * @param r_proxy_admin An already opened connection to ProxySQL. - * @param queries Queries to be executed that should return a **non-zero** value after the sync has taken place. - * @param sync_timeout Timeout for the sync to happen. - * - * @return EXIT_SUCCESS in case of success, otherwise: - * - '-1' if a query against Admin fails to be performed (failure is logged). - * - '-2' if timeout expired without sync being completed. - */ -int sync_checker(MYSQL* r_proxy_admin, const std::vector& queries, uint32_t sync_timeout) { - bool not_synced_query = false; - uint waited = 0; - - while (waited < sync_timeout) { - not_synced_query = false; - - // Check that all the entries have been synced - for (const auto& query : queries) { - int q_res = mysql_query(r_proxy_admin, query.c_str()); - if (q_res != EXIT_SUCCESS) { - fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(r_proxy_admin)); - return -1; - } - - MYSQL_RES* proxysql_servers_res = mysql_store_result(r_proxy_admin); - MYSQL_ROW row = mysql_fetch_row(proxysql_servers_res); - int row_value = atoi(row[0]); - mysql_free_result(proxysql_servers_res); - - if (row_value == 0) { - not_synced_query = true; - break; - } - } - - if (not_synced_query) { - waited += 1; - sleep(1); - } else { - break; - } - } - - if (not_synced_query) { - return -2; - } else { - return EXIT_SUCCESS; - } -} - -int check_nodes_sync( - const CommandLine& cl, const std::vector& core_nodes, const std::string& check_query, uint32_t sync_timeout -) { - int ret_status = EXIT_FAILURE; - - for (const auto& node : core_nodes) { - const std::string host { node[0] }; - const int port = std::stol(node[1]); - - MYSQL* c_node_admin = mysql_init(NULL); - if (!mysql_real_connect(c_node_admin, host.c_str(), cl.admin_username, cl.admin_password, NULL, port, NULL, 0)) { - fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(c_node_admin)); - goto __exit; - } - - int not_synced = sync_checker(c_node_admin, { check_query }, sync_timeout); - if (not_synced != EXIT_SUCCESS) { - const std::string err_msg { "Node '" + host + ":" + std::to_string(port) + "' sync timed out" }; - fprintf(stderr, "File %s, line %d, Error: `%s`\n", __FILE__, __LINE__, err_msg.c_str()); - goto __exit; - } - } - - ret_status = EXIT_SUCCESS; - -__exit: - return ret_status; -} - int insert_mysql_servers_records(MYSQL* proxy_admin, const std::vector& insert_mysql_servers_values, const std::vector& insert_replication_hostgroups_values) { @@ -535,7 +457,7 @@ cleanup: int test_read_only_offline_hard_servers(MYSQL* proxy_admin, const CommandLine& cl, bool isolate_primary_node) { - std::vector core_nodes; + pair> core_nodes; std::string check_no_primary_query; if (isolate_primary_node) { @@ -553,10 +475,9 @@ int test_read_only_offline_hard_servers(MYSQL* proxy_admin, const CommandLine& c // 2. Remove primary from Core nodes MYSQL_QUERY__(proxy_admin, "DELETE FROM proxysql_servers WHERE hostname=='127.0.0.1' AND PORT==6032"); MYSQL_QUERY__(proxy_admin, "LOAD PROXYSQL SERVERS TO RUNTIME"); - MYSQL_QUERY__(proxy_admin, "SELECT hostname,port FROM proxysql_servers"); - MYSQL_RES* my_res = mysql_store_result(proxy_admin); - core_nodes = { extract_mysql_rows(my_res) }; - mysql_free_result(my_res); + + core_nodes = fetch_cluster_nodes(proxy_admin); + if (core_nodes.first) { goto cleanup; } // 3. Wait for all Core nodes to sync (confirm primary out of core nodes) string_format( @@ -564,7 +485,7 @@ int test_read_only_offline_hard_servers(MYSQL* proxy_admin, const CommandLine& c check_no_primary_query, cl.host, cl.admin_port ); - int check_res = check_nodes_sync(cl, core_nodes, check_no_primary_query, SYNC_TIMEOUT); + int check_res = check_nodes_sync(cl, core_nodes.second, check_no_primary_query, SYNC_TIMEOUT); if (check_res != EXIT_SUCCESS) { goto cleanup; } @@ -601,9 +522,9 @@ cleanup: insert_query, cl.host, cl.admin_port ); - for (const auto& row : core_nodes) { - const std::string host{ row[0] }; - const int port = std::stol(row[1]); + for (const auto& row : core_nodes.second) { + const std::string host{ row.host }; + const int port = row.port; MYSQL* c_node_admin = mysql_init(NULL); diag("RESTORING: Inserting into node '%s:%d'", host.c_str(), port); @@ -637,7 +558,7 @@ cleanup: ); // Wait for the other nodes to sync ProxySQL servers to include Primary - int check_res = check_nodes_sync(cl, core_nodes, check_no_primary_query, SYNC_TIMEOUT); + int check_res = check_nodes_sync(cl, core_nodes.second, check_no_primary_query, SYNC_TIMEOUT); if (check_res != EXIT_SUCCESS) { return EXIT_FAILURE; } // Recover the old ProxySQL servers from backup in primary diff --git a/test/tap/tests/test_session_status_flags-t.cpp b/test/tap/tests/test_session_status_flags-t.cpp index 181c2561b..2849fa49e 100644 --- a/test/tap/tests/test_session_status_flags-t.cpp +++ b/test/tap/tests/test_session_status_flags-t.cpp @@ -172,7 +172,7 @@ int prepare_stmt_queries(const CommandLine& cl, const vector& p_queries // 1. Prepare the stmt in a connection MYSQL* proxy_mysql = mysql_init(NULL); - diag("%s: Openning INITIAL connection...", tap_curtime().c_str()); + diag("Openning INITIAL connection..."); if (!mysql_real_connect(proxy_mysql, cl.root_host, cl.root_username, cl.root_password, NULL, cl.root_port, NULL, 0)) { fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxy_mysql)); return EXIT_FAILURE; @@ -216,7 +216,7 @@ int prepare_stmt_queries(const CommandLine& cl, const vector& p_queries return EXIT_FAILURE; } - diag("%s: Issuing PREPARE for `%s` in INIT conn", tap_curtime().c_str(), query.c_str()); + diag("Issuing PREPARE for `%s` in INIT conn", query.c_str()); int my_err = mysql_stmt_prepare(stmt, query.c_str(), strlen(query.c_str())); if (my_err) { diag( @@ -229,7 +229,7 @@ int prepare_stmt_queries(const CommandLine& cl, const vector& p_queries mysql_stmt_close(stmt); } - diag("%s: Closing PREPARING connection...", tap_curtime().c_str()); + diag("Closing PREPARING connection..."); mysql_close(proxy_mysql); return EXIT_SUCCESS; @@ -326,7 +326,7 @@ int exec_stmt_queries(MYSQL* proxy_mysql, const vector& test_queries) { } else { MYSQL_STMT* stmt = mysql_stmt_init(proxy_mysql); - diag("%s: Issuing PREPARE for `%s` in new conn", tap_curtime().c_str(), query.c_str()); + diag("Issuing PREPARE for `%s` in new conn", query.c_str()); int my_err = mysql_stmt_prepare(stmt, query.c_str(), strlen(query.c_str())); if (my_err) { diag( @@ -339,7 +339,7 @@ int exec_stmt_queries(MYSQL* proxy_mysql, const vector& test_queries) { // TODO: Remember to DOC requiring to execute { if (rep_check.first == 0 || rep_check.second == 0) { - diag("%s: Issuing EXECUTE for `%s` in new conn", tap_curtime().c_str(), query.c_str()); + diag("Issuing EXECUTE for `%s` in new conn", query.c_str()); my_err = mysql_stmt_execute(stmt); if (my_err) { diag("'mysql_stmt_execute' at line %d failed: %s", __LINE__, mysql_stmt_error(stmt)); diff --git a/test/tap/tests/test_thread_conn_dist-t.cpp b/test/tap/tests/test_thread_conn_dist-t.cpp new file mode 100644 index 000000000..047498edc --- /dev/null +++ b/test/tap/tests/test_thread_conn_dist-t.cpp @@ -0,0 +1,132 @@ +#include +#include +#include +#include +#include + +#include +#include + +#include "tap.h" +#include "command_line.h" +#include "utils.h" + +#include "mysql.h" +#include "json.hpp" + +using std::map; +using std::pair; +using std::string; +using std::vector; + +using nlohmann::json; + +#define TAP_NAME "TAP_THREAD_CONN_DIST___" + +const int TEST_DURATION_SEC = get_env_int(TAP_NAME"TEST_DURATION_SEC", 20); +const int ITER_CONN_COUNT = get_env_int(TAP_NAME"ITER_CONN_COUNT", 256); + +void incr_proc_limits(uint32_t MAX_CONN_COUNT) { + diag("Elevating process limits if required for conns creation"); + + struct rlimit limits { 0, 0 }; + getrlimit(RLIMIT_NOFILE, &limits); + diag("Old process limits rlim_cur=%ld rlim_max=%ld", limits.rlim_cur, limits.rlim_max); + + if (limits.rlim_cur < MAX_CONN_COUNT * 2) { + diag("Updating process max FD limit"); + limits.rlim_cur = MAX_CONN_COUNT * 2; + setrlimit(RLIMIT_NOFILE, &limits); + } + + diag("New process limits rlim_cur=%ld rlim_max=%ld", limits.rlim_cur, limits.rlim_max); +} + +pair> create_frontend_conns(CommandLine& cl, uint32_t CONNS_TOTAL) { + vector conns {}; + + for (int i = 0; i < CONNS_TOTAL; i++) { + MYSQL* myconn = mysql_init(NULL); + + if (!mysql_real_connect(myconn, cl.host, cl.username, cl.password, NULL, cl.port, NULL, 0)) { + diag( + "Failed to connect addr=%s port=%d user=%s pass=%s err=%s", + cl.host, cl.port, cl.username, cl.password, mysql_error(myconn) + ); + return { EXIT_FAILURE, {} }; + } + + conns.push_back(myconn); + } + + return { EXIT_SUCCESS, conns }; +} + +void check_thread_conn_dist(const map>& m_thread_conns) { + size_t lo_count = 0; + size_t hg_count = 0; + + diag("Dumping per-thread conn count:"); + for (const pair>& thread_conns : m_thread_conns) { + if (lo_count == 0 || thread_conns.second.size() < lo_count) { + lo_count = thread_conns.second.size(); + } + if (hg_count == 0 || thread_conns.second.size() > hg_count) { + hg_count = thread_conns.second.size(); + } + fprintf(stderr, "Map entry thread=%s count=%ld\n", thread_conns.first.c_str(), thread_conns.second.size()); + } + + ok( + hg_count / 2 < lo_count, + "Half the highest conn count shouldn't be higher than lowest conn count" + " hg_count=%ld lo_count=%ld", + hg_count, lo_count + ); +} + +void update_conn_thread_map(vector& conns, map>& m_thread_conns) { + for (MYSQL* myconn : conns) { + json j_session = fetch_internal_session(myconn, false); + string thread_addr { j_session["thread"] }; + + m_thread_conns[thread_addr].push_back(myconn); + } +} + +int main(int argc, char** argv) { + CommandLine cl; + + if (cl.getEnv()) { + diag("Failed to get the required environmental variables."); + return EXIT_FAILURE; + } + + plan(1); + + auto start = std::chrono::system_clock::now(); + std::chrono::duration elapsed {}; + + incr_proc_limits(ITER_CONN_COUNT); + + map> m_thread_conns {}; + + while (elapsed.count() < TEST_DURATION_SEC) { + pair> p_err_conns { create_frontend_conns(cl, ITER_CONN_COUNT) }; + if (p_err_conns.first) { + diag("Frontend conn creation failed; aborting further testing err=%d", p_err_conns.first); + return EXIT_FAILURE; + } + + update_conn_thread_map(p_err_conns.second, m_thread_conns); + + for (MYSQL* conn : p_err_conns.second) { + mysql_close(conn); + } + + auto it_end = std::chrono::system_clock::now(); + elapsed = it_end - start; + } + + check_thread_conn_dist(m_thread_conns); +} diff --git a/test/tap/tests/test_unshun_algorithm-t.cpp b/test/tap/tests/test_unshun_algorithm-t.cpp index 4a5714eaa..7b424a7fd 100644 --- a/test/tap/tests/test_unshun_algorithm-t.cpp +++ b/test/tap/tests/test_unshun_algorithm-t.cpp @@ -59,12 +59,12 @@ #include #include -#include #include #include +#include +#include #include "mysql.h" -#include "mysqld_error.h" #include "proxysql_utils.h" #include "tap.h" @@ -74,14 +74,17 @@ const uint32_t SHUN_RECOVERY_TIME = 1; const uint32_t VALID_RANGE = 1; const uint32_t SERVERS_COUNT = 10; +const uint32_t SYNC_TIMEOUT = 10; +using std::pair; using std::string; +using std::vector; int shunn_server(MYSQL* proxysql_admin, uint32_t i, uint32_t j) { std::string t_simulator_error_query { "PROXYSQL_SIMULATOR mysql_error %d 127.0.0.1:330%d 1234" }; std::string simulator_error_q_i {}; string_format(t_simulator_error_query, simulator_error_q_i, i, j); - diag("%s: running query: %s", tap_curtime().c_str(), simulator_error_q_i.c_str()); + diag("running query: %s", simulator_error_q_i.c_str()); MYSQL_QUERY(proxysql_admin, simulator_error_q_i.c_str()); return EXIT_SUCCESS; @@ -113,7 +116,7 @@ int wakup_target_server(MYSQL* proxysql_mysql, uint32_t i) { string_format(t_simple_do_query, simple_do_query, i); mysql_query(proxysql_mysql, simple_do_query.c_str()); - diag("%s: running query: %s", tap_curtime().c_str(), simple_do_query.c_str()); + diag("running query: %s", simple_do_query.c_str()); return EXIT_SUCCESS; } @@ -124,7 +127,7 @@ int server_status_checker(MYSQL* admin, const string& f_st, const string& n_st, }; std::string server_status_query {}; string_format(t_server_status_query, server_status_query, i); - diag("%s: running query: %s", tap_curtime().c_str(), server_status_query.c_str()); + diag("running query: %s", server_status_query.c_str()); MYSQL_QUERY(admin, server_status_query.c_str()); MYSQL_RES* status_res = mysql_store_result(admin); @@ -190,9 +193,9 @@ int test_unshun_algorithm_variable(MYSQL* proxysql_admin) { }; MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES FROM DISK"); - diag("%s: Line:%d running admin query to reload variables: LOAD MYSQL VARIABLES FROM DISK", tap_curtime().c_str(), __LINE__); + diag("Line:%d running admin query to reload variables: LOAD MYSQL VARIABLES FROM DISK", __LINE__); MYSQL_QUERY(proxysql_admin, "SET mysql-hostgroup_manager_verbose=3"); - diag("%s: Line:%d running admin query: SET mysql-hostgroup_manager_verbose=3", tap_curtime().c_str(), __LINE__); + diag("Line:%d running admin query: SET mysql-hostgroup_manager_verbose=3", __LINE__); MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); int32_t def_unshun_value = get_current_unshun_algorithm_val(proxysql_admin); ok(def_unshun_value == 0, "Default 'mysql-unshun_algorithm' should be '0', actual: %d", def_unshun_value); @@ -203,7 +206,7 @@ int test_unshun_algorithm_variable(MYSQL* proxysql_admin) { std::string set_unshun {}; string_format(t_set_unshun, set_unshun, i); MYSQL_QUERY(proxysql_admin, set_unshun.c_str()); - diag("%s: Line:%d running admin query: %s", tap_curtime().c_str(), __LINE__, set_unshun.c_str()); + diag("Line:%d running admin query: %s", __LINE__, set_unshun.c_str()); MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); int32_t cur_unshun_val = get_current_unshun_algorithm_val(proxysql_admin); @@ -214,7 +217,7 @@ int test_unshun_algorithm_variable(MYSQL* proxysql_admin) { std::string set_unshun {}; string_format(t_set_unshun, set_unshun, VALID_RANGE + 1); MYSQL_QUERY(proxysql_admin, set_unshun.c_str()); - diag("%s: Line:%d running admin query: %s", tap_curtime().c_str(), __LINE__, set_unshun.c_str()); + diag("Line:%d running admin query: %s", __LINE__, set_unshun.c_str()); MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); int32_t cur_unshun_val = get_current_unshun_algorithm_val(proxysql_admin); @@ -273,13 +276,13 @@ int test_proxysql_simulator_error(MYSQL* proxysql_admin) { */ int configure_mysql_shunning_variables(MYSQL* proxysql_admin) { MYSQL_QUERY(proxysql_admin, "SET mysql-shun_on_failures=3"); - diag("%s: Line:%d running admin query: SET mysql-shun_on_failures=3", tap_curtime().c_str(), __LINE__); + diag("Line:%d running admin query: SET mysql-shun_on_failures=3", __LINE__); MYSQL_QUERY(proxysql_admin, "SET mysql-connect_retries_on_failure=3"); - diag("%s: Line:%d running admin query: SET mysql-connect_retries_on_failure=3", tap_curtime().c_str(), __LINE__); + diag("Line:%d running admin query: SET mysql-connect_retries_on_failure=3", __LINE__); MYSQL_QUERY(proxysql_admin, "SET mysql-connect_retries_delay=1000"); - diag("%s: Line:%d running admin query: SET mysql-connect_retries_delay=1000", tap_curtime().c_str(), __LINE__); + diag("Line:%d running admin query: SET mysql-connect_retries_delay=1000", __LINE__); return EXIT_SUCCESS; } @@ -287,11 +290,11 @@ int configure_mysql_shunning_variables(MYSQL* proxysql_admin) { int test_unshun_algorithm_behavior(MYSQL* proxysql_mysql, MYSQL* proxysql_admin) { // Configure Admin variables with lower thresholds MYSQL_QUERY(proxysql_admin, "SET mysql-shun_recovery_time_sec=1"); - diag("%s: Line:%d running admin query: SET mysql-shun_recovery_time_sec=1", tap_curtime().c_str(), __LINE__); + diag("Line:%d running admin query: SET mysql-shun_recovery_time_sec=1", __LINE__); // Set verbosity up for extra information in ProxySQL log MYSQL_QUERY(proxysql_admin, "SET mysql-hostgroup_manager_verbose=3"); - diag("%s: Line:%d running admin query: SET mysql-hostgroup_manager_verbose=3", tap_curtime().c_str(), __LINE__); + diag("Line:%d running admin query: SET mysql-hostgroup_manager_verbose=3", __LINE__); // Configure the relevant variables for the desired UNSHUNNING behavior if (configure_mysql_shunning_variables(proxysql_admin)) { @@ -324,7 +327,7 @@ int test_unshun_algorithm_behavior(MYSQL* proxysql_mysql, MYSQL* proxysql_admin) { MYSQL_QUERY(proxysql_admin, "SET mysql-unshun_algorithm=0"); - diag("%s: Line:%d running admin query: SET mysql-unshun_algorithm=0", tap_curtime().c_str(), __LINE__); + diag("Line:%d running admin query: SET mysql-unshun_algorithm=0", __LINE__); MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); int shunn_err = shunn_all_servers(proxysql_admin); @@ -344,7 +347,7 @@ int test_unshun_algorithm_behavior(MYSQL* proxysql_mysql, MYSQL* proxysql_admin) { MYSQL_QUERY(proxysql_admin, "SET mysql-unshun_algorithm=1"); - diag("%s: Line:%d running admin query: SET mysql-unshun_algorithm=1", tap_curtime().c_str(), __LINE__); + diag("Line:%d running admin query: SET mysql-unshun_algorithm=1", __LINE__); MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); int shunn_err = shunn_all_servers(proxysql_admin); @@ -364,7 +367,7 @@ int test_unshun_algorithm_behavior(MYSQL* proxysql_mysql, MYSQL* proxysql_admin) { MYSQL_QUERY(proxysql_admin, "SET mysql-unshun_algorithm=0"); - diag("%s: Line:%d running admin query: SET mysql-unshun_algorithm=0", tap_curtime().c_str(), __LINE__); + diag("Line:%d running admin query: SET mysql-unshun_algorithm=0", __LINE__); MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); int shunn_err = shunn_all_servers(proxysql_admin); @@ -376,7 +379,7 @@ int test_unshun_algorithm_behavior(MYSQL* proxysql_mysql, MYSQL* proxysql_admin) diag(" "); // empty line MYSQL_QUERY(proxysql_admin, "SET mysql-unshun_algorithm=1"); - diag("%s: Line:%d running admin query: SET mysql-unshun_algorithm=1", tap_curtime().c_str(), __LINE__); + diag("Line:%d running admin query: SET mysql-unshun_algorithm=1", __LINE__); MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); for (uint32_t i = 0; i < SERVERS_COUNT; i++) { @@ -426,8 +429,43 @@ int main(int argc, char** argv) { } // Disable Monitor for the following tests - MYSQL_QUERY(proxysql_admin, "SET mysql-monitor_enabled=0"); - MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + MYSQL_QUERY_T(proxysql_admin, "SET mysql-monitor_enabled=0"); + MYSQL_QUERY_T(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + // We need to wait for synchronization in case 'admin-cluster_mysql_servers_sync_algorithm=1' to prevent + // circular fetches. If config is set to '2' we DO NOT wait, as circular fetches shouldn't be possible, + // this way a misbehavior **could** be detected by the test. We do not enforce failure either. + { + const char sync_algo_query[] { + "SELECT variable_value FROM global_variables" + " WHERE variable_name='admin-cluster_mysql_servers_sync_algorithm'", + }; + ext_val_t servers_sync_algo { mysql_query_ext_val(proxysql_admin, sync_algo_query, 1) }; + if (servers_sync_algo.err != EXIT_SUCCESS) { + const string err { get_ext_val_err(proxysql_admin, servers_sync_algo) }; + diag("Failed getting variable query:`%s`, err:`%s`", sync_algo_query, err.c_str()); + goto cleanup; + } + + // NOTE: '3' is a configuration thought for replicas, this test assumes it's connecting to a + // primary, so the same protection should be applied. + if (servers_sync_algo.val == 1 || servers_sync_algo.val == 3) { + const pair> core_nodes { fetch_cluster_nodes(proxysql_admin) }; + if (core_nodes.first) { + diag("Cluster node fetching failed with mysql_errno=%d", core_nodes.first); + goto cleanup; + } + + const char sync_query[] { + "SELECT CASE WHEN " + "(SELECT variable_value FROM runtime_global_variables WHERE" + " variable_name='mysql-monitor_enabled')='false'" + " THEN 1 ELSE 0 END" + }; + int check_res = check_nodes_sync(cl, core_nodes.second, sync_query, SYNC_TIMEOUT); + if (check_res != EXIT_SUCCESS) { goto cleanup; } + } + } { int simulator_err = test_proxysql_simulator_error(proxysql_admin);