From e2dd0a30cd79700f7da3ced1ecdaf2c1387006c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Tue, 10 Aug 2021 21:00:06 +0200 Subject: [PATCH 01/10] Initial implementation for 'client_error_limit' Add initial support for FR limiting the number of connection errors that can be initiated from a particular address before deniying future connections from that address. Feature is controlled by new introduced variables: - mysql-client_host_error_counts - mysql-client_host_cache_size --- include/MySQL_Thread.h | 80 ++++++++++++++++++++ include/proxysql_structs.h | 4 + lib/MySQL_Session.cpp | 7 ++ lib/MySQL_Thread.cpp | 147 ++++++++++++++++++++++++++++++++++++- lib/ProxySQL_Admin.cpp | 11 +++ 5 files changed, 247 insertions(+), 2 deletions(-) diff --git a/include/MySQL_Thread.h b/include/MySQL_Thread.h index 4afa51740..7b9f7d015 100644 --- a/include/MySQL_Thread.h +++ b/include/MySQL_Thread.h @@ -118,6 +118,7 @@ enum MySQL_Thread_status_variable { st_var_aws_aurora_replicas_skipped_during_query, st_var_automatic_detected_sqli, st_var_whitelisted_sqli_fingerprint, + st_var_client_host_error_killed_connections, st_var_END }; @@ -320,6 +321,7 @@ struct p_th_counter { whitelisted_sqli_fingerprint, mysql_killed_backend_connections, mysql_killed_backend_queries, + client_host_error_killed_connections, __size }; }; @@ -359,6 +361,20 @@ struct th_metrics_map_idx { }; }; +/** + * @brief Structure holding the data for a Client_Host_Cache entry. + */ +typedef struct _MySQL_Client_Host_Cache_Entry { + /** + * @brief Last time the entry was updated. + */ + uint64_t updated_at; + /** + * @brief Error count associated with the entry. + */ + uint32_t error_count; +} MySQL_Client_Host_Cache_Entry; + class MySQL_Threads_Handler { private: @@ -382,6 +398,22 @@ class MySQL_Threads_Handler // variable address // special variable : if true, further input validation is required std::unordered_map> VariablesPointers_bool; + /** + * @brief Holds the clients host cache. It keeps track of the number of + * errors associated to a specific client: + * - Key: client identifier, based on 'clientaddr'. + * - Value: Structure of type 'MySQL_Client_Host_Cache_Entry' holding + * the last time the entry was updated and the error count associated + * with the client. + */ + std::unordered_map client_host_cache; + /** + * @brief Holds the mutex for accessing 'client_host_cache', since every + * access can potentially perform 'read/write' operations, a regular mutex + * is enough. + */ + pthread_mutex_t mutex_client_host_cache; + public: struct { int monitor_history; @@ -429,6 +461,8 @@ class MySQL_Threads_Handler int query_retries_on_failure; bool client_multi_statements; bool connection_warming; + int client_host_cache_size; + int client_host_error_counts; int connect_retries_on_failure; int connect_retries_delay; int connection_delay_multiplex_ms; @@ -545,6 +579,52 @@ class MySQL_Threads_Handler std::array p_counter_array {}; std::array p_gauge_array {}; } status_variables; + /** + * @brief Update the client host cache with the supplied 'client_sockaddr', + * and the supplied 'error' parameter specifying if there was a connection + * error or not. + * + * NOTE: This function is not safe, the supplied 'client_sockaddr' should + * have been initialized by 'accept' or 'getpeername'. NULL checks are not + * performed. + * + * @details The 'client_sockaddr' parameter is inspected, and the + * 'client_host_cache' map is only updated in case of: + * - 'address_family' is either 'AF_INET' or 'AF_INET6'. + * - The address obtained from it isn't '127.0.0.1'. + * + * @param client_sockaddr A 'sockaddr' holding the required client information + * to update the 'client_host_cache_map'. + * @param error 'true' if there was an error in the connection that should be + * register, 'false' otherwise. + */ + void update_client_host_cache(struct sockaddr* client_sockaddr, bool error); + /** + * @brief Retrieves the entry of the underlying 'client_host_cache' map for + * the supplied 'client_sockaddr' in case of existing. In case it doesn't + * exist or the supplied 'client_sockaddr' doesn't met the requirements + * for being registered in the map, and zeroed 'MySQL_Client_Host_Cache_Entry' + * is returned. + * + * NOTE: This function is not safe, the supplied 'client_sockaddr' should + * have been initialized by 'accept' or 'getpeername'. NULL checks are not + * performed. + * + * @details The 'client_sockaddr' parameter is inspected, and the + * 'client_host_cache' map is only searched in case of: + * - 'address_family' is either 'AF_INET' or 'AF_INET6'. + * - The address obtained from it isn't '127.0.0.1'. + * + * @param client_sockaddr A 'sockaddr' holding the required client information + * to update the 'client_host_cache_map'. + * @return If found, the corresponding entry for the supplied 'client_sockaddr', + * a zeroed 'MySQL_Client_Host_Cache_Entry' otherwise. + */ + MySQL_Client_Host_Cache_Entry find_client_host_cache(struct sockaddr* client_sockaddr); + /** + * @brief Delete all the entries in the 'client_host_cache' internal map. + */ + void flush_client_host_cache(); /** * @brief Callback to update the metrics. */ diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index 57a47eb81..6b08ec1f2 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -799,6 +799,8 @@ __thread bool mysql_thread___enable_client_deprecate_eof; __thread bool mysql_thread___enable_server_deprecate_eof; __thread bool mysql_thread___log_mysql_warnings_enabled; __thread bool mysql_thread___enable_load_data_local_infile; +__thread int mysql_thread___client_host_cache_size; +__thread int mysql_thread___client_host_error_counts; /* variables used for Query Cache */ __thread int mysql_thread___query_cache_size_MB; @@ -951,6 +953,8 @@ extern __thread bool mysql_thread___enable_client_deprecate_eof; extern __thread bool mysql_thread___enable_server_deprecate_eof; extern __thread bool mysql_thread___log_mysql_warnings_enabled; extern __thread bool mysql_thread___enable_load_data_local_infile; +extern __thread int mysql_thread___client_host_cache_size; +extern __thread int mysql_thread___client_host_error_counts; /* variables used for Query Cache */ extern __thread int mysql_thread___query_cache_size_MB; diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index 9a9f583ea..62ba5b3c4 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -4760,6 +4760,7 @@ void MySQL_Session::handler___status_CHANGING_USER_CLIENT___STATE_CLIENT_HANDSHA void MySQL_Session::handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE(PtrSize_t *pkt, bool *wrong_pass) { bool is_encrypted = client_myds->encrypted; bool handshake_response_return = client_myds->myprot.process_pkt_handshake_response((unsigned char *)pkt->ptr,pkt->size); + bool handshake_err = true; proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION,8,"Session=%p , DS=%p , handshake_response=%d , switching_auth_stage=%d , is_encrypted=%d , client_encrypted=%d\n", this, client_myds, handshake_response_return, client_myds->switching_auth_stage, is_encrypted, client_myds->encrypted); if ( @@ -4943,6 +4944,7 @@ void MySQL_Session::handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE( ) { // we are good! client_myds->myprot.generate_pkt_OK(true,NULL,NULL, _pid, 0,0,0,0,NULL); + handshake_err = false; GloMyLogger->log_audit_entry(PROXYSQL_MYSQL_AUTH_OK, this, NULL); status=WAITING_CLIENT_DATA; client_myds->DSS=STATE_CLIENT_AUTH_OK; @@ -4979,6 +4981,7 @@ void MySQL_Session::handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE( proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION,8,"Session=%p , DS=%p . STATE_CLIENT_AUTH_OK\n", this, client_myds); GloMyLogger->log_audit_entry(PROXYSQL_MYSQL_AUTH_OK, this, NULL); client_myds->myprot.generate_pkt_OK(true,NULL,NULL, _pid, 0,0,0,0,NULL); + handshake_err = false; status=WAITING_CLIENT_DATA; client_myds->DSS=STATE_CLIENT_AUTH_OK; } @@ -5045,6 +5048,10 @@ void MySQL_Session::handler___status_CONNECTING_CLIENT___STATE_SERVER_HANDSHAKE( __sync_add_and_fetch(&MyHGM->status.client_connections_aborted,1); client_myds->DSS=STATE_SLEEP; } + + if (mysql_thread___client_host_cache_size) { + GloMTH->update_client_host_cache(client_myds->client_addr, handshake_err); + } } // Note: as commented in issue #546 and #547 , some clients ignore the status of CLIENT_MULTI_STATEMENTS diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index 946942294..8b90af25a 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -88,6 +88,7 @@ mythr_st_vars_t MySQL_Thread_status_variables_counter_array[] { { st_var_whitelisted_sqli_fingerprint,p_th_counter::whitelisted_sqli_fingerprint, (char *)"whitelisted_sqli_fingerprint" }, { st_var_max_connect_timeout_err, p_th_counter::max_connect_timeouts, (char *)"max_connect_timeouts" }, { st_var_generated_pkt_err, p_th_counter::generated_error_packets, (char *)"generated_error_packets" }, + { st_var_client_host_error_killed_connections, p_th_counter::client_host_error_killed_connections, (char *)"client_host_error_killed_connections" }, }; mythr_g_st_vars_t MySQL_Thread_status_variables_gauge_array[] { @@ -412,6 +413,8 @@ static char * mysql_thread_variables_names[]= { (char *)"shun_recovery_time_sec", (char *)"query_retries_on_failure", (char *)"client_multi_statements", + (char *)"client_host_cache_size", + (char *)"client_host_error_counts", (char *)"connect_retries_on_failure", (char *)"connect_retries_delay", (char *)"connection_delay_multiplex_ms", @@ -863,6 +866,12 @@ th_metrics_map = std::make_tuple( "proxysql_mysql_killed_backend_queries_total", "Killed backend queries.", metric_tags {} + ), + std::make_tuple ( + p_th_counter::client_host_error_killed_connections, + "proxysql_client_host_error_killed_connections", + "Killed client connections because address exceeded 'client_host_error_counts'.", + metric_tags {} ) }, th_gauge_vector { @@ -1169,8 +1178,10 @@ MySQL_Threads_Handler::MySQL_Threads_Handler() { // Initialize prometheus metrics init_prometheus_counter_array(th_metrics_map, this->status_variables.p_counter_array); init_prometheus_gauge_array(th_metrics_map, this->status_variables.p_gauge_array); -} + // Init client_host_cache mutex + pthread_mutex_init(&mutex_client_host_cache, NULL); +} unsigned int MySQL_Threads_Handler::get_global_version() { return __sync_fetch_and_add(&__global_MySQL_Thread_Variables_version,0); @@ -2090,6 +2101,9 @@ char ** MySQL_Threads_Handler::get_variables_list() { VariablesPointers_int["handle_unknown_charset"] = make_tuple(&variables.handle_unknown_charset, 0, HANDLE_UNKNOWN_CHARSET__MAX_HANDLE_VALUE, false); VariablesPointers_int["ping_interval_server_msec"] = make_tuple(&variables.ping_interval_server_msec, 1000, 7*24*3600*1000, false); VariablesPointers_int["ping_timeout_server"] = make_tuple(&variables.ping_timeout_server, 10, 600*1000, false); + VariablesPointers_int["client_host_cache_size"] = make_tuple(&variables.client_host_cache_size, 0, 1024*1024, false); + VariablesPointers_int["client_host_error_counts"] = make_tuple(&variables.client_host_error_counts, 0, 1024*1024, false); + // logs VariablesPointers_int["auditlog_filesize"] = make_tuple(&variables.auditlog_filesize, 1024*1024, 1*1024*1024*1024, false); VariablesPointers_int["eventslog_filesize"] = make_tuple(&variables.eventslog_filesize, 1024*1024, 1*1024*1024*1024, false); @@ -2285,6 +2299,120 @@ void MySQL_Threads_Handler::stop_listeners() { free_tokenizer( &tok ); } +/** + * @brief Gets the client address stored in 'client_addr' member as + * an string if available. If member 'client_addr' is NULL, returns an + * empty string. + * + * @return Either an string holding the string representation of internal + * member 'client_addr', or empty string if this member is NULL. + */ +std::string get_client_addr(struct sockaddr* client_addr) { + char buf[512]; + std::string str_client_addr {}; + + if (client_addr == NULL) { + return str_client_addr; + } + + switch (client_addr->sa_family) { + case AF_INET: { + struct sockaddr_in *ipv4 = (struct sockaddr_in *)client_addr; + if (ipv4->sin_port) { + inet_ntop(client_addr->sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); + str_client_addr = std::string { buf }; + } else { + str_client_addr = std::string { "localhost" }; + } + break; + } + case AF_INET6: { + struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)client_addr; + inet_ntop(client_addr->sa_family, &ipv6->sin6_addr, buf, INET6_ADDRSTRLEN); + str_client_addr = std::string { buf }; + break; + } + default: + str_client_addr = std::string { "localhost" }; + break; + } + + return str_client_addr; +} + +MySQL_Client_Host_Cache_Entry MySQL_Threads_Handler::find_client_host_cache(struct sockaddr* client_sockaddr) { + MySQL_Client_Host_Cache_Entry entry { 0, 0 }; + if (client_sockaddr->sa_family != AF_INET && client_sockaddr->sa_family != AF_INET6) { + return entry; + } + std::string client_addr = get_client_addr(client_sockaddr); + if (client_addr == "127.0.0.1") { + return entry; + } + + pthread_mutex_lock(&mutex_client_host_cache); + auto found_entry = client_host_cache.find(client_addr); + if (found_entry != client_host_cache.end()) { + entry = found_entry->second; + } + pthread_mutex_unlock(&mutex_client_host_cache); + + return entry; +} + +void MySQL_Threads_Handler::update_client_host_cache(struct sockaddr* client_sockaddr, bool error) { + if (client_sockaddr->sa_family != AF_INET && client_sockaddr->sa_family != AF_INET6) { + return; + } + std::string client_addr = get_client_addr(client_sockaddr); + if (client_addr == "127.0.0.1") { + return; + } + + if (error) { + pthread_mutex_lock(&mutex_client_host_cache); + // If the cache is full, find the latest entry on it, and update/remove it. + if (client_host_cache.size() == static_cast(mysql_thread___client_host_cache_size)) { + auto older_elem = std::min_element( + client_host_cache.begin(), + client_host_cache.end(), + [] (const std::pair& f_entry, + const std::pair& s_entry) + { + return f_entry.second.updated_at < s_entry.second.updated_at; + } + ); + if (older_elem->first == client_addr) { + older_elem->second.error_count += 1; + older_elem->second.updated_at = monotonic_time(); + } else { + client_host_cache.erase(older_elem); + } + } + + // Find the entry for the client, and update/insert it. + auto cache_entry = client_host_cache.find(client_addr); + if (cache_entry != client_host_cache.end()) { + cache_entry->second.error_count += 1; + cache_entry->second.updated_at = monotonic_time(); + } else { + MySQL_Client_Host_Cache_Entry new_entry { monotonic_time(), 1 }; + client_host_cache.insert({client_addr, new_entry}); + } + pthread_mutex_unlock(&mutex_client_host_cache); + } else { + pthread_mutex_lock(&mutex_client_host_cache); + client_host_cache.erase(client_addr); + pthread_mutex_unlock(&mutex_client_host_cache); + } +} + +void MySQL_Threads_Handler::flush_client_host_cache() { + pthread_mutex_lock(&mutex_client_host_cache); + client_host_cache.clear(); + pthread_mutex_unlock(&mutex_client_host_cache); +} + MySQL_Threads_Handler::~MySQL_Threads_Handler() { if (variables.monitor_username) { free(variables.monitor_username); variables.monitor_username=NULL; } if (variables.monitor_password) { free(variables.monitor_password); variables.monitor_password=NULL; } @@ -3401,6 +3529,7 @@ void MySQL_Thread::process_all_sessions() { if (sess_time/1000 > (unsigned long long)mysql_thread___connect_timeout_client) { proxy_warning("Closing not established client connection %s:%d after %llums\n",sess->client_myds->addr.addr,sess->client_myds->addr.port, sess_time/1000); sess->healthy = 0; + GloMTH->update_client_host_cache(sess->client_myds->client_addr, true); } } if (maintenance_loop) { @@ -3660,6 +3789,8 @@ void MySQL_Thread::refresh_variables() { mysql_thread___enable_server_deprecate_eof=(bool)GloMTH->get_variable_int((char *)"enable_server_deprecate_eof"); mysql_thread___enable_load_data_local_infile=(bool)GloMTH->get_variable_int((char *)"enable_load_data_local_infile"); mysql_thread___log_mysql_warnings_enabled=(bool)GloMTH->get_variable_int((char *)"log_mysql_warnings_enabled"); + mysql_thread___client_host_cache_size=GloMTH->get_variable_int((char *)"client_host_cache_size"); + mysql_thread___client_host_error_counts=GloMTH->get_variable_int((char *)"client_host_error_counts"); #ifdef DEBUG mysql_thread___session_debug=(bool)GloMTH->get_variable_int((char *)"session_debug"); #endif /* DEBUG */ @@ -3736,7 +3867,6 @@ void MySQL_Thread::unregister_session_connection_handler(int idx, bool _new) { mysql_sessions->remove_index_fast(idx); } - void MySQL_Thread::listener_handle_new_connection(MySQL_Data_Stream *myds, unsigned int n) { int c; union { @@ -3758,6 +3888,19 @@ void MySQL_Thread::listener_handle_new_connection(MySQL_Data_Stream *myds, unsig } c=accept(myds->fd, addr, &addrlen); if (c>-1) { // accept() succeeded + if (mysql_thread___client_host_cache_size) { + MySQL_Client_Host_Cache_Entry client_host_entry = + GloMTH->find_client_host_cache(addr); + if ( + client_host_entry.updated_at != 0 && + client_host_entry.error_count == static_cast(mysql_thread___client_host_error_counts) + ) { + close(c); + status_variables.stvar[st_var_client_host_error_killed_connections] += 1; + return; + } + } + // create a new client connection mypolls.fds[n].revents=0; MySQL_Session *sess=create_new_session_and_client_data_stream(c); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index ef20bdc7e..42227a51d 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -1523,6 +1523,17 @@ bool admin_handler_command_proxysql(char *query_no_space, unsigned int query_no_ return false; } + if (query_no_space_length==strlen("PROXYSQL FLUSH MYSQL CLIENT HOSTS") && !strncasecmp("PROXYSQL FLUSH MYSQL CLIENT HOSTS",query_no_space, query_no_space_length)) { + proxy_info("Received PROXYSQL FLUSH MYSQL CLIENT HOSTS command\n"); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + if (GloMTH) { + GloMTH->flush_client_host_cache(); + } + SPA->flush_error_log(); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); + return false; + } + if ( (query_no_space_length==strlen("PROXYSQL FLUSH CONFIGDB") && !strncasecmp("PROXYSQL FLUSH CONFIGDB",query_no_space, query_no_space_length)) // see #923 ) { From 3addb9a1941699ed37a56dc641675a627c8c92c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Thu, 9 Sep 2021 18:20:05 +0200 Subject: [PATCH 02/10] Improved implementation of helper function 'get_client_addr' using 'handler___status_NONE_or_default' as reference --- lib/MySQL_Thread.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index 3b6d7a6c1..be65a250c 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -2395,7 +2395,7 @@ void MySQL_Threads_Handler::stop_listeners() { * member 'client_addr', or empty string if this member is NULL. */ std::string get_client_addr(struct sockaddr* client_addr) { - char buf[512]; + char buf[INET6_ADDRSTRLEN]; std::string str_client_addr {}; if (client_addr == NULL) { @@ -2405,12 +2405,8 @@ std::string get_client_addr(struct sockaddr* client_addr) { switch (client_addr->sa_family) { case AF_INET: { struct sockaddr_in *ipv4 = (struct sockaddr_in *)client_addr; - if (ipv4->sin_port) { - inet_ntop(client_addr->sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); - str_client_addr = std::string { buf }; - } else { - str_client_addr = std::string { "localhost" }; - } + inet_ntop(client_addr->sa_family, &ipv4->sin_addr, buf, INET_ADDRSTRLEN); + str_client_addr = std::string { buf }; break; } case AF_INET6: { From 625daa9cd323c2864cf5ee19e67f0c253c2c9174 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Thu, 9 Sep 2021 18:27:09 +0200 Subject: [PATCH 03/10] Fixed check for 'client_host' error count and added missing free --- lib/MySQL_Thread.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index be65a250c..9e2a1f406 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -3993,9 +3993,10 @@ void MySQL_Thread::listener_handle_new_connection(MySQL_Data_Stream *myds, unsig GloMTH->find_client_host_cache(addr); if ( client_host_entry.updated_at != 0 && - client_host_entry.error_count == static_cast(mysql_thread___client_host_error_counts) + client_host_entry.error_count >= static_cast(mysql_thread___client_host_error_counts) ) { close(c); + free(addr); status_variables.stvar[st_var_client_host_error_killed_connections] += 1; return; } From e7d9a4c74dffd30a76fe2b488787cdc1f2daa5b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Thu, 9 Sep 2021 18:28:18 +0200 Subject: [PATCH 04/10] Added extra logging when a connection is closed due to 'client_host_error_counts' being reached --- lib/MySQL_Thread.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index 9e2a1f406..5b878034e 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -3995,6 +3995,11 @@ void MySQL_Thread::listener_handle_new_connection(MySQL_Data_Stream *myds, unsig client_host_entry.updated_at != 0 && client_host_entry.error_count >= static_cast(mysql_thread___client_host_error_counts) ) { + std::string client_addr = get_client_addr(addr); + proxy_error( + "Closing connection because client '%s' reached 'mysql-client_host_error_counts': %d\n", + client_addr.c_str(), mysql_thread___client_host_error_counts + ); close(c); free(addr); status_variables.stvar[st_var_client_host_error_killed_connections] += 1; From 2d3da0fca393507f13e14f6639e057efd1e96906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Thu, 9 Sep 2021 18:30:21 +0200 Subject: [PATCH 05/10] Simplified check for query 'PROXYSQL FLUSH MYSQL CLIENT HOSTS' via 'strcasecmp' --- lib/ProxySQL_Admin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 3579d3068..ed74787a6 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -1530,7 +1530,7 @@ bool admin_handler_command_proxysql(char *query_no_space, unsigned int query_no_ return false; } - if (query_no_space_length==strlen("PROXYSQL FLUSH MYSQL CLIENT HOSTS") && !strncasecmp("PROXYSQL FLUSH MYSQL CLIENT HOSTS",query_no_space, query_no_space_length)) { + if (!strcasecmp("PROXYSQL FLUSH MYSQL CLIENT HOSTS", query_no_space)) { proxy_info("Received PROXYSQL FLUSH MYSQL CLIENT HOSTS command\n"); ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; if (GloMTH) { From 514ffe939ae36ac86ad667f94368c43198aa899f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Thu, 9 Sep 2021 18:35:53 +0200 Subject: [PATCH 06/10] Fixed double increment in 'update_client_host_cache' when oldest entry is updated --- lib/MySQL_Thread.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index 5b878034e..442c4772d 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -2465,10 +2465,7 @@ void MySQL_Threads_Handler::update_client_host_cache(struct sockaddr* client_soc return f_entry.second.updated_at < s_entry.second.updated_at; } ); - if (older_elem->first == client_addr) { - older_elem->second.error_count += 1; - older_elem->second.updated_at = monotonic_time(); - } else { + if (older_elem->first != client_addr) { client_host_cache.erase(older_elem); } } From 416a0470a53fe74b4ab429f6f403b195362cd2fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Thu, 9 Sep 2021 18:41:21 +0200 Subject: [PATCH 07/10] Added tables 'stats_mysql_client_host_cache' and 'stats_mysql_client_host_cache_reset' exposing entries in 'client_host_cache' --- include/MySQL_Thread.h | 25 ++++++++++++++ include/proxysql_admin.h | 1 + lib/MySQL_Thread.cpp | 71 ++++++++++++++++++++++++++++++++++++++++ lib/ProxySQL_Admin.cpp | 62 +++++++++++++++++++++++++++++++++++ 4 files changed, 159 insertions(+) diff --git a/include/MySQL_Thread.h b/include/MySQL_Thread.h index 658553124..9589c33e3 100644 --- a/include/MySQL_Thread.h +++ b/include/MySQL_Thread.h @@ -596,6 +596,15 @@ class MySQL_Threads_Handler * - 'address_family' is either 'AF_INET' or 'AF_INET6'. * - The address obtained from it isn't '127.0.0.1'. * + * In case 'client_sockaddr' matches the previous description, the update + * of the client host cache is performed in the following way: + * 1. If the cache is full, the oldest element in the cache is searched. + * In case the oldest element address doesn't match the supplied + * address, the oldest element is removed. + * 2. The cache is searched looking for the supplied address, in case of + * being found, the entry is updated, otherwise the entry is inserted in + * the cache. + * * @param client_sockaddr A 'sockaddr' holding the required client information * to update the 'client_host_cache_map'. * @param error 'true' if there was an error in the connection that should be @@ -628,6 +637,22 @@ class MySQL_Threads_Handler * @brief Delete all the entries in the 'client_host_cache' internal map. */ void flush_client_host_cache(); + /** + * @brief Returns the current entries of 'client_host_cache' in a + * 'SQLite3_result'. In case the param 'reset' is specified, the structure + * is cleaned after being queried. + * + * @param reset If 'true' the entries of the internal structure + * 'client_host_cache' will be cleaned after scrapping. + * + * @return SQLite3_result holding the current entries of the + * 'client_host_cache'. In the following format: + * + * [ 'client_address', 'error_num', 'last_updated' ] + * + * Where 'last_updated' is the last updated time expressed in 'ns'. + */ + SQLite3_result* get_client_host_cache(bool reset); /** * @brief Callback to update the metrics. */ diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 421715b9e..54927da24 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -355,6 +355,7 @@ class ProxySQL_Admin { void stats___proxysql_servers_metrics(); void stats___mysql_prepared_statements_info(); void stats___mysql_gtid_executed(); + void stats___mysql_client_host_cache(bool reset); // Update prometheus metrics void p_stats___memory_metrics(); diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index 442c4772d..66ab0f945 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -2443,6 +2443,77 @@ MySQL_Client_Host_Cache_Entry MySQL_Threads_Handler::find_client_host_cache(stru return entry; } +/** + * @brief Number of columns for representing a 'MySQL_Client_Host_Cache_Entry' + * in a 'SQLite3_result'. + */ +const int CLIENT_HOST_CACHE_COLUMNS = 3; + +/** + * @brief Helper function that converts a given client address and a + * 'MySQL_Client_Host_Cache_Entry', into a row for a 'SQLite3_result' for + * table 'STATS_SQLITE_TABLE_MYSQL_CLIENT_HOST_CACHE'. + * + * @param address The client address to be added to the resulset row. + * @param entry The 'MySQL_Client_Host_Cache_Entry' to be added to the resulset + * row. + * + * @return A pointer array holding the values for each of the columns of the + * row. It should be freed through helper function 'free_client_host_cache_row'. + */ +char** client_host_cache_entry_row( + const std::string address, const MySQL_Client_Host_Cache_Entry& entry +) { + // INET6_ADDRSTRLEN length should be enough for holding any member: + // { address: MAX INET6_ADDRSTRLEN, updated_at: uint64_t, error_count: uint32_t } + char buff[INET6_ADDRSTRLEN]; + char** row = + static_cast(malloc(sizeof(char*)*CLIENT_HOST_CACHE_COLUMNS)); + + row[0]=strdup(address.c_str()); + sprintf(buff, "%u", entry.error_count); + row[1]=strdup(buff); + sprintf(buff, "%lu", entry.updated_at); + row[2]=strdup(buff); + + return row; +} + +/** + * @brief Helper function to free the row returned by + * 'client_host_cache_entry_row'. + * + * @param row The pointer array holding the row values to be freed. + */ +void free_client_host_cache_row(char** row) { + for (int i = 0; i < CLIENT_HOST_CACHE_COLUMNS; i++) { + free(row[i]); + } + free(row); +} + +SQLite3_result* MySQL_Threads_Handler::get_client_host_cache(bool reset) { + SQLite3_result *result = new SQLite3_result(CLIENT_HOST_CACHE_COLUMNS); + + pthread_mutex_lock(&mutex_client_host_cache); + result->add_column_definition(SQLITE_TEXT,"client_address"); + result->add_column_definition(SQLITE_TEXT,"error_count"); + result->add_column_definition(SQLITE_TEXT,"last_updated"); + + for (const auto& cache_entry : client_host_cache) { + char** row = client_host_cache_entry_row(cache_entry.first, cache_entry.second); + result->add_row(row); + free_client_host_cache_row(row); + } + + if (reset) { + client_host_cache.clear(); + } + + pthread_mutex_unlock(&mutex_client_host_cache); + return result; +} + void MySQL_Threads_Handler::update_client_host_cache(struct sockaddr* client_sockaddr, bool error) { if (client_sockaddr->sa_family != AF_INET && client_sockaddr->sa_family != AF_INET6) { return; diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index ed74787a6..73cea52da 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -490,6 +490,9 @@ static int http_handler(void *cls, struct MHD_Connection *connection, const char #define STATS_SQLITE_TABLE_MYSQL_ERRORS "CREATE TABLE stats_mysql_errors (hostgroup INT NOT NULL , hostname VARCHAR NOT NULL , port INT NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , schemaname VARCHAR NOT NULL , errno INT NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , last_error VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup, hostname, port, username, schemaname, errno) )" #define STATS_SQLITE_TABLE_MYSQL_ERRORS_RESET "CREATE TABLE stats_mysql_errors_reset (hostgroup INT NOT NULL , hostname VARCHAR NOT NULL , port INT NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , schemaname VARCHAR NOT NULL , errno INT NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , last_error VARCHAR NOT NULL DEFAULT '' , PRIMARY KEY (hostgroup, hostname, port, username, schemaname, errno) )" +#define STATS_SQLITE_TABLE_MYSQL_CLIENT_HOST_CACHE "CREATE TABLE stats_mysql_client_host_cache (client_address VARCHAR NOT NULL, error_count INT NOT NULL, last_updated BIGINT NOT NULL)" +#define STATS_SQLITE_TABLE_MYSQL_CLIENT_HOST_CACHE_RESET "CREATE TABLE stats_mysql_client_host_cache_reset (client_address VARCHAR NOT NULL, error_count INT NOT NULL, last_updated BIGINT NOT NULL)" + #ifdef DEBUG #define ADMIN_SQLITE_TABLE_DEBUG_LEVELS "CREATE TABLE debug_levels (module VARCHAR NOT NULL PRIMARY KEY , verbosity INT NOT NULL DEFAULT 0)" #define ADMIN_SQLITE_TABLE_DEBUG_FILTERS "CREATE TABLE debug_filters (filename VARCHAR NOT NULL , line INT NOT NULL , funct VARCHAR NOT NULL , PRIMARY KEY (filename, line, funct) )" @@ -2936,6 +2939,8 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign bool stats_mysql_query_rules=false; bool stats_mysql_users=false; bool stats_mysql_gtid_executed=false; + bool stats_mysql_client_host_cache=false; + bool stats_mysql_client_host_cache_reset=false; bool dump_global_variables=false; bool runtime_scheduler=false; @@ -3023,6 +3028,10 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign { stats_mysql_users=true; refresh=true; } if (strstr(query_no_space,"stats_mysql_gtid_executed")) { stats_mysql_gtid_executed=true; refresh=true; } + if (strstr(query_no_space,"stats_mysql_client_host_cache")) + { stats_mysql_client_host_cache=true; refresh=true; } + if (strstr(query_no_space,"stats_mysql_client_host_cache_reset")) + { stats_mysql_client_host_cache_reset=true; refresh=true; } if (strstr(query_no_space,"stats_proxysql_servers_checksums")) { stats_proxysql_servers_checksums = true; refresh = true; } @@ -3163,6 +3172,13 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign stats___mysql_prepared_statements_info(); } + if (stats_mysql_client_host_cache) { + stats___mysql_client_host_cache(false); + } + if (stats_mysql_client_host_cache_reset) { + stats___mysql_client_host_cache(true); + } + if (admin) { if (dump_global_variables) { pthread_mutex_lock(&GloVars.checksum_mutex); @@ -5537,6 +5553,8 @@ bool ProxySQL_Admin::init() { insert_into_tables_defs(tables_defs_stats,"stats_mysql_users", STATS_SQLITE_TABLE_MYSQL_USERS); insert_into_tables_defs(tables_defs_stats,"global_variables", ADMIN_SQLITE_TABLE_GLOBAL_VARIABLES); // workaround for issue #708 insert_into_tables_defs(tables_defs_stats,"stats_mysql_prepared_statements_info", ADMIN_SQLITE_TABLE_STATS_MYSQL_PREPARED_STATEMENTS_INFO); + insert_into_tables_defs(tables_defs_stats,"stats_mysql_client_host_cache", STATS_SQLITE_TABLE_MYSQL_CLIENT_HOST_CACHE); + insert_into_tables_defs(tables_defs_stats,"stats_mysql_client_host_cache_reset", STATS_SQLITE_TABLE_MYSQL_CLIENT_HOST_CACHE_RESET); // ProxySQL Cluster insert_into_tables_defs(tables_defs_admin,"proxysql_servers", ADMIN_SQLITE_TABLE_PROXYSQL_SERVERS); @@ -8770,6 +8788,50 @@ void ProxySQL_Admin::stats___mysql_query_digests(bool reset, bool copy) { delete resultset; } +void ProxySQL_Admin::stats___mysql_client_host_cache(bool reset) { + if (!GloQPro) return; + + SQLite3_result* resultset = GloMTH->get_client_host_cache(reset); + if (resultset==NULL) return; + + statsdb->execute("BEGIN"); + + int rc = 0; + sqlite3_stmt* statement=NULL; + char* query = NULL; + + if (reset) { + query=(char*)"INSERT INTO stats_mysql_client_host_cache_reset VALUES (?1, ?2, ?3)"; + } else { + query=(char*)"INSERT INTO stats_mysql_client_host_cache VALUES (?1, ?2, ?3)"; + } + + if (reset) { + statsdb->execute("DELETE FROM stats_mysql_client_host_cache_reset"); + } else { + statsdb->execute("DELETE FROM stats_mysql_client_host_cache"); + } + + rc = statsdb->prepare_v2(query, &statement); + ASSERT_SQLITE_OK(rc, statsdb); + + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *row = *it; + + rc=(*proxy_sqlite3_bind_text)(statement, 1, row->fields[0], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement, 2, atoll(row->fields[1])); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement, 3, atoll(row->fields[2])); ASSERT_SQLITE_OK(rc, statsdb); + + SAFE_SQLITE3_STEP2(statement); + rc=(*proxy_sqlite3_clear_bindings)(statement); + rc=(*proxy_sqlite3_reset)(statement); + } + + (*proxy_sqlite3_finalize)(statement); + statsdb->execute("COMMIT"); + delete resultset; +} + void ProxySQL_Admin::stats___mysql_errors(bool reset) { if (!GloQPro) return; SQLite3_result * resultset=NULL; From d115e20ed4ed76fc6a93e57a0da9ac4e1e957b85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Thu, 9 Sep 2021 19:17:58 +0200 Subject: [PATCH 08/10] Fixed size check for 'client_host_cache' and fixed logic preventing the cache from reducing it's size in certain conditions --- lib/MySQL_Thread.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index 66ab0f945..4b166cf2b 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -2525,8 +2525,8 @@ void MySQL_Threads_Handler::update_client_host_cache(struct sockaddr* client_soc if (error) { pthread_mutex_lock(&mutex_client_host_cache); - // If the cache is full, find the latest entry on it, and update/remove it. - if (client_host_cache.size() == static_cast(mysql_thread___client_host_cache_size)) { + // If the cache is full, find the oldest entry on it, and update/remove it. + if (client_host_cache.size() >= static_cast(mysql_thread___client_host_cache_size)) { auto older_elem = std::min_element( client_host_cache.begin(), client_host_cache.end(), @@ -2547,8 +2547,14 @@ void MySQL_Threads_Handler::update_client_host_cache(struct sockaddr* client_soc cache_entry->second.error_count += 1; cache_entry->second.updated_at = monotonic_time(); } else { - MySQL_Client_Host_Cache_Entry new_entry { monotonic_time(), 1 }; - client_host_cache.insert({client_addr, new_entry}); + // Notice than the value of 'mysql_thread___client_host_cache_size' can + // change at runtime. Due to this, we should only insert when the size of the + // cache is smaller than this value, otherwise we could end in situations in + // which cache doesn't shrink after it's size is reduced at runtime. + if (client_host_cache.size() < static_cast(mysql_thread___client_host_cache_size)) { + MySQL_Client_Host_Cache_Entry new_entry { monotonic_time(), 1 }; + client_host_cache.insert({client_addr, new_entry}); + } } pthread_mutex_unlock(&mutex_client_host_cache); } else { From 9aef676973a6fc04b3943e34a17de2d36537cf1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Fri, 10 Sep 2021 17:31:26 +0200 Subject: [PATCH 09/10] Added population of 'stats_mysql_client_host_cache' when 'stats_mysql_client_host_cache_reset' is queried --- lib/ProxySQL_Admin.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 73cea52da..67b24b81f 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -8806,11 +8806,8 @@ void ProxySQL_Admin::stats___mysql_client_host_cache(bool reset) { query=(char*)"INSERT INTO stats_mysql_client_host_cache VALUES (?1, ?2, ?3)"; } - if (reset) { - statsdb->execute("DELETE FROM stats_mysql_client_host_cache_reset"); - } else { - statsdb->execute("DELETE FROM stats_mysql_client_host_cache"); - } + statsdb->execute("DELETE FROM stats_mysql_client_host_cache_reset"); + statsdb->execute("DELETE FROM stats_mysql_client_host_cache"); rc = statsdb->prepare_v2(query, &statement); ASSERT_SQLITE_OK(rc, statsdb); @@ -8828,6 +8825,11 @@ void ProxySQL_Admin::stats___mysql_client_host_cache(bool reset) { } (*proxy_sqlite3_finalize)(statement); + + if (reset) { + statsdb->execute("INSERT INTO stats_mysql_client_host_cache SELECT * FROM stats_mysql_client_host_cache_reset"); + } + statsdb->execute("COMMIT"); delete resultset; } From 5caed323ec020b947e548da14d77ab81fcdf3e6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jaramago=20Fern=C3=A1ndez?= Date: Fri, 10 Sep 2021 22:03:24 +0200 Subject: [PATCH 10/10] Added initial version of test 'test_client_limit_error.cpp' checking 'client limit error' feature --- .../tests/client_host_err/create_netns_n.sh | 25 + .../tests/client_host_err/delete_netns_n.sh | 14 + test/tap/tests/test_client_limit_error.cpp | 776 ++++++++++++++++++ 3 files changed, 815 insertions(+) create mode 100755 test/tap/tests/client_host_err/create_netns_n.sh create mode 100755 test/tap/tests/client_host_err/delete_netns_n.sh create mode 100644 test/tap/tests/test_client_limit_error.cpp diff --git a/test/tap/tests/client_host_err/create_netns_n.sh b/test/tap/tests/client_host_err/create_netns_n.sh new file mode 100755 index 000000000..4128030d6 --- /dev/null +++ b/test/tap/tests/client_host_err/create_netns_n.sh @@ -0,0 +1,25 @@ +#!/bin/bash +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root" + exit 1 +fi + +if [ ! "$1" ] || [ ! -z "${1//[0-9]}" ]; then + echo "Parameter should be a number" + exit 1 +fi + +ip netns add ns$1 + +ip netns exec ns$1 ip link set lo up +ip netns exec ns$1 ip link set dev lo up + +ip link add v-eth$1 type veth peer name v-peer1 +ip link set v-peer1 netns ns$1 +ip netns exec ns$1 ip addr +ip netns exec ns$1 ip addr add 10.200.$1.2/24 dev v-peer1 +ip netns exec ns$1 ip link set v-peer1 up +ip addr add 10.200.$1.1/24 dev v-eth$1 +ip link set v-eth$1 up +ip netns exec ns$1 ip route add default via 10.200.$1.1 + diff --git a/test/tap/tests/client_host_err/delete_netns_n.sh b/test/tap/tests/client_host_err/delete_netns_n.sh new file mode 100755 index 000000000..80dfbcc5f --- /dev/null +++ b/test/tap/tests/client_host_err/delete_netns_n.sh @@ -0,0 +1,14 @@ +#!/bin/bash +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root" + exit 1 +fi + +if [ ! "$1" ] || [ ! -z "${1//[0-9]}" ]; then + echo "Parameter should be a number" + exit 1 +fi + +ip netns delete ns$1 +ip link delete v-eth$1 + diff --git a/test/tap/tests/test_client_limit_error.cpp b/test/tap/tests/test_client_limit_error.cpp new file mode 100644 index 000000000..ace8bdd76 --- /dev/null +++ b/test/tap/tests/test_client_limit_error.cpp @@ -0,0 +1,776 @@ +/** + * @file test_client_limit_error.cpp + * @brief This test aims to verify the logic for the 'client_limit_error' feature. + * NOTE: This test isn't executed automatically because it requires elevated privileges + * for being able to run the scripts 'create_netns_n' and 'delete_netns_n' that creates + * and delete the networks namespaces required for it. + * + * @details Right now the test verifies the following cases: + * 1. Enable the feature and checks that the error count is incremented when a single + * client tries to connect and that the cache entry values match expected ones. + * 2. Flush the entries, and check that counting is performed properly for a + * single cache entry. + * 3. Flush the entries, and check that counting is performed properly for + * multiple cache entries. + * 4. Flush the entries, and check: + * 1. That counting is performed properly for multiple cache entries. + * 2. Connections fail after the limit for one client. + * 3. Clients are deleted after a succesfull connection is performed. + * 5. Flush the entries, fill the cache and check that when the + * 'mysql-client_host_error_counts' is changed at runtime, connections are denied + * to a client exceeding the new limit. + * 6. Flush the entries, fill the cache and check that the when the + * 'mysql-client_host_cache_size' is changed at runtime: + * 1. The exceeding elements are cleaned with each new connection. + * 2. Check that is not relevant if the element was or not present + * in the cache. + * 3. Checks that a proper connection is performed succesfully and the element is + * removed from the cache + * + * @date 2021-09-10 + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "tap.h" +#include "command_line.h" +#include "utils.h" + +const uint32_t NUM_NETWORK_NAMESPACES = 5; + +using host_cache_entry = std::tuple; + +inline unsigned long long monotonic_time() { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (((unsigned long long) ts.tv_sec) * 1000000) + (ts.tv_nsec / 1000); +} + +std::vector get_client_host_cache_entries(MYSQL* proxysql_admin) { + int rc = mysql_query( + proxysql_admin, + "SELECT * FROM stats.stats_mysql_client_host_cache ORDER BY client_address" + ); + + MYSQL_ROW row = NULL; + MYSQL_RES* proxy_res = mysql_store_result(proxysql_admin); + std::vector host_cache_entries {}; + + while ((row = mysql_fetch_row(proxy_res))) { + std::string client_address = row[0]; + uint32_t error_count = atoi(row[1]); + uint64_t last_update = atoll(row[2]); + + host_cache_entries.push_back({client_address, error_count, last_update}); + } + + mysql_free_result(proxy_res); + + return host_cache_entries; +} + +int main(int, char**) { + int res = 0; + CommandLine cl; + + if (cl.getEnv()) { + diag("Failed to get the required environmental variables."); + return -1; + } + + plan(27); + + MYSQL* proxysql_admin = mysql_init(NULL); + + // Initialize connections + if (!proxysql_admin) { + fprintf( + stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, + mysql_error(proxysql_admin) + ); + return -1; + } + + // Connect to ProxySQL Admin + if ( + !mysql_real_connect( + proxysql_admin, cl.host, cl.admin_username, cl.admin_password, + NULL, cl.admin_port, NULL, 0 + ) + ) { + fprintf( + stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, + mysql_error(proxysql_admin) + ); + return -1; + } + + // Setup the virtual namespaces to be used by the test + diag(" Setting up testing network namespaces "); + diag("*********************************************************************"); + printf("\n"); + + int setup_ns_i = 0; + const std::string t_create_ns_command = + std::string { cl.workdir } + "/client_host_err/create_netns_n.sh %d"; + for (setup_ns_i = 1; setup_ns_i < NUM_NETWORK_NAMESPACES; setup_ns_i++) { + std::string create_ns_command {}; + string_format(t_create_ns_command, create_ns_command, setup_ns_i); + + int c_err = system(create_ns_command.c_str()); + if (c_err) { + diag( + "Failed to create netns number '%d' with err: '%d'", + setup_ns_i, c_err + ); + goto cleanup; + } + } + + printf("\n"); + diag("*********************************************************************"); + printf("\n"); + + diag(" Performing queries and checks "); + diag("*********************************************************************"); + printf("\n"); + + { + const std::string t_inv_user_command = + "ip netns exec ns%d mysql -h10.200.%d.1 -uinv_user -pinv_pass -P6033"; + const std::string t_valid_connection_command { + "ip netns exec ns%d mysql -h10.200.%d.1 -uroot -proot -P6033 -e'DO 1' 2>&1" + }; + + // 1. Enable the feature check that error count is incremented when a + // new client fails to connect, and that the cache entry values are the + // expected ones. + { + printf("\n"); + diag(" START TEST NUMBER 1 "); + diag("-------------------------------------------------------------"); + + MYSQL_QUERY(proxysql_admin, "SET mysql-client_host_cache_size=1"); + MYSQL_QUERY(proxysql_admin, "SET mysql-client_host_error_counts=5"); + MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + // There shouldn't be any other entries in the cache for this test. + MYSQL_QUERY(proxysql_admin, "PROXYSQL FLUSH MYSQL CLIENT HOSTS"); + + std::string inv_user_command {}; + string_format(t_inv_user_command, inv_user_command, 1, 1); + uint64_t pre_command_time = monotonic_time(); + + diag("Performing connections to fill 'client_host_cache'"); + + printf("\n"); + int inv_user_errno = system(inv_user_command.c_str()); + diag("Client connection failed with error: %d", inv_user_errno); + printf("\n"); + + diag("Performing checks over 'client_host_cache'"); + + std::vector entries = + get_client_host_cache_entries(proxysql_admin); + + ok( + entries.size() == 1, + "'client_host_cache' entries should be '1' after issuing 'PROXYSQL FLUSH" + " MYSQL CLIENT HOSTS' and one failed connection." + ); + + if (entries.size() == 1) { + host_cache_entry unique_entry { entries.back() }; + const std::string client_addr { std::get<0>(unique_entry) }; + const uint32_t error_count { std::get<1>(unique_entry) }; + const uint64_t last_updated { std::get<2>(unique_entry) }; + uint64_t post_command_time = monotonic_time(); + + ok( + client_addr == "10.200.1.2" && error_count == 1 && + (pre_command_time < last_updated < post_command_time), + "Entry should match expected values - exp(addr: %s, err_count: %d, last_updated: %ld < %ld < %ld)," + " act(addr: %s, err_count: %d, last_updated: %ld < %ld < %ld)", + "10.200.1.2", 1, pre_command_time, last_updated, post_command_time, + client_addr.c_str(), error_count, pre_command_time, last_updated, post_command_time + ); + } + } + + // 2. Flush the entries, and check that counting is performed properly for a + // single cache entry. + { + printf("\n"); + diag(" START TEST NUMBER 2 "); + diag("-------------------------------------------------------------"); + + + // There shouldn't be any other entries in the cache for this test. + MYSQL_QUERY(proxysql_admin, "PROXYSQL FLUSH MYSQL CLIENT HOSTS"); + int errors = 0; + + std::string inv_user_command {}; + string_format(t_inv_user_command, inv_user_command, 1, 1); + uint64_t pre_command_time = monotonic_time(); + + printf("\n"); + diag("Performing connections to fill 'client_host_cache'"); + for (errors = 0; errors < 5; errors++) { + printf("\n"); + int inv_user_errno = system(inv_user_command.c_str()); + diag("Client connection failed with error: %d", inv_user_errno); + } + printf("\n"); + + diag("Performing checks over 'client_host_cache'"); + + std::vector entries = + get_client_host_cache_entries(proxysql_admin); + + ok( + entries.size() == 1, + "'client_host_cache' entries should be '1' after issuing 'PROXYSQL FLUSH" + " MYSQL CLIENT HOSTS' and one failed connection." + ); + + if (entries.size() == 1) { + host_cache_entry unique_entry { entries.back() }; + const std::string client_addr { std::get<0>(unique_entry) }; + const uint32_t error_count { std::get<1>(unique_entry) }; + const uint64_t last_updated { std::get<2>(unique_entry) }; + uint64_t post_command_time = monotonic_time(); + + ok( + client_addr == "10.200.1.2" && error_count == errors && + (pre_command_time < last_updated < post_command_time), + "Entry should match expected values - exp(addr: %s, err_count: %d, last_updated: %ld < %ld < %ld)," + " act(addr: %s, err_count: %d, last_updated: %ld < %ld < %ld)", + "10.200.1.2", 1, pre_command_time, last_updated, post_command_time, + client_addr.c_str(), error_count, pre_command_time, last_updated, post_command_time + ); + } + } + + // 3. Flush the entries, and check that counting is performed properly for + // multiple cache entries. + { + printf("\n"); + diag(" START TEST NUMBER 3 "); + diag("-------------------------------------------------------------"); + + // Increase cache size + MYSQL_QUERY(proxysql_admin, "SET mysql-client_host_cache_size=5"); + MYSQL_QUERY(proxysql_admin, "SET mysql-client_host_error_counts=5"); + MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + // There shouldn't be any other entries in the cache for this test. + MYSQL_QUERY(proxysql_admin, "PROXYSQL FLUSH MYSQL CLIENT HOSTS"); + int errors = 0; + + // Prepare several commands from different network namespaces + std::vector inv_connection_commands {}; + for (int i = 1; i < NUM_NETWORK_NAMESPACES; i++) { + std::string inv_user_command {}; + string_format(t_inv_user_command, inv_user_command, i, i); + inv_connection_commands.push_back(inv_user_command); + } + + uint64_t pre_command_time = monotonic_time(); + + printf("\n"); + diag("Performing connections to fill 'client_host_cache'"); + for (const auto inv_conn_command : inv_connection_commands) { + for (errors = 0; errors < 2; errors++) { + printf("\n"); + int inv_user_errno = system(inv_conn_command.c_str()); + diag("Client connection failed with error: %d", inv_user_errno); + } + printf("\n"); + } + + diag("Performing checks over 'client_host_cache'"); + + std::vector entries = + get_client_host_cache_entries(proxysql_admin); + + ok( + entries.size() == NUM_NETWORK_NAMESPACES - 1, + "'client_host_cache' entries should be 'NUM_NETWORK_NAMESPACES' after issuing 'PROXYSQL FLUSH" + " MYSQL CLIENT HOSTS' and 'NUM_NETWORK_NAMESPACES' failed connections. Entries: '%ld'", + entries.size() + ); + + if (entries.size() == NUM_NETWORK_NAMESPACES - 1) { + uint32_t entry_num = 1; + + for (const auto& entry : entries) { + const std::string client_addr { std::get<0>(entry) }; + const uint32_t error_count { std::get<1>(entry) }; + const uint64_t last_updated { std::get<2>(entry) }; + uint64_t post_command_time = monotonic_time(); + + std::string t_exp_client_addr { "10.200.%d.2" }; + std::string exp_client_addr {}; + string_format(t_exp_client_addr, exp_client_addr, entry_num); + + ok( + client_addr == exp_client_addr && error_count == errors && + (pre_command_time < last_updated < post_command_time), + "Entry should match expected values - exp(addr: %s, err_count: %d, last_updated: %ld < %ld < %ld)," + " act(addr: %s, err_count: %d, last_updated: %ld < %ld < %ld)", + exp_client_addr.c_str(), errors, pre_command_time, last_updated, post_command_time, + client_addr.c_str(), error_count, pre_command_time, last_updated, post_command_time + ); + + entry_num += 1; + } + } + } + + // 4. Flush the entries, and check: + // 1. That counting is performed properly for multiple cache entries. + // 2. Connections fail after the limit for one client. + // 3. Clients are deleted after a succesfull connection is performed. + { + printf("\n"); + diag(" START TEST NUMBER 4 "); + diag("-------------------------------------------------------------"); + + // Increase cache size + MYSQL_QUERY(proxysql_admin, "SET mysql-client_host_cache_size=5"); + MYSQL_QUERY(proxysql_admin, "SET mysql-client_host_error_counts=5"); + MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + // There shouldn't be any other entries in the cache for this test. + MYSQL_QUERY(proxysql_admin, "PROXYSQL FLUSH MYSQL CLIENT HOSTS"); + int errors = 0; + + // Prepare several commands from different network namespaces + std::vector inv_connection_commands {}; + for (int i = 1; i < NUM_NETWORK_NAMESPACES; i++) { + std::string inv_user_command {}; + string_format(t_inv_user_command, inv_user_command, i, i); + inv_connection_commands.push_back(inv_user_command); + } + + uint64_t pre_command_time = monotonic_time(); + + printf("\n"); + diag("Performing connections to fill 'client_host_cache'"); + for (const auto inv_conn_command : inv_connection_commands) { + for (errors = 0; errors < 3; errors++) { + printf("\n"); + int inv_user_errno = system(inv_conn_command.c_str()); + diag("Client connection failed with error: %d", inv_user_errno); + } + printf("\n"); + } + + printf("\n"); + diag("1. Check that counting is perfomred properly over multiple 'client_host_cache'"); + + std::vector entries = + get_client_host_cache_entries(proxysql_admin); + + ok( + entries.size() == NUM_NETWORK_NAMESPACES - 1, + "'client_host_cache' entries should be 'NUM_NETWORK_NAMESPACES' after issuing 'PROXYSQL FLUSH" + " MYSQL CLIENT HOSTS' and 'NUM_NETWORK_NAMESPACES' failed connections. Entries: '%ld'", + entries.size() + ); + + if (entries.size() == NUM_NETWORK_NAMESPACES - 1) { + uint32_t entry_num = 1; + + for (const auto& entry : entries) { + const std::string client_addr { std::get<0>(entry) }; + const uint32_t error_count { std::get<1>(entry) }; + const uint64_t last_updated { std::get<2>(entry) }; + uint64_t post_command_time = monotonic_time(); + + std::string t_exp_client_addr { "10.200.%d.2" }; + std::string exp_client_addr {}; + string_format(t_exp_client_addr, exp_client_addr, entry_num); + + ok( + client_addr == exp_client_addr && error_count == errors && + (pre_command_time < last_updated < post_command_time), + "Entry should match expected values - exp(addr: %s, err_count: %d, last_updated: %ld < %ld < %ld)," + " act(addr: %s, err_count: %d, last_updated: %ld < %ld < %ld)", + exp_client_addr.c_str(), errors, pre_command_time, last_updated, post_command_time, + client_addr.c_str(), error_count, pre_command_time, last_updated, post_command_time + ); + + entry_num += 1; + } + } + + printf("\n"); + diag("Performing connections to fill 'client_host_cache'"); + + uint32_t expected_ns = 4; + const std::string expected_address { "10.200.4.2" }; + uint32_t expected_errors = 5; + + int limits = errors; + std::string inv_user_command_limit {}; + const std::string t_stderr_inv_user_command { + "ip netns exec ns%d mysql -h10.200.%d.1 -uinv_user -pinv_pass -P6033 2>&1" + }; + string_format(t_stderr_inv_user_command, inv_user_command_limit, 4, 4); + + std::string command_res {}; + for (int limits = errors; limits < 5 + 1; limits++) { + printf("\n"); + int inv_user_limit_err = exec(inv_user_command_limit.c_str(), command_res); + diag("Client connection failed with error: (%d, %s)", inv_user_limit_err, command_res.c_str()); + } + printf("\n"); + + diag("2. Checking the connection is denied when the limit is reached."); + + auto limit_error = command_res.find("ERROR 2013 (HY000)"); + ok( + limit_error != std::string::npos, + "Last connection should fail with 'ERROR 2013', it exceeded the error limit. ErrMsg: '%s'", + command_res.c_str() + ); + + std::vector new_entries { + get_client_host_cache_entries(proxysql_admin) + }; + + auto cache_entry = + std::find_if( + std::begin(new_entries), + std::end(new_entries), + [&] (const host_cache_entry& elem) -> bool { + return std::get<0>(elem) == expected_address; + } + ); + + bool found_exp_values = false; + std::string client_address {}; + uint32_t found_errors = 0; + + if (cache_entry != std::end(new_entries)) { + client_address = std::get<0>(*cache_entry); + found_errors = std::get<1>(*cache_entry); + + found_exp_values = + std::get<0>(*cache_entry) == expected_address && + std::get<1>(*cache_entry) == expected_errors; + } + + ok( + found_exp_values, + "Entry should match expected values - exp(addr: %s, err_count: %d), act(addr: %s, err_count: %d)", + expected_address.c_str(), expected_errors, client_address.c_str(), found_errors + ); + + diag("3. Check that clients are deleted from the cache when the connections are succesfully performed"); + + for (int i = 1; i < NUM_NETWORK_NAMESPACES; i++) { + // This client as exceeded the max failures + + std::string valid_connection_command {}; + string_format(t_valid_connection_command, valid_connection_command, i, i); + + // Client has exceeded maximum connections failure is expected + if (i == 4) { + std::string command_res {}; + exec(valid_connection_command, command_res); + auto limit_error = command_res.find("ERROR 2013 (HY000)"); + + ok( + limit_error != std::string::npos, + "Connection should fail due to limit exceeded. ErrMsg: '%s'", command_res.c_str() + ); + } else { + int command_err = system(valid_connection_command.c_str()); + ok( + command_err == 0, + "Connection should succeed for clients which limit haven't been exceeded." + ); + } + } + + new_entries = get_client_host_cache_entries(proxysql_admin); + ok( + new_entries.size() == 1 && + std::get<0>(new_entries.back()) == "10.200.4.2", + "Only client address exceeding the limit should remain in the cache -" + " exp('10.200.4.2'), act('%s')", std::get<0>(new_entries.back()).c_str() + ); + } + + // 5. Flush the entries, fill the cache and check that the when the + // 'mysql-client_host_error_counts' is changed at runtime, connections are denied + // to a client exceeding the new limit. + { + printf("\n"); + diag(" START TEST NUMBER 5 "); + diag("-------------------------------------------------------------"); + + // Increase cache size + printf("\n"); + diag("Setting the value of 'mysql-client_host_cache_size' to '5'"); + + MYSQL_QUERY(proxysql_admin, "SET mysql-client_host_cache_size=5"); + MYSQL_QUERY(proxysql_admin, "SET mysql-client_host_error_counts=5"); + MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + // There shouldn't be any other entries in the cache for this test. + MYSQL_QUERY(proxysql_admin, "PROXYSQL FLUSH MYSQL CLIENT HOSTS"); + int errors = 0; + + std::string inv_user_command {}; + string_format(t_inv_user_command, inv_user_command, 1, 1); + uint64_t pre_command_time = monotonic_time(); + + diag("Performing connections to fill 'client_host_cache'"); + + for (int i = 0; i < 4; i++) { + printf("\n"); + int inv_user_errno = system(inv_user_command.c_str()); + diag("Client connection failed with error: %d", inv_user_errno); + printf("\n"); + } + + diag("Decreasing the value of 'mysql-client_host_error_counts' to '3'"); + MYSQL_QUERY(proxysql_admin, "SET mysql-client_host_error_counts=3"); + MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + { + printf("\n"); + std::string valid_user_command {}; + string_format(t_valid_connection_command, valid_user_command, 1, 1); + + std::string command_res {}; + int valid_user_err = exec(valid_user_command.c_str(), command_res); + diag("Client connection failed with error: (%d, %s)", valid_user_err, command_res.c_str()); + + auto limit_error = command_res.find("ERROR 2013 (HY000)"); + ok( + limit_error != std::string::npos, + "Last connection should fail with 'ERROR 2013', it exceeded the error limit. ErrMsg: '%s'", + command_res.c_str() + ); + } + } + + // 6. Flush the entries, fill the cache and check that the when the + // 'mysql-client_host_cache_size' is changed at runtime, the exceeding + // elements are cleaned with each new connection. Without being relevant if was + // present or not in the cache. + { + printf("\n"); + diag(" START TEST NUMBER 6 "); + diag("-------------------------------------------------------------"); + + // Increase cache size + printf("\n"); + diag("Setting the value of 'mysql-client_host_cache_size' to '5'"); + + MYSQL_QUERY(proxysql_admin, "SET mysql-client_host_cache_size=5"); + MYSQL_QUERY(proxysql_admin, "SET mysql-client_host_error_counts=5"); + MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + // There shouldn't be any other entries in the cache for this test. + MYSQL_QUERY(proxysql_admin, "PROXYSQL FLUSH MYSQL CLIENT HOSTS"); + int errors = 0; + + // Fill the cache with the entries from all the created namespaces + { + std::vector inv_connection_commands {}; + for (int i = 1; i < NUM_NETWORK_NAMESPACES; i++) { + std::string inv_user_command {}; + string_format(t_inv_user_command, inv_user_command, i, i); + inv_connection_commands.push_back(inv_user_command); + } + + printf("\n"); + diag("Performing connections to fill 'client_host_cache'"); + for (const auto inv_conn_command : inv_connection_commands) { + for (errors = 0; errors < 3; errors++) { + printf("\n"); + int inv_user_errno = system(inv_conn_command.c_str()); + diag("Client connection failed with error: %d", inv_user_errno); + } + printf("\n"); + } + printf("\n"); + } + + diag("Decreasing the value of 'mysql-client_host_cache_size' to '3'"); + MYSQL_QUERY(proxysql_admin, "SET mysql-client_host_cache_size=1"); + MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + // Update the latest entry in the cache, oldest member "10.200.1.2" should go away. + { + uint64_t pre_command_time = monotonic_time(); + + diag("1. Checking that the connection updates the entry and the oldest entry is removed"); + + std::string inv_user_command {}; + string_format(t_inv_user_command, inv_user_command, 4, 4); + + printf("\n"); + int inv_user_err = system(inv_user_command.c_str()); + diag("Client connection failed with error: %d", inv_user_err); + printf("\n"); + + std::vector updated_entries { + get_client_host_cache_entries(proxysql_admin) + }; + + std::string exp_client_addr { "10.200.4.2" }; + + auto entry = std::find_if( + std::begin(updated_entries), + std::end(updated_entries), + [&] (const host_cache_entry& entry) -> bool { + return std::get<0>(entry) == exp_client_addr; + } + ); + + std::string act_client_addr {}; + uint64_t last_updated = 0; + + if (entry != std::end(updated_entries)) { + act_client_addr = std::get<0>(*entry); + last_updated = std::get<2>(*entry); + } + + ok( + exp_client_addr == act_client_addr && last_updated > pre_command_time, + "Entry should be present and updated with the following values -" + " exp('%s', %ld > %ld), act('%s', %ld > %ld)", exp_client_addr.c_str(), + last_updated, pre_command_time, act_client_addr.c_str(), last_updated, + pre_command_time + ); + + // Oldest member shouldn't be present + std::string oldest_member { "10.200.1.2" }; + + auto oldest_entry = std::find_if( + std::begin(updated_entries), + std::end(updated_entries), + [&] (const host_cache_entry& entry) -> bool { + return std::get<0>(entry) == oldest_member; + } + ); + + ok( + oldest_entry == std::end(updated_entries), + "Oldest entry '%s' shouldn't be present in the cache.", oldest_member.c_str() + ); + + printf("\n"); + diag("2. Checking that the same behavior is observed if connection comes from a non-cached address"); + + string_format(t_inv_user_command, inv_user_command, 1, 1); + + printf("\n"); + inv_user_err = system(inv_user_command.c_str()); + diag("Client connection failed with error: %d", inv_user_err); + printf("\n"); + + diag("2.1 Checking that the address hasn't been added"); + + const std::string new_member { "10.200.1.2" }; + + updated_entries = get_client_host_cache_entries(proxysql_admin); + auto new_entry = std::find_if( + std::begin(updated_entries), + std::end(updated_entries), + [&] (const host_cache_entry& entry) -> bool { + return std::get<0>(entry) == new_member; + } + ); + + ok( + new_entry == std::end(updated_entries), + "New entry from address '10.200.1.2' shouldn't be present in the cache" + ); + + printf("\n"); + diag("2.1 Checking that the oldest address has been removed"); + oldest_member = "10.200.2.2"; + + oldest_entry = std::find_if( + std::begin(updated_entries), + std::end(updated_entries), + [&] (const host_cache_entry& entry) -> bool { + return std::get<0>(entry) == oldest_member; + } + ); + + ok( + oldest_entry == std::end(updated_entries), + "Oldest entry '%s' shouldn't be present in the cache.", oldest_member.c_str() + ); + + printf("\n"); + diag("2.2 Checking that a successful connection gets a client removed"); + printf("\n"); + + std::string valid_connection_command {}; + string_format(t_valid_connection_command, valid_connection_command, 3, 3); + system(valid_connection_command.c_str()); + + const std::string forgotten_address { "10.200.3.2" }; + + updated_entries = get_client_host_cache_entries(proxysql_admin); + auto forgot_entry = std::find_if( + std::begin(updated_entries), + std::end(updated_entries), + [&] (const host_cache_entry& entry) -> bool { + return std::get<0>(entry) == forgotten_address; + } + ); + + ok( + forgot_entry == std::end(updated_entries), + "Entry '%s' should have been forgotten due to successful connection.", + forgotten_address.c_str() + ); + } + } + } + +cleanup: + // Cleanup the virtual namespaces to be used by the test + printf("\n"); + diag(" Cleanup of testing network namespaces "); + diag("*********************************************************************"); + printf("\n"); + + const std::string t_delete_ns_command = + std::string { cl.workdir } + "/client_host_err/delete_netns_n.sh %d"; + for (int i = 1; i < setup_ns_i; i++) { + std::string delete_ns_command {}; + string_format(t_delete_ns_command, delete_ns_command, i); + system(delete_ns_command.c_str()); + } + + mysql_close(proxysql_admin); + + return exit_status(); +}