From bd6674c3839eb235bf32ed9ac0eec344573816c5 Mon Sep 17 00:00:00 2001 From: Rahim Kanji Date: Fri, 8 Nov 2024 12:48:29 +0500 Subject: [PATCH] * Introduced a 5% jitter to cache_ttl to increase the likelihood of distributing DNS resolution across multiple intervals, reducing the chance of simultaneous expiry times. * Added a small delay after queuing DNS records, allowing DNS resolver thread time to process as many hostnames as possible before assessing the need for additional helper threads. This helps prevent 'DNS resolver queue too big' warnings in the logs. --- include/MySQL_Monitor.hpp | 3 ++- lib/MySQL_Monitor.cpp | 26 ++++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/include/MySQL_Monitor.hpp b/include/MySQL_Monitor.hpp index 08901f561..f85aa5804 100644 --- a/include/MySQL_Monitor.hpp +++ b/include/MySQL_Monitor.hpp @@ -436,7 +436,8 @@ struct DNS_Resolve_Data { std::shared_ptr dns_cache; std::string hostname; std::set cached_ips; - unsigned int ttl; + unsigned int ttl = 0; + unsigned int refresh_intv = 0; }; diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 707ae7e7c..0273ef8a1 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -18,6 +18,7 @@ using json = nlohmann::json; #include #include #include +#include #include "prometheus/counter.h" #include "MySQL_Protocol.h" #include "MySQL_HostGroups_Manager.h" @@ -4678,6 +4679,13 @@ void* monitor_dns_resolver_thread(void* args) { if (!ips.empty()) { bool to_update_cache = false; + int cache_ttl = dns_resolve_data->ttl; + if (dns_resolve_data->ttl > dns_resolve_data->refresh_intv) { + thread_local std::mt19937 gen(std::random_device{}()); + const int jitter = static_cast(dns_resolve_data->ttl * 0.025); + std::uniform_int_distribution dis(-jitter, jitter); + cache_ttl += dis(gen); + } if (!dns_resolve_data->cached_ips.empty()) { @@ -4696,14 +4704,14 @@ void* monitor_dns_resolver_thread(void* args) { // only update dns_records_bookkeeping if (!to_update_cache) { proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "DNS cache record already up-to-date. (Hostname:[%s] IP:[%s])\n", dns_resolve_data->hostname.c_str(), debug_iplisttostring(ips).c_str()); - dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, std::move(dns_resolve_data->cached_ips), monotonic_time() + (1000 * dns_resolve_data->ttl)))); + dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, std::move(dns_resolve_data->cached_ips), monotonic_time() + (1000 * cache_ttl)))); } } else to_update_cache = true; if (to_update_cache) { - dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, ips, monotonic_time() + (1000 * dns_resolve_data->ttl)))); + dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, ips, monotonic_time() + (1000 * cache_ttl)))); dns_resolve_data->dns_cache->add(dns_resolve_data->hostname, std::move(ips)); } @@ -4851,6 +4859,16 @@ void* MySQL_Monitor::monitor_dns_cache() { std::list>> dns_resolve_result; + int delay_us = 100; + if (hostnames.empty() == false) { + delay_us = mysql_thread___monitor_local_dns_cache_refresh_interval / 2 / hostnames.size(); + delay_us *= 40; + if (delay_us > 1000000 || delay_us <= 0) { + delay_us = 10000; + } + delay_us = delay_us + rand() % delay_us; + } + if (dns_records_bookkeeping.empty() == false) { unsigned long long current_time = monotonic_time(); @@ -4871,12 +4889,14 @@ void* MySQL_Monitor::monitor_dns_cache() { dns_resolve_data->hostname = std::move(itr->hostname_); dns_resolve_data->cached_ips = std::move(itr->ips_); dns_resolve_data->ttl = mysql_thread___monitor_local_dns_cache_ttl; + dns_resolve_data->refresh_intv = mysql_thread___monitor_local_dns_cache_refresh_interval; dns_resolve_data->dns_cache = dns_cache; dns_resolve_result.emplace_back(dns_resolve_data->result.get_future()); proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Removing expired DNS record from bookkeeper. (Hostname:[%s] IP:[%s])\n", itr->hostname_.c_str(), debug_iplisttostring(dns_resolve_data->cached_ips).c_str()); dns_resolver_queue.add(new WorkItem(dns_resolve_data.release(), monitor_dns_resolver_thread)); itr = dns_records_bookkeeping.erase(itr); + usleep(delay_us); continue; } @@ -4921,9 +4941,11 @@ void* MySQL_Monitor::monitor_dns_cache() { std::unique_ptr dns_resolve_data(new DNS_Resolve_Data()); dns_resolve_data->hostname = hostname; dns_resolve_data->ttl = mysql_thread___monitor_local_dns_cache_ttl; + dns_resolve_data->refresh_intv = mysql_thread___monitor_local_dns_cache_refresh_interval; dns_resolve_data->dns_cache = dns_cache; dns_resolve_result.emplace_back(dns_resolve_data->result.get_future()); dns_resolver_queue.add(new WorkItem(dns_resolve_data.release(), monitor_dns_resolver_thread)); + usleep(delay_us); } }