* Introduced a 5% jitter to cache_ttl to increase the likelihood of distributing DNS resolution across multiple intervals, reducing the chance of simultaneous expiry times.

* Added a small delay after queuing DNS records, allowing DNS resolver thread time to process as many hostnames as possible before assessing the need for additional helper threads. This helps prevent 'DNS resolver queue too big' warnings in the logs.

include/MySQL_Monitor.hpp

@@ -436,7 +436,8 @@ struct DNS_Resolve_Data {
 	std::shared_ptr<DNS_Cache> dns_cache;
 	std::string hostname;
 	std::set<std::string> cached_ips;
-	unsigned int ttl;
+	unsigned int ttl = 0;
+	unsigned int refresh_intv = 0;
 };
 
 

lib/MySQL_Monitor.cpp

@@ -18,6 +18,7 @@ using json = nlohmann::json;
 #include <thread>
 #include <future>
 #include <sstream>
+#include <random>
 #include "prometheus/counter.h"
 #include "MySQL_Protocol.h"
 #include "MySQL_HostGroups_Manager.h"
@@ -4678,6 +4679,13 @@ void* monitor_dns_resolver_thread(void* args) {
 		if (!ips.empty()) {
 
 			bool to_update_cache = false;
+			int cache_ttl = dns_resolve_data->ttl;
+			if (dns_resolve_data->ttl > dns_resolve_data->refresh_intv) {
+				thread_local std::mt19937 gen(std::random_device{}());
+				const int jitter = static_cast<int>(dns_resolve_data->ttl * 0.025);
+				std::uniform_int_distribution<int> dis(-jitter, jitter);
+				cache_ttl += dis(gen);
+			}
 
 			if (!dns_resolve_data->cached_ips.empty()) {
 
@@ -4696,14 +4704,14 @@ void* monitor_dns_resolver_thread(void* args) {
 				// only update dns_records_bookkeeping
 				if (!to_update_cache) {
 					proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "DNS cache record already up-to-date. (Hostname:[%s] IP:[%s])\n", dns_resolve_data->hostname.c_str(), debug_iplisttostring(ips).c_str());
-					dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, std::move(dns_resolve_data->cached_ips), monotonic_time() + (1000 * dns_resolve_data->ttl))));
+					dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, std::move(dns_resolve_data->cached_ips), monotonic_time() + (1000 * cache_ttl))));
 				}
 			}
 			else
 				to_update_cache = true;
 
 			if (to_update_cache) {
-				dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, ips, monotonic_time() + (1000 * dns_resolve_data->ttl))));
+				dns_resolve_data->result.set_value(std::make_tuple<>(true, DNS_Cache_Record(dns_resolve_data->hostname, ips, monotonic_time() + (1000 * cache_ttl))));
 				dns_resolve_data->dns_cache->add(dns_resolve_data->hostname, std::move(ips));
 			}
 
@@ -4851,6 +4859,16 @@ void* MySQL_Monitor::monitor_dns_cache() {
 
 			std::list<std::future<std::tuple<bool, DNS_Cache_Record>>> dns_resolve_result;
 
+			int delay_us = 100;
+			if (hostnames.empty() == false) {
+				delay_us = mysql_thread___monitor_local_dns_cache_refresh_interval / 2 / hostnames.size();
+				delay_us *= 40;
+				if (delay_us > 1000000 || delay_us <= 0) {
+					delay_us = 10000;
+				}
+				delay_us = delay_us + rand() % delay_us;
+			}
+
 			if (dns_records_bookkeeping.empty() == false) {
 				unsigned long long current_time = monotonic_time();
 
@@ -4871,12 +4889,14 @@ void* MySQL_Monitor::monitor_dns_cache() {
 							dns_resolve_data->hostname = std::move(itr->hostname_);
 							dns_resolve_data->cached_ips = std::move(itr->ips_);
 							dns_resolve_data->ttl = mysql_thread___monitor_local_dns_cache_ttl;
+							dns_resolve_data->refresh_intv = mysql_thread___monitor_local_dns_cache_refresh_interval;
 							dns_resolve_data->dns_cache = dns_cache;
 							dns_resolve_result.emplace_back(dns_resolve_data->result.get_future());
 
 							proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Removing expired DNS record from bookkeeper. (Hostname:[%s] IP:[%s])\n", itr->hostname_.c_str(), debug_iplisttostring(dns_resolve_data->cached_ips).c_str());
 							dns_resolver_queue.add(new WorkItem<DNS_Resolve_Data>(dns_resolve_data.release(), monitor_dns_resolver_thread));
 							itr = dns_records_bookkeeping.erase(itr);
+							usleep(delay_us);
 							continue;
 						}
 
@@ -4921,9 +4941,11 @@ void* MySQL_Monitor::monitor_dns_cache() {
 					std::unique_ptr<DNS_Resolve_Data> dns_resolve_data(new DNS_Resolve_Data());
 					dns_resolve_data->hostname = hostname;
 					dns_resolve_data->ttl = mysql_thread___monitor_local_dns_cache_ttl;
+					dns_resolve_data->refresh_intv = mysql_thread___monitor_local_dns_cache_refresh_interval;
 					dns_resolve_data->dns_cache = dns_cache;
 					dns_resolve_result.emplace_back(dns_resolve_data->result.get_future());
 					dns_resolver_queue.add(new WorkItem<DNS_Resolve_Data>(dns_resolve_data.release(), monitor_dns_resolver_thread));
+					usleep(delay_us);
 				}
 			}