From 782aae072efdb4f76b6a716e163699b47c6b06dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Thu, 10 Jan 2019 21:03:52 +1100 Subject: [PATCH] Implement automatic vacuum of stats schema #1011 --- include/proxysql_admin.h | 4 ++- lib/ProxySQL_Admin.cpp | 74 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 24c9482b1..617624f6b 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -83,6 +83,7 @@ class ProxySQL_Admin { char *telnet_stats_ifaces; bool admin_read_only; bool hash_passwords; + bool vacuum_stats; char * admin_version; char * cluster_username; char * cluster_password; @@ -255,7 +256,7 @@ class ProxySQL_Admin { int Read_ProxySQL_Servers_from_configfile(); void flush_error_log(); - void GenericRefreshStatistics(const char *query_no_space, unsigned int query_no_space_length, bool admin); + bool GenericRefreshStatistics(const char *query_no_space, unsigned int query_no_space_length, bool admin); SQLite3_result * generate_show_table_status(const char *, char **err); SQLite3_result * generate_show_fields_from(const char *tablename, char **err); @@ -294,5 +295,6 @@ class ProxySQL_Admin { void save_clickhouse_users_runtime_to_database(bool _runtime); #endif /* PROXYSQLCLICKHOUSE */ + void vacuum_stats(bool); }; #endif /* __CLASS_PROXYSQL_ADMIN_H */ diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 3b0b47c81..642217d4a 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -373,6 +373,7 @@ static char * admin_variables_names[]= { (char *)"refresh_interval", (char *)"read_only", (char *)"hash_passwords", + (char *)"vacuum_stats", (char *)"version", (char *)"cluster_username", (char *)"cluster_password", @@ -1910,7 +1911,8 @@ void ProxySQL_Admin::flush_configdb() { // see #923 wrunlock(); } -void ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsigned int query_no_space_length, bool admin) { +bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsigned int query_no_space_length, bool admin) { + bool ret=false; bool refresh=false; bool stats_mysql_processlist=false; bool stats_mysql_connection_pool=false; @@ -2147,6 +2149,17 @@ void ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign } pthread_mutex_unlock(&admin_mutex); } + if ( + stats_mysql_processlist || stats_mysql_connection_pool || stats_mysql_connection_pool_reset || + stats_mysql_query_digest || stats_mysql_query_digest_reset || stats_mysql_errors || + stats_mysql_errors_reset || stats_mysql_global || stats_memory_metrics || + stats_mysql_commands_counters || stats_mysql_query_rules || stats_mysql_users || + stats_mysql_gtid_executed + ) { + ret = true; + } + + return ret; } @@ -2312,6 +2325,7 @@ SQLite3_result * ProxySQL_Admin::generate_show_table_status(const char *tablenam void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { ProxySQL_Admin *pa=(ProxySQL_Admin *)_pa; + bool needs_vacuum = false; char *error=NULL; int cols; int affected_rows; @@ -2371,7 +2385,7 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { } { ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; - SPA->GenericRefreshStatistics(query_no_space,query_no_space_length, ( sess->session_type == PROXYSQL_SESSION_ADMIN ? true : false ) ); + needs_vacuum = SPA->GenericRefreshStatistics(query_no_space,query_no_space_length, ( sess->session_type == PROXYSQL_SESSION_ADMIN ? true : false ) ); } @@ -3108,10 +3122,16 @@ __run_query: } else { SPA->admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); } + if (needs_vacuum) { + SPA->vacuum_stats(true); + } } else { SPA->statsdb->execute("PRAGMA query_only = ON"); SPA->statsdb->execute_statement(query, &error , &cols , &affected_rows , &resultset); SPA->statsdb->execute("PRAGMA query_only = OFF"); + if (needs_vacuum) { + SPA->vacuum_stats(false); + } } sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot); delete resultset; @@ -3121,6 +3141,41 @@ __run_query: pthread_mutex_unlock(&pa->sql_query_global_mutex); } +void ProxySQL_Admin::vacuum_stats(bool is_admin) { + if (variables.vacuum_stats==false) { + return; + } + if (is_admin) { + admindb->execute("DELETE FROM stats.stats_mysql_commands_counters"); + admindb->execute("DELETE FROM stats.stats_mysql_connection_pool"); + admindb->execute("DELETE FROM stats.stats_mysql_connection_pool_reset"); + admindb->execute("DELETE FROM stats.stats_mysql_prepared_statements_info"); + admindb->execute("DELETE FROM stats.stats_mysql_processlist"); + admindb->execute("DELETE FROM stats.stats_mysql_query_digest"); + admindb->execute("DELETE FROM stats.stats_mysql_query_digest_reset"); + admindb->execute("DELETE FROM stats.stats_mysql_query_rules"); + admindb->execute("DELETE FROM stats.stats_mysql_users"); + admindb->execute("DELETE FROM stats.stats_proxysql_servers_checksums"); + admindb->execute("DELETE FROM stats.stats_proxysql_servers_metrics"); + admindb->execute("DELETE FROM stats.stats_proxysql_servers_status"); + statsdb->execute("VACUUM stats"); + } else { + statsdb->execute("DELETE stats_mysql_commands_counters"); + statsdb->execute("DELETE stats_mysql_connection_pool"); + statsdb->execute("DELETE stats_mysql_connection_pool_reset"); + statsdb->execute("DELETE stats_mysql_prepared_statements_info"); + statsdb->execute("DELETE stats_mysql_processlist"); + statsdb->execute("DELETE stats_mysql_query_digest"); + statsdb->execute("DELETE stats_mysql_query_digest_reset"); + statsdb->execute("DELETE stats_mysql_query_rules"); + statsdb->execute("DELETE stats_mysql_users"); + statsdb->execute("DELETE stats_proxysql_servers_checksums"); + statsdb->execute("DELETE stats_proxysql_servers_metrics"); + statsdb->execute("DELETE stats_proxysql_servers_status"); + statsdb->execute("VACUUM"); + } +} + void *child_mysql(void *arg) { @@ -3453,6 +3508,7 @@ ProxySQL_Admin::ProxySQL_Admin() { variables.telnet_stats_ifaces=NULL; variables.refresh_interval=2000; variables.hash_passwords=true; // issue #676 + variables.vacuum_stats=true; // issue #1011 variables.admin_read_only=false; // by default, the admin interface accepts writes variables.admin_version=(char *)PROXYSQL_VERSION; variables.cluster_username=strdup((char *)""); @@ -4500,6 +4556,9 @@ char * ProxySQL_Admin::get_variable(char *name) { if (!strcasecmp(name,"hash_passwords")) { return strdup((variables.hash_passwords ? "true" : "false")); } + if (!strcasecmp(name,"vacuum_stats")) { + return strdup((variables.vacuum_stats ? "true" : "false")); + } if (!strcasecmp(name,"checksum_mysql_query_rules")) { return strdup((checksum_variables.checksum_mysql_query_rules ? "true" : "false")); } @@ -4946,6 +5005,17 @@ bool ProxySQL_Admin::set_variable(char *name, char *value) { // this is the pub } return false; } + if (!strcasecmp(name,"vacuum_stats")) { + if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { + variables.vacuum_stats=true; + return true; + } + if (strcasecmp(value,"false")==0 || strcasecmp(value,"0")==0) { + variables.vacuum_stats=false; + return true; + } + return false; + } if (!strcasecmp(name,"web_enabled")) { if (strcasecmp(value,"true")==0 || strcasecmp(value,"1")==0) { variables.web_enabled=true;