From 2a46c239becd850ade4756e1164cb2aebd9a284a Mon Sep 17 00:00:00 2001 From: Rene Cannao Date: Wed, 18 Feb 2026 09:28:45 +0000 Subject: [PATCH] Add PostgreSQL advanced eventslog schema and variable scaffolding --- include/PgSQL_Thread.h | 6 ++++++ include/ProxySQL_Admin_Tables_Definitions.h | 4 ++++ include/ProxySQL_Statistics.hpp | 13 +++++++++++++ include/proxysql_admin.h | 2 ++ include/proxysql_structs.h | 6 ++++++ lib/Admin_Bootstrap.cpp | 1 + lib/PgSQL_Thread.cpp | 12 ++++++++++++ lib/ProxySQL_Admin.cpp | 20 ++++++++++++++++++++ lib/ProxySQL_Statistics.cpp | 15 +++++++++++++++ 9 files changed, 79 insertions(+) diff --git a/include/PgSQL_Thread.h b/include/PgSQL_Thread.h index bb874463f..5a4b174f3 100644 --- a/include/PgSQL_Thread.h +++ b/include/PgSQL_Thread.h @@ -1026,6 +1026,12 @@ public: int poll_timeout_on_failure; char* eventslog_filename; int eventslog_filesize; + /** @brief Circular buffer size for PostgreSQL advanced events logging. */ + int eventslog_buffer_history_size; + /** @brief Maximum rows retained in stats_pgsql_query_events in-memory table. */ + int eventslog_table_memory_size; + /** @brief Maximum query length copied into PostgreSQL eventslog circular buffer. */ + int eventslog_buffer_max_query_length; int eventslog_default_log; int eventslog_format; char* auditlog_filename; diff --git a/include/ProxySQL_Admin_Tables_Definitions.h b/include/ProxySQL_Admin_Tables_Definitions.h index 37135c347..aa3c6f98c 100644 --- a/include/ProxySQL_Admin_Tables_Definitions.h +++ b/include/ProxySQL_Admin_Tables_Definitions.h @@ -338,6 +338,10 @@ #define STATS_SQLITE_TABLE_PGSQL_CLIENT_HOST_CACHE "CREATE TABLE stats_pgsql_client_host_cache (client_address VARCHAR NOT NULL , error_count INT NOT NULL , last_updated BIGINT NOT NULL)" #define STATS_SQLITE_TABLE_PGSQL_CLIENT_HOST_CACHE_RESET "CREATE TABLE stats_pgsql_client_host_cache_reset (client_address VARCHAR NOT NULL , error_count INT NOT NULL , last_updated BIGINT NOT NULL)" #define STATS_SQLITE_TABLE_PGSQL_QUERY_RULES "CREATE TABLE stats_pgsql_query_rules (rule_id INTEGER PRIMARY KEY , hits INT NOT NULL)" +/** + * @brief In-memory PostgreSQL query events table used by advanced events logging. + */ +#define STATS_SQLITE_TABLE_PGSQL_QUERY_EVENTS "CREATE TABLE stats_pgsql_query_events (id INTEGER PRIMARY KEY AUTOINCREMENT , thread_id INTEGER , username TEXT , database TEXT , start_time INTEGER , end_time INTEGER , query_digest TEXT , query TEXT , server TEXT , client TEXT , event_type INTEGER , hid INTEGER , extra_info TEXT , affected_rows INTEGER , rows_sent INTEGER , client_stmt_name TEXT , sqlstate TEXT , error TEXT)" #define STATS_SQLITE_TABLE_PGSQL_COMMANDS_COUNTERS "CREATE TABLE stats_pgsql_commands_counters (Command VARCHAR NOT NULL PRIMARY KEY , Total_Time_us INT NOT NULL , Total_cnt INT NOT NULL , cnt_100us INT NOT NULL , cnt_500us INT NOT NULL , cnt_1ms INT NOT NULL , cnt_5ms INT NOT NULL , cnt_10ms INT NOT NULL , cnt_50ms INT NOT NULL , cnt_100ms INT NOT NULL , cnt_500ms INT NOT NULL , cnt_1s INT NOT NULL , cnt_5s INT NOT NULL , cnt_10s INT NOT NULL , cnt_INFs)" #define STATS_SQLITE_TABLE_PGSQL_QUERY_DIGEST "CREATE TABLE stats_pgsql_query_digest (hostgroup INT , database VARCHAR NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , digest VARCHAR NOT NULL , digest_text VARCHAR NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , sum_time INTEGER NOT NULL , min_time INTEGER NOT NULL , max_time INTEGER NOT NULL , sum_rows_affected INTEGER NOT NULL , sum_rows_sent INTEGER NOT NULL , PRIMARY KEY(hostgroup, database, username, client_address, digest))" #define STATS_SQLITE_TABLE_PGSQL_QUERY_DIGEST_RESET "CREATE TABLE stats_pgsql_query_digest_reset (hostgroup INT , database VARCHAR NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , digest VARCHAR NOT NULL , digest_text VARCHAR NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , sum_time INTEGER NOT NULL , min_time INTEGER NOT NULL , max_time INTEGER NOT NULL , sum_rows_affected INTEGER NOT NULL , sum_rows_sent INTEGER NOT NULL , PRIMARY KEY(hostgroup, database, username, client_address, digest))" diff --git a/include/ProxySQL_Statistics.hpp b/include/ProxySQL_Statistics.hpp index 2dc2618e6..56ea214cd 100644 --- a/include/ProxySQL_Statistics.hpp +++ b/include/ProxySQL_Statistics.hpp @@ -92,6 +92,10 @@ #define STATSDB_SQLITE_TABLE_HISTORY_MYSQL_QUERY_EVENTS "CREATE TABLE history_mysql_query_events (id INTEGER PRIMARY KEY AUTOINCREMENT , thread_id INTEGER , username TEXT , schemaname TEXT , start_time INTEGER , end_time INTEGER , query_digest TEXT , query TEXT , server TEXT , client TEXT , event_type INTEGER , hid INTEGER , extra_info TEXT , affected_rows INTEGER , last_insert_id INTEGER , rows_sent INTEGER , client_stmt_id INTEGER , gtid TEXT , errno INT , error TEXT)" +/** + * @brief On-disk PostgreSQL query events table used by advanced events logging. + */ +#define STATSDB_SQLITE_TABLE_HISTORY_PGSQL_QUERY_EVENTS "CREATE TABLE history_pgsql_query_events (id INTEGER PRIMARY KEY AUTOINCREMENT , thread_id INTEGER , username TEXT , database TEXT , start_time INTEGER , end_time INTEGER , query_digest TEXT , query TEXT , server TEXT , client TEXT , event_type INTEGER , hid INTEGER , extra_info TEXT , affected_rows INTEGER , rows_sent INTEGER , client_stmt_name TEXT , sqlstate TEXT , error TEXT)" class ProxySQL_Statistics { SQLite3DB *statsdb_mem; // internal statistics DB @@ -105,6 +109,7 @@ class ProxySQL_Statistics { unsigned long long next_timer_mysql_query_digest_to_disk; unsigned long long next_timer_system_cpu; unsigned long long last_timer_mysql_dump_eventslog_to_disk = 0; + unsigned long long last_timer_pgsql_dump_eventslog_to_disk = 0; #ifndef NOJEM unsigned long long next_timer_system_memory; #endif @@ -121,6 +126,8 @@ class ProxySQL_Statistics { int stats_system_cpu; int stats_mysql_query_digest_to_disk; int stats_mysql_eventslog_sync_buffer_to_disk; + /** @brief Periodic disk sync interval (seconds) for PostgreSQL eventslog buffer. */ + int stats_pgsql_eventslog_sync_buffer_to_disk; #ifndef NOJEM int stats_system_memory; #endif @@ -142,6 +149,12 @@ class ProxySQL_Statistics { * The dump interval is retrieved from the ProxySQL configuration. If the dump interval is 0, no dumping is performed. */ bool MySQL_Logger_dump_eventslog_timetoget(unsigned long long currentTimeMicros); + /** + * @brief Checks if it's time to dump PostgreSQL eventslog buffer to disk. + * @param currentTimeMicros The current time in microseconds. + * @return True when periodic PostgreSQL events dump should run. + */ + bool PgSQL_Logger_dump_eventslog_timetoget(unsigned long long currentTimeMicros); #ifndef NOJEM bool system_memory_timetoget(unsigned long long); diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 75152c4ae..49d0440e8 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -326,6 +326,8 @@ class ProxySQL_Admin { int stats_mysql_query_cache; int stats_mysql_query_digest_to_disk; int stats_mysql_eventslog_sync_buffer_to_disk; + /** @brief Periodic disk sync interval (seconds) for PostgreSQL eventslog buffer. */ + int stats_pgsql_eventslog_sync_buffer_to_disk; int stats_system_cpu; int stats_system_memory; bool restapi_enabled; diff --git a/include/proxysql_structs.h b/include/proxysql_structs.h index 0efb79576..9ed1b4fea 100644 --- a/include/proxysql_structs.h +++ b/include/proxysql_structs.h @@ -1174,6 +1174,9 @@ __thread char* pgsql_thread___auditlog_filename; __thread int pgsql_thread___auditlog_filesize; __thread char* pgsql_thread___eventslog_filename; __thread int pgsql_thread___eventslog_filesize; +__thread int pgsql_thread___eventslog_buffer_history_size; +__thread int pgsql_thread___eventslog_table_memory_size; +__thread int pgsql_thread___eventslog_buffer_max_query_length; __thread int pgsql_thread___eventslog_default_log; __thread int pgsql_thread___eventslog_format; __thread char* pgsql_thread___firewall_whitelist_errormsg; @@ -1478,6 +1481,9 @@ extern __thread char* pgsql_thread___auditlog_filename; extern __thread int pgsql_thread___auditlog_filesize; extern __thread char* pgsql_thread___eventslog_filename; extern __thread int pgsql_thread___eventslog_filesize; +extern __thread int pgsql_thread___eventslog_buffer_history_size; +extern __thread int pgsql_thread___eventslog_table_memory_size; +extern __thread int pgsql_thread___eventslog_buffer_max_query_length; extern __thread int pgsql_thread___eventslog_default_log; extern __thread int pgsql_thread___eventslog_format; extern __thread char* pgsql_thread___firewall_whitelist_errormsg; diff --git a/lib/Admin_Bootstrap.cpp b/lib/Admin_Bootstrap.cpp index 7ff81e759..e8c66df44 100644 --- a/lib/Admin_Bootstrap.cpp +++ b/lib/Admin_Bootstrap.cpp @@ -910,6 +910,7 @@ bool ProxySQL_Admin::init(const bootstrap_info_t& bootstrap_info) { insert_into_tables_defs(tables_defs_stats,"stats_pgsql_query_digest", STATS_SQLITE_TABLE_PGSQL_QUERY_DIGEST); insert_into_tables_defs(tables_defs_stats,"stats_pgsql_query_digest_reset", STATS_SQLITE_TABLE_PGSQL_QUERY_DIGEST_RESET); insert_into_tables_defs(tables_defs_stats,"stats_pgsql_prepared_statements_info", STATS_SQLITE_TABLE_PGSQL_PREPARED_STATEMENTS_INFO); + insert_into_tables_defs(tables_defs_stats,"stats_pgsql_query_events", STATS_SQLITE_TABLE_PGSQL_QUERY_EVENTS); // ProxySQL Cluster insert_into_tables_defs(tables_defs_admin,"proxysql_servers", ADMIN_SQLITE_TABLE_PROXYSQL_SERVERS); diff --git a/lib/PgSQL_Thread.cpp b/lib/PgSQL_Thread.cpp index c3aa74136..c5a415bf3 100644 --- a/lib/PgSQL_Thread.cpp +++ b/lib/PgSQL_Thread.cpp @@ -291,6 +291,9 @@ static char* pgsql_thread_variables_names[] = { (char*)"connect_timeout_server_max", (char*)"eventslog_filename", (char*)"eventslog_filesize", + (char*)"eventslog_buffer_history_size", + (char*)"eventslog_table_memory_size", + (char*)"eventslog_buffer_max_query_length", (char*)"eventslog_default_log", (char*)"eventslog_format", (char*)"auditlog_filename", @@ -1129,6 +1132,9 @@ PgSQL_Threads_Handler::PgSQL_Threads_Handler() { variables.interfaces = strdup((char*)""); variables.eventslog_filename = strdup((char*)""); // proxysql-mysql-eventslog is recommended variables.eventslog_filesize = 100 * 1024 * 1024; + variables.eventslog_buffer_history_size = 0; + variables.eventslog_table_memory_size = 10000; + variables.eventslog_buffer_max_query_length = 32 * 1024; variables.eventslog_default_log = 0; variables.eventslog_format = 1; variables.auditlog_filename = strdup((char*)""); @@ -2240,6 +2246,9 @@ char** PgSQL_Threads_Handler::get_variables_list() { // logs VariablesPointers_int["auditlog_filesize"] = make_tuple(&variables.auditlog_filesize, 1024 * 1024, 1 * 1024 * 1024 * 1024, false); VariablesPointers_int["eventslog_filesize"] = make_tuple(&variables.eventslog_filesize, 1024 * 1024, 1 * 1024 * 1024 * 1024, false); + VariablesPointers_int["eventslog_buffer_history_size"] = make_tuple(&variables.eventslog_buffer_history_size, 0, 8 * 1024 * 1024, false); + VariablesPointers_int["eventslog_table_memory_size"] = make_tuple(&variables.eventslog_table_memory_size, 0, 8 * 1024 * 1024, false); + VariablesPointers_int["eventslog_buffer_max_query_length"] = make_tuple(&variables.eventslog_buffer_max_query_length, 128, 32 * 1024 * 1024, false); VariablesPointers_int["eventslog_default_log"] = make_tuple(&variables.eventslog_default_log, 0, 1, false); // various VariablesPointers_int["long_query_time"] = make_tuple(&variables.long_query_time, 0, 20 * 24 * 3600 * 1000, false); @@ -4011,6 +4020,9 @@ void PgSQL_Thread::refresh_variables() { if (pgsql_thread___eventslog_filename) free(pgsql_thread___eventslog_filename); pgsql_thread___eventslog_filesize = GloPTH->get_variable_int((char*)"eventslog_filesize"); + pgsql_thread___eventslog_buffer_history_size = GloPTH->get_variable_int((char*)"eventslog_buffer_history_size"); + pgsql_thread___eventslog_table_memory_size = GloPTH->get_variable_int((char*)"eventslog_table_memory_size"); + pgsql_thread___eventslog_buffer_max_query_length = GloPTH->get_variable_int((char*)"eventslog_buffer_max_query_length"); pgsql_thread___eventslog_default_log = GloPTH->get_variable_int((char*)"eventslog_default_log"); pgsql_thread___eventslog_format = GloPTH->get_variable_int((char*)"eventslog_format"); pgsql_thread___eventslog_filename = GloPTH->get_variable_string((char*)"eventslog_filename"); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 4543a5359..7523aaace 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -377,6 +377,8 @@ static char * admin_variables_names[]= { (char *)"stats_mysql_connection_pool", (char *)"stats_mysql_query_cache", (char *)"stats_mysql_query_digest_to_disk", + (char *)"stats_mysql_eventslog_sync_buffer_to_disk", + (char *)"stats_pgsql_eventslog_sync_buffer_to_disk", (char *)"stats_system_cpu", (char *)"stats_system_memory", (char *)"mysql_ifaces", @@ -2041,6 +2043,8 @@ void ProxySQL_Admin::vacuum_stats(bool is_admin) { "stats_mysql_query_digest_reset", "stats_pgsql_query_digest", "stats_pgsql_query_digest_reset", + "stats_mysql_query_events", + "stats_pgsql_query_events", "stats_mysql_query_rules", "stats_pgsql_query_rules", "stats_mysql_users", @@ -2802,6 +2806,7 @@ ProxySQL_Admin::ProxySQL_Admin() : variables.stats_mysql_query_cache = 60; variables.stats_mysql_query_digest_to_disk = 0; variables.stats_mysql_eventslog_sync_buffer_to_disk = 0; + variables.stats_pgsql_eventslog_sync_buffer_to_disk = 0; variables.stats_system_cpu = 60; variables.stats_system_memory = 60; GloProxyStats->variables.stats_mysql_connection_pool = 60; @@ -2809,6 +2814,7 @@ ProxySQL_Admin::ProxySQL_Admin() : GloProxyStats->variables.stats_mysql_query_cache = 60; GloProxyStats->variables.stats_mysql_query_digest_to_disk = 0; GloProxyStats->variables.stats_mysql_eventslog_sync_buffer_to_disk = 0; + GloProxyStats->variables.stats_pgsql_eventslog_sync_buffer_to_disk = 0; GloProxyStats->variables.stats_system_cpu = 60; #ifndef NOJEM GloProxyStats->variables.stats_system_memory = 60; @@ -3646,6 +3652,10 @@ char * ProxySQL_Admin::get_variable(char *name) { snprintf(intbuf, sizeof(intbuf),"%d",variables.stats_mysql_eventslog_sync_buffer_to_disk); return strdup(intbuf); } + if (!strcasecmp(name,"stats_pgsql_eventslog_sync_buffer_to_disk")) { + snprintf(intbuf, sizeof(intbuf),"%d",variables.stats_pgsql_eventslog_sync_buffer_to_disk); + return strdup(intbuf); + } if (!strcasecmp(name,"stats_system_cpu")) { snprintf(intbuf, sizeof(intbuf),"%d",variables.stats_system_cpu); return strdup(intbuf); @@ -3988,6 +3998,16 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) { // this return false; } } + if (!strcasecmp(name,"stats_pgsql_eventslog_sync_buffer_to_disk")) { + int intv=atoi(value); + if (intv >= 0 && intv <= 24*3600) { + variables.stats_pgsql_eventslog_sync_buffer_to_disk=intv; + GloProxyStats->variables.stats_pgsql_eventslog_sync_buffer_to_disk=intv; + return true; + } else { + return false; + } + } if (!strcasecmp(name,"stats_system_cpu")) { int intv=atoi(value); if (intv >= 0 && intv <= 600) { diff --git a/lib/ProxySQL_Statistics.cpp b/lib/ProxySQL_Statistics.cpp index 59634d36e..447b78a81 100644 --- a/lib/ProxySQL_Statistics.cpp +++ b/lib/ProxySQL_Statistics.cpp @@ -102,6 +102,7 @@ void ProxySQL_Statistics::init() { insert_into_tables_defs(tables_defs_statsdb_disk,"history_mysql_query_events", STATSDB_SQLITE_TABLE_HISTORY_MYSQL_QUERY_EVENTS); + insert_into_tables_defs(tables_defs_statsdb_disk,"history_pgsql_query_events", STATSDB_SQLITE_TABLE_HISTORY_PGSQL_QUERY_EVENTS); disk_upgrade_mysql_connections(); @@ -123,6 +124,8 @@ void ProxySQL_Statistics::init() { statsdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_history_mysql_query_events_start_time ON history_mysql_query_events(start_time)"); statsdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_history_mysql_query_events_query_digest ON history_mysql_query_events(query_digest)"); + statsdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_history_pgsql_query_events_start_time ON history_pgsql_query_events(start_time)"); + statsdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_history_pgsql_query_events_query_digest ON history_pgsql_query_events(query_digest)"); } void ProxySQL_Statistics::disk_upgrade_mysql_connections() { @@ -199,6 +202,18 @@ bool ProxySQL_Statistics::MySQL_Logger_dump_eventslog_timetoget(unsigned long lo return false; } +bool ProxySQL_Statistics::PgSQL_Logger_dump_eventslog_timetoget(unsigned long long currentTimeMicros) { + if (variables.stats_pgsql_eventslog_sync_buffer_to_disk) { // only proceed if not zero + unsigned long long t = variables.stats_pgsql_eventslog_sync_buffer_to_disk; // originally in seconds + t = t * 1000 * 1000; + if (currentTimeMicros > last_timer_pgsql_dump_eventslog_to_disk + t) { + last_timer_pgsql_dump_eventslog_to_disk = currentTimeMicros; + return true; + } + } + return false; +} + bool ProxySQL_Statistics::MySQL_Threads_Handler_timetoget(unsigned long long curtime) { unsigned int i = (unsigned int)variables.stats_mysql_connections; if (i) {