Add PostgreSQL advanced eventslog schema and variable scaffolding

include/PgSQL_Thread.h

@@ -1026,6 +1026,12 @@ public:
 		int poll_timeout_on_failure;
 		char* eventslog_filename;
 		int eventslog_filesize;
+		/** @brief Circular buffer size for PostgreSQL advanced events logging. */
+		int eventslog_buffer_history_size;
+		/** @brief Maximum rows retained in stats_pgsql_query_events in-memory table. */
+		int eventslog_table_memory_size;
+		/** @brief Maximum query length copied into PostgreSQL eventslog circular buffer. */
+		int eventslog_buffer_max_query_length;
 		int eventslog_default_log;
 		int eventslog_format;
 		char* auditlog_filename;

include/ProxySQL_Admin_Tables_Definitions.h

@@ -338,6 +338,10 @@
 #define STATS_SQLITE_TABLE_PGSQL_CLIENT_HOST_CACHE "CREATE TABLE stats_pgsql_client_host_cache (client_address VARCHAR NOT NULL , error_count INT NOT NULL , last_updated BIGINT NOT NULL)"
 #define STATS_SQLITE_TABLE_PGSQL_CLIENT_HOST_CACHE_RESET "CREATE TABLE stats_pgsql_client_host_cache_reset (client_address VARCHAR NOT NULL , error_count INT NOT NULL , last_updated BIGINT NOT NULL)"
 #define STATS_SQLITE_TABLE_PGSQL_QUERY_RULES "CREATE TABLE stats_pgsql_query_rules (rule_id INTEGER PRIMARY KEY , hits INT NOT NULL)"
+/**
+ * @brief In-memory PostgreSQL query events table used by advanced events logging.
+ */
+#define STATS_SQLITE_TABLE_PGSQL_QUERY_EVENTS "CREATE TABLE stats_pgsql_query_events (id INTEGER PRIMARY KEY AUTOINCREMENT , thread_id INTEGER , username TEXT , database TEXT , start_time INTEGER , end_time INTEGER , query_digest TEXT , query TEXT , server TEXT , client TEXT , event_type INTEGER , hid INTEGER , extra_info TEXT , affected_rows INTEGER , rows_sent INTEGER , client_stmt_name TEXT , sqlstate TEXT , error TEXT)"
 #define STATS_SQLITE_TABLE_PGSQL_COMMANDS_COUNTERS "CREATE TABLE stats_pgsql_commands_counters (Command VARCHAR NOT NULL PRIMARY KEY , Total_Time_us INT NOT NULL , Total_cnt INT NOT NULL , cnt_100us INT NOT NULL , cnt_500us INT NOT NULL , cnt_1ms INT NOT NULL , cnt_5ms INT NOT NULL , cnt_10ms INT NOT NULL , cnt_50ms INT NOT NULL , cnt_100ms INT NOT NULL , cnt_500ms INT NOT NULL , cnt_1s INT NOT NULL , cnt_5s INT NOT NULL , cnt_10s INT NOT NULL , cnt_INFs)"
 #define STATS_SQLITE_TABLE_PGSQL_QUERY_DIGEST "CREATE TABLE stats_pgsql_query_digest (hostgroup INT , database VARCHAR NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , digest VARCHAR NOT NULL , digest_text VARCHAR NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , sum_time INTEGER NOT NULL , min_time INTEGER NOT NULL , max_time INTEGER NOT NULL , sum_rows_affected INTEGER NOT NULL , sum_rows_sent INTEGER NOT NULL , PRIMARY KEY(hostgroup, database, username, client_address, digest))"
 #define STATS_SQLITE_TABLE_PGSQL_QUERY_DIGEST_RESET "CREATE TABLE stats_pgsql_query_digest_reset (hostgroup INT , database VARCHAR NOT NULL , username VARCHAR NOT NULL , client_address VARCHAR NOT NULL , digest VARCHAR NOT NULL , digest_text VARCHAR NOT NULL , count_star INTEGER NOT NULL , first_seen INTEGER NOT NULL , last_seen INTEGER NOT NULL , sum_time INTEGER NOT NULL , min_time INTEGER NOT NULL , max_time INTEGER NOT NULL , sum_rows_affected INTEGER NOT NULL , sum_rows_sent INTEGER NOT NULL , PRIMARY KEY(hostgroup, database, username, client_address, digest))"

include/ProxySQL_Statistics.hpp

@@ -92,6 +92,10 @@
 
 
 #define STATSDB_SQLITE_TABLE_HISTORY_MYSQL_QUERY_EVENTS "CREATE TABLE history_mysql_query_events (id INTEGER PRIMARY KEY AUTOINCREMENT , thread_id INTEGER , username TEXT , schemaname TEXT , start_time INTEGER , end_time INTEGER , query_digest TEXT , query TEXT , server TEXT , client TEXT , event_type INTEGER , hid INTEGER , extra_info TEXT , affected_rows INTEGER , last_insert_id INTEGER , rows_sent INTEGER , client_stmt_id INTEGER , gtid TEXT , errno INT , error TEXT)"
+/**
+ * @brief On-disk PostgreSQL query events table used by advanced events logging.
+ */
+#define STATSDB_SQLITE_TABLE_HISTORY_PGSQL_QUERY_EVENTS "CREATE TABLE history_pgsql_query_events (id INTEGER PRIMARY KEY AUTOINCREMENT , thread_id INTEGER , username TEXT , database TEXT , start_time INTEGER , end_time INTEGER , query_digest TEXT , query TEXT , server TEXT , client TEXT , event_type INTEGER , hid INTEGER , extra_info TEXT , affected_rows INTEGER , rows_sent INTEGER , client_stmt_name TEXT , sqlstate TEXT , error TEXT)"
 
 class ProxySQL_Statistics {
 	SQLite3DB *statsdb_mem; // internal statistics DB
@@ -105,6 +109,7 @@ class ProxySQL_Statistics {
 	unsigned long long next_timer_mysql_query_digest_to_disk;
 	unsigned long long next_timer_system_cpu;
 	unsigned long long last_timer_mysql_dump_eventslog_to_disk = 0;
+	unsigned long long last_timer_pgsql_dump_eventslog_to_disk = 0;
 #ifndef NOJEM
 	unsigned long long next_timer_system_memory;
 #endif
@@ -121,6 +126,8 @@ class ProxySQL_Statistics {
 		int stats_system_cpu;
 		int stats_mysql_query_digest_to_disk;
 		int stats_mysql_eventslog_sync_buffer_to_disk;
+		/** @brief Periodic disk sync interval (seconds) for PostgreSQL eventslog buffer. */
+		int stats_pgsql_eventslog_sync_buffer_to_disk;
 #ifndef NOJEM
 		int stats_system_memory;
 #endif
@@ -142,6 +149,12 @@ class ProxySQL_Statistics {
 	 * The dump interval is retrieved from the ProxySQL configuration.  If the dump interval is 0, no dumping is performed.
 	 */
 	bool MySQL_Logger_dump_eventslog_timetoget(unsigned long long currentTimeMicros);
+	/**
+	 * @brief Checks if it's time to dump PostgreSQL eventslog buffer to disk.
+	 * @param currentTimeMicros The current time in microseconds.
+	 * @return True when periodic PostgreSQL events dump should run.
+	 */
+	bool PgSQL_Logger_dump_eventslog_timetoget(unsigned long long currentTimeMicros);
 
 #ifndef NOJEM
 	bool system_memory_timetoget(unsigned long long);

include/proxysql_admin.h

@@ -326,6 +326,8 @@ class ProxySQL_Admin {
 		int stats_mysql_query_cache;
 		int stats_mysql_query_digest_to_disk;
 		int stats_mysql_eventslog_sync_buffer_to_disk;
+		/** @brief Periodic disk sync interval (seconds) for PostgreSQL eventslog buffer. */
+		int stats_pgsql_eventslog_sync_buffer_to_disk;
 		int stats_system_cpu;
 		int stats_system_memory;
 		bool restapi_enabled;

include/proxysql_structs.h

@@ -1174,6 +1174,9 @@ __thread char* pgsql_thread___auditlog_filename;
 __thread int pgsql_thread___auditlog_filesize;
 __thread char* pgsql_thread___eventslog_filename;
 __thread int pgsql_thread___eventslog_filesize;
+__thread int pgsql_thread___eventslog_buffer_history_size;
+__thread int pgsql_thread___eventslog_table_memory_size;
+__thread int pgsql_thread___eventslog_buffer_max_query_length;
 __thread int pgsql_thread___eventslog_default_log;
 __thread int pgsql_thread___eventslog_format;
 __thread char* pgsql_thread___firewall_whitelist_errormsg;
@@ -1478,6 +1481,9 @@ extern __thread char* pgsql_thread___auditlog_filename;
 extern __thread int pgsql_thread___auditlog_filesize;
 extern __thread char* pgsql_thread___eventslog_filename;
 extern __thread int pgsql_thread___eventslog_filesize;
+extern __thread int pgsql_thread___eventslog_buffer_history_size;
+extern __thread int pgsql_thread___eventslog_table_memory_size;
+extern __thread int pgsql_thread___eventslog_buffer_max_query_length;
 extern __thread int pgsql_thread___eventslog_default_log;
 extern __thread int pgsql_thread___eventslog_format;
 extern __thread char* pgsql_thread___firewall_whitelist_errormsg;

lib/Admin_Bootstrap.cpp

@@ -910,6 +910,7 @@ bool ProxySQL_Admin::init(const bootstrap_info_t& bootstrap_info) {
 	insert_into_tables_defs(tables_defs_stats,"stats_pgsql_query_digest", STATS_SQLITE_TABLE_PGSQL_QUERY_DIGEST);
 	insert_into_tables_defs(tables_defs_stats,"stats_pgsql_query_digest_reset", STATS_SQLITE_TABLE_PGSQL_QUERY_DIGEST_RESET);
 	insert_into_tables_defs(tables_defs_stats,"stats_pgsql_prepared_statements_info", STATS_SQLITE_TABLE_PGSQL_PREPARED_STATEMENTS_INFO);
+	insert_into_tables_defs(tables_defs_stats,"stats_pgsql_query_events", STATS_SQLITE_TABLE_PGSQL_QUERY_EVENTS);
 
 	// ProxySQL Cluster
 	insert_into_tables_defs(tables_defs_admin,"proxysql_servers", ADMIN_SQLITE_TABLE_PROXYSQL_SERVERS);

lib/PgSQL_Thread.cpp

@@ -291,6 +291,9 @@ static char* pgsql_thread_variables_names[] = {
 	(char*)"connect_timeout_server_max",
 	(char*)"eventslog_filename",
 	(char*)"eventslog_filesize",
+	(char*)"eventslog_buffer_history_size",
+	(char*)"eventslog_table_memory_size",
+	(char*)"eventslog_buffer_max_query_length",
 	(char*)"eventslog_default_log",
 	(char*)"eventslog_format",
 	(char*)"auditlog_filename",
@@ -1129,6 +1132,9 @@ PgSQL_Threads_Handler::PgSQL_Threads_Handler() {
 	variables.interfaces = strdup((char*)"");
 	variables.eventslog_filename = strdup((char*)""); // proxysql-mysql-eventslog is recommended
 	variables.eventslog_filesize = 100 * 1024 * 1024;
+	variables.eventslog_buffer_history_size = 0;
+	variables.eventslog_table_memory_size = 10000;
+	variables.eventslog_buffer_max_query_length = 32 * 1024;
 	variables.eventslog_default_log = 0;
 	variables.eventslog_format = 1;
 	variables.auditlog_filename = strdup((char*)"");
@@ -2240,6 +2246,9 @@ char** PgSQL_Threads_Handler::get_variables_list() {
 		// logs
 		VariablesPointers_int["auditlog_filesize"] = make_tuple(&variables.auditlog_filesize, 1024 * 1024, 1 * 1024 * 1024 * 1024, false);
 		VariablesPointers_int["eventslog_filesize"] = make_tuple(&variables.eventslog_filesize, 1024 * 1024, 1 * 1024 * 1024 * 1024, false);
+		VariablesPointers_int["eventslog_buffer_history_size"] = make_tuple(&variables.eventslog_buffer_history_size, 0, 8 * 1024 * 1024, false);
+		VariablesPointers_int["eventslog_table_memory_size"] = make_tuple(&variables.eventslog_table_memory_size, 0, 8 * 1024 * 1024, false);
+		VariablesPointers_int["eventslog_buffer_max_query_length"] = make_tuple(&variables.eventslog_buffer_max_query_length, 128, 32 * 1024 * 1024, false);
 		VariablesPointers_int["eventslog_default_log"] = make_tuple(&variables.eventslog_default_log, 0, 1, false);
 		// various
 		VariablesPointers_int["long_query_time"] = make_tuple(&variables.long_query_time, 0, 20 * 24 * 3600 * 1000, false);
@@ -4011,6 +4020,9 @@ void PgSQL_Thread::refresh_variables() {
 
 	if (pgsql_thread___eventslog_filename) free(pgsql_thread___eventslog_filename);
 	pgsql_thread___eventslog_filesize = GloPTH->get_variable_int((char*)"eventslog_filesize");
+	pgsql_thread___eventslog_buffer_history_size = GloPTH->get_variable_int((char*)"eventslog_buffer_history_size");
+	pgsql_thread___eventslog_table_memory_size = GloPTH->get_variable_int((char*)"eventslog_table_memory_size");
+	pgsql_thread___eventslog_buffer_max_query_length = GloPTH->get_variable_int((char*)"eventslog_buffer_max_query_length");
 	pgsql_thread___eventslog_default_log = GloPTH->get_variable_int((char*)"eventslog_default_log");
 	pgsql_thread___eventslog_format = GloPTH->get_variable_int((char*)"eventslog_format");
 	pgsql_thread___eventslog_filename = GloPTH->get_variable_string((char*)"eventslog_filename");

lib/ProxySQL_Admin.cpp

@@ -377,6 +377,8 @@ static char * admin_variables_names[]= {
 	(char *)"stats_mysql_connection_pool",
 	(char *)"stats_mysql_query_cache",
 	(char *)"stats_mysql_query_digest_to_disk",
+	(char *)"stats_mysql_eventslog_sync_buffer_to_disk",
+	(char *)"stats_pgsql_eventslog_sync_buffer_to_disk",
 	(char *)"stats_system_cpu",
 	(char *)"stats_system_memory",
 	(char *)"mysql_ifaces",
@@ -2041,6 +2043,8 @@ void ProxySQL_Admin::vacuum_stats(bool is_admin) {
 		"stats_mysql_query_digest_reset",
 		"stats_pgsql_query_digest",
 		"stats_pgsql_query_digest_reset",
+		"stats_mysql_query_events",
+		"stats_pgsql_query_events",
 		"stats_mysql_query_rules",
 		"stats_pgsql_query_rules",
 		"stats_mysql_users",
@@ -2802,6 +2806,7 @@ ProxySQL_Admin::ProxySQL_Admin() :
 	variables.stats_mysql_query_cache = 60;
 	variables.stats_mysql_query_digest_to_disk = 0;
 	variables.stats_mysql_eventslog_sync_buffer_to_disk = 0;
+	variables.stats_pgsql_eventslog_sync_buffer_to_disk = 0;
 	variables.stats_system_cpu = 60;
 	variables.stats_system_memory = 60;
 	GloProxyStats->variables.stats_mysql_connection_pool = 60;
@@ -2809,6 +2814,7 @@ ProxySQL_Admin::ProxySQL_Admin() :
 	GloProxyStats->variables.stats_mysql_query_cache = 60;
 	GloProxyStats->variables.stats_mysql_query_digest_to_disk = 0;
 	GloProxyStats->variables.stats_mysql_eventslog_sync_buffer_to_disk = 0;
+	GloProxyStats->variables.stats_pgsql_eventslog_sync_buffer_to_disk = 0;
 	GloProxyStats->variables.stats_system_cpu = 60;
 #ifndef NOJEM
 	GloProxyStats->variables.stats_system_memory = 60;
@@ -3646,6 +3652,10 @@ char * ProxySQL_Admin::get_variable(char *name) {
 			snprintf(intbuf, sizeof(intbuf),"%d",variables.stats_mysql_eventslog_sync_buffer_to_disk);
 			return strdup(intbuf);
 		}
+		if (!strcasecmp(name,"stats_pgsql_eventslog_sync_buffer_to_disk")) {
+			snprintf(intbuf, sizeof(intbuf),"%d",variables.stats_pgsql_eventslog_sync_buffer_to_disk);
+			return strdup(intbuf);
+		}
 		if (!strcasecmp(name,"stats_system_cpu")) {
 			snprintf(intbuf, sizeof(intbuf),"%d",variables.stats_system_cpu);
 			return strdup(intbuf);
@@ -3988,6 +3998,16 @@ bool ProxySQL_Admin::set_variable(char *name, char *value, bool lock) {  // this
 				return false;
 			}
 		}
+		if (!strcasecmp(name,"stats_pgsql_eventslog_sync_buffer_to_disk")) {
+			int intv=atoi(value);
+			if (intv >= 0 && intv <= 24*3600) {
+				variables.stats_pgsql_eventslog_sync_buffer_to_disk=intv;
+				GloProxyStats->variables.stats_pgsql_eventslog_sync_buffer_to_disk=intv;
+				return true;
+			} else {
+				return false;
+			}
+		}
 		if (!strcasecmp(name,"stats_system_cpu")) {
 			int intv=atoi(value);
 			if (intv >= 0 && intv <= 600) {

lib/ProxySQL_Statistics.cpp

@@ -102,6 +102,7 @@ void ProxySQL_Statistics::init() {
 
 
 	insert_into_tables_defs(tables_defs_statsdb_disk,"history_mysql_query_events", STATSDB_SQLITE_TABLE_HISTORY_MYSQL_QUERY_EVENTS);
+	insert_into_tables_defs(tables_defs_statsdb_disk,"history_pgsql_query_events", STATSDB_SQLITE_TABLE_HISTORY_PGSQL_QUERY_EVENTS);
 
 	disk_upgrade_mysql_connections();
 
@@ -123,6 +124,8 @@ void ProxySQL_Statistics::init() {
 
 	statsdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_history_mysql_query_events_start_time ON history_mysql_query_events(start_time)");
 	statsdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_history_mysql_query_events_query_digest ON history_mysql_query_events(query_digest)");
+	statsdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_history_pgsql_query_events_start_time ON history_pgsql_query_events(start_time)");
+	statsdb_disk->execute("CREATE INDEX IF NOT EXISTS idx_history_pgsql_query_events_query_digest ON history_pgsql_query_events(query_digest)");
 }
 
 void ProxySQL_Statistics::disk_upgrade_mysql_connections() {
@@ -199,6 +202,18 @@ bool ProxySQL_Statistics::MySQL_Logger_dump_eventslog_timetoget(unsigned long lo
 	return false;
 }
 
+bool ProxySQL_Statistics::PgSQL_Logger_dump_eventslog_timetoget(unsigned long long currentTimeMicros) {
+	if (variables.stats_pgsql_eventslog_sync_buffer_to_disk) { // only proceed if not zero
+		unsigned long long t = variables.stats_pgsql_eventslog_sync_buffer_to_disk; // originally in seconds
+		t = t * 1000 * 1000;
+		if (currentTimeMicros > last_timer_pgsql_dump_eventslog_to_disk + t) {
+			last_timer_pgsql_dump_eventslog_to_disk = currentTimeMicros;
+			return true;
+		}
+	}
+	return false;
+}
+
 bool ProxySQL_Statistics::MySQL_Threads_Handler_timetoget(unsigned long long curtime) {
 	unsigned int i = (unsigned int)variables.stats_mysql_connections;
 	if (i) {