Further improvement for handling GTID

Statistics collected aboout GTID queries
Statistics displayed on HTTP server
Count number of GTID events per server
Online upgrade of all mysql_connections tables
Fixed path for libev

include/MySQL_HostGroups_Manager.h

@@ -62,6 +62,7 @@ class GTID_Server_Data {
 	size_t pos;
 	struct ev_io *w;
 	char uuid_server[64];
+	unsigned long long events_read;
 	gtid_set_t gtid_executed;
 	bool active;
 	GTID_Server_Data(struct ev_io *_w, char *_address, uint16_t _port, uint16_t _mysql_port);

include/MySQL_Thread.h

@@ -216,6 +216,7 @@ class MySQL_Thread
 		unsigned long long frontend_stmt_close;
 		unsigned long long queries;
 		unsigned long long queries_slow;
+		unsigned long long queries_gtid;
 		unsigned long long queries_backends_bytes_sent;
 		unsigned long long queries_backends_bytes_recv;
 		unsigned long long query_processor_time;
@@ -226,6 +227,8 @@ class MySQL_Thread
 		unsigned long long ConnPool_get_conn_immediate;
 		unsigned long long ConnPool_get_conn_success;
 		unsigned long long ConnPool_get_conn_failure;
+		unsigned long long gtid_binlog_collected;
+		unsigned long long gtid_session_collected;
 		unsigned int active_transactions;
 	} status_variables;
 
@@ -459,6 +462,8 @@ class MySQL_Threads_Handler
 	unsigned long long get_total_frontend_stmt_close();
 	unsigned long long get_total_queries();
 	unsigned long long get_slow_queries();
+	unsigned long long get_gtid_queries();
+	unsigned long long get_gtid_session_collected();
 	unsigned long long get_queries_backends_bytes_recv();
 	unsigned long long get_queries_backends_bytes_sent();
 	unsigned int get_active_transations();

include/ProxySQL_Statistics.hpp

@@ -6,7 +6,11 @@
 //#include "wqueue.h"
 #include <vector>
 
-#define STATSDB_SQLITE_TABLE_MYSQL_CONNECTIONS "CREATE TABLE mysql_connections (timestamp INT NOT NULL, Client_Connections_aborted INT NOT NULL, Client_Connections_connected INT NOT NULL, Client_Connections_created INT NOT NULL, Server_Connections_aborted INT NOT NULL, Server_Connections_connected INT NOT NULL, Server_Connections_created INT NOT NULL, ConnPool_get_conn_failure INT NOT NULL, ConnPool_get_conn_immediate INT NOT NULL, ConnPool_get_conn_success INT NOT NULL, Questions INT NOT NULL, Slow_queries INT NOT NULL, PRIMARY KEY (timestamp))"
+#define STATSDB_SQLITE_TABLE_MYSQL_CONNECTIONS_V1_4 "CREATE TABLE mysql_connections (timestamp INT NOT NULL, Client_Connections_aborted INT NOT NULL, Client_Connections_connected INT NOT NULL, Client_Connections_created INT NOT NULL, Server_Connections_aborted INT NOT NULL, Server_Connections_connected INT NOT NULL, Server_Connections_created INT NOT NULL, ConnPool_get_conn_failure INT NOT NULL, ConnPool_get_conn_immediate INT NOT NULL, ConnPool_get_conn_success INT NOT NULL, Questions INT NOT NULL, Slow_queries INT NOT NULL, PRIMARY KEY (timestamp))"
+
+#define STATSDB_SQLITE_TABLE_MYSQL_CONNECTIONS_V2_0 "CREATE TABLE mysql_connections (timestamp INT NOT NULL, Client_Connections_aborted INT NOT NULL, Client_Connections_connected INT NOT NULL, Client_Connections_created INT NOT NULL, Server_Connections_aborted INT NOT NULL, Server_Connections_connected INT NOT NULL, Server_Connections_created INT NOT NULL, ConnPool_get_conn_failure INT NOT NULL, ConnPool_get_conn_immediate INT NOT NULL, ConnPool_get_conn_success INT NOT NULL, Questions INT NOT NULL, Slow_queries INT NOT NULL, GTID_consistent_queries INT NOT NULL, PRIMARY KEY (timestamp))"
+
+#define STATSDB_SQLITE_TABLE_MYSQL_CONNECTIONS STATSDB_SQLITE_TABLE_MYSQL_CONNECTIONS_V2_0
 
 #define STATSDB_SQLITE_TABLE_SYSTEM_CPU "CREATE TABLE system_cpu (timestamp INT NOT NULL, tms_utime INT NOT NULL, tms_stime INT NOT NULL, PRIMARY KEY (timestamp))"
 
@@ -14,7 +18,11 @@
 #define STATSDB_SQLITE_TABLE_SYSTEM_MEMORY "CREATE TABLE system_memory (timestamp INT NOT NULL, allocated INT NOT NULL, resident INT NOT NULL, active INT NOT NULL, mapped INT NOT NULL, metadata INT NOT NULL, retained INT NOT NULL, PRIMARY KEY (timestamp))"
 #endif
 
-#define STATSDB_SQLITE_TABLE_MYSQL_CONNECTIONS_HOUR "CREATE TABLE mysql_connections_hour (timestamp INT NOT NULL, Client_Connections_aborted INT NOT NULL, Client_Connections_connected INT NOT NULL, Client_Connections_created INT NOT NULL, Server_Connections_aborted INT NOT NULL, Server_Connections_connected INT NOT NULL, Server_Connections_created INT NOT NULL, ConnPool_get_conn_failure INT NOT NULL, ConnPool_get_conn_immediate INT NOT NULL, ConnPool_get_conn_success INT NOT NULL, Questions INT NOT NULL, Slow_queries INT NOT NULL, PRIMARY KEY (timestamp))"
+#define STATSDB_SQLITE_TABLE_MYSQL_CONNECTIONS_HOUR_V1_4 "CREATE TABLE mysql_connections_hour (timestamp INT NOT NULL, Client_Connections_aborted INT NOT NULL, Client_Connections_connected INT NOT NULL, Client_Connections_created INT NOT NULL, Server_Connections_aborted INT NOT NULL, Server_Connections_connected INT NOT NULL, Server_Connections_created INT NOT NULL, ConnPool_get_conn_failure INT NOT NULL, ConnPool_get_conn_immediate INT NOT NULL, ConnPool_get_conn_success INT NOT NULL, Questions INT NOT NULL, Slow_queries INT NOT NULL, PRIMARY KEY (timestamp))"
+
+#define STATSDB_SQLITE_TABLE_MYSQL_CONNECTIONS_HOUR_V2_0 "CREATE TABLE mysql_connections_hour (timestamp INT NOT NULL, Client_Connections_aborted INT NOT NULL, Client_Connections_connected INT NOT NULL, Client_Connections_created INT NOT NULL, Server_Connections_aborted INT NOT NULL, Server_Connections_connected INT NOT NULL, Server_Connections_created INT NOT NULL, ConnPool_get_conn_failure INT NOT NULL, ConnPool_get_conn_immediate INT NOT NULL, ConnPool_get_conn_success INT NOT NULL, Questions INT NOT NULL, Slow_queries INT NOT NULL, GTID_consistent_queries INT NOT NULL, PRIMARY KEY (timestamp))"
+
+#define STATSDB_SQLITE_TABLE_MYSQL_CONNECTIONS_HOUR STATSDB_SQLITE_TABLE_MYSQL_CONNECTIONS_HOUR_V2_0
 
 #define STATSDB_SQLITE_TABLE_SYSTEM_CPU_HOUR "CREATE TABLE system_cpu_hour (timestamp INT NOT NULL, tms_utime INT NOT NULL, tms_stime INT NOT NULL, PRIMARY KEY (timestamp))"
 
@@ -22,7 +30,11 @@
 #define STATSDB_SQLITE_TABLE_SYSTEM_MEMORY_HOUR "CREATE TABLE system_memory_hour (timestamp INT NOT NULL, allocated INT NOT NULL, resident INT NOT NULL, active INT NOT NULL, mapped INT NOT NULL, metadata INT NOT NULL, retained INT NOT NULL, PRIMARY KEY (timestamp))"
 #endif
 
-#define STATSDB_SQLITE_TABLE_MYSQL_CONNECTIONS_DAY "CREATE TABLE mysql_connections_day (timestamp INT NOT NULL, Client_Connections_aborted INT NOT NULL, Client_Connections_connected INT NOT NULL, Client_Connections_created INT NOT NULL, Server_Connections_aborted INT NOT NULL, Server_Connections_connected INT NOT NULL, Server_Connections_created INT NOT NULL, ConnPool_get_conn_failure INT NOT NULL, ConnPool_get_conn_immediate INT NOT NULL, ConnPool_get_conn_success INT NOT NULL, Questions INT NOT NULL, Slow_queries INT NOT NULL, PRIMARY KEY (timestamp))"
+#define STATSDB_SQLITE_TABLE_MYSQL_CONNECTIONS_DAY_V1_4 "CREATE TABLE mysql_connections_day (timestamp INT NOT NULL, Client_Connections_aborted INT NOT NULL, Client_Connections_connected INT NOT NULL, Client_Connections_created INT NOT NULL, Server_Connections_aborted INT NOT NULL, Server_Connections_connected INT NOT NULL, Server_Connections_created INT NOT NULL, ConnPool_get_conn_failure INT NOT NULL, ConnPool_get_conn_immediate INT NOT NULL, ConnPool_get_conn_success INT NOT NULL, Questions INT NOT NULL, Slow_queries INT NOT NULL, PRIMARY KEY (timestamp))"
+
+#define STATSDB_SQLITE_TABLE_MYSQL_CONNECTIONS_DAY_V2_0 "CREATE TABLE mysql_connections_day (timestamp INT NOT NULL, Client_Connections_aborted INT NOT NULL, Client_Connections_connected INT NOT NULL, Client_Connections_created INT NOT NULL, Server_Connections_aborted INT NOT NULL, Server_Connections_connected INT NOT NULL, Server_Connections_created INT NOT NULL, ConnPool_get_conn_failure INT NOT NULL, ConnPool_get_conn_immediate INT NOT NULL, ConnPool_get_conn_success INT NOT NULL, Questions INT NOT NULL, Slow_queries INT NOT NULL, GTID_consistent_queries INT NOT NULL, PRIMARY KEY (timestamp))"
+
+#define STATSDB_SQLITE_TABLE_MYSQL_CONNECTIONS_DAY STATSDB_SQLITE_TABLE_MYSQL_CONNECTIONS_DAY_V2_0
 
 #define STATSDB_SQLITE_TABLE_SYSTEM_CPU_DAY "CREATE TABLE system_cpu_day (timestamp INT NOT NULL, tms_utime INT NOT NULL, tms_stime INT NOT NULL, PRIMARY KEY (timestamp))"
 
@@ -83,6 +95,7 @@ class ProxySQL_Statistics {
 	SQLite3_result * get_system_memory_metrics(int interval);
 #endif
 	SQLite3_result * get_MySQL_Query_Cache_metrics(int interval);
+	void disk_upgrade_mysql_connections();
 };
 
 #endif /* CLASS_PROXYSQL_STATISTICS_H */

lib/MySQL_HostGroups_Manager.cpp

@@ -179,6 +179,7 @@ GTID_Server_Data::GTID_Server_Data(struct ev_io *_w, char *_address, uint16_t _p
 	address = strdup(_address);
 	port = _port;
 	mysql_port = _mysql_port;
+	events_read = 0;
 }
 
 void GTID_Server_Data::resize(size_t _s) {
@@ -221,18 +222,18 @@ bool GTID_Server_Data::readall() {
 bool GTID_Server_Data::gtid_exists(char *gtid_uuid, uint64_t gtid_trxid) {
 	std::string s = gtid_uuid;
 	auto it = gtid_executed.find(s);
-	fprintf(stderr,"Checking if server %s:%d has GTID %s:%lu ... ", address, port, gtid_uuid, gtid_trxid);
+//	fprintf(stderr,"Checking if server %s:%d has GTID %s:%lu ... ", address, port, gtid_uuid, gtid_trxid);
 	if (it == gtid_executed.end()) {
-		fprintf(stderr,"NO\n");
+//		fprintf(stderr,"NO\n");
 		return false;
 	}
 	for (auto itr = it->second.begin(); itr != it->second.end(); ++itr) {
 		if ((int64_t)gtid_trxid >= itr->first && (int64_t)gtid_trxid <= itr->second) {
-			fprintf(stderr,"YES\n");
+//			fprintf(stderr,"YES\n");
 			return true;
 		}
 	}
-	fprintf(stderr,"NO\n");
+//	fprintf(stderr,"NO\n");
 	return false;
 }
 
@@ -359,14 +360,15 @@ bool GTID_Server_Data::read_next_gtid() {
 				default:
 					break;
 			}
-			fprintf(stdout,"%s:%lu\n", uuid_server, rec_trxid);
+			//fprintf(stdout,"%s:%lu\n", uuid_server, rec_trxid);
 			std::string s = uuid_server;
 			gtid_t new_gtid = std::make_pair(s,rec_trxid);
 			addGtid(new_gtid,gtid_executed);
+			events_read++;
 			//return true;
 		}
 	}
-	std::cout << "current pos " << gtid_executed_to_string(gtid_executed) << std::endl << std::endl;
+	//std::cout << "current pos " << gtid_executed_to_string(gtid_executed) << std::endl << std::endl;
 	return true;
 }
 
@@ -1339,7 +1341,7 @@ bool MySQL_HostGroups_Manager::gtid_exists(MySrvC *mysrvc, char * gtid_uuid, uin
 			ret = gtid_is->gtid_exists(gtid_uuid,gtid_trxid);
 		}
 	}	
-	proxy_info("Checking if server %s has GTID %s:%lu . %s\n", s1.c_str(), gtid_uuid, gtid_trxid, (ret ? "YES" : "NO"));
+	//proxy_info("Checking if server %s has GTID %s:%lu . %s\n", s1.c_str(), gtid_uuid, gtid_trxid, (ret ? "YES" : "NO"));
 	pthread_rwlock_unlock(&gtid_rwlock);
 	return ret;
 }
@@ -3182,11 +3184,12 @@ void MySQL_HostGroups_Manager::converge_group_replication_config(int _writer_hos
 }
 
 SQLite3_result * MySQL_HostGroups_Manager::get_stats_mysql_gtid_executed() {
-	const int colnum = 3;
+	const int colnum = 4;
 	SQLite3_result * result = new SQLite3_result(colnum);
 	result->add_column_definition(SQLITE_TEXT,"hostname");
 	result->add_column_definition(SQLITE_TEXT,"port");
 	result->add_column_definition(SQLITE_TEXT,"gtid_executed");
+	result->add_column_definition(SQLITE_TEXT,"events");
 	int k;
 	pthread_rwlock_wrlock(&gtid_rwlock);
 	std::unordered_map<string, GTID_Server_Data *>::iterator it = gtid_map.begin();
@@ -3200,6 +3203,8 @@ SQLite3_result * MySQL_HostGroups_Manager::get_stats_mysql_gtid_executed() {
 		//sprintf(buf,"%d", mysrvc->port);
 		string s1 = gtid_executed_to_string(gtid_si->gtid_executed);
 		pta[2]=strdup(s1.c_str());
+		sprintf(buf,"%llu", (int)gtid_si->events_read);
+		pta[3]=strdup(buf);
 		result->add_row(pta);
 		for (k=0; k<colnum; k++) {
 			if (pta[k])

lib/MySQL_Session.cpp

@@ -145,6 +145,9 @@ void Query_Info::end() {
 	if ((end_time-start_time) > (unsigned int)mysql_thread___long_query_time*1000) {
 		__sync_add_and_fetch(&sess->thread->status_variables.queries_slow,1);
 	}
+	if (sess->with_gtid) {
+		__sync_add_and_fetch(&sess->thread->status_variables.queries_gtid,1);
+	}
 	assert(mysql_stmt==NULL);
 	if (stmt_info) {
 		stmt_info=NULL;

lib/MySQL_Thread.cpp

@@ -3409,6 +3409,7 @@ MySQL_Thread::MySQL_Thread() {
 
 	status_variables.queries=0;
 	status_variables.queries_slow=0;
+	status_variables.queries_gtid=0;
 	status_variables.queries_backends_bytes_sent=0;
 	status_variables.queries_backends_bytes_recv=0;
 	status_variables.query_processor_time=0;
@@ -3420,6 +3421,7 @@ MySQL_Thread::MySQL_Thread() {
 	status_variables.ConnPool_get_conn_success=0;
 	status_variables.ConnPool_get_conn_failure=0;
 	status_variables.active_transactions=0;
+	status_variables.gtid_session_collected = 0;
 
 	match_regexes=NULL;
 
@@ -3763,6 +3765,18 @@ SQLite3_result * MySQL_Threads_Handler::SQL3_GlobalStatus(bool _memory) {
 		pta[1]=buf;
 		result->add_row(pta);
 	}
+	{	// Queries with GTID consistent read
+		pta[0]=(char *)"GTID_consistent_queries";
+		sprintf(buf,"%llu",get_gtid_queries());
+		pta[1]=buf;
+		result->add_row(pta);
+	}
+	{	// Queries with GTID session state
+		pta[0]=(char *)"GTID_session_collected";
+		sprintf(buf,"%llu",get_gtid_session_collected());
+		pta[1]=buf;
+		result->add_row(pta);
+	}
 	{	// Servers_table_version
 		pta[0]=(char *)"Servers_table_version";
 		sprintf(buf,"%u",MyHGM->get_servers_table_version());
@@ -4342,6 +4356,32 @@ unsigned long long MySQL_Threads_Handler::get_slow_queries() {
 	return q;
 }
 
+unsigned long long MySQL_Threads_Handler::get_gtid_queries() {
+	unsigned long long q=0;
+	unsigned int i;
+	for (i=0;i<num_threads;i++) {
+		if (mysql_threads) {
+			MySQL_Thread *thr=(MySQL_Thread *)mysql_threads[i].worker;
+			if (thr)
+				q+=__sync_fetch_and_add(&thr->status_variables.queries_gtid,0);
+		}
+	}
+	return q;
+}
+
+unsigned long long MySQL_Threads_Handler::get_gtid_session_collected() {
+	unsigned long long q=0;
+	unsigned int i;
+	for (i=0;i<num_threads;i++) {
+		if (mysql_threads) {
+			MySQL_Thread *thr=(MySQL_Thread *)mysql_threads[i].worker;
+			if (thr)
+				q+=__sync_fetch_and_add(&thr->status_variables.gtid_session_collected,0);
+		}
+	}
+	return q;
+}
+
 unsigned long long MySQL_Threads_Handler::get_queries_backends_bytes_recv() {
 	unsigned long long q=0;
 	unsigned int i;

lib/ProxySQL_Admin.cpp

@@ -278,7 +278,7 @@ static int http_handler(void *cls, struct MHD_Connection *connection, const char
 
 #define STATS_SQLITE_TABLE_MEMORY_METRICS "CREATE TABLE stats_memory_metrics (Variable_Name VARCHAR NOT NULL PRIMARY KEY , Variable_Value VARCHAR NOT NULL)"
 
-#define STATS_SQLITE_TABLE_MYSQL_GTID_EXECUTED "CREATE TABLE stats_mysql_gtid_executed (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_executed VARCHAR)"
+#define STATS_SQLITE_TABLE_MYSQL_GTID_EXECUTED "CREATE TABLE stats_mysql_gtid_executed (hostname VARCHAR NOT NULL , port INT NOT NULL DEFAULT 3306 , gtid_executed VARCHAR , events INT NOT NULL)"
 
 #ifdef DEBUG
 #define ADMIN_SQLITE_TABLE_DEBUG_LEVELS "CREATE TABLE debug_levels (module VARCHAR NOT NULL PRIMARY KEY , verbosity INT NOT NULL DEFAULT 0)"
@@ -6506,8 +6506,8 @@ void ProxySQL_Admin::stats___mysql_gtid_executed() {
 		sqlite3 *mydb3=statsdb->get_db();
 		char *query1=NULL;
 		char *query32=NULL;
-		query1=(char *)"INSERT INTO stats_mysql_gtid_executed VALUES (?1, ?2, ?3)";
-		query32=(char *)"INSERT INTO stats_mysql_gtid_executed VALUES (?1, ?2, ?3), (?4, ?5, ?6), (?7, ?8, ?9), (?10, ?11, ?12), (?13, ?14, ?15), (?16, ?17, ?18), (?19, ?20, ?21), (?22, ?23, ?24), (?25, ?26, ?27), (?28, ?29, ?30), (?31, ?32, ?33), (?34, ?35, ?36), (?37, ?38, ?39), (?40, ?41, ?42), (?43, ?44, ?45), (?46, ?47, ?48), (?49, ?50, ?51), (?52, ?53, ?54), (?55, ?56, ?57), (?58, ?59, ?60), (?61, ?62, ?63), (?64, ?65, ?66), (?67, ?68, ?69), (?70, ?71, ?72), (?73, ?74, ?75), (?76, ?77, ?78), (?79, ?80, ?81), (?82, ?83, ?84), (?85, ?86, ?87), (?88, ?89, ?90), (?91, ?92, ?93), (?94, ?95, ?96)";
+		query1=(char *)"INSERT INTO stats_mysql_gtid_executed VALUES (?1, ?2, ?3, ?4)";
+		query32=(char *)"INSERT INTO stats_mysql_gtid_executed VALUES (?1, ?2, ?3, ?4), (?5, ?6, ?7, ?8), (?9, ?10, ?11, ?12), (?13, ?14, ?15, ?16), (?17, ?18, ?19, ?20), (?21, ?22, ?23, ?24), (?25, ?26, ?27, ?28), (?29, ?30, ?31, ?32), (?33, ?34, ?35, ?36), (?37, ?38, ?39, ?40), (?41, ?42, ?43, ?44), (?45, ?46, ?47, ?48), (?49, ?50, ?51, ?52), (?53, ?54, ?55, ?56), (?57, ?58, ?59, ?60), (?61, ?62, ?63, ?64), (?65, ?66, ?67, ?68), (?69, ?70, ?71, ?72), (?73, ?74, ?75, ?76), (?77, ?78, ?79, ?80), (?81, ?82, ?83, ?84), (?85, ?86, ?87, ?88), (?89, ?90, ?91, ?92), (?93, ?94, ?95, ?96), (?97, ?98, ?99, ?100), (?101, ?102, ?103, ?104), (?105, ?106, ?107, ?108), (?109, ?110, ?111, ?112), (?113, ?114, ?115, ?116), (?117, ?118, ?119, ?120), (?121, ?122, ?123, ?124), (?125, ?126, ?127, ?128), (?129, ?130, ?131, ?132)";
 
 		rc=sqlite3_prepare_v2(mydb3, query1, -1, &statement1, 0);
 		assert(rc==SQLITE_OK);
@@ -6520,9 +6520,10 @@ void ProxySQL_Admin::stats___mysql_gtid_executed() {
 			SQLite3_row *r1=*it;
 			int idx=row_idx%32;
 			if (row_idx<max_bulk_row_idx) { // bulk
-				rc=sqlite3_bind_text(statement32, (idx*3)+1, r1->fields[0], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK);
-				rc=sqlite3_bind_int64(statement32, (idx*3)+2, atoi(r1->fields[1])); assert(rc==SQLITE_OK);
-				rc=sqlite3_bind_text(statement32, (idx*3)+3, r1->fields[2], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK);
+				rc=sqlite3_bind_text(statement32, (idx*4)+1, r1->fields[0], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK);
+				rc=sqlite3_bind_int64(statement32, (idx*4)+2, atoi(r1->fields[1])); assert(rc==SQLITE_OK);
+				rc=sqlite3_bind_text(statement32, (idx*4)+3, r1->fields[2], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK);
+				rc=sqlite3_bind_int64(statement32, (idx*4)+4, atoll(r1->fields[3])); assert(rc==SQLITE_OK);
 				if (idx==31) {
 					SAFE_SQLITE3_STEP(statement32);
 					rc=sqlite3_clear_bindings(statement32); assert(rc==SQLITE_OK);
@@ -6532,6 +6533,7 @@ void ProxySQL_Admin::stats___mysql_gtid_executed() {
 				rc=sqlite3_bind_text(statement1, 1, r1->fields[0], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK);
 				rc=sqlite3_bind_int64(statement1, 2, atoi(r1->fields[1])); assert(rc==SQLITE_OK);
 				rc=sqlite3_bind_text(statement1, 3, r1->fields[2], -1, SQLITE_TRANSIENT); assert(rc==SQLITE_OK);
+				rc=sqlite3_bind_int64(statement1, 4, atoll(r1->fields[3])); assert(rc==SQLITE_OK);
 				SAFE_SQLITE3_STEP(statement1);
 				rc=sqlite3_clear_bindings(statement1); assert(rc==SQLITE_OK);
 				rc=sqlite3_reset(statement1); assert(rc==SQLITE_OK);

lib/ProxySQL_HTTP_Server.cpp

@@ -634,27 +634,33 @@ int ProxySQL_HTTP_Server::handler(void *cls, struct MHD_Connection *connection,
 			free(ts);
 
 
-			nm = (char **)malloc(sizeof(char *)*4);
+			nm = (char **)malloc(sizeof(char *)*6);
 			nm[0] = (char *)"ConnPool_get_conn_failure";
 			nm[1] = (char *)"ConnPool_get_conn_immediate";
 			nm[2] = (char *)"ConnPool_get_conn_success";
 			nm[3] = (char *)"Questions";
-			nl = (char **)malloc(sizeof(char *)*4);
+			nm[4] = (char *)"Slow_queries";
+			nm[5] = (char *)"GTID_consistent_queries";
+			nl = (char **)malloc(sizeof(char *)*6);
 			nl[0] = (char *)"ConnPool failure";
 			nl[1] = (char *)"ConnPool immediate";
 			nl[2] = (char *)"ConnPool success";
 			nl[3] = (char *)"Questions";
-			nv = (char **)malloc(sizeof(char *)*4);
+			nl[4] = (char *)"Slow Queries";
+			nl[5] = (char *)"GTID Consistent Queries";
+			nv = (char **)malloc(sizeof(char *)*6);
 			nv[0] = extract_values(mysql_metrics_sqlite,8,true);
 			nv[1] = extract_values(mysql_metrics_sqlite,9,true);
 			nv[2] = extract_values(mysql_metrics_sqlite,10,true);
 			nv[3] = extract_values(mysql_metrics_sqlite,11,true);
+			nv[4] = extract_values(mysql_metrics_sqlite,12,true);
+			nv[5] = extract_values(mysql_metrics_sqlite,13,true);
 			ts = extract_ts(mysql_metrics_sqlite,true);
-			s1 = generate_chart((char *)"myChart2",ts,4,nm,nl,nv);
+			s1 = generate_chart((char *)"myChart2",ts,6,nm,nl,nv);
 			s->append(s1->c_str());
 			free(nm);
 			free(nl);
-			for (int aa=0 ; aa<4 ; aa++) {
+			for (int aa=0 ; aa<6 ; aa++) {
 				free(nv[aa]);
 			}
 			free(nv);

lib/ProxySQL_Statistics.cpp

@@ -96,10 +96,37 @@ void ProxySQL_Statistics::init() {
 	insert_into_tables_defs(tables_defs_statsdb_disk,"mysql_query_cache_hour", STATSDB_SQLITE_TABLE_MYSQL_QUERY_CACHE_HOUR);
 	insert_into_tables_defs(tables_defs_statsdb_disk,"mysql_query_cache_day", STATSDB_SQLITE_TABLE_MYSQL_QUERY_CACHE_DAY);
 
+	disk_upgrade_mysql_connections();
+
 	check_and_build_standard_tables(statsdb_mem, tables_defs_statsdb_disk);
 	check_and_build_standard_tables(statsdb_disk, tables_defs_statsdb_disk);
 }
 
+void ProxySQL_Statistics::disk_upgrade_mysql_connections() {
+	int rci;
+	rci=statsdb_disk->check_table_structure((char *)"mysql_connections",(char *)STATSDB_SQLITE_TABLE_MYSQL_CONNECTIONS_V1_4);
+	if (rci) {
+		proxy_warning("Detected version v1.4 of table mysql_connections\n");
+		proxy_warning("ONLINE UPGRADE of table mysql_connections in progress\n");
+		statsdb_disk->execute("ALTER TABLE mysql_connections ADD COLUMN GTID_consistent_queries INT NOT NULL");
+		proxy_warning("ONLINE UPGRADE of table mysql_connections completed\n");
+	}
+	rci=statsdb_disk->check_table_structure((char *)"mysql_connections_hour",(char *)STATSDB_SQLITE_TABLE_MYSQL_CONNECTIONS_HOUR_V1_4);
+	if (rci) {
+		proxy_warning("Detected version v1.4 of table mysql_connections_hour\n");
+		proxy_warning("ONLINE UPGRADE of table mysql_connections_hour in progress\n");
+		statsdb_disk->execute("ALTER TABLE mysql_connections_hour ADD COLUMN GTID_consistent_queries INT NOT NULL");
+		proxy_warning("ONLINE UPGRADE of table mysql_connections_hour completed\n");
+	}
+	rci=statsdb_disk->check_table_structure((char *)"mysql_connections_day",(char *)STATSDB_SQLITE_TABLE_MYSQL_CONNECTIONS_DAY_V1_4);
+	if (rci) {
+		proxy_warning("Detected version v1.4 of table mysql_connections_day\n");
+		proxy_warning("ONLINE UPGRADE of table mysql_connections_day in progress\n");
+		statsdb_disk->execute("ALTER TABLE mysql_connections_day ADD COLUMN GTID_consistent_queries INT NOT NULL");
+		proxy_warning("ONLINE UPGRADE of table mysql_connections_day completed\n");
+	}
+}
+
 void ProxySQL_Statistics::print_version() {
   fprintf(stderr,"Standard ProxySQL Statistics rev. %s -- %s -- %s\n", PROXYSQL_STATISTICS_VERSION, __FILE__, __TIMESTAMP__);
 }
@@ -210,8 +237,8 @@ SQLite3_result * ProxySQL_Statistics::get_mysql_metrics(int interval) {
 	int affected_rows;
 	char *error = NULL;
 	char *query = NULL;
-	char *query1 = (char *)"SELECT SUBSTR(FROM_UNIXTIME(timestamp),0,20) ts, timestamp, Client_Connections_aborted, Client_Connections_connected, Client_Connections_created, Server_Connections_aborted, Server_Connections_connected, Server_Connections_created, ConnPool_get_conn_failure, ConnPool_get_conn_immediate, ConnPool_get_conn_success, Questions FROM mysql_connections WHERE timestamp BETWEEN %d AND %d ORDER BY timestamp";
-	char *query2 = (char *)"SELECT SUBSTR(FROM_UNIXTIME(timestamp),0,20) ts, timestamp, Client_Connections_aborted, Client_Connections_connected, Client_Connections_created, Server_Connections_aborted, Server_Connections_connected, Server_Connections_created, ConnPool_get_conn_failure, ConnPool_get_conn_immediate, ConnPool_get_conn_success, Questions FROM mysql_connections_hour WHERE timestamp BETWEEN %d AND %d ORDER BY timestamp";
+	char *query1 = (char *)"SELECT SUBSTR(FROM_UNIXTIME(timestamp),0,20) ts, timestamp, Client_Connections_aborted, Client_Connections_connected, Client_Connections_created, Server_Connections_aborted, Server_Connections_connected, Server_Connections_created, ConnPool_get_conn_failure, ConnPool_get_conn_immediate, ConnPool_get_conn_success, Questions, Slow_queries, GTID_consistent_queries FROM mysql_connections WHERE timestamp BETWEEN %d AND %d ORDER BY timestamp";
+	char *query2 = (char *)"SELECT SUBSTR(FROM_UNIXTIME(timestamp),0,20) ts, timestamp, Client_Connections_aborted, Client_Connections_connected, Client_Connections_created, Server_Connections_aborted, Server_Connections_connected, Server_Connections_created, ConnPool_get_conn_failure, ConnPool_get_conn_immediate, ConnPool_get_conn_success, Questions, Slow_queries, GTID_consistent_queries FROM mysql_connections_hour WHERE timestamp BETWEEN %d AND %d ORDER BY timestamp";
 	time_t ts = time(NULL);
 	switch (interval) {
 		case 1800:
@@ -535,7 +562,7 @@ void ProxySQL_Statistics::MySQL_Threads_Handler_sets(SQLite3_result *resultset)
 	char *query1=NULL;
 	//char *query2=NULL;
 	//char *query3=NULL;
-	query1=(char *)"INSERT INTO mysql_connections VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)";
+	query1=(char *)"INSERT INTO mysql_connections VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)";
 	rc=sqlite3_prepare_v2(mydb3, query1, -1, &statement1, 0);
 	assert(rc==SQLITE_OK);
 	//rc=sqlite3_prepare_v2(mydb3, query2, -1, &statement2, 0);
@@ -544,8 +571,8 @@ void ProxySQL_Statistics::MySQL_Threads_Handler_sets(SQLite3_result *resultset)
 	//assert(rc==SQLITE_OK);
 
 
-	uint64_t mysql_connections_values[12];
-	for (int i=0; i<12; i++) {
+	uint64_t mysql_connections_values[13];
+	for (int i=0; i<13; i++) {
 		mysql_connections_values[i]=0;
 	}
 	mysql_connections_values[0] = time(NULL);
@@ -596,9 +623,13 @@ void ProxySQL_Statistics::MySQL_Threads_Handler_sets(SQLite3_result *resultset)
 			mysql_connections_values[11]=atoi(r1->fields[1]);
 			continue;
 		}
+		if (!strcasecmp(r1->fields[0],"GTID_consistent_queries")) {
+			mysql_connections_values[12]=atoi(r1->fields[1]);
+			continue;
+		}
 	}
 
-	for (int i=0; i<12; i++) {
+	for (int i=0; i<13; i++) {
 		rc=sqlite3_bind_int64(statement1, i+1, mysql_connections_values[i]); assert(rc==SQLITE_OK);
 	}
 
@@ -621,20 +652,20 @@ void ProxySQL_Statistics::MySQL_Threads_Handler_sets(SQLite3_result *resultset)
 		}
 		free(error);
 	} else {
-		char buf[512];
+		char buf[1024];
 		if (resultset2->rows_count == 0) {
-			sprintf(buf,"INSERT INTO mysql_connections_hour SELECT timestamp/3600*3600 , MAX(Client_Connections_aborted), AVG(Client_Connections_connected), MAX(Client_Connections_created), MAX(Server_Connections_aborted), AVG(Server_Connections_connected), MAX(Server_Connections_created), MAX(ConnPool_get_conn_failure), MAX(ConnPool_get_conn_immediate), MAX(ConnPool_get_conn_success), MAX(Questions), MAX(Slow_queries) FROM mysql_connections WHERE timestamp < %ld GROUP BY timestamp/3600", (ts/3600)*3600);
+			sprintf(buf,"INSERT INTO mysql_connections_hour SELECT timestamp/3600*3600 , MAX(Client_Connections_aborted), AVG(Client_Connections_connected), MAX(Client_Connections_created), MAX(Server_Connections_aborted), AVG(Server_Connections_connected), MAX(Server_Connections_created), MAX(ConnPool_get_conn_failure), MAX(ConnPool_get_conn_immediate), MAX(ConnPool_get_conn_success), MAX(Questions), MAX(Slow_queries), MAX(GTID_consistent_queries) FROM mysql_connections WHERE timestamp < %ld GROUP BY timestamp/3600", (ts/3600)*3600);
 			statsdb_disk->execute(buf);
 		} else {
 			SQLite3_row *r = resultset2->rows[0];
 			if (r->fields[0]) {
 				time_t t = atol(r->fields[0]);
 				if (ts >= t + 3600) {
-					sprintf(buf,"INSERT INTO mysql_connections_hour SELECT timestamp/3600*3600 , MAX(Client_Connections_aborted), AVG(Client_Connections_connected), MAX(Client_Connections_created), MAX(Server_Connections_aborted), AVG(Server_Connections_connected), MAX(Server_Connections_created), MAX(ConnPool_get_conn_failure), MAX(ConnPool_get_conn_immediate), MAX(ConnPool_get_conn_success), MAX(Questions), MAX(Slow_queries) FROM mysql_connections WHERE timestamp >= %ld AND timestamp < %ld GROUP BY timestamp/3600", t+3600 , (ts/3600)*3600);
+					sprintf(buf,"INSERT INTO mysql_connections_hour SELECT timestamp/3600*3600 , MAX(Client_Connections_aborted), AVG(Client_Connections_connected), MAX(Client_Connections_created), MAX(Server_Connections_aborted), AVG(Server_Connections_connected), MAX(Server_Connections_created), MAX(ConnPool_get_conn_failure), MAX(ConnPool_get_conn_immediate), MAX(ConnPool_get_conn_success), MAX(Questions), MAX(Slow_queries), MAX(GTID_consistent_queries) FROM mysql_connections WHERE timestamp >= %ld AND timestamp < %ld GROUP BY timestamp/3600", t+3600 , (ts/3600)*3600);
 					statsdb_disk->execute(buf);
 				}
 			} else {
-				sprintf(buf,"INSERT INTO mysql_connections_hour SELECT timestamp/3600*3600 , MAX(Client_Connections_aborted), AVG(Client_Connections_connected), MAX(Client_Connections_created), MAX(Server_Connections_aborted), AVG(Server_Connections_connected), MAX(Server_Connections_created), MAX(ConnPool_get_conn_failure), MAX(ConnPool_get_conn_immediate), MAX(ConnPool_get_conn_success), MAX(Questions), MAX(Slow_queries) FROM mysql_connections WHERE timestamp < %ld GROUP BY timestamp/3600", (ts/3600)*3600);
+				sprintf(buf,"INSERT INTO mysql_connections_hour SELECT timestamp/3600*3600 , MAX(Client_Connections_aborted), AVG(Client_Connections_connected), MAX(Client_Connections_created), MAX(Server_Connections_aborted), AVG(Server_Connections_connected), MAX(Server_Connections_created), MAX(ConnPool_get_conn_failure), MAX(ConnPool_get_conn_immediate), MAX(ConnPool_get_conn_success), MAX(Questions), MAX(Slow_queries), MAX(GTID_consistent_queries) FROM mysql_connections WHERE timestamp < %ld GROUP BY timestamp/3600", (ts/3600)*3600);
 				statsdb_disk->execute(buf);
 			}
 		}
@@ -723,7 +754,7 @@ void ProxySQL_Statistics::MySQL_Query_Cache_sets(SQLite3_result *resultset) {
 		}
 		free(error);
 	} else {
-		char buf[512];
+		char buf[1024];
 		if (resultset2->rows_count == 0) {
 			sprintf(buf,"INSERT INTO mysql_query_cache_hour SELECT timestamp/3600*3600 , MAX(count_GET), MAX(count_GET_OK), MAX(count_SET), MAX(bytes_IN), MAX(bytes_OUT), MAX(Entries_Purged), AVG(Entries_In_Cache), AVG(Memory_bytes) FROM mysql_query_cache WHERE timestamp < %ld GROUP BY timestamp/3600", (ts/3600)*3600);
 			statsdb_disk->execute(buf);

lib/mysql_connection.cpp

@@ -1716,7 +1716,8 @@ bool MySQL_Connection::get_gtid(char *buff, uint64_t *trx_id) {
 		if (mysql_session_track_get_first(mysql, SESSION_TRACK_GTIDS, &data, &length) == 0) {
 			memcpy(buff,data,length);
 			buff[length]=0;
-			fprintf(stderr,"GTID=%s\n",buff);
+			//fprintf(stderr,"GTID=%s\n",buff);
+			__sync_fetch_and_add(&myds->sess->thread->status_variables.gtid_session_collected,1);
 			ret = true;
 		}
 	}

src/Makefile

@@ -51,7 +51,7 @@ EV_LDIR=$(EV_DIR)/.libs
 IDIR=../include
 LDIR=../lib
 IDIRS=-I$(IDIR) -I$(JEMALLOC_IDIR) -I$(MARIADB_IDIR) $(LIBCONFIG_IDIR) -I$(DAEMONPATH_IDIR) -I$(SQLITE3_DIR) -I$(CLICKHOUSE_CPP_DIR) -I$(MICROHTTPD_IDIR) -I$(CURL_IDIR) -I$(EV_IDIR)
-LDIRS=-L$(LDIR) -L$(JEMALLOC_LDIR) $(LIBCONFIG_LDIR) -L$(RE2_PATH)/obj -L$(MARIADB_LDIR) -L$(DAEMONPATH_LDIR) -L$(PCRE_LDIR) -L$(MICROHTTPD_LDIR) -L$(CURL_LDIR) -L$(EV_DIR)
+LDIRS=-L$(LDIR) -L$(JEMALLOC_LDIR) $(LIBCONFIG_LDIR) -L$(RE2_PATH)/obj -L$(MARIADB_LDIR) -L$(DAEMONPATH_LDIR) -L$(PCRE_LDIR) -L$(MICROHTTPD_LDIR) -L$(CURL_LDIR) -L$(EV_LDIR)
 
 UNAME_S := $(shell uname -s)