Export query cache metrics in WebUI

include/ProxySQL_Statistics.hpp

@@ -30,6 +30,12 @@
 #define STATSDB_SQLITE_TABLE_SYSTEM_MEMORY_DAY "CREATE TABLE system_memory_day (timestamp INT NOT NULL, allocated INT NOT NULL, resident INT NOT NULL, active INT NOT NULL, mapped INT NOT NULL, metadata INT NOT NULL, retained INT NOT NULL, PRIMARY KEY (timestamp))"
 #endif
 
+
+#define STATSDB_SQLITE_TABLE_MYSQL_QUERY_CACHE "CREATE TABLE mysql_query_cache (timestamp INT NOT NULL, count_GET INT NOT NULL, count_GET_OK INT NOT NULL, count_SET INT NOT NULL, bytes_IN INT NOT NULL, bytes_OUT INT NOT NULL, Entries_Purged INT NOT NULL, Entries_In_Cache INT NOT NULL, Memory_Bytes INT NOT NULL, PRIMARY KEY (timestamp))"
+#define STATSDB_SQLITE_TABLE_MYSQL_QUERY_CACHE_HOUR "CREATE TABLE mysql_query_cache_hour (timestamp INT NOT NULL, count_GET INT NOT NULL, count_GET_OK INT NOT NULL, count_SET INT NOT NULL, bytes_IN INT NOT NULL, bytes_OUT INT NOT NULL, Entries_Purged INT NOT NULL, Entries_In_Cache INT NOT NULL, Memory_Bytes INT NOT NULL, PRIMARY KEY (timestamp))"
+#define STATSDB_SQLITE_TABLE_MYSQL_QUERY_CACHE_DAY "CREATE TABLE mysql_query_cache_day (timestamp INT NOT NULL, count_GET INT NOT NULL, count_GET_OK INT NOT NULL, count_SET INT NOT NULL, bytes_IN INT NOT NULL, bytes_OUT INT NOT NULL, Entries_Purged INT NOT NULL, Entries_In_Cache INT NOT NULL, Memory_Bytes INT NOT NULL, PRIMARY KEY (timestamp))"
+
+
 class ProxySQL_Statistics {
 	SQLite3DB *statsdb_mem; // internal statistics DB
 	std::vector<table_def_t *> *tables_defs_statsdb_mem;
@@ -43,6 +49,7 @@ class ProxySQL_Statistics {
 #ifndef NOJEM
 	unsigned long long next_timer_system_memory;
 #endif
+	unsigned long long next_timer_MySQL_Query_Cache;
 	public:
 	struct {
 		int stats_mysql_connection_pool;
@@ -63,16 +70,19 @@ class ProxySQL_Statistics {
 #ifndef NOJEM
 	bool system_memory_timetoget(unsigned long long);
 #endif
+	bool MySQL_Query_Cache_timetoget(unsigned long long);
 	void MySQL_Threads_Handler_sets(SQLite3_result *);
 	void system_cpu_sets();
 #ifndef NOJEM
 	void system_memory_sets();
 #endif
+	void MySQL_Query_Cache_sets(SQLite3_result *);
 	SQLite3_result * get_mysql_metrics(int interval);
 	SQLite3_result * get_system_cpu_metrics(int interval);
 #ifndef NOJEM
 	SQLite3_result * get_system_memory_metrics(int interval);
 #endif
+	SQLite3_result * get_MySQL_Query_Cache_metrics(int interval);
 };
 
 #endif /* CLASS_PROXYSQL_STATISTICS_H */

lib/ProxySQL_Admin.cpp

@@ -3134,6 +3134,15 @@ __end_while_pool:
 					}
 				}
 			}
+			if (GloProxyStats->MySQL_Query_Cache_timetoget(curtime)) {
+				if (GloMTH) {
+					SQLite3_result * resultset=GloQC->SQL3_getStats();
+					if (resultset) {
+						GloProxyStats->MySQL_Query_Cache_sets(resultset);
+						delete resultset;
+					}
+				}
+			}
 			if (GloProxyStats->system_cpu_timetoget(curtime)) {
 				GloProxyStats->system_cpu_sets();
 			}

lib/ProxySQL_HTTP_Server.cpp

@@ -599,7 +599,7 @@ int ProxySQL_HTTP_Server::handler(void *cls, struct MHD_Connection *connection,
 			}
 			free(nv);
 			free(ts);
-			
+
 
 			nm = (char **)malloc(sizeof(char *)*4);
 			nm[0] = (char *)"ConnPool_get_conn_failure";
@@ -629,8 +629,96 @@ int ProxySQL_HTTP_Server::handler(void *cls, struct MHD_Connection *connection,
 
 			s->append("</body></html>");
 			response = MHD_create_response_from_buffer(s->length(), (void *) s->c_str(), MHD_RESPMEM_MUST_COPY);
-  			ret = MHD_queue_response (connection, MHD_HTTP_OK, response);
-  			MHD_destroy_response (response);
+			ret = MHD_queue_response (connection, MHD_HTTP_OK, response);
+			MHD_destroy_response (response);
+			delete s;
+			return ret;
+		}
+		if (strcmp(valmetric,"cache")==0) {
+			string *s = generate_header((char *)"ProxySQL Graphs");
+			char *buttons = generate_buttons((char *)"cache");
+			s->append(buttons);
+			free(buttons);
+			s->append("<div class=\"graphs\" style=\"clear: both; height: auto;\">\n");
+			string *s1 = generate_canvas((char *)"myChart1");
+			s->append(s1->c_str());
+			s->append("<p></p>\n");
+			s1 = generate_canvas((char *)"myChart2");
+			s->append(s1->c_str());
+			s->append("<p></p>\n");
+			s1 = generate_canvas((char *)"myChart3");
+			s->append(s1->c_str());
+			s->append("<p></p>\n");
+			s1 = generate_canvas((char *)"myChart4");
+			s->append(s1->c_str());
+			s->append("</div>\n");
+			//SQLite3_result *cpu_sqlite = GloProxyStats->get_system_cpu_metrics();
+			//SQLite3_result *memory_sqlite = GloProxyStats->get_system_memory_metrics();
+			SQLite3_result *mysql_metrics_sqlite = GloProxyStats->get_MySQL_Query_Cache_metrics(interval_i);
+			char **nm = NULL;
+			char **nl = NULL;
+			char **nv = NULL;
+			char *ts = NULL;
+
+//Client_Connections_aborted, Client_Connections_connected, Client_Connections_created, Server_Connections_aborted, Server_Connections_connected, Server_Connections_created, ConnPool_get_conn_immediate, ConnPool_get_conn_success, Questions
+
+			nm = (char **)malloc(sizeof(char *)*5);
+			nm[0] = (char *)"count_GET";
+			nm[1] = (char *)"count_GET_OK";
+			nm[2] = (char *)"count_SET";
+			nm[3] = (char *)"Entries_Purged";
+			nm[4] = (char *)"Entries_In_Cache";
+			nl = (char **)malloc(sizeof(char *)*5);
+			nl[0] = (char *)"Count GET";
+			nl[1] = (char *)"Count GET OK";
+			nl[2] = (char *)"Count SET";
+			nl[3] = (char *)"Entries Purged";
+			nl[4] = (char *)"Entries In Cache";
+			nv = (char **)malloc(sizeof(char *)*6);
+			nv[0] = extract_values(mysql_metrics_sqlite,2,true,(double)1);
+			nv[1] = extract_values(mysql_metrics_sqlite,3,true,(double)1);
+			nv[2] = extract_values(mysql_metrics_sqlite,4,true,(double)1);
+			nv[3] = extract_values(mysql_metrics_sqlite,7,true,(double)1);
+			nv[4] = extract_values(mysql_metrics_sqlite,8,false,(double)1);
+			ts = extract_ts(mysql_metrics_sqlite,true);
+			s1 = generate_chart((char *)"myChart1",ts,5,nm,nl,nv);
+			s->append(s1->c_str());
+			free(nm);
+			free(nl);
+			for (int aa=0 ; aa<5 ; aa++) {
+				free(nv[aa]);
+			}
+			free(nv);
+			free(ts);
+
+
+			nm = (char **)malloc(sizeof(char *)*3);
+			nm[0] = (char *)"bytes_IN";
+			nm[1] = (char *)"bytes_OUT";
+			nm[2] = (char *)"Memory_Bytes";
+			nl = (char **)malloc(sizeof(char *)*3);
+			nl[0] = (char *)"KB IN";
+			nl[1] = (char *)"KB OUT";
+			nl[2] = (char *)"QC size MB";
+			nv = (char **)malloc(sizeof(char *)*3);
+			nv[0] = extract_values(mysql_metrics_sqlite,5,true,(double)1/1024);
+			nv[1] = extract_values(mysql_metrics_sqlite,6,true,(double)1/1024);
+			nv[2] = extract_values(mysql_metrics_sqlite,9,false,(double)1/1024/1024);
+			ts = extract_ts(mysql_metrics_sqlite,true);
+			s1 = generate_chart((char *)"myChart2",ts,3,nm,nl,nv);
+			s->append(s1->c_str());
+			free(nm);
+			free(nl);
+			for (int aa=0 ; aa<3 ; aa++) {
+				free(nv[aa]);
+			}
+			free(nv);
+			free(ts);
+
+			s->append("</body></html>");
+			response = MHD_create_response_from_buffer(s->length(), (void *) s->c_str(), MHD_RESPMEM_MUST_COPY);
+			ret = MHD_queue_response (connection, MHD_HTTP_OK, response);
+			MHD_destroy_response (response);
 			delete s;
 			return ret;
 		}

lib/ProxySQL_Statistics.cpp

@@ -59,6 +59,7 @@ ProxySQL_Statistics::ProxySQL_Statistics() {
 #ifndef NOJEM
 	next_timer_system_memory = 0;
 #endif
+	next_timer_MySQL_Query_Cache = 0;
 }
 
 ProxySQL_Statistics::~ProxySQL_Statistics() {
@@ -91,6 +92,10 @@ void ProxySQL_Statistics::init() {
 	insert_into_tables_defs(tables_defs_statsdb_disk,"system_memory_day", STATSDB_SQLITE_TABLE_SYSTEM_MEMORY_DAY);
 #endif
 
+	insert_into_tables_defs(tables_defs_statsdb_disk,"mysql_query_cache", STATSDB_SQLITE_TABLE_MYSQL_QUERY_CACHE);
+	insert_into_tables_defs(tables_defs_statsdb_disk,"mysql_query_cache_hour", STATSDB_SQLITE_TABLE_MYSQL_QUERY_CACHE_HOUR);
+	insert_into_tables_defs(tables_defs_statsdb_disk,"mysql_query_cache_day", STATSDB_SQLITE_TABLE_MYSQL_QUERY_CACHE_DAY);
+
 	check_and_build_standard_tables(statsdb_mem, tables_defs_statsdb_disk);
 	check_and_build_standard_tables(statsdb_disk, tables_defs_statsdb_disk);
 }
@@ -149,6 +154,22 @@ bool ProxySQL_Statistics::MySQL_Threads_Handler_timetoget(unsigned long long cur
 	return false;
 }
 
+bool ProxySQL_Statistics::MySQL_Query_Cache_timetoget(unsigned long long curtime) {
+	unsigned int i = (unsigned int)variables.stats_mysql_query_cache;
+	if (i) {
+		if (
+			( curtime > next_timer_MySQL_Query_Cache )
+			||
+			( curtime + i*1000*1000 < next_timer_MySQL_Query_Cache )
+		) {
+			next_timer_MySQL_Query_Cache = curtime/1000/1000 + i;
+			next_timer_MySQL_Query_Cache = next_timer_MySQL_Query_Cache * 1000 * 1000;
+			return true;
+		}
+	}
+	return false;
+}
+
 bool ProxySQL_Statistics::system_cpu_timetoget(unsigned long long curtime) {
 	unsigned int i = (unsigned int)variables.stats_system_cpu;
 	if (i) {
@@ -236,6 +257,48 @@ SQLite3_result * ProxySQL_Statistics::get_mysql_metrics(int interval) {
 	return resultset;
 }
 
+SQLite3_result * ProxySQL_Statistics::get_MySQL_Query_Cache_metrics(int interval) {
+	SQLite3_result *resultset = NULL;
+	int cols;
+	int affected_rows;
+	char *error = NULL;
+	char *query = NULL;
+	char *query1 = (char *)"SELECT SUBSTR(FROM_UNIXTIME(timestamp),0,20) ts, timestamp, count_GET, count_GET_OK, count_SET, bytes_IN, bytes_OUT, Entries_Purged, Entries_In_Cache, Memory_Bytes FROM mysql_query_cache WHERE timestamp BETWEEN %d AND %d ORDER BY timestamp";
+	char *query2 = (char *)"SELECT SUBSTR(FROM_UNIXTIME(timestamp),0,20) ts, timestamp, count_GET, count_GET_OK, count_SET, bytes_IN, bytes_OUT, Entries_Purged, Entries_In_Cache, Memory_Bytes FROM mysql_query_cache_hour WHERE timestamp BETWEEN %d AND %d ORDER BY timestamp";
+	time_t ts = time(NULL);
+	switch (interval) {
+		case 1800:
+		case 3600:
+		case 7200:
+			query = (char *)malloc(strlen(query1)+128);
+			sprintf(query, query1, ts-interval, ts);
+			break;
+		case 28800:
+		case 86400:
+		case 259200:
+		case 604800:
+		case 2592000:
+		case 7776000:
+			query = (char *)malloc(strlen(query2)+128);
+			sprintf(query, query2, ts-interval, ts);
+			break;
+		default:
+			assert(0);
+			break;
+	}
+	//fprintf(stderr,"%s\n", query);
+	statsdb_disk->execute_statement(query, &error , &cols , &affected_rows , &resultset);
+	free(query);
+	if (error) {
+		if (resultset) {
+			delete resultset;
+			resultset = NULL;
+		}
+		free(error);
+	}
+	return resultset;
+}
+
 #ifndef NOJEM
 SQLite3_result * ProxySQL_Statistics::get_system_memory_metrics(int interval) {
 	SQLite3_result *resultset = NULL;
@@ -480,7 +543,7 @@ void ProxySQL_Statistics::MySQL_Threads_Handler_sets(SQLite3_result *resultset)
 	//assert(rc==SQLITE_OK);
 
 
-	int mysql_connections_values[12];
+	uint64_t mysql_connections_values[12];
 	for (int i=0; i<12; i++) {
 		mysql_connections_values[i]=0;
 	}
@@ -582,3 +645,105 @@ void ProxySQL_Statistics::MySQL_Threads_Handler_sets(SQLite3_result *resultset)
 		statsdb_disk->execute(buf);
 	}
 }
+
+void ProxySQL_Statistics::MySQL_Query_Cache_sets(SQLite3_result *resultset) {
+	int rc;
+	if (resultset == NULL)
+		return;
+	sqlite3 *mydb3=statsdb_disk->get_db();
+	sqlite3_stmt *statement1=NULL;
+	char *query1=NULL;
+	query1=(char *)"INSERT INTO mysql_query_cache VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)";
+	rc=sqlite3_prepare_v2(mydb3, query1, -1, &statement1, 0);
+	assert(rc==SQLITE_OK);
+
+	uint64_t qc_values[9];
+	for (int i=0; i<9; i++) {
+		qc_values[i]=0;
+	}
+	qc_values[0] = time(NULL);
+
+	for (std::vector<SQLite3_row *>::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) {
+		SQLite3_row *r1=*it;
+		if (!strcasecmp(r1->fields[0],"Query_Cache_count_GET")) {
+			qc_values[1]=strtoull(r1->fields[1], NULL, 10);
+			continue;
+		}
+		if (!strcasecmp(r1->fields[0],"Query_Cache_count_GET_OK")) {
+			qc_values[2]=strtoull(r1->fields[1], NULL, 10);
+			continue;
+		}
+		if (!strcasecmp(r1->fields[0],"Query_Cache_count_SET")) {
+			qc_values[3]=strtoull(r1->fields[1], NULL, 10);
+			continue;
+		}
+		if (!strcasecmp(r1->fields[0],"Query_Cache_bytes_IN")) {
+			qc_values[4]=strtoull(r1->fields[1], NULL, 10);
+			continue;
+		}
+		if (!strcasecmp(r1->fields[0],"Query_Cache_bytes_OUT")) {
+			qc_values[5]=strtoull(r1->fields[1], NULL, 10);
+			continue;
+		}
+		if (!strcasecmp(r1->fields[0],"Query_Cache_Purged")) {
+			qc_values[6]=strtoull(r1->fields[1], NULL, 10);
+			continue;
+		}
+		if (!strcasecmp(r1->fields[0],"Query_Cache_Entries")) {
+			qc_values[7]=strtoull(r1->fields[1], NULL, 10);
+			continue;
+		}
+		if (!strcasecmp(r1->fields[0],"Query_Cache_Memory_bytes")) {
+			qc_values[8]=strtoull(r1->fields[1], NULL, 10);
+			continue;
+		}
+	}
+
+	for (int i=0; i<9; i++) {
+		rc=sqlite3_bind_int64(statement1, i+1, qc_values[i]); assert(rc==SQLITE_OK);
+	}
+
+	SAFE_SQLITE3_STEP2(statement1);
+	rc=sqlite3_clear_bindings(statement1); assert(rc==SQLITE_OK);
+	rc=sqlite3_reset(statement1); //assert(rc==SQLITE_OK);
+	sqlite3_finalize(statement1);
+
+	SQLite3_result *resultset2 = NULL;
+	int cols;
+	int affected_rows;
+	char *error = NULL;
+	time_t ts = time(NULL);
+	char *query = (char *)"SELECT MAX(timestamp) FROM mysql_query_cache_hour";
+	statsdb_disk->execute_statement(query, &error , &cols , &affected_rows , &resultset2);
+	if (error) {
+		if (resultset2) {
+			delete resultset2;
+			resultset2 = NULL;
+		}
+		free(error);
+	} else {
+		char buf[512];
+		if (resultset2->rows_count == 0) {
+			sprintf(buf,"INSERT INTO mysql_query_cache_hour SELECT timestamp/3600*3600 , MAX(count_GET), MAX(count_GET_OK), MAX(count_SET), MAX(bytes_IN), MAX(bytes_OUT), MAX(Entries_Purged), AVG(Entries_In_Cache), AVG(Memory_bytes) FROM mysql_query_cache WHERE timestamp < %ld GROUP BY timestamp/3600", (ts/3600)*3600);
+			statsdb_disk->execute(buf);
+		} else {
+			SQLite3_row *r = resultset2->rows[0];
+			if (r->fields[0]) {
+				time_t t = atol(r->fields[0]);
+				if (ts >= t + 3600) {
+					sprintf(buf,"INSERT INTO mysql_query_cache_hour SELECT timestamp/3600*3600 , MAX(count_GET), MAX(count_GET_OK), MAX(count_SET), MAX(bytes_IN), MAX(bytes_OUT), MAX(Entries_Purged), AVG(Entries_In_Cache), AVG(Memory_bytes) FROM mysql_query_cache WHERE timestamp >= %ld AND timestamp < %ld GROUP BY timestamp/3600", t+3600 , (ts/3600)*3600);
+					statsdb_disk->execute(buf);
+				}
+			} else {
+				sprintf(buf,"INSERT INTO mysql_query_cache_hour SELECT timestamp/3600*3600 , MAX(count_GET), MAX(count_GET_OK), MAX(count_SET), MAX(bytes_IN), MAX(bytes_OUT), MAX(Entries_Purged), AVG(Entries_In_Cache), AVG(Memory_bytes) FROM mysql_query_cache WHERE timestamp < %ld GROUP BY timestamp/3600", (ts/3600)*3600);
+				statsdb_disk->execute(buf);
+			}
+		}
+		delete resultset2;
+		resultset2 = NULL;
+		sprintf(buf,"DELETE FROM mysql_query_cache WHERE timestamp < %ld", ts - 3600*24*7);
+		statsdb_disk->execute(buf);
+		sprintf(buf,"DELETE FROM mysql_query_cache_hour WHERE timestamp < %ld", ts - 3600*24*365);
+		statsdb_disk->execute(buf);
+	}
+}