From 263d645baddbc1ef94791f60b0871e349efd2303 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20S=C3=A1nchez=20Parra?= Date: Mon, 30 Jan 2023 10:05:28 +0100 Subject: [PATCH 01/21] Add auxiliary maps in get_query_digests() to improve performance Otherwise, all queries processed by ProxySQL have to wait for get_query_digest() to finish reading the maps. With this patch, ProxySQL stores query digests in the auxiliary maps while the main maps are being used by get_query_digests(). Once get_query_digests() finish reading the main maps, it merge the auxiliary maps in to the main maps and clear the content of the auxiliary maps. --- include/query_processor.h | 6 ++- lib/Query_Processor.cpp | 111 +++++++++++++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 3 deletions(-) diff --git a/include/query_processor.h b/include/query_processor.h index 15f4ceeca..567d53b37 100644 --- a/include/query_processor.h +++ b/include/query_processor.h @@ -61,7 +61,10 @@ class QP_query_digest_stats { unsigned long long rows_sent; int hid; QP_query_digest_stats(char *u, char *s, uint64_t d, char *dt, int h, char *ca); - void add_time(unsigned long long t, unsigned long long n, unsigned long long ra, unsigned long long rs); + void add_time( + unsigned long long t, unsigned long long n, unsigned long long ra, unsigned long long rs, + unsigned long long cnt = 1 + ); ~QP_query_digest_stats(); char **get_row(umap_query_digest_text *digest_text_umap, query_digest_stats_pointers_t *qdsp); }; @@ -319,6 +322,7 @@ class Query_Processor { SQLite3_result * get_stats_commands_counters(); SQLite3_result * get_query_digests(); SQLite3_result * get_query_digests_reset(); + SQLite3_result * get_query_digests_v2(); void get_query_digests_reset(umap_query_digest *uqd, umap_query_digest_text *uqdt); unsigned long long purge_query_digests(bool async_purge, bool parallel, char **msg); unsigned long long purge_query_digests_async(char **msg); diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index debba9d22..5923ce0ed 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -186,8 +186,11 @@ QP_query_digest_stats::QP_query_digest_stats(char *u, char *s, uint64_t d, char rows_sent=0; hid=h; } -void QP_query_digest_stats::add_time(unsigned long long t, unsigned long long n, unsigned long long ra, unsigned long long rs) { - count_star++; +void QP_query_digest_stats::add_time( + unsigned long long t, unsigned long long n, unsigned long long ra, unsigned long long rs, + unsigned long long cnt +) { + count_star += cnt; sum_time+=t; rows_affected+=ra; rows_sent+=rs; @@ -1173,6 +1176,110 @@ unsigned long long Query_Processor::get_query_digests_total_size() { return ret; } +SQLite3_result * Query_Processor::get_query_digests_v2() { + proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping current query digest\n"); + SQLite3_result *result = NULL; + // Create two auxiliary maps and swap its content with the main maps. This + // way, this function can read query digests stored until now while other + // threads write in the other map. We need to lock while swapping. + umap_query_digest digest_umap_aux; + umap_query_digest_text digest_text_umap_aux; + pthread_rwlock_wrlock(&digest_rwlock); + digest_umap.swap(digest_umap_aux); + digest_text_umap.swap(digest_text_umap_aux); + pthread_rwlock_unlock(&digest_rwlock); + unsigned long long curtime1; + unsigned long long curtime2; + size_t map_size = digest_umap_aux.size(); + if (map_size >= DIGEST_STATS_FAST_MINSIZE) { + result = new SQLite3_result(14, true); + curtime1 = monotonic_time(); + } else { + result = new SQLite3_result(14); + } + result->add_column_definition(SQLITE_TEXT,"hid"); + result->add_column_definition(SQLITE_TEXT,"schemaname"); + result->add_column_definition(SQLITE_TEXT,"username"); + result->add_column_definition(SQLITE_TEXT,"client_address"); + result->add_column_definition(SQLITE_TEXT,"digest"); + result->add_column_definition(SQLITE_TEXT,"digest_text"); + result->add_column_definition(SQLITE_TEXT,"count_star"); + result->add_column_definition(SQLITE_TEXT,"first_seen"); + result->add_column_definition(SQLITE_TEXT,"last_seen"); + result->add_column_definition(SQLITE_TEXT,"sum_time"); + result->add_column_definition(SQLITE_TEXT,"min_time"); + result->add_column_definition(SQLITE_TEXT,"max_time"); + result->add_column_definition(SQLITE_TEXT,"rows_affected"); + result->add_column_definition(SQLITE_TEXT,"rows_sent"); + if (map_size >= DIGEST_STATS_FAST_MINSIZE) { + int n=DIGEST_STATS_FAST_THREADS; + get_query_digests_parallel_args args[n]; + for (int i=0; i::iterator it = digest_umap_aux.begin(); + it != digest_umap_aux.end(); + ++it + ) { + QP_query_digest_stats *qds=(QP_query_digest_stats *)it->second; + query_digest_stats_pointers_t *a = (query_digest_stats_pointers_t *)malloc(sizeof(query_digest_stats_pointers_t)); + char **pta=qds->get_row(&digest_text_umap_aux, a); + result->add_row(pta); + free(a); + } + } + if (map_size >= DIGEST_STATS_FAST_MINSIZE) { + curtime2=monotonic_time(); + curtime1 = curtime1/1000; + curtime2 = curtime2/1000; + proxy_info("Running query on stats_mysql_query_digest: %llums to retrieve %lu entries\n", curtime2-curtime1, map_size); + } + + // Once the reading finishes, we lock and swap again both maps. Then, we + // merge the content of the auxiliary maps in the main maps and clear the + // content of the auxiliary maps. + pthread_rwlock_wrlock(&digest_rwlock); + digest_umap_aux.swap(digest_umap); + digest_text_umap_aux.swap(digest_text_umap); + for (const auto& element : digest_umap_aux) { + uint64_t digest = element.first; + QP_query_digest_stats *qds = (QP_query_digest_stats *)element.second; + std::unordered_map::iterator it = digest_umap.find(digest); + if (it != digest_umap.end()) { + // found + QP_query_digest_stats *qds_equal = (QP_query_digest_stats *)it->second; + qds_equal->add_time( + qds->min_time, qds->last_seen, qds->rows_affected, qds->rows_sent, qds->count_star + ); + } else { + digest_umap.insert(element); + } + } + digest_text_umap.insert(digest_text_umap_aux.begin(), digest_text_umap_aux.end()); + digest_umap_aux.clear(); + digest_text_umap_aux.clear(); + pthread_rwlock_unlock(&digest_rwlock); + + return result; +} + SQLite3_result * Query_Processor::get_query_digests() { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping current query digest\n"); SQLite3_result *result = NULL; From a7153d2637f3f040ba3ce7e4c528dd8735dda331 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20S=C3=A1nchez=20Parra?= Date: Tue, 31 Jan 2023 10:42:54 +0100 Subject: [PATCH 02/21] Add testing for the auxiliary map in get_query_digests() --- test/tap/tests/test_digest_umap_aux-t.cpp | 291 ++++++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100644 test/tap/tests/test_digest_umap_aux-t.cpp diff --git a/test/tap/tests/test_digest_umap_aux-t.cpp b/test/tap/tests/test_digest_umap_aux-t.cpp new file mode 100644 index 000000000..4f9168b87 --- /dev/null +++ b/test/tap/tests/test_digest_umap_aux-t.cpp @@ -0,0 +1,291 @@ +/** + * @file test_digest_umap_aux-t.cpp + * @brief This tests that the auxiliary digest map is working correctly. + * @details This test sends dummy queries to ProxySQL while also sending + * queries to read table stats_mysql_query_digest. Then, it checks that the + * execution time of the dummy queries has no been afected by the execution + * time of the queries that read from table stats_mysql_query_digest. Finally, + * check that the data stored in stats_mysql_query_digest is correct. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "proxysql_utils.h" +#include "command_line.h" +#include "utils.h" +#include "tap.h" + +using std::vector; +using std::string; + +CommandLine cl; +double slowest_query = 0.0; +double fastest_query = 0.0; +std::atomic_bool stop(false); + +vector DUMMY_QUERIES = { + "SELECT 1", + "SELECT 1 UNION SELECT 2 UNION SELECT 3", + "SELECT 1 UNION SELECT 2", +}; +int num_dummy_queries_executed = 0; + +struct digest_stats { + int hostgroup; + string schemaname; + string username; + string client_address; + string digest; + string digest_text; + int count_star; + int first_seen; + int last_seen; + int sum_time; + int min_time; + int max_time; + int sum_rows_affected; + int sum_rows_sent; +}; + +class timer { +public: + std::chrono::time_point lastTime; + timer() : lastTime(std::chrono::high_resolution_clock::now()) {} + inline double elapsed() { + std::chrono::time_point thisTime = std::chrono::high_resolution_clock::now(); + double deltaTime = std::chrono::duration(thisTime-lastTime).count(); + lastTime = thisTime; + return deltaTime; + } +}; + +vector get_digest_stats(MYSQL* proxy_admin) { + const char* get_digest_stats_query = + "SELECT * FROM stats_mysql_query_digest WHERE username='root' AND " + "digest_text IN ('SELECT ?', 'SELECT ? UNION SELECT ?', 'SELECT ? UNION SELECT ? UNION SELECT ?') " + "ORDER BY hostgroup, schemaname, username, client_address, digest"; + diag("Running: %s", get_digest_stats_query); + vector ds_vector; + + int err = mysql_query(proxy_admin, get_digest_stats_query); + if (err) { + diag("Failed to executed query `%s`. Error: `%s`", get_digest_stats_query, mysql_error(proxy_admin)); + return ds_vector; + } + + MYSQL_RES *res = NULL; + res = mysql_store_result(proxy_admin); + MYSQL_ROW row; + while (row = mysql_fetch_row(res)) { + digest_stats ds = {}; + ds.hostgroup = atoi(row[0]); + ds.schemaname = row[1]; + ds.username = row[2]; + ds.client_address = row[3]; + ds.digest = row[4]; + ds.digest_text = row[5]; + ds.count_star = atoi(row[6]); + ds.first_seen = atoi(row[7]); + ds.last_seen = atoi(row[8]); + ds.sum_time = atoi(row[9]); + ds.min_time = atoi(row[10]); + ds.max_time = atoi(row[11]); + ds.sum_rows_affected = atoi(row[12]); + ds.sum_rows_sent = atoi(row[13]); + ds_vector.push_back(ds); + } + mysql_free_result(res); + + return ds_vector; +} + +void run_dummy_queries() { + MYSQL* proxy_mysql = mysql_init(NULL); + + if (!mysql_real_connect(proxy_mysql, cl.host, cl.username, cl.password, NULL, cl.port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxy_mysql)); + slowest_query = -1.0; + return; + } + + vector execution_times = {}; + MYSQL_RES *res = NULL; + while (!stop) { + for (int i = 0; i < DUMMY_QUERIES.size(); i++) { + timer stopwatch; + int err = mysql_query(proxy_mysql, DUMMY_QUERIES[i]); + execution_times.push_back(stopwatch.elapsed()); + if (err) { + diag( + "Failed to executed query `%s`. Error: `%s`", + DUMMY_QUERIES[i], mysql_error(proxy_mysql) + ); + slowest_query = -1.0; + mysql_close(proxy_mysql); + return; + } + res = mysql_store_result(proxy_mysql); + mysql_free_result(res); + } + num_dummy_queries_executed++; + } + mysql_close(proxy_mysql); + + slowest_query = *std::max_element(execution_times.begin(), execution_times.end()); +} + +void run_stats_digest_query(MYSQL* proxy_admin) { + const char *count_digest_stats_query = "SELECT COUNT(*) FROM stats_mysql_query_digest"; + vector execution_times = {}; + const int num_queries = 3; + MYSQL_RES *res; + + for (int i; i < num_queries; i++) { + diag("Running: %s", count_digest_stats_query); + timer stopwatch; + int err = mysql_query(proxy_admin, count_digest_stats_query); + execution_times.push_back(stopwatch.elapsed()); + if (err) { + diag( + "Failed to executed query `%s`. Error: `%s`", + count_digest_stats_query, mysql_error(proxy_admin) + ); + fastest_query = -1.0; + return; + } + res = mysql_store_result(proxy_admin); + mysql_free_result(res); + } + + fastest_query = *std::min_element(execution_times.begin(), execution_times.end()); +} + +int main(int argc, char** argv) { + + if (cl.getEnv()) { + diag("Failed to get the required environmental variables."); + return EXIT_FAILURE; + } + + plan(1 + DUMMY_QUERIES.size() * 3); // always specify the number of tests that are going to be performed + + MYSQL *proxy_admin = mysql_init(NULL); + if (!mysql_real_connect(proxy_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxy_admin)); + return EXIT_FAILURE; + } + + vector admin_queries = { + "DELETE FROM mysql_query_rules", + "LOAD MYSQL QUERY RULES TO RUNTIME", + "PROXYSQLTEST 1 1000", + }; + for (const auto &query : admin_queries) { + diag("Running: %s", query); + MYSQL_QUERY(proxy_admin, query); + } + + MYSQL *proxy_mysql = mysql_init(NULL); + if (!mysql_real_connect(proxy_mysql, cl.host, cl.username, cl.password, NULL, cl.port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxy_mysql)); + mysql_close(proxy_admin); + return EXIT_FAILURE; + } + + MYSQL_RES *res = NULL; + for (const auto &query : DUMMY_QUERIES) { + diag("Running: %s", query); + MYSQL_QUERY(proxy_mysql, query); + res = mysql_store_result(proxy_mysql); + mysql_free_result(res); + } + mysql_close(proxy_mysql); + + vector ds_vector_before = get_digest_stats(proxy_admin); + + std::thread run_dummy_queries_thread(run_dummy_queries); + std::thread run_stats_digest_query_thread(run_stats_digest_query, proxy_admin); + + run_stats_digest_query_thread.join(); + if (fastest_query == -1.0) { + fprintf( + stderr, "File %s, line %d, Error: " + "thread run_stats_digest_query_thread finished with errors", __FILE__, __LINE__ + ); + mysql_close(proxy_admin); + return EXIT_FAILURE; + } + + stop = true; + run_dummy_queries_thread.join(); + if (slowest_query == -1.0) { + fprintf( + stderr, "File %s, line %d, Error: " + "thread run_dummy_queries_thread finished with errors", __FILE__, __LINE__ + ); + mysql_close(proxy_admin); + return EXIT_FAILURE; + } + + ok( + slowest_query < fastest_query, + "The slowest dummy query must be faster than the fastest digests stats query.\n" + " Slowest dummy query time: %f.\n" + " Fastest count digest stats query time: %f.", + slowest_query, fastest_query + ); + + vector ds_vector_after = get_digest_stats(proxy_admin); + for (int i = 0; i < DUMMY_QUERIES.size(); i++) { + ok( + ds_vector_before[i].hostgroup == ds_vector_after[i].hostgroup && + ds_vector_before[i].schemaname == ds_vector_after[i].schemaname && + ds_vector_before[i].username == ds_vector_after[i].username && + ds_vector_before[i].client_address == ds_vector_after[i].client_address && + ds_vector_before[i].digest == ds_vector_after[i].digest && + ds_vector_before[i].digest_text == ds_vector_after[i].digest_text && + ds_vector_before[i].first_seen - 1 <= ds_vector_after[i].first_seen && + ds_vector_after[i].first_seen <= ds_vector_before[i].first_seen + 1, + "Hostgroup, schemaname, username, client_address, digest, digest_test and first_seen " + "should be equal in both digest stats.\n" + " Hostgroup -> before:`%d` - after:`%d`.\n" + " Schemaname -> before:`%s` - after:`%s`.\n" + " Username -> before:`%s` - after:`%s`.\n" + " Client_address -> before:`%s` - after:`%s`.\n" + " Digests -> before:`%s` - after:`%s`.\n" + " Digests_text -> before:`%s` - after:`%s`.\n" + " First_seen -> before:`%d` - after:`%d`.", + ds_vector_before[i].hostgroup, ds_vector_after[i].hostgroup, + ds_vector_before[i].schemaname.c_str(), ds_vector_after[i].schemaname.c_str(), + ds_vector_before[i].username.c_str(), ds_vector_after[i].username.c_str(), + ds_vector_before[i].client_address.c_str(), ds_vector_after[i].client_address.c_str(), + ds_vector_before[i].digest.c_str(), ds_vector_after[i].digest.c_str(), + ds_vector_before[i].digest_text.c_str(), ds_vector_after[i].digest_text.c_str(), + ds_vector_before[i].first_seen, ds_vector_after[i].first_seen + ); + ok( + ds_vector_after[i].count_star - ds_vector_before[i].count_star == num_dummy_queries_executed, + "Query `%s` should be executed %d times. Act:'%d'", + ds_vector_after[i].digest_text.c_str(), num_dummy_queries_executed, + ds_vector_after[i].count_star - ds_vector_before[i].count_star + ); + ok( + ds_vector_before[i].last_seen < ds_vector_after[i].last_seen && + ds_vector_before[i].sum_time < ds_vector_after[i].sum_time, + "Last_seen and sum_time must have increased.\n" + " Last_seen -> before:`%d` - after:`%d`.\n" + " Sum_time -> before:`%d` - after:`%d`.", + ds_vector_before[i].last_seen, ds_vector_after[i].last_seen, + ds_vector_before[i].sum_time, ds_vector_after[i].sum_time + ); + } + + return exit_status(); +} From 28095e3109f42564a5a2791f87fc2ce1652cd73a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20S=C3=A1nchez=20Parra?= Date: Mon, 6 Feb 2023 18:21:27 +0100 Subject: [PATCH 03/21] Add auxiliary maps in get_query_digests_reset() to improve performance --- include/query_processor.h | 1 + lib/Query_Processor.cpp | 97 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/include/query_processor.h b/include/query_processor.h index 567d53b37..fc0566c63 100644 --- a/include/query_processor.h +++ b/include/query_processor.h @@ -323,6 +323,7 @@ class Query_Processor { SQLite3_result * get_query_digests(); SQLite3_result * get_query_digests_reset(); SQLite3_result * get_query_digests_v2(); + SQLite3_result * get_query_digests_reset_v2(); void get_query_digests_reset(umap_query_digest *uqd, umap_query_digest_text *uqdt); unsigned long long purge_query_digests(bool async_purge, bool parallel, char **msg); unsigned long long purge_query_digests_async(char **msg); diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 5923ce0ed..96daa9c6e 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -1347,6 +1347,103 @@ SQLite3_result * Query_Processor::get_query_digests() { return result; } +SQLite3_result * Query_Processor::get_query_digests_reset_v2() { + SQLite3_result *result = NULL; + umap_query_digest digest_umap_aux; + umap_query_digest_text digest_text_umap_aux; + pthread_rwlock_wrlock(&digest_rwlock); + digest_umap.swap(digest_umap_aux); + digest_text_umap.swap(digest_text_umap_aux); + pthread_rwlock_unlock(&digest_rwlock); + unsigned long long curtime1; + unsigned long long curtime2; + bool free_me = true; + bool defer_free = true; + int n=DIGEST_STATS_FAST_THREADS; + get_query_digests_parallel_args args[n]; + size_t map_size = digest_umap_aux.size(); + if (map_size >= DIGEST_STATS_FAST_MINSIZE) { + curtime1=monotonic_time(); + result = new SQLite3_result(14, true); + } else { + result = new SQLite3_result(14); + } + result->add_column_definition(SQLITE_TEXT,"hid"); + result->add_column_definition(SQLITE_TEXT,"schemaname"); + result->add_column_definition(SQLITE_TEXT,"username"); + result->add_column_definition(SQLITE_TEXT,"client_address"); + result->add_column_definition(SQLITE_TEXT,"digest"); + result->add_column_definition(SQLITE_TEXT,"digest_text"); + result->add_column_definition(SQLITE_TEXT,"count_star"); + result->add_column_definition(SQLITE_TEXT,"first_seen"); + result->add_column_definition(SQLITE_TEXT,"last_seen"); + result->add_column_definition(SQLITE_TEXT,"sum_time"); + result->add_column_definition(SQLITE_TEXT,"min_time"); + result->add_column_definition(SQLITE_TEXT,"max_time"); + result->add_column_definition(SQLITE_TEXT,"rows_affected"); + result->add_column_definition(SQLITE_TEXT,"rows_sent"); + if (map_size >= DIGEST_STATS_FAST_MINSIZE) { + for (int i=0; i::iterator it=digest_umap_aux.begin(); it!=digest_umap_aux.end(); ++it) { + QP_query_digest_stats *qds=(QP_query_digest_stats *)it->second; + delete qds; + } + } + } else { + for (std::unordered_map::iterator it=digest_umap_aux.begin(); it!=digest_umap_aux.end(); ++it) { + QP_query_digest_stats *qds=(QP_query_digest_stats *)it->second; + query_digest_stats_pointers_t *a = (query_digest_stats_pointers_t *)malloc(sizeof(query_digest_stats_pointers_t)); + char **pta=qds->get_row(&digest_text_umap_aux, a); + result->add_row(pta); + //qds->free_row(pta); + free(a); + delete qds; + } + } + digest_umap_aux.clear(); + // this part is always single-threaded + for (std::unordered_map::iterator it=digest_text_umap_aux.begin(); it!=digest_text_umap_aux.end(); ++it) { + free(it->second); + } + digest_text_umap_aux.clear(); + if (map_size >= DIGEST_STATS_FAST_MINSIZE) { + curtime2=monotonic_time(); + curtime1 = curtime1/1000; + curtime2 = curtime2/1000; + proxy_info("Running query on stats_mysql_query_digest_reset: %llums to retrieve %lu entries\n", curtime2-curtime1, map_size); + if (free_me) { + if (defer_free) { + for (int i=0; i Date: Fri, 10 Feb 2023 09:41:50 +0100 Subject: [PATCH 04/21] Move get digest text code to its own function --- include/query_processor.h | 1 + lib/Query_Processor.cpp | 38 +++++++++++++++++++++++++------------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/include/query_processor.h b/include/query_processor.h index fc0566c63..d9110d13f 100644 --- a/include/query_processor.h +++ b/include/query_processor.h @@ -66,6 +66,7 @@ class QP_query_digest_stats { unsigned long long cnt = 1 ); ~QP_query_digest_stats(); + char *get_digest_text(const umap_query_digest_text *digest_text_umap); char **get_row(umap_query_digest_text *digest_text_umap, query_digest_stats_pointers_t *qdsp); }; diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 96daa9c6e..2bc89256a 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -232,6 +232,30 @@ QP_query_digest_stats::~QP_query_digest_stats() { client_address=NULL; } } + +// Funtion to get the digest text associated to a QP_query_digest_stats. +// QP_query_digest_stats member type "char *digest_text" may by NULL, so we +// have to get the digest text from "digest_text_umap". +char *QP_query_digest_stats::get_digest_text(const umap_query_digest_text *digest_text_umap) { + char *digest_text_str = NULL; + + if (digest_text) { + digest_text_str = digest_text; + } else { + std::unordered_map::const_iterator it; + it = digest_text_umap->find(digest); + if (it != digest_text_umap->end()) { + digest_text_str = it->second; + } else { + // LCOV_EXCL_START + assert(0); + // LCOV_EXCL_STOP + } + } + + return digest_text_str; +} + char **QP_query_digest_stats::get_row(umap_query_digest_text *digest_text_umap, query_digest_stats_pointers_t *qdsp) { char **pta=qdsp->pta; @@ -247,19 +271,7 @@ char **QP_query_digest_stats::get_row(umap_query_digest_text *digest_text_umap, sprintf(qdsp->digest,"0x%016llX", (long long unsigned int)digest); pta[3]=qdsp->digest; - if (digest_text) { - pta[4]=digest_text; - } else { - std::unordered_map::iterator it; - it=digest_text_umap->find(digest); - if (it != digest_text_umap->end()) { - pta[4] = it->second; - } else { - // LCOV_EXCL_START - assert(0); - // LCOV_EXCL_STOP - } - } + pta[4] = get_digest_text(digest_text_umap); //sprintf(qdsp->count_star,"%u",count_star); my_itoa(qdsp->count_star, count_star); From 14ba7475d400594680fe061cf6f903371785bfda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20S=C3=A1nchez=20Parra?= Date: Fri, 10 Feb 2023 10:52:44 +0100 Subject: [PATCH 05/21] Create stats_mysql_query_digest statements directly from the digest_umap --- include/proxysql_admin.h | 5 + include/query_processor.h | 4 +- lib/ProxySQL_Admin.cpp | 100 +++++++++++++++++ lib/Query_Processor.cpp | 222 ++++++++++++++++++++------------------ 4 files changed, 223 insertions(+), 108 deletions(-) diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index ce15efa67..81576bbc0 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -5,6 +5,7 @@ #include #include +#include "query_processor.h" #include "proxy_defines.h" #include "proxysql.h" #include "cpp.h" @@ -417,6 +418,10 @@ class ProxySQL_Admin { void p_update_metrics(); void stats___mysql_query_rules(); + void stats___save_mysql_query_digest_to_sqlite( + const bool reset, const bool copy, const SQLite3_result *resultset, + const umap_query_digest *digest_umap, const umap_query_digest_text *digest_text_umap + ); void stats___mysql_query_digests(bool reset, bool copy=false); //void stats___mysql_query_digests_reset(); void stats___mysql_commands_counters(); diff --git a/include/query_processor.h b/include/query_processor.h index d9110d13f..2e8cf247d 100644 --- a/include/query_processor.h +++ b/include/query_processor.h @@ -323,8 +323,8 @@ class Query_Processor { SQLite3_result * get_stats_commands_counters(); SQLite3_result * get_query_digests(); SQLite3_result * get_query_digests_reset(); - SQLite3_result * get_query_digests_v2(); - SQLite3_result * get_query_digests_reset_v2(); + SQLite3_result * get_query_digests_v2(const bool use_resultset = true); + SQLite3_result * get_query_digests_reset_v2(const bool use_resultset = true); void get_query_digests_reset(umap_query_digest *uqd, umap_query_digest_text *uqdt); unsigned long long purge_query_digests(bool async_purge, bool parallel, char **msg); unsigned long long purge_query_digests_async(char **msg); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index feafdcefd..abe51f9bd 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -1,4 +1,5 @@ #include // std::cout +#include // std::stringstream #include #include // std::sort #include @@ -9412,6 +9413,105 @@ void ProxySQL_Admin::stats___proxysql_message_metrics(bool reset) { delete resultset; } +void ProxySQL_Admin::stats___save_mysql_query_digest_to_sqlite( + const bool reset, const bool copy, const SQLite3_result *resultset, const umap_query_digest *digest_umap, + const umap_query_digest_text *digest_text_umap +) { + statsdb->execute("BEGIN"); + int rc; + sqlite3_stmt *statement1=NULL; + sqlite3_stmt *statement32=NULL; + char *query1=NULL; + char *query32=NULL; + std::string query32s = ""; + statsdb->execute("DELETE FROM stats_mysql_query_digest_reset"); + statsdb->execute("DELETE FROM stats_mysql_query_digest"); + if (reset) { + query1=(char *)"INSERT INTO stats_mysql_query_digest_reset VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)"; + query32s = "INSERT INTO stats_mysql_query_digest_reset VALUES " + generate_multi_rows_query(32,14); + query32 = (char *)query32s.c_str(); + } else { + query1=(char *)"INSERT INTO stats_mysql_query_digest VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)"; + query32s = "INSERT INTO stats_mysql_query_digest VALUES " + generate_multi_rows_query(32,14); + query32 = (char *)query32s.c_str(); + } + + rc = statsdb->prepare_v2(query1, &statement1); + ASSERT_SQLITE_OK(rc, statsdb); + rc = statsdb->prepare_v2(query32, &statement32); + ASSERT_SQLITE_OK(rc, statsdb); + int row_idx=0; + int num_rows = resultset ? resultset->rows_count : digest_umap->size(); + int max_bulk_row_idx = num_rows/32; + max_bulk_row_idx=max_bulk_row_idx*32; + auto it = digest_umap->cbegin(); + int i = 0; + // If the function do not receives a resultset, it gets the values directly from the digest_umap + while (resultset ? i != resultset->rows_count : it != digest_umap->end()) { + QP_query_digest_stats *qds = (QP_query_digest_stats *)it->second; + SQLite3_row *row = resultset ? resultset->rows[i] : NULL; i++; + string digest_hex_str; + if (!resultset) { + std::ostringstream digest_stream; + digest_stream << "0x" << std::hex << qds->digest; + digest_hex_str = digest_stream.str(); + } + int idx=row_idx%32; + if (row_idxfields[11]) : qds->hid); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+2, resultset ? row->fields[0] : qds->schemaname, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+3, resultset ? row->fields[1] : qds->username, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+4, resultset ? row->fields[2] : qds->client_address, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+5, resultset ? row->fields[3] : digest_hex_str.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement32, (idx*14)+6, resultset ? row->fields[4] : qds->get_digest_text(digest_text_umap), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+7, resultset ? atoll(row->fields[5]) : qds->count_star); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+8, resultset ? atoll(row->fields[6]) : qds->first_seen); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+9, resultset ? atoll(row->fields[7]) : qds->last_seen); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+10, resultset ? atoll(row->fields[8]) : qds->sum_time); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+11, resultset ? atoll(row->fields[9]) : qds->min_time); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+12, resultset ? atoll(row->fields[10]) : qds->max_time); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+13, resultset ? atoll(row->fields[12]) : qds->rows_affected); ASSERT_SQLITE_OK(rc, statsdb); // rows affected + rc=(*proxy_sqlite3_bind_int64)(statement32, (idx*14)+14, resultset ? atoll(row->fields[13]) : qds->rows_sent); ASSERT_SQLITE_OK(rc, statsdb); // rows sent + if (idx==31) { + SAFE_SQLITE3_STEP2(statement32); + rc=(*proxy_sqlite3_clear_bindings)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement32); ASSERT_SQLITE_OK(rc, statsdb); + } + } else { // single row + rc=(*proxy_sqlite3_bind_int64)(statement1, 1, resultset ? atoll(row->fields[11]) : qds->hid); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 2, resultset ? row->fields[0] : qds->schemaname, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 3, resultset ? row->fields[1] : qds->username, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 4, resultset ? row->fields[2] : qds->client_address, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 5, resultset ? row->fields[3] : digest_hex_str.c_str(), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_text)(statement1, 6, resultset ? row->fields[4] : qds->get_digest_text(digest_text_umap), -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 7, resultset ? atoll(row->fields[5]) : qds->count_star); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 8, resultset ? atoll(row->fields[6]) : qds->first_seen); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 9, resultset ? atoll(row->fields[7]) : qds->last_seen); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 10, resultset ? atoll(row->fields[8]) : qds->sum_time); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 11, resultset ? atoll(row->fields[9]) : qds->min_time); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 12, resultset ? atoll(row->fields[10]) : qds->max_time); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_bind_int64)(statement1, 13, resultset ? atoll(row->fields[12]) : qds->rows_affected); ASSERT_SQLITE_OK(rc, statsdb); // rows affected + rc=(*proxy_sqlite3_bind_int64)(statement1, 14, resultset ? atoll(row->fields[13]) : qds->rows_sent); ASSERT_SQLITE_OK(rc, statsdb); // rows sent + SAFE_SQLITE3_STEP2(statement1); + rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); + } + row_idx++; + if (resultset) + i++; + else + it++; + } + (*proxy_sqlite3_finalize)(statement1); + (*proxy_sqlite3_finalize)(statement32); + if (reset) { + if (copy) { + statsdb->execute("INSERT INTO stats_mysql_query_digest SELECT * FROM stats_mysql_query_digest_reset"); + } + } + statsdb->execute("COMMIT"); +} + void ProxySQL_Admin::stats___mysql_query_digests(bool reset, bool copy) { if (!GloQPro) return; SQLite3_result * resultset=NULL; diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 2bc89256a..ac2ba66cb 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -31,6 +31,7 @@ #include #include extern MySQL_Threads_Handler *GloMTH; +extern ProxySQL_Admin *GloAdmin; static int int_cmp(const void *a, const void *b) { const unsigned long long *ia = (const unsigned long long *)a; @@ -1188,7 +1189,7 @@ unsigned long long Query_Processor::get_query_digests_total_size() { return ret; } -SQLite3_result * Query_Processor::get_query_digests_v2() { +SQLite3_result * Query_Processor::get_query_digests_v2(const bool use_resultset) { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping current query digest\n"); SQLite3_result *result = NULL; // Create two auxiliary maps and swap its content with the main maps. This @@ -1203,60 +1204,64 @@ SQLite3_result * Query_Processor::get_query_digests_v2() { unsigned long long curtime1; unsigned long long curtime2; size_t map_size = digest_umap_aux.size(); - if (map_size >= DIGEST_STATS_FAST_MINSIZE) { - result = new SQLite3_result(14, true); - curtime1 = monotonic_time(); - } else { - result = new SQLite3_result(14); - } - result->add_column_definition(SQLITE_TEXT,"hid"); - result->add_column_definition(SQLITE_TEXT,"schemaname"); - result->add_column_definition(SQLITE_TEXT,"username"); - result->add_column_definition(SQLITE_TEXT,"client_address"); - result->add_column_definition(SQLITE_TEXT,"digest"); - result->add_column_definition(SQLITE_TEXT,"digest_text"); - result->add_column_definition(SQLITE_TEXT,"count_star"); - result->add_column_definition(SQLITE_TEXT,"first_seen"); - result->add_column_definition(SQLITE_TEXT,"last_seen"); - result->add_column_definition(SQLITE_TEXT,"sum_time"); - result->add_column_definition(SQLITE_TEXT,"min_time"); - result->add_column_definition(SQLITE_TEXT,"max_time"); - result->add_column_definition(SQLITE_TEXT,"rows_affected"); - result->add_column_definition(SQLITE_TEXT,"rows_sent"); - if (map_size >= DIGEST_STATS_FAST_MINSIZE) { - int n=DIGEST_STATS_FAST_THREADS; - get_query_digests_parallel_args args[n]; - for (int i=0; i= DIGEST_STATS_FAST_MINSIZE) { + result = new SQLite3_result(14, true); + curtime1 = monotonic_time(); + } else { + result = new SQLite3_result(14); + } + result->add_column_definition(SQLITE_TEXT,"hid"); + result->add_column_definition(SQLITE_TEXT,"schemaname"); + result->add_column_definition(SQLITE_TEXT,"username"); + result->add_column_definition(SQLITE_TEXT,"client_address"); + result->add_column_definition(SQLITE_TEXT,"digest"); + result->add_column_definition(SQLITE_TEXT,"digest_text"); + result->add_column_definition(SQLITE_TEXT,"count_star"); + result->add_column_definition(SQLITE_TEXT,"first_seen"); + result->add_column_definition(SQLITE_TEXT,"last_seen"); + result->add_column_definition(SQLITE_TEXT,"sum_time"); + result->add_column_definition(SQLITE_TEXT,"min_time"); + result->add_column_definition(SQLITE_TEXT,"max_time"); + result->add_column_definition(SQLITE_TEXT,"rows_affected"); + result->add_column_definition(SQLITE_TEXT,"rows_sent"); + if (map_size >= DIGEST_STATS_FAST_MINSIZE) { + int n=DIGEST_STATS_FAST_THREADS; + get_query_digests_parallel_args args[n]; + for (int i=0; i::iterator it = digest_umap_aux.begin(); + it != digest_umap_aux.end(); + ++it + ) { + QP_query_digest_stats *qds=(QP_query_digest_stats *)it->second; + query_digest_stats_pointers_t *a = (query_digest_stats_pointers_t *)malloc(sizeof(query_digest_stats_pointers_t)); + char **pta=qds->get_row(&digest_text_umap_aux, a); + result->add_row(pta); + free(a); } - } - for (int i=0; i::iterator it = digest_umap_aux.begin(); - it != digest_umap_aux.end(); - ++it - ) { - QP_query_digest_stats *qds=(QP_query_digest_stats *)it->second; - query_digest_stats_pointers_t *a = (query_digest_stats_pointers_t *)malloc(sizeof(query_digest_stats_pointers_t)); - char **pta=qds->get_row(&digest_text_umap_aux, a); - result->add_row(pta); - free(a); } } + GloAdmin->stats___save_mysql_query_digest_to_sqlite( + false, false, result, &digest_umap_aux, &digest_text_umap_aux + ); if (map_size >= DIGEST_STATS_FAST_MINSIZE) { curtime2=monotonic_time(); curtime1 = curtime1/1000; @@ -1359,7 +1364,7 @@ SQLite3_result * Query_Processor::get_query_digests() { return result; } -SQLite3_result * Query_Processor::get_query_digests_reset_v2() { +SQLite3_result * Query_Processor::get_query_digests_reset_v2(const bool use_resultset) { SQLite3_result *result = NULL; umap_query_digest digest_umap_aux; umap_query_digest_text digest_text_umap_aux; @@ -1369,68 +1374,73 @@ SQLite3_result * Query_Processor::get_query_digests_reset_v2() { pthread_rwlock_unlock(&digest_rwlock); unsigned long long curtime1; unsigned long long curtime2; - bool free_me = true; - bool defer_free = true; + size_t map_size = digest_umap.size(); + bool free_me = false; + bool defer_free = false; int n=DIGEST_STATS_FAST_THREADS; get_query_digests_parallel_args args[n]; - size_t map_size = digest_umap_aux.size(); - if (map_size >= DIGEST_STATS_FAST_MINSIZE) { - curtime1=monotonic_time(); - result = new SQLite3_result(14, true); - } else { - result = new SQLite3_result(14); - } - result->add_column_definition(SQLITE_TEXT,"hid"); - result->add_column_definition(SQLITE_TEXT,"schemaname"); - result->add_column_definition(SQLITE_TEXT,"username"); - result->add_column_definition(SQLITE_TEXT,"client_address"); - result->add_column_definition(SQLITE_TEXT,"digest"); - result->add_column_definition(SQLITE_TEXT,"digest_text"); - result->add_column_definition(SQLITE_TEXT,"count_star"); - result->add_column_definition(SQLITE_TEXT,"first_seen"); - result->add_column_definition(SQLITE_TEXT,"last_seen"); - result->add_column_definition(SQLITE_TEXT,"sum_time"); - result->add_column_definition(SQLITE_TEXT,"min_time"); - result->add_column_definition(SQLITE_TEXT,"max_time"); - result->add_column_definition(SQLITE_TEXT,"rows_affected"); - result->add_column_definition(SQLITE_TEXT,"rows_sent"); - if (map_size >= DIGEST_STATS_FAST_MINSIZE) { - for (int i=0; i= DIGEST_STATS_FAST_MINSIZE) { + curtime1=monotonic_time(); + result = new SQLite3_result(14, true); + } else { + result = new SQLite3_result(14); + } + result->add_column_definition(SQLITE_TEXT,"hid"); + result->add_column_definition(SQLITE_TEXT,"schemaname"); + result->add_column_definition(SQLITE_TEXT,"username"); + result->add_column_definition(SQLITE_TEXT,"client_address"); + result->add_column_definition(SQLITE_TEXT,"digest"); + result->add_column_definition(SQLITE_TEXT,"digest_text"); + result->add_column_definition(SQLITE_TEXT,"count_star"); + result->add_column_definition(SQLITE_TEXT,"first_seen"); + result->add_column_definition(SQLITE_TEXT,"last_seen"); + result->add_column_definition(SQLITE_TEXT,"sum_time"); + result->add_column_definition(SQLITE_TEXT,"min_time"); + result->add_column_definition(SQLITE_TEXT,"max_time"); + result->add_column_definition(SQLITE_TEXT,"rows_affected"); + result->add_column_definition(SQLITE_TEXT,"rows_sent"); + if (map_size >= DIGEST_STATS_FAST_MINSIZE) { + for (int i=0; i::iterator it=digest_umap_aux.begin(); it!=digest_umap_aux.end(); ++it) { + QP_query_digest_stats *qds=(QP_query_digest_stats *)it->second; + delete qds; + } + } + } else { for (std::unordered_map::iterator it=digest_umap_aux.begin(); it!=digest_umap_aux.end(); ++it) { QP_query_digest_stats *qds=(QP_query_digest_stats *)it->second; + query_digest_stats_pointers_t *a = (query_digest_stats_pointers_t *)malloc(sizeof(query_digest_stats_pointers_t)); + char **pta=qds->get_row(&digest_text_umap_aux, a); + result->add_row(pta); + free(a); delete qds; } } - } else { - for (std::unordered_map::iterator it=digest_umap_aux.begin(); it!=digest_umap_aux.end(); ++it) { - QP_query_digest_stats *qds=(QP_query_digest_stats *)it->second; - query_digest_stats_pointers_t *a = (query_digest_stats_pointers_t *)malloc(sizeof(query_digest_stats_pointers_t)); - char **pta=qds->get_row(&digest_text_umap_aux, a); - result->add_row(pta); - //qds->free_row(pta); - free(a); - delete qds; - } } + GloAdmin->stats___save_mysql_query_digest_to_sqlite( + false, false, result, &digest_umap_aux, &digest_text_umap_aux + ); digest_umap_aux.clear(); // this part is always single-threaded for (std::unordered_map::iterator it=digest_text_umap_aux.begin(); it!=digest_text_umap_aux.end(); ++it) { From 81bab9e878aeb9e8520c4321be43eb284aa048f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sat, 11 Feb 2023 22:58:23 +0000 Subject: [PATCH 06/21] Various bug fixes in v2.x-digest_umap_aux --- lib/ProxySQL_Admin.cpp | 6 +++++- lib/Query_Processor.cpp | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index abe51f9bd..4b73d46f4 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -9449,7 +9449,7 @@ void ProxySQL_Admin::stats___save_mysql_query_digest_to_sqlite( // If the function do not receives a resultset, it gets the values directly from the digest_umap while (resultset ? i != resultset->rows_count : it != digest_umap->end()) { QP_query_digest_stats *qds = (QP_query_digest_stats *)it->second; - SQLite3_row *row = resultset ? resultset->rows[i] : NULL; i++; + SQLite3_row *row = resultset ? resultset->rows[i] : NULL; string digest_hex_str; if (!resultset) { std::ostringstream digest_stream; @@ -9496,6 +9496,10 @@ void ProxySQL_Admin::stats___save_mysql_query_digest_to_sqlite( rc=(*proxy_sqlite3_clear_bindings)(statement1); ASSERT_SQLITE_OK(rc, statsdb); rc=(*proxy_sqlite3_reset)(statement1); ASSERT_SQLITE_OK(rc, statsdb); } +#ifdef DEBUG + if (resultset) + assert(row_idx == i); +#endif row_idx++; if (resultset) i++; diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index ac2ba66cb..62df1f8bb 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -1374,7 +1374,7 @@ SQLite3_result * Query_Processor::get_query_digests_reset_v2(const bool use_resu pthread_rwlock_unlock(&digest_rwlock); unsigned long long curtime1; unsigned long long curtime2; - size_t map_size = digest_umap.size(); + size_t map_size = digest_umap_aux.size(); // we need to use the new map bool free_me = false; bool defer_free = false; int n=DIGEST_STATS_FAST_THREADS; From ba41bad72c5582c016cad78296fa693e6fc661a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20S=C3=A1nchez=20Parra?= Date: Tue, 14 Feb 2023 15:53:40 +0100 Subject: [PATCH 07/21] Add PROXYSQLTESTs to get the digest map with old and new algorithms --- lib/ProxySQL_Admin.cpp | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 4b73d46f4..024dce87e 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -1073,6 +1073,21 @@ incoming_servers_t::incoming_servers_t( incoming_hostgroup_attributes(incoming_hostgroup_attributes) {} +int ProxySQL_Test___GetDigestTable_v2(bool reset, bool use_resultset) { + int r = 0; + if (!GloQPro) return 0; + SQLite3_result * resultset=NULL; + if (reset==true) { + resultset=GloQPro->get_query_digests_reset_v2(use_resultset); + } else { + resultset=GloQPro->get_query_digests_v2(use_resultset); + } + if (resultset==NULL) return 0; + r = resultset->rows_count; + delete resultset; + return r; +} + int ProxySQL_Test___GetDigestTable(bool reset, bool use_swap) { int r = 0; if (!GloQPro) return 0; @@ -4043,6 +4058,28 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { run_query=false; free(msg); break; + case 22: + // get all the entries from the digest map, but WRITING to DB + // it uses multiple threads + // It locks the maps while generating the resultset + SPA->stats___mysql_query_digests(false, true); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, 0); + run_query=false; + break; + case 23: + // get all the entries from the digest map, but WRITING to DB + // it uses multiple threads for creating the resultset + r1 = ProxySQL_Test___GetDigestTable_v2(false, true); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); + run_query=false; + break; + case 24: + // get all the entries from the digest map, but WRITING to DB + // Do not create a resultset, uses the digest_umap + r1 = ProxySQL_Test___GetDigestTable_v2(false, false); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); + run_query=false; + break; case 31: { if (test_arg1==0) { From 2c13dfdc520be9dbf71fd1d1df65a8293ffa4361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20S=C3=A1nchez=20Parra?= Date: Tue, 14 Feb 2023 18:13:47 +0100 Subject: [PATCH 08/21] Make PROXYSLQTEST 22 and 24 return the number of row affected --- include/proxysql_admin.h | 4 ++-- include/query_processor.h | 4 ++-- lib/ProxySQL_Admin.cpp | 28 ++++++++++++++++------------ lib/Query_Processor.cpp | 15 +++++++++------ 4 files changed, 29 insertions(+), 22 deletions(-) diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 81576bbc0..b2cf4d722 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -418,11 +418,11 @@ class ProxySQL_Admin { void p_update_metrics(); void stats___mysql_query_rules(); - void stats___save_mysql_query_digest_to_sqlite( + int stats___save_mysql_query_digest_to_sqlite( const bool reset, const bool copy, const SQLite3_result *resultset, const umap_query_digest *digest_umap, const umap_query_digest_text *digest_text_umap ); - void stats___mysql_query_digests(bool reset, bool copy=false); + int stats___mysql_query_digests(bool reset, bool copy=false); //void stats___mysql_query_digests_reset(); void stats___mysql_commands_counters(); void stats___mysql_processlist(); diff --git a/include/query_processor.h b/include/query_processor.h index 2e8cf247d..25112afac 100644 --- a/include/query_processor.h +++ b/include/query_processor.h @@ -323,8 +323,8 @@ class Query_Processor { SQLite3_result * get_stats_commands_counters(); SQLite3_result * get_query_digests(); SQLite3_result * get_query_digests_reset(); - SQLite3_result * get_query_digests_v2(const bool use_resultset = true); - SQLite3_result * get_query_digests_reset_v2(const bool use_resultset = true); + std::pair get_query_digests_v2(const bool use_resultset = true); + std::pair get_query_digests_reset_v2(const bool use_resultset = true); void get_query_digests_reset(umap_query_digest *uqd, umap_query_digest_text *uqdt); unsigned long long purge_query_digests(bool async_purge, bool parallel, char **msg); unsigned long long purge_query_digests_async(char **msg); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 024dce87e..97abd958e 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -1076,15 +1076,15 @@ incoming_servers_t::incoming_servers_t( int ProxySQL_Test___GetDigestTable_v2(bool reset, bool use_resultset) { int r = 0; if (!GloQPro) return 0; - SQLite3_result * resultset=NULL; + std::pair res; if (reset==true) { - resultset=GloQPro->get_query_digests_reset_v2(use_resultset); + res = GloQPro->get_query_digests_reset_v2(use_resultset); } else { - resultset=GloQPro->get_query_digests_v2(use_resultset); + res = GloQPro->get_query_digests_v2(use_resultset); } - if (resultset==NULL) return 0; - r = resultset->rows_count; - delete resultset; + if (res.first == NULL) return res.second; + r = res.first->rows_count; + delete res.first; return r; } @@ -4062,8 +4062,8 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { // get all the entries from the digest map, but WRITING to DB // it uses multiple threads // It locks the maps while generating the resultset - SPA->stats___mysql_query_digests(false, true); - SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, 0); + r1 = SPA->stats___mysql_query_digests(false, true); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); run_query=false; break; case 23: @@ -9450,7 +9450,7 @@ void ProxySQL_Admin::stats___proxysql_message_metrics(bool reset) { delete resultset; } -void ProxySQL_Admin::stats___save_mysql_query_digest_to_sqlite( +int ProxySQL_Admin::stats___save_mysql_query_digest_to_sqlite( const bool reset, const bool copy, const SQLite3_result *resultset, const umap_query_digest *digest_umap, const umap_query_digest_text *digest_text_umap ) { @@ -9551,17 +9551,19 @@ void ProxySQL_Admin::stats___save_mysql_query_digest_to_sqlite( } } statsdb->execute("COMMIT"); + + return row_idx; } -void ProxySQL_Admin::stats___mysql_query_digests(bool reset, bool copy) { - if (!GloQPro) return; +int ProxySQL_Admin::stats___mysql_query_digests(bool reset, bool copy) { + if (!GloQPro) return 0; SQLite3_result * resultset=NULL; if (reset==true) { resultset=GloQPro->get_query_digests_reset(); } else { resultset=GloQPro->get_query_digests(); } - if (resultset==NULL) return; + if (resultset==NULL) return 0; statsdb->execute("BEGIN"); int rc; sqlite3_stmt *statement1=NULL; @@ -9656,6 +9658,8 @@ void ProxySQL_Admin::stats___mysql_query_digests(bool reset, bool copy) { } statsdb->execute("COMMIT"); delete resultset; + + return row_idx; } void ProxySQL_Admin::stats___mysql_client_host_cache(bool reset) { diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 62df1f8bb..992401f3a 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -1189,7 +1189,7 @@ unsigned long long Query_Processor::get_query_digests_total_size() { return ret; } -SQLite3_result * Query_Processor::get_query_digests_v2(const bool use_resultset) { +std::pair Query_Processor::get_query_digests_v2(const bool use_resultset) { proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 4, "Dumping current query digest\n"); SQLite3_result *result = NULL; // Create two auxiliary maps and swap its content with the main maps. This @@ -1259,7 +1259,7 @@ SQLite3_result * Query_Processor::get_query_digests_v2(const bool use_resultset) } } } - GloAdmin->stats___save_mysql_query_digest_to_sqlite( + int num_rows = GloAdmin->stats___save_mysql_query_digest_to_sqlite( false, false, result, &digest_umap_aux, &digest_text_umap_aux ); if (map_size >= DIGEST_STATS_FAST_MINSIZE) { @@ -1294,7 +1294,8 @@ SQLite3_result * Query_Processor::get_query_digests_v2(const bool use_resultset) digest_text_umap_aux.clear(); pthread_rwlock_unlock(&digest_rwlock); - return result; + std::pair res{result, num_rows}; + return res; } SQLite3_result * Query_Processor::get_query_digests() { @@ -1364,7 +1365,7 @@ SQLite3_result * Query_Processor::get_query_digests() { return result; } -SQLite3_result * Query_Processor::get_query_digests_reset_v2(const bool use_resultset) { +std::pair Query_Processor::get_query_digests_reset_v2(const bool use_resultset) { SQLite3_result *result = NULL; umap_query_digest digest_umap_aux; umap_query_digest_text digest_text_umap_aux; @@ -1438,7 +1439,7 @@ SQLite3_result * Query_Processor::get_query_digests_reset_v2(const bool use_resu } } } - GloAdmin->stats___save_mysql_query_digest_to_sqlite( + int num_rows = GloAdmin->stats___save_mysql_query_digest_to_sqlite( false, false, result, &digest_umap_aux, &digest_text_umap_aux ); digest_umap_aux.clear(); @@ -1464,7 +1465,9 @@ SQLite3_result * Query_Processor::get_query_digests_reset_v2(const bool use_resu } } } - return result; + + std::pair res{result, num_rows}; + return res; } void Query_Processor::get_query_digests_reset(umap_query_digest *uqd, umap_query_digest_text *uqdt) { From 222d8b576b183baa22703268e4c8e03930181d19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20S=C3=A1nchez=20Parra?= Date: Wed, 15 Feb 2023 10:49:30 +0100 Subject: [PATCH 09/21] Add PROXYSQLTESTs to get and reset the digest map with old and new algorithms --- lib/ProxySQL_Admin.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 97abd958e..66f4e6913 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -4080,6 +4080,28 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); run_query=false; break; + case 25: + // get all the entries from the digest map AND RESET, but WRITING to DB + // it uses multiple threads + // It locks the maps while generating the resultset + r1 = SPA->stats___mysql_query_digests(true, true); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); + run_query=false; + break; + case 26: + // get all the entries from the digest map AND RESET, but WRITING to DB + // it uses multiple threads for creating the resultset + r1 = ProxySQL_Test___GetDigestTable_v2(true, true); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); + run_query=false; + break; + case 27: + // get all the entries from the digest map AND RESET, but WRITING to DB + // Do not create a resultset, uses the digest_umap + r1 = ProxySQL_Test___GetDigestTable_v2(true, false); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); + run_query=false; + break; case 31: { if (test_arg1==0) { From 45522e47a4d5fa62624966465b10800e686eedd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20S=C3=A1nchez=20Parra?= Date: Wed, 15 Feb 2023 16:25:24 +0100 Subject: [PATCH 10/21] Add copy parameter to ProxySQL_Test___GetDigestTable_v2() --- lib/ProxySQL_Admin.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 66f4e6913..97218f2f9 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -1073,7 +1073,7 @@ incoming_servers_t::incoming_servers_t( incoming_hostgroup_attributes(incoming_hostgroup_attributes) {} -int ProxySQL_Test___GetDigestTable_v2(bool reset, bool use_resultset) { +int ProxySQL_Test___GetDigestTable_v2(bool reset, bool copy, bool use_resultset) { int r = 0; if (!GloQPro) return 0; std::pair res; @@ -4069,14 +4069,14 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { case 23: // get all the entries from the digest map, but WRITING to DB // it uses multiple threads for creating the resultset - r1 = ProxySQL_Test___GetDigestTable_v2(false, true); + r1 = ProxySQL_Test___GetDigestTable_v2(false, false, true); SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); run_query=false; break; case 24: // get all the entries from the digest map, but WRITING to DB // Do not create a resultset, uses the digest_umap - r1 = ProxySQL_Test___GetDigestTable_v2(false, false); + r1 = ProxySQL_Test___GetDigestTable_v2(false, false, false); SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); run_query=false; break; @@ -4091,14 +4091,14 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { case 26: // get all the entries from the digest map AND RESET, but WRITING to DB // it uses multiple threads for creating the resultset - r1 = ProxySQL_Test___GetDigestTable_v2(true, true); + r1 = ProxySQL_Test___GetDigestTable_v2(true, true, true); SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); run_query=false; break; case 27: // get all the entries from the digest map AND RESET, but WRITING to DB // Do not create a resultset, uses the digest_umap - r1 = ProxySQL_Test___GetDigestTable_v2(true, false); + r1 = ProxySQL_Test___GetDigestTable_v2(true, true, false); SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); run_query=false; break; From baa70fc6fdd26158514d387e5628671e380af380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20S=C3=A1nchez=20Parra?= Date: Wed, 15 Feb 2023 16:41:18 +0100 Subject: [PATCH 11/21] Revert always writing to DB in Query_Processor --- lib/ProxySQL_Admin.cpp | 16 ++++++++++------ lib/Query_Processor.cpp | 16 ++++++++++------ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 97218f2f9..29db38603 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -1077,15 +1077,19 @@ int ProxySQL_Test___GetDigestTable_v2(bool reset, bool copy, bool use_resultset) int r = 0; if (!GloQPro) return 0; std::pair res; - if (reset==true) { + if (reset == true) { res = GloQPro->get_query_digests_reset_v2(use_resultset); } else { res = GloQPro->get_query_digests_v2(use_resultset); } - if (res.first == NULL) return res.second; - r = res.first->rows_count; + + if (res.first == NULL) + return res.second; + + int num_rows = GloAdmin->stats___save_mysql_query_digest_to_sqlite(reset, copy, res.first, NULL, NULL); delete res.first; - return r; + + return num_rows; } int ProxySQL_Test___GetDigestTable(bool reset, bool use_swap) { @@ -9503,11 +9507,11 @@ int ProxySQL_Admin::stats___save_mysql_query_digest_to_sqlite( int num_rows = resultset ? resultset->rows_count : digest_umap->size(); int max_bulk_row_idx = num_rows/32; max_bulk_row_idx=max_bulk_row_idx*32; - auto it = digest_umap->cbegin(); + auto it = resultset ? (std::unordered_map::iterator)NULL : digest_umap->cbegin(); int i = 0; // If the function do not receives a resultset, it gets the values directly from the digest_umap while (resultset ? i != resultset->rows_count : it != digest_umap->end()) { - QP_query_digest_stats *qds = (QP_query_digest_stats *)it->second; + QP_query_digest_stats *qds = (QP_query_digest_stats *)(resultset ? NULL : it->second); SQLite3_row *row = resultset ? resultset->rows[i] : NULL; string digest_hex_str; if (!resultset) { diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 992401f3a..42e03ec5c 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -1201,6 +1201,7 @@ std::pair Query_Processor::get_query_digests_v2(const boo digest_umap.swap(digest_umap_aux); digest_text_umap.swap(digest_text_umap_aux); pthread_rwlock_unlock(&digest_rwlock); + int num_rows = 0; unsigned long long curtime1; unsigned long long curtime2; size_t map_size = digest_umap_aux.size(); @@ -1258,10 +1259,11 @@ std::pair Query_Processor::get_query_digests_v2(const boo free(a); } } + } else { + num_rows = GloAdmin->stats___save_mysql_query_digest_to_sqlite( + false, false, NULL, &digest_umap_aux, &digest_text_umap_aux + ); } - int num_rows = GloAdmin->stats___save_mysql_query_digest_to_sqlite( - false, false, result, &digest_umap_aux, &digest_text_umap_aux - ); if (map_size >= DIGEST_STATS_FAST_MINSIZE) { curtime2=monotonic_time(); curtime1 = curtime1/1000; @@ -1373,6 +1375,7 @@ std::pair Query_Processor::get_query_digests_reset_v2(con digest_umap.swap(digest_umap_aux); digest_text_umap.swap(digest_text_umap_aux); pthread_rwlock_unlock(&digest_rwlock); + int num_rows = 0; unsigned long long curtime1; unsigned long long curtime2; size_t map_size = digest_umap_aux.size(); // we need to use the new map @@ -1438,10 +1441,11 @@ std::pair Query_Processor::get_query_digests_reset_v2(con delete qds; } } + } else { + num_rows = GloAdmin->stats___save_mysql_query_digest_to_sqlite( + false, false, result, &digest_umap_aux, &digest_text_umap_aux + ); } - int num_rows = GloAdmin->stats___save_mysql_query_digest_to_sqlite( - false, false, result, &digest_umap_aux, &digest_text_umap_aux - ); digest_umap_aux.clear(); // this part is always single-threaded for (std::unordered_map::iterator it=digest_text_umap_aux.begin(); it!=digest_text_umap_aux.end(); ++it) { From 79fdc6817e5911b9529193e864f94c1f0e845ffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20S=C3=A1nchez=20Parra?= Date: Wed, 15 Feb 2023 17:43:40 +0100 Subject: [PATCH 12/21] Fix memory leak when merging digests umaps --- lib/Query_Processor.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 42e03ec5c..13e8fb7a9 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -1287,6 +1287,7 @@ std::pair Query_Processor::get_query_digests_v2(const boo qds_equal->add_time( qds->min_time, qds->last_seen, qds->rows_affected, qds->rows_sent, qds->count_star ); + delete qds_equal; } else { digest_umap.insert(element); } From a5bd44fccf5bee9574588aa5c9b039b83b725d49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20S=C3=A1nchez=20Parra?= Date: Wed, 15 Feb 2023 17:44:20 +0100 Subject: [PATCH 13/21] Clear auxiliary maps outside the lock --- lib/Query_Processor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 13e8fb7a9..5e8c49a3f 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -1293,9 +1293,9 @@ std::pair Query_Processor::get_query_digests_v2(const boo } } digest_text_umap.insert(digest_text_umap_aux.begin(), digest_text_umap_aux.end()); + pthread_rwlock_unlock(&digest_rwlock); digest_umap_aux.clear(); digest_text_umap_aux.clear(); - pthread_rwlock_unlock(&digest_rwlock); std::pair res{result, num_rows}; return res; From 1644c9a87093423a5bbd56c97296d4bfacc6b469 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20S=C3=A1nchez=20Parra?= Date: Thu, 16 Feb 2023 10:28:30 +0100 Subject: [PATCH 14/21] Add a second group of auxiliary maps This way, we can merge previous maps without locking the mutex. We move locking the mutex for merging the second group of auxiliary maps. This second merge it is faster than the previous merge, because the second group of auxiliary maps is not going to grow large. --- lib/Query_Processor.cpp | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 5e8c49a3f..1f6329e38 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -1195,8 +1195,8 @@ std::pair Query_Processor::get_query_digests_v2(const boo // Create two auxiliary maps and swap its content with the main maps. This // way, this function can read query digests stored until now while other // threads write in the other map. We need to lock while swapping. - umap_query_digest digest_umap_aux; - umap_query_digest_text digest_text_umap_aux; + umap_query_digest digest_umap_aux, digest_umap_aux_2; + umap_query_digest_text digest_text_umap_aux, digest_text_umap_aux_2; pthread_rwlock_wrlock(&digest_rwlock); digest_umap.swap(digest_umap_aux); digest_text_umap.swap(digest_text_umap_aux); @@ -1271,7 +1271,41 @@ std::pair Query_Processor::get_query_digests_v2(const boo proxy_info("Running query on stats_mysql_query_digest: %llums to retrieve %lu entries\n", curtime2-curtime1, map_size); } - // Once the reading finishes, we lock and swap again both maps. Then, we + // Once we finish creating the resultset or writing to SQLite, we use a + // second group of auxiliary maps to swap it with the first group of + // auxiliary maps. This way, we can merge the main maps and the first + // auxiliary maps without locking the mutex during the process. This is + // useful because writing to SQLite can take a lot of time, so the first + // group of auxiliary maps could grow large. + pthread_rwlock_wrlock(&digest_rwlock); + digest_umap.swap(digest_umap_aux_2); + digest_text_umap.swap(digest_text_umap_aux_2); + pthread_rwlock_unlock(&digest_rwlock); + + // Once we do the swap, we merge the content of the first auxiliary maps + // in the main maps and clear the content of the auxiliary maps. + digest_text_umap_aux.swap(digest_text_umap); + for (const auto& element : digest_umap_aux) { + uint64_t digest = element.first; + QP_query_digest_stats *qds = (QP_query_digest_stats *)element.second; + std::unordered_map::iterator it = digest_umap_aux_2.find(digest); + if (it != digest_umap_aux_2.end()) { + // found + QP_query_digest_stats *qds_equal = (QP_query_digest_stats *)it->second; + qds_equal->add_time( + qds->min_time, qds->last_seen, qds->rows_affected, qds->rows_sent, qds->count_star + ); + delete qds_equal; + } else { + digest_umap_aux_2.insert(element); + } + } + digest_text_umap.insert(digest_text_umap_aux.begin(), digest_text_umap_aux.end()); + digest_umap_aux_2.clear(); + digest_text_umap_aux_2.clear(); + + // Once we finish merging the main maps and the first auxiliary maps, we + // lock and swap the main maps with the second auxiliary maps. Then, we // merge the content of the auxiliary maps in the main maps and clear the // content of the auxiliary maps. pthread_rwlock_wrlock(&digest_rwlock); From baddb46fe308bbfd83f15d13f9590f18edc61d05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20S=C3=A1nchez=20Parra?= Date: Thu, 16 Feb 2023 18:31:58 +0100 Subject: [PATCH 15/21] Fix memory leak in get_query_digests_reset_v2 not using resultset --- lib/Query_Processor.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 1f6329e38..9d0ae6928 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -1480,6 +1480,14 @@ std::pair Query_Processor::get_query_digests_reset_v2(con num_rows = GloAdmin->stats___save_mysql_query_digest_to_sqlite( false, false, result, &digest_umap_aux, &digest_text_umap_aux ); + for ( + std::unordered_map::iterator it = digest_umap_aux.begin(); + it != digest_umap_aux.end(); + ++it + ) { + QP_query_digest_stats *qds = (QP_query_digest_stats *)it->second; + delete qds; + } } digest_umap_aux.clear(); // this part is always single-threaded From 12a064313f4e42f9a8e80ad228cc610fa9f9b68d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20S=C3=A1nchez=20Parra?= Date: Fri, 17 Feb 2023 08:54:13 +0100 Subject: [PATCH 16/21] Make default new algorithm to get query digest map --- include/proxysql_admin.h | 1 + lib/ProxySQL_Admin.cpp | 49 ++++++++++++++++++++-------------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index b2cf4d722..bdd1b26c7 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -423,6 +423,7 @@ class ProxySQL_Admin { const umap_query_digest *digest_umap, const umap_query_digest_text *digest_text_umap ); int stats___mysql_query_digests(bool reset, bool copy=false); + int stats___mysql_query_digests_v2(bool reset, bool copy, bool use_resultset); //void stats___mysql_query_digests_reset(); void stats___mysql_commands_counters(); void stats___mysql_processlist(); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 29db38603..9d3bf39dd 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -1073,25 +1073,6 @@ incoming_servers_t::incoming_servers_t( incoming_hostgroup_attributes(incoming_hostgroup_attributes) {} -int ProxySQL_Test___GetDigestTable_v2(bool reset, bool copy, bool use_resultset) { - int r = 0; - if (!GloQPro) return 0; - std::pair res; - if (reset == true) { - res = GloQPro->get_query_digests_reset_v2(use_resultset); - } else { - res = GloQPro->get_query_digests_v2(use_resultset); - } - - if (res.first == NULL) - return res.second; - - int num_rows = GloAdmin->stats___save_mysql_query_digest_to_sqlite(reset, copy, res.first, NULL, NULL); - delete res.first; - - return num_rows; -} - int ProxySQL_Test___GetDigestTable(bool reset, bool use_swap) { int r = 0; if (!GloQPro) return 0; @@ -3280,10 +3261,10 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign if (stats_mysql_processlist) stats___mysql_processlist(); if (stats_mysql_query_digest_reset) { - stats___mysql_query_digests(true, stats_mysql_query_digest); + stats___mysql_query_digests_v2(true, stats_mysql_query_digest, false); } else { if (stats_mysql_query_digest) { - stats___mysql_query_digests(false); + stats___mysql_query_digests_v2(false, false, false); } } if (stats_mysql_errors) @@ -4073,14 +4054,14 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { case 23: // get all the entries from the digest map, but WRITING to DB // it uses multiple threads for creating the resultset - r1 = ProxySQL_Test___GetDigestTable_v2(false, false, true); + r1 = SPA->stats___mysql_query_digests_v2(false, false, true); SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); run_query=false; break; case 24: // get all the entries from the digest map, but WRITING to DB // Do not create a resultset, uses the digest_umap - r1 = ProxySQL_Test___GetDigestTable_v2(false, false, false); + r1 = SPA->stats___mysql_query_digests_v2(false, false, false); SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); run_query=false; break; @@ -4095,14 +4076,14 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { case 26: // get all the entries from the digest map AND RESET, but WRITING to DB // it uses multiple threads for creating the resultset - r1 = ProxySQL_Test___GetDigestTable_v2(true, true, true); + r1 = SPA->stats___mysql_query_digests_v2(true, true, true); SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); run_query=false; break; case 27: // get all the entries from the digest map AND RESET, but WRITING to DB // Do not create a resultset, uses the digest_umap - r1 = ProxySQL_Test___GetDigestTable_v2(true, true, false); + r1 = SPA->stats___mysql_query_digests_v2(true, true, false); SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, r1); run_query=false; break; @@ -9688,6 +9669,24 @@ int ProxySQL_Admin::stats___mysql_query_digests(bool reset, bool copy) { return row_idx; } +int ProxySQL_Admin::stats___mysql_query_digests_v2(bool reset, bool copy, bool use_resultset) { + if (!GloQPro) return 0; + std::pair res; + if (reset == true) { + res = GloQPro->get_query_digests_reset_v2(use_resultset); + } else { + res = GloQPro->get_query_digests_v2(use_resultset); + } + + if (res.first == NULL) + return res.second; + + int num_rows = GloAdmin->stats___save_mysql_query_digest_to_sqlite(reset, copy, res.first, NULL, NULL); + delete res.first; + + return num_rows; +} + void ProxySQL_Admin::stats___mysql_client_host_cache(bool reset) { if (!GloQPro) return; From a19edfd42888044a73fe35ed0a3c7f64b9cbf9c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20S=C3=A1nchez=20Parra?= Date: Mon, 20 Feb 2023 13:08:20 +0100 Subject: [PATCH 17/21] Various bug fixes in get_query_digests_v2() --- lib/Query_Processor.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 9d0ae6928..1debd1a4c 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -1284,20 +1284,19 @@ std::pair Query_Processor::get_query_digests_v2(const boo // Once we do the swap, we merge the content of the first auxiliary maps // in the main maps and clear the content of the auxiliary maps. - digest_text_umap_aux.swap(digest_text_umap); - for (const auto& element : digest_umap_aux) { + for (const auto& element : digest_umap_aux_2) { uint64_t digest = element.first; QP_query_digest_stats *qds = (QP_query_digest_stats *)element.second; - std::unordered_map::iterator it = digest_umap_aux_2.find(digest); - if (it != digest_umap_aux_2.end()) { + std::unordered_map::iterator it = digest_umap_aux.find(digest); + if (it != digest_umap_aux.end()) { // found QP_query_digest_stats *qds_equal = (QP_query_digest_stats *)it->second; qds_equal->add_time( qds->min_time, qds->last_seen, qds->rows_affected, qds->rows_sent, qds->count_star ); - delete qds_equal; + delete qds; } else { - digest_umap_aux_2.insert(element); + digest_umap_aux.insert(element); } } digest_text_umap.insert(digest_text_umap_aux.begin(), digest_text_umap_aux.end()); @@ -1321,7 +1320,7 @@ std::pair Query_Processor::get_query_digests_v2(const boo qds_equal->add_time( qds->min_time, qds->last_seen, qds->rows_affected, qds->rows_sent, qds->count_star ); - delete qds_equal; + delete qds; } else { digest_umap.insert(element); } From 0da59a662045cba483bb9eb81709c8fe124f7f69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20S=C3=A1nchez=20Parra?= Date: Mon, 20 Feb 2023 16:23:57 +0100 Subject: [PATCH 18/21] Honor reset and copy options in get_query_digests_reset_v2() --- include/query_processor.h | 4 +++- lib/ProxySQL_Admin.cpp | 2 +- lib/Query_Processor.cpp | 6 ++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/include/query_processor.h b/include/query_processor.h index 25112afac..5e17903c0 100644 --- a/include/query_processor.h +++ b/include/query_processor.h @@ -324,7 +324,9 @@ class Query_Processor { SQLite3_result * get_query_digests(); SQLite3_result * get_query_digests_reset(); std::pair get_query_digests_v2(const bool use_resultset = true); - std::pair get_query_digests_reset_v2(const bool use_resultset = true); + std::pair get_query_digests_reset_v2( + const bool copy, const bool use_resultset = true + ); void get_query_digests_reset(umap_query_digest *uqd, umap_query_digest_text *uqdt); unsigned long long purge_query_digests(bool async_purge, bool parallel, char **msg); unsigned long long purge_query_digests_async(char **msg); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 9d3bf39dd..3d87fc382 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -9673,7 +9673,7 @@ int ProxySQL_Admin::stats___mysql_query_digests_v2(bool reset, bool copy, bool u if (!GloQPro) return 0; std::pair res; if (reset == true) { - res = GloQPro->get_query_digests_reset_v2(use_resultset); + res = GloQPro->get_query_digests_reset_v2(copy, use_resultset); } else { res = GloQPro->get_query_digests_v2(use_resultset); } diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 1debd1a4c..04fba0aaf 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -1401,7 +1401,9 @@ SQLite3_result * Query_Processor::get_query_digests() { return result; } -std::pair Query_Processor::get_query_digests_reset_v2(const bool use_resultset) { +std::pair Query_Processor::get_query_digests_reset_v2( + const bool copy, const bool use_resultset +) { SQLite3_result *result = NULL; umap_query_digest digest_umap_aux; umap_query_digest_text digest_text_umap_aux; @@ -1477,7 +1479,7 @@ std::pair Query_Processor::get_query_digests_reset_v2(con } } else { num_rows = GloAdmin->stats___save_mysql_query_digest_to_sqlite( - false, false, result, &digest_umap_aux, &digest_text_umap_aux + true, copy, result, &digest_umap_aux, &digest_text_umap_aux ); for ( std::unordered_map::iterator it = digest_umap_aux.begin(); From be4ee20dd5556bdf4042ca9907657de7013e8f30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sun, 16 Apr 2023 20:49:47 +0000 Subject: [PATCH 19/21] Always initialize curtime1 in QP timers --- lib/Query_Processor.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 04fba0aaf..1856ec94a 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -1205,10 +1205,10 @@ std::pair Query_Processor::get_query_digests_v2(const boo unsigned long long curtime1; unsigned long long curtime2; size_t map_size = digest_umap_aux.size(); + curtime1 = monotonic_time(); // curtime1 must always be initialized if (use_resultset) { if (map_size >= DIGEST_STATS_FAST_MINSIZE) { result = new SQLite3_result(14, true); - curtime1 = monotonic_time(); } else { result = new SQLite3_result(14); } @@ -1268,7 +1268,7 @@ std::pair Query_Processor::get_query_digests_v2(const boo curtime2=monotonic_time(); curtime1 = curtime1/1000; curtime2 = curtime2/1000; - proxy_info("Running query on stats_mysql_query_digest: %llums to retrieve %lu entries\n", curtime2-curtime1, map_size); + proxy_info("Running query on stats_mysql_query_digest: (not locked) %llums to retrieve %lu entries\n", curtime2-curtime1, map_size); } // Once we finish creating the resultset or writing to SQLite, we use a @@ -1341,9 +1341,9 @@ SQLite3_result * Query_Processor::get_query_digests() { unsigned long long curtime1; unsigned long long curtime2; size_t map_size = digest_umap.size(); + curtime1 = monotonic_time(); // curtime1 must always be initialized if (map_size >= DIGEST_STATS_FAST_MINSIZE) { result = new SQLite3_result(14, true); - curtime1 = monotonic_time(); } else { result = new SQLite3_result(14); } @@ -1419,11 +1419,11 @@ std::pair Query_Processor::get_query_digests_reset_v2( bool defer_free = false; int n=DIGEST_STATS_FAST_THREADS; get_query_digests_parallel_args args[n]; + curtime1 = monotonic_time(); // curtime1 must always be initialized if (use_resultset) { free_me = true; defer_free = true; if (map_size >= DIGEST_STATS_FAST_MINSIZE) { - curtime1=monotonic_time(); result = new SQLite3_result(14, true); } else { result = new SQLite3_result(14); @@ -1500,7 +1500,7 @@ std::pair Query_Processor::get_query_digests_reset_v2( curtime2=monotonic_time(); curtime1 = curtime1/1000; curtime2 = curtime2/1000; - proxy_info("Running query on stats_mysql_query_digest_reset: %llums to retrieve %lu entries\n", curtime2-curtime1, map_size); + proxy_info("Running query on stats_mysql_query_digest: (not locked) %llums to retrieve %lu entries\n", curtime2-curtime1, map_size); if (free_me) { if (defer_free) { for (int i=0; i= DIGEST_STATS_FAST_MINSIZE) { - curtime1=monotonic_time(); result = new SQLite3_result(14, true); } else { result = new SQLite3_result(14); From a1de300a8b8f3ee123223a4081422a85db21f5e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sun, 16 Apr 2023 21:43:25 +0000 Subject: [PATCH 20/21] Adding TAP test with new PROXYSQLTEST commands --- test/tap/tests/admin_various_commands3-t.cpp | 132 +++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 test/tap/tests/admin_various_commands3-t.cpp diff --git a/test/tap/tests/admin_various_commands3-t.cpp b/test/tap/tests/admin_various_commands3-t.cpp new file mode 100644 index 000000000..4a80f16b0 --- /dev/null +++ b/test/tap/tests/admin_various_commands3-t.cpp @@ -0,0 +1,132 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "tap.h" +#include "command_line.h" +#include "utils.h" + +using std::string; + +/* this test: + * enables mysql-have_ssl + * execute various command +*/ + +std::vector queries_t = { + "PROXYSQLTEST 22", + "PROXYSQLTEST 23", + "PROXYSQLTEST 24", + "PROXYSQLTEST 25", + "PROXYSQLTEST 26", + "PROXYSQLTEST 27", + "SELECT COUNT(*) FROM stats_mysql_query_digest" + }; + + +//std::vector vals = { 100, 345, 800, 999, 2037, 12345 }; +//std::vector vals = { 100, 345, 800, 999, 2037 }; +std::vector vals = { 100, 345, 800 }; + +std::vector queries = {}; + +int run_q(MYSQL *mysql, const char *q) { + MYSQL_QUERY(mysql,q); + return 0; +} +int main() { + CommandLine cl; + + if (cl.getEnv()) { + diag("Failed to get the required environmental variables."); + return -1; + } + + srandom(123); + + + for (auto it = vals.begin() ; it != vals.end() ; it++) { + std::string q = "PROXYSQLTEST 1 " + std::to_string(*it); + queries.push_back(q); + for (int i=0; i<5; i++) { + queries.push_back(queries_t[rand()%queries_t.size()]); + } + queries.push_back("SELECT COUNT(*) FROM stats_mysql_query_digest"); + for (int i=0; i<5; i++) { + queries.push_back(queries_t[rand()%queries_t.size()]); + } + if (rand()%2 == 0) { + queries.push_back("SELECT COUNT(*) FROM stats_mysql_query_digest_reset"); + } else { + queries.push_back("TRUNCATE TABLE stats.stats_mysql_query_digest"); + } + } + queries.push_back("TRUNCATE TABLE stats.stats_mysql_query_digest"); + + + + MYSQL* proxysql_admin = mysql_init(NULL); + // Initialize connections + if (!proxysql_admin) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql_admin)); + return -1; + } + + if (!mysql_real_connect(proxysql_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql_admin)); + return -1; + } + + MYSQL_QUERY(proxysql_admin, "SET mysql-have_ssl='true'"); + MYSQL_QUERY(proxysql_admin, "SET mysql-have_compress='true'"); + MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME"); + + + + unsigned int p = queries.size(); + for (std::vector::iterator it2 = queries.begin(); it2 != queries.end(); it2++) { + if ( + (strncasecmp(it2->c_str(), "SELECT ", 7)==0) + ) { + // extra test for each queries returning a resultset + p++; + } + } + plan(p); + diag("Running test with %lu queries", queries.size()); + + + for (std::vector::iterator it2 = queries.begin(); it2 != queries.end(); it2++) { + MYSQL* proxysql_admin = mysql_init(NULL); // local scope + if (!proxysql_admin) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql_admin)); + return -1; + } + mysql_ssl_set(proxysql_admin, NULL, NULL, NULL, NULL, NULL); + if (!mysql_real_connect(proxysql_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, CLIENT_SSL|CLIENT_COMPRESS)) { + fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql_admin)); + return -1; + } + int rc = run_q(proxysql_admin, it2->c_str()); + ok(rc==0, "Query: %s" , it2->c_str()); + if ( + (strncasecmp(it2->c_str(), "SELECT ", 7)==0) + ) { + MYSQL_RES* proxy_res = mysql_store_result(proxysql_admin); + unsigned long long num_rows = mysql_num_rows(proxy_res); + ok(num_rows != 0 , "Returned rows: %llu" , num_rows); + mysql_free_result(proxy_res); + } + mysql_close(proxysql_admin); + } + mysql_close(proxysql_admin); + + return exit_status(); +} From 4c3dab1496470f9181f19f4d8c8c233a084fca55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Sun, 16 Apr 2023 22:09:56 +0000 Subject: [PATCH 21/21] Speed up on purge_query_digests_async() Responsible for the various TRUNCATE commands on stats_mysql_query_digest(_reset) --- lib/Query_Processor.cpp | 65 +++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 35 deletions(-) diff --git a/lib/Query_Processor.cpp b/lib/Query_Processor.cpp index 1856ec94a..22cc0367d 100644 --- a/lib/Query_Processor.cpp +++ b/lib/Query_Processor.cpp @@ -1049,51 +1049,46 @@ unsigned long long Query_Processor::purge_query_digests(bool async_purge, bool p unsigned long long Query_Processor::purge_query_digests_async(char **msg) { unsigned long long ret = 0; pthread_rwlock_wrlock(&digest_rwlock); + + + umap_query_digest digest_umap_aux; + umap_query_digest_text digest_text_umap_aux; + pthread_rwlock_wrlock(&digest_rwlock); + digest_umap.swap(digest_umap_aux); + digest_text_umap.swap(digest_text_umap_aux); + pthread_rwlock_unlock(&digest_rwlock); + int num_rows = 0; unsigned long long curtime1=monotonic_time(); - size_t map1_size = digest_umap.size(); - size_t map2_size = digest_text_umap.size(); + size_t map1_size = digest_umap_aux.size(); + size_t map2_size = digest_text_umap_aux.size(); ret = map1_size + map2_size; - unsigned long long i = 0; - QP_query_digest_stats **array1 = (QP_query_digest_stats **)malloc(sizeof(QP_query_digest_stats *)*map1_size); - char **array2 = (char **)malloc(sizeof(char *)*map2_size); - i=0; - for (std::unordered_map::iterator it=digest_umap.begin(); it!=digest_umap.end(); ++it) { - array1[i]=(QP_query_digest_stats *)it->second; - i++; - //delete qds; + + for ( + std::unordered_map::iterator it = digest_umap_aux.begin(); + it != digest_umap_aux.end(); + ++it + ) { + QP_query_digest_stats *qds = (QP_query_digest_stats *)it->second; + delete qds; } - i=0; - for (std::unordered_map::iterator it=digest_text_umap.begin(); it!=digest_text_umap.end(); ++it) { - array2[i] = it->second; - //free(it->second); - i++; + digest_umap_aux.clear(); + for (std::unordered_map::iterator it=digest_text_umap_aux.begin(); it!=digest_text_umap_aux.end(); ++it) { + free(it->second); } - digest_umap.erase(digest_umap.begin(),digest_umap.end()); - digest_text_umap.erase(digest_text_umap.begin(),digest_text_umap.end()); - pthread_rwlock_unlock(&digest_rwlock); - unsigned long long curtime2=monotonic_time(); - curtime1 = curtime1/1000; - curtime2 = curtime2/1000; + digest_text_umap_aux.clear(); + + if (map1_size >= DIGEST_STATS_FAST_MINSIZE) { - proxy_info("Purging stats_mysql_query_digest: locked for %llums to remove %lu entries\n", curtime2-curtime1, map1_size); - } - char buf[128]; - sprintf(buf, "Query digest map locked for %llums", curtime2-curtime1); - *msg = strdup(buf); - for (i=0; i