From 9c2f0bf3acdd5d0604946bb3df958795aa4fab05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 31 Mar 2020 21:31:10 +0200 Subject: [PATCH] Adding tests to benchmark multiple lookups in mysql_query_rules_fast_routing --- include/proxysql_admin.h | 4 +-- include/query_processor.h | 1 + lib/ProxySQL_Admin.cpp | 53 +++++++++++++++++++++++++++++---------- lib/Query_Processor.cpp | 46 +++++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 15 deletions(-) diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 2fa79bcdc..add57955f 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -339,7 +339,7 @@ class ProxySQL_Admin { void enable_grouprep_testing(); #endif // TEST_GROUPREP - unsigned int ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(unsigned int); - bool ProxySQL_Test___Verify_mysql_query_rules_fast_routing(int *ret1, int *ret2, int cnt); + unsigned int ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(unsigned int, bool); + bool ProxySQL_Test___Verify_mysql_query_rules_fast_routing(int *ret1, int *ret2, int cnt, int dual); }; #endif /* __CLASS_PROXYSQL_ADMIN_H */ diff --git a/include/query_processor.h b/include/query_processor.h index 01efd1fd2..0fe43edb3 100644 --- a/include/query_processor.h +++ b/include/query_processor.h @@ -326,6 +326,7 @@ class Query_Processor { void load_fast_routing(SQLite3_result *resultset); SQLite3_result * get_current_query_rules_fast_routing(); int testing___find_HG_in_mysql_query_rules_fast_routing(char *username, char *schemaname, int flagIN); + int testing___find_HG_in_mysql_query_rules_fast_routing_dual(char *username, char *schemaname, int flagIN); // firewall void load_mysql_firewall(SQLite3_result *u, SQLite3_result *r, SQLite3_result *sf); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 5b85bc64a..04c78d75e 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -3321,7 +3321,11 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { if (test_arg1==0) { test_arg1=10000; } - r1 = SPA->ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(test_arg1); + if (test_arg2) { + r1 = SPA->ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(test_arg1, true); + } else { + r1 = SPA->ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(test_arg1, false); + } SPA->send_MySQL_OK(&sess->client_myds->myprot, (char *)"Generated new mysql_query_rules_fast_routing table", r1); run_query=false; break; @@ -3330,7 +3334,11 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { if (test_arg1==0) { test_arg1=10000; } - r1 = SPA->ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(test_arg1); + if (test_arg2) { + r1 = SPA->ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(test_arg1, true); + } else { + r1 = SPA->ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(test_arg1, false); + } msg = SPA->load_mysql_query_rules_to_runtime(); if (msg==NULL) { SPA->send_MySQL_OK(&sess->client_myds->myprot, (char *)"Generated new mysql_query_rules_fast_routing table and loaded to runtime", r1); @@ -3340,7 +3348,7 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { run_query=false; break; case 13: - // generate random mysql_query_rules_fast_routing and LOAD TO RUNTIME + // LOAD MYSQL QUERY RULES TO RUNTIME for N times if (test_arg1==0) { test_arg1=1; } @@ -3360,7 +3368,7 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { } { int ret1, ret2; - bool bret = SPA->ProxySQL_Test___Verify_mysql_query_rules_fast_routing(&ret1, &ret2, test_arg1); + bool bret = SPA->ProxySQL_Test___Verify_mysql_query_rules_fast_routing(&ret1, &ret2, test_arg1, test_arg2); if (bret) { SPA->send_MySQL_OK(&sess->client_myds->myprot, (char *)"Verified all rules in mysql_query_rules_fast_routing", ret1); } else { @@ -5882,7 +5890,8 @@ bool ProxySQL_Admin::ProxySQL_Test___Load_MySQL_Whitelist(int *ret1, int *ret2, return ret; } -bool ProxySQL_Admin::ProxySQL_Test___Verify_mysql_query_rules_fast_routing(int *ret1, int *ret2, int cnt) { +// if dual is not 0 , we call the new search algorithm +bool ProxySQL_Admin::ProxySQL_Test___Verify_mysql_query_rules_fast_routing(int *ret1, int *ret2, int cnt, int dual) { char *q = (char *)"SELECT username, schemaname, flagIN, destination_hostgroup FROM mysql_query_rules_fast_routing ORDER BY RANDOM()"; char *error=NULL; int cols=0; @@ -5900,7 +5909,13 @@ bool ProxySQL_Admin::ProxySQL_Test___Verify_mysql_query_rules_fast_routing(int * for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; int dest_HG = atoi(r->fields[3]); - int ret_HG = GloQPro->testing___find_HG_in_mysql_query_rules_fast_routing(r->fields[0], r->fields[1], atoi(r->fields[2])); + int ret_HG; + if (dual==0) { + // legacy algorithm + ret_HG = GloQPro->testing___find_HG_in_mysql_query_rules_fast_routing(r->fields[0], r->fields[1], atoi(r->fields[2])); + } else { + ret_HG = GloQPro->testing___find_HG_in_mysql_query_rules_fast_routing_dual(r->fields[0], r->fields[1], atoi(r->fields[2])); + } if (dest_HG == ret_HG) { matching_rows++; } @@ -5916,7 +5931,13 @@ bool ProxySQL_Admin::ProxySQL_Test___Verify_mysql_query_rules_fast_routing(int * for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { SQLite3_row *r=*it; int dest_HG = atoi(r->fields[3]); - int ret_HG = GloQPro->testing___find_HG_in_mysql_query_rules_fast_routing(r->fields[0], r->fields[1], atoi(r->fields[2])); + int ret_HG; + if (dual==0) { + // legacy algorithm + ret_HG = GloQPro->testing___find_HG_in_mysql_query_rules_fast_routing(r->fields[0], r->fields[1], atoi(r->fields[2])); + } else { + ret_HG = GloQPro->testing___find_HG_in_mysql_query_rules_fast_routing_dual(r->fields[0], r->fields[1], atoi(r->fields[2])); + } assert(dest_HG==ret_HG); } } @@ -5926,7 +5947,7 @@ bool ProxySQL_Admin::ProxySQL_Test___Verify_mysql_query_rules_fast_routing(int * return ret; } -unsigned int ProxySQL_Admin::ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(unsigned int cnt) { +unsigned int ProxySQL_Admin::ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(unsigned int cnt, bool empty) { char *a = (char *)"INSERT OR IGNORE INTO mysql_query_rules_fast_routing VALUES (?1, ?2, ?3, ?4, '')"; int rc; sqlite3_stmt *statement1=NULL; @@ -5937,16 +5958,22 @@ unsigned int ProxySQL_Admin::ProxySQL_Test___GenerateRandom_mysql_query_rules_fa char * schemaname_buf = (char *)malloc(64); //ui.username = username_buf; //ui.schemaname = schemaname_buf; - strcpy(username_buf,"user_name_"); + if (empty==false) { + strcpy(username_buf,"user_name_"); + } else { + strcpy(username_buf,""); + } strcpy(schemaname_buf,"shard_name_"); int _k; for (unsigned int i=0; iget_size(); }; +// this testing function doesn't care if the user exists or not +// the arguments are coming from this query: +// SELECT username, schemaname, flagIN, destination_hostgroup FROM mysql_query_rules_fast_routing ORDER BY RANDOM() int Query_Processor::testing___find_HG_in_mysql_query_rules_fast_routing(char *username, char *schemaname, int flagIN) { int ret = -1; pthread_rwlock_rdlock(&rwlock); @@ -2708,6 +2711,49 @@ int Query_Processor::testing___find_HG_in_mysql_query_rules_fast_routing(char *u return ret; } +// this testing function implement the dual search: with and without username +// if the length of username is 0 , it will search for random username (that shouldn't exist!) +int Query_Processor::testing___find_HG_in_mysql_query_rules_fast_routing_dual(char *username, char *schemaname, int flagIN) { + int ret = -1; + const char * random_user = (char *)"my_ReaLLy_Rand_User_123456"; + char * u = NULL; + if (strlen(username)) { + u = username; + } else { + u = (char *)random_user; + } + pthread_rwlock_rdlock(&rwlock); + if (rules_fast_routing) { + char keybuf[256]; + char * keybuf_ptr = keybuf; + size_t keylen = strlen(u)+strlen(rand_del)+strlen(schemaname)+30; // 30 is a big number + if (keylen > 250) { + keybuf_ptr = (char *)malloc(keylen); + } + sprintf(keybuf_ptr,"%s%s%s---%d", username, rand_del, schemaname, flagIN); + khiter_t k = kh_get(khStrInt, rules_fast_routing, keybuf_ptr); + if (k == kh_end(rules_fast_routing)) { + } else { + ret = kh_val(rules_fast_routing,k); + } + if (ret == -1) { // we didn't find it + if (strlen(username)==0) { // we need to search for empty username + sprintf(keybuf_ptr,"%s%s---%d", rand_del, schemaname, flagIN); // no username here + khiter_t k = kh_get(khStrInt, rules_fast_routing, keybuf_ptr); + if (k == kh_end(rules_fast_routing)) { + } else { + ret = kh_val(rules_fast_routing,k); + } + } + } + if (keylen > 250) { + free(keybuf_ptr); + } + } + pthread_rwlock_unlock(&rwlock); + return ret; +} + void Query_Processor::get_current_mysql_firewall_whitelist(SQLite3_result **u, SQLite3_result **r, SQLite3_result **sf) { pthread_mutex_lock(&global_mysql_firewall_whitelist_mutex); if (global_mysql_firewall_whitelist_rules_runtime) {