mirror of https://github.com/sysown/proxysql
Merge pull request #4096 from sysown/v2.x-digest_umap_aux
Add several optimizations to stats_mysql_query_digest table fetchingpull/3693/merge
René Cannaò
3 years ago
committed by
GitHub
commit
810f86a2f9
@ -0,0 +1,132 @@
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
#include <unistd.h>
|
||||
#include <vector>
|
||||
#include <tuple>
|
||||
|
||||
#include <mysql.h>
|
||||
#include <mysql/mysqld_error.h>
|
||||
|
||||
#include "tap.h"
|
||||
#include "command_line.h"
|
||||
#include "utils.h"
|
||||
|
||||
using std::string;
|
||||
|
||||
/* this test:
|
||||
* enables mysql-have_ssl
|
||||
* execute various command
|
||||
*/
|
||||
|
||||
std::vector<std::string> queries_t = {
|
||||
"PROXYSQLTEST 22",
|
||||
"PROXYSQLTEST 23",
|
||||
"PROXYSQLTEST 24",
|
||||
"PROXYSQLTEST 25",
|
||||
"PROXYSQLTEST 26",
|
||||
"PROXYSQLTEST 27",
|
||||
"SELECT COUNT(*) FROM stats_mysql_query_digest"
|
||||
};
|
||||
|
||||
|
||||
//std::vector<unsigned int> vals = { 100, 345, 800, 999, 2037, 12345 };
|
||||
//std::vector<unsigned int> vals = { 100, 345, 800, 999, 2037 };
|
||||
std::vector<unsigned int> vals = { 100, 345, 800 };
|
||||
|
||||
std::vector<std::string> queries = {};
|
||||
|
||||
int run_q(MYSQL *mysql, const char *q) {
|
||||
MYSQL_QUERY(mysql,q);
|
||||
return 0;
|
||||
}
|
||||
int main() {
|
||||
CommandLine cl;
|
||||
|
||||
if (cl.getEnv()) {
|
||||
diag("Failed to get the required environmental variables.");
|
||||
return -1;
|
||||
}
|
||||
|
||||
srandom(123);
|
||||
|
||||
|
||||
for (auto it = vals.begin() ; it != vals.end() ; it++) {
|
||||
std::string q = "PROXYSQLTEST 1 " + std::to_string(*it);
|
||||
queries.push_back(q);
|
||||
for (int i=0; i<5; i++) {
|
||||
queries.push_back(queries_t[rand()%queries_t.size()]);
|
||||
}
|
||||
queries.push_back("SELECT COUNT(*) FROM stats_mysql_query_digest");
|
||||
for (int i=0; i<5; i++) {
|
||||
queries.push_back(queries_t[rand()%queries_t.size()]);
|
||||
}
|
||||
if (rand()%2 == 0) {
|
||||
queries.push_back("SELECT COUNT(*) FROM stats_mysql_query_digest_reset");
|
||||
} else {
|
||||
queries.push_back("TRUNCATE TABLE stats.stats_mysql_query_digest");
|
||||
}
|
||||
}
|
||||
queries.push_back("TRUNCATE TABLE stats.stats_mysql_query_digest");
|
||||
|
||||
|
||||
|
||||
MYSQL* proxysql_admin = mysql_init(NULL);
|
||||
// Initialize connections
|
||||
if (!proxysql_admin) {
|
||||
fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql_admin));
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (!mysql_real_connect(proxysql_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) {
|
||||
fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql_admin));
|
||||
return -1;
|
||||
}
|
||||
|
||||
MYSQL_QUERY(proxysql_admin, "SET mysql-have_ssl='true'");
|
||||
MYSQL_QUERY(proxysql_admin, "SET mysql-have_compress='true'");
|
||||
MYSQL_QUERY(proxysql_admin, "LOAD MYSQL VARIABLES TO RUNTIME");
|
||||
|
||||
|
||||
|
||||
unsigned int p = queries.size();
|
||||
for (std::vector<std::string>::iterator it2 = queries.begin(); it2 != queries.end(); it2++) {
|
||||
if (
|
||||
(strncasecmp(it2->c_str(), "SELECT ", 7)==0)
|
||||
) {
|
||||
// extra test for each queries returning a resultset
|
||||
p++;
|
||||
}
|
||||
}
|
||||
plan(p);
|
||||
diag("Running test with %lu queries", queries.size());
|
||||
|
||||
|
||||
for (std::vector<std::string>::iterator it2 = queries.begin(); it2 != queries.end(); it2++) {
|
||||
MYSQL* proxysql_admin = mysql_init(NULL); // local scope
|
||||
if (!proxysql_admin) {
|
||||
fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql_admin));
|
||||
return -1;
|
||||
}
|
||||
mysql_ssl_set(proxysql_admin, NULL, NULL, NULL, NULL, NULL);
|
||||
if (!mysql_real_connect(proxysql_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, CLIENT_SSL|CLIENT_COMPRESS)) {
|
||||
fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql_admin));
|
||||
return -1;
|
||||
}
|
||||
int rc = run_q(proxysql_admin, it2->c_str());
|
||||
ok(rc==0, "Query: %s" , it2->c_str());
|
||||
if (
|
||||
(strncasecmp(it2->c_str(), "SELECT ", 7)==0)
|
||||
) {
|
||||
MYSQL_RES* proxy_res = mysql_store_result(proxysql_admin);
|
||||
unsigned long long num_rows = mysql_num_rows(proxy_res);
|
||||
ok(num_rows != 0 , "Returned rows: %llu" , num_rows);
|
||||
mysql_free_result(proxy_res);
|
||||
}
|
||||
mysql_close(proxysql_admin);
|
||||
}
|
||||
mysql_close(proxysql_admin);
|
||||
|
||||
return exit_status();
|
||||
}
|
||||
@ -0,0 +1,291 @@
|
||||
/**
|
||||
* @file test_digest_umap_aux-t.cpp
|
||||
* @brief This tests that the auxiliary digest map is working correctly.
|
||||
* @details This test sends dummy queries to ProxySQL while also sending
|
||||
* queries to read table stats_mysql_query_digest. Then, it checks that the
|
||||
* execution time of the dummy queries has no been afected by the execution
|
||||
* time of the queries that read from table stats_mysql_query_digest. Finally,
|
||||
* check that the data stored in stats_mysql_query_digest is correct.
|
||||
*/
|
||||
|
||||
#include <unistd.h>
|
||||
#include <iostream>
|
||||
#include <mysql.h>
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <chrono>
|
||||
#include <thread>
|
||||
#include <atomic>
|
||||
|
||||
#include "proxysql_utils.h"
|
||||
#include "command_line.h"
|
||||
#include "utils.h"
|
||||
#include "tap.h"
|
||||
|
||||
using std::vector;
|
||||
using std::string;
|
||||
|
||||
CommandLine cl;
|
||||
double slowest_query = 0.0;
|
||||
double fastest_query = 0.0;
|
||||
std::atomic_bool stop(false);
|
||||
|
||||
vector<const char*> DUMMY_QUERIES = {
|
||||
"SELECT 1",
|
||||
"SELECT 1 UNION SELECT 2 UNION SELECT 3",
|
||||
"SELECT 1 UNION SELECT 2",
|
||||
};
|
||||
int num_dummy_queries_executed = 0;
|
||||
|
||||
struct digest_stats {
|
||||
int hostgroup;
|
||||
string schemaname;
|
||||
string username;
|
||||
string client_address;
|
||||
string digest;
|
||||
string digest_text;
|
||||
int count_star;
|
||||
int first_seen;
|
||||
int last_seen;
|
||||
int sum_time;
|
||||
int min_time;
|
||||
int max_time;
|
||||
int sum_rows_affected;
|
||||
int sum_rows_sent;
|
||||
};
|
||||
|
||||
class timer {
|
||||
public:
|
||||
std::chrono::time_point<std::chrono::high_resolution_clock> lastTime;
|
||||
timer() : lastTime(std::chrono::high_resolution_clock::now()) {}
|
||||
inline double elapsed() {
|
||||
std::chrono::time_point<std::chrono::high_resolution_clock> thisTime = std::chrono::high_resolution_clock::now();
|
||||
double deltaTime = std::chrono::duration<double>(thisTime-lastTime).count();
|
||||
lastTime = thisTime;
|
||||
return deltaTime;
|
||||
}
|
||||
};
|
||||
|
||||
vector<digest_stats> get_digest_stats(MYSQL* proxy_admin) {
|
||||
const char* get_digest_stats_query =
|
||||
"SELECT * FROM stats_mysql_query_digest WHERE username='root' AND "
|
||||
"digest_text IN ('SELECT ?', 'SELECT ? UNION SELECT ?', 'SELECT ? UNION SELECT ? UNION SELECT ?') "
|
||||
"ORDER BY hostgroup, schemaname, username, client_address, digest";
|
||||
diag("Running: %s", get_digest_stats_query);
|
||||
vector<digest_stats> ds_vector;
|
||||
|
||||
int err = mysql_query(proxy_admin, get_digest_stats_query);
|
||||
if (err) {
|
||||
diag("Failed to executed query `%s`. Error: `%s`", get_digest_stats_query, mysql_error(proxy_admin));
|
||||
return ds_vector;
|
||||
}
|
||||
|
||||
MYSQL_RES *res = NULL;
|
||||
res = mysql_store_result(proxy_admin);
|
||||
MYSQL_ROW row;
|
||||
while (row = mysql_fetch_row(res)) {
|
||||
digest_stats ds = {};
|
||||
ds.hostgroup = atoi(row[0]);
|
||||
ds.schemaname = row[1];
|
||||
ds.username = row[2];
|
||||
ds.client_address = row[3];
|
||||
ds.digest = row[4];
|
||||
ds.digest_text = row[5];
|
||||
ds.count_star = atoi(row[6]);
|
||||
ds.first_seen = atoi(row[7]);
|
||||
ds.last_seen = atoi(row[8]);
|
||||
ds.sum_time = atoi(row[9]);
|
||||
ds.min_time = atoi(row[10]);
|
||||
ds.max_time = atoi(row[11]);
|
||||
ds.sum_rows_affected = atoi(row[12]);
|
||||
ds.sum_rows_sent = atoi(row[13]);
|
||||
ds_vector.push_back(ds);
|
||||
}
|
||||
mysql_free_result(res);
|
||||
|
||||
return ds_vector;
|
||||
}
|
||||
|
||||
void run_dummy_queries() {
|
||||
MYSQL* proxy_mysql = mysql_init(NULL);
|
||||
|
||||
if (!mysql_real_connect(proxy_mysql, cl.host, cl.username, cl.password, NULL, cl.port, NULL, 0)) {
|
||||
fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxy_mysql));
|
||||
slowest_query = -1.0;
|
||||
return;
|
||||
}
|
||||
|
||||
vector<double> execution_times = {};
|
||||
MYSQL_RES *res = NULL;
|
||||
while (!stop) {
|
||||
for (int i = 0; i < DUMMY_QUERIES.size(); i++) {
|
||||
timer stopwatch;
|
||||
int err = mysql_query(proxy_mysql, DUMMY_QUERIES[i]);
|
||||
execution_times.push_back(stopwatch.elapsed());
|
||||
if (err) {
|
||||
diag(
|
||||
"Failed to executed query `%s`. Error: `%s`",
|
||||
DUMMY_QUERIES[i], mysql_error(proxy_mysql)
|
||||
);
|
||||
slowest_query = -1.0;
|
||||
mysql_close(proxy_mysql);
|
||||
return;
|
||||
}
|
||||
res = mysql_store_result(proxy_mysql);
|
||||
mysql_free_result(res);
|
||||
}
|
||||
num_dummy_queries_executed++;
|
||||
}
|
||||
mysql_close(proxy_mysql);
|
||||
|
||||
slowest_query = *std::max_element(execution_times.begin(), execution_times.end());
|
||||
}
|
||||
|
||||
void run_stats_digest_query(MYSQL* proxy_admin) {
|
||||
const char *count_digest_stats_query = "SELECT COUNT(*) FROM stats_mysql_query_digest";
|
||||
vector<double> execution_times = {};
|
||||
const int num_queries = 3;
|
||||
MYSQL_RES *res;
|
||||
|
||||
for (int i; i < num_queries; i++) {
|
||||
diag("Running: %s", count_digest_stats_query);
|
||||
timer stopwatch;
|
||||
int err = mysql_query(proxy_admin, count_digest_stats_query);
|
||||
execution_times.push_back(stopwatch.elapsed());
|
||||
if (err) {
|
||||
diag(
|
||||
"Failed to executed query `%s`. Error: `%s`",
|
||||
count_digest_stats_query, mysql_error(proxy_admin)
|
||||
);
|
||||
fastest_query = -1.0;
|
||||
return;
|
||||
}
|
||||
res = mysql_store_result(proxy_admin);
|
||||
mysql_free_result(res);
|
||||
}
|
||||
|
||||
fastest_query = *std::min_element(execution_times.begin(), execution_times.end());
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
||||
if (cl.getEnv()) {
|
||||
diag("Failed to get the required environmental variables.");
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
plan(1 + DUMMY_QUERIES.size() * 3); // always specify the number of tests that are going to be performed
|
||||
|
||||
MYSQL *proxy_admin = mysql_init(NULL);
|
||||
if (!mysql_real_connect(proxy_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) {
|
||||
fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxy_admin));
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
vector<const char*> admin_queries = {
|
||||
"DELETE FROM mysql_query_rules",
|
||||
"LOAD MYSQL QUERY RULES TO RUNTIME",
|
||||
"PROXYSQLTEST 1 1000",
|
||||
};
|
||||
for (const auto &query : admin_queries) {
|
||||
diag("Running: %s", query);
|
||||
MYSQL_QUERY(proxy_admin, query);
|
||||
}
|
||||
|
||||
MYSQL *proxy_mysql = mysql_init(NULL);
|
||||
if (!mysql_real_connect(proxy_mysql, cl.host, cl.username, cl.password, NULL, cl.port, NULL, 0)) {
|
||||
fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxy_mysql));
|
||||
mysql_close(proxy_admin);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
MYSQL_RES *res = NULL;
|
||||
for (const auto &query : DUMMY_QUERIES) {
|
||||
diag("Running: %s", query);
|
||||
MYSQL_QUERY(proxy_mysql, query);
|
||||
res = mysql_store_result(proxy_mysql);
|
||||
mysql_free_result(res);
|
||||
}
|
||||
mysql_close(proxy_mysql);
|
||||
|
||||
vector<digest_stats> ds_vector_before = get_digest_stats(proxy_admin);
|
||||
|
||||
std::thread run_dummy_queries_thread(run_dummy_queries);
|
||||
std::thread run_stats_digest_query_thread(run_stats_digest_query, proxy_admin);
|
||||
|
||||
run_stats_digest_query_thread.join();
|
||||
if (fastest_query == -1.0) {
|
||||
fprintf(
|
||||
stderr, "File %s, line %d, Error: "
|
||||
"thread run_stats_digest_query_thread finished with errors", __FILE__, __LINE__
|
||||
);
|
||||
mysql_close(proxy_admin);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
stop = true;
|
||||
run_dummy_queries_thread.join();
|
||||
if (slowest_query == -1.0) {
|
||||
fprintf(
|
||||
stderr, "File %s, line %d, Error: "
|
||||
"thread run_dummy_queries_thread finished with errors", __FILE__, __LINE__
|
||||
);
|
||||
mysql_close(proxy_admin);
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
ok(
|
||||
slowest_query < fastest_query,
|
||||
"The slowest dummy query must be faster than the fastest digests stats query.\n"
|
||||
" Slowest dummy query time: %f.\n"
|
||||
" Fastest count digest stats query time: %f.",
|
||||
slowest_query, fastest_query
|
||||
);
|
||||
|
||||
vector<digest_stats> ds_vector_after = get_digest_stats(proxy_admin);
|
||||
for (int i = 0; i < DUMMY_QUERIES.size(); i++) {
|
||||
ok(
|
||||
ds_vector_before[i].hostgroup == ds_vector_after[i].hostgroup &&
|
||||
ds_vector_before[i].schemaname == ds_vector_after[i].schemaname &&
|
||||
ds_vector_before[i].username == ds_vector_after[i].username &&
|
||||
ds_vector_before[i].client_address == ds_vector_after[i].client_address &&
|
||||
ds_vector_before[i].digest == ds_vector_after[i].digest &&
|
||||
ds_vector_before[i].digest_text == ds_vector_after[i].digest_text &&
|
||||
ds_vector_before[i].first_seen - 1 <= ds_vector_after[i].first_seen &&
|
||||
ds_vector_after[i].first_seen <= ds_vector_before[i].first_seen + 1,
|
||||
"Hostgroup, schemaname, username, client_address, digest, digest_test and first_seen "
|
||||
"should be equal in both digest stats.\n"
|
||||
" Hostgroup -> before:`%d` - after:`%d`.\n"
|
||||
" Schemaname -> before:`%s` - after:`%s`.\n"
|
||||
" Username -> before:`%s` - after:`%s`.\n"
|
||||
" Client_address -> before:`%s` - after:`%s`.\n"
|
||||
" Digests -> before:`%s` - after:`%s`.\n"
|
||||
" Digests_text -> before:`%s` - after:`%s`.\n"
|
||||
" First_seen -> before:`%d` - after:`%d`.",
|
||||
ds_vector_before[i].hostgroup, ds_vector_after[i].hostgroup,
|
||||
ds_vector_before[i].schemaname.c_str(), ds_vector_after[i].schemaname.c_str(),
|
||||
ds_vector_before[i].username.c_str(), ds_vector_after[i].username.c_str(),
|
||||
ds_vector_before[i].client_address.c_str(), ds_vector_after[i].client_address.c_str(),
|
||||
ds_vector_before[i].digest.c_str(), ds_vector_after[i].digest.c_str(),
|
||||
ds_vector_before[i].digest_text.c_str(), ds_vector_after[i].digest_text.c_str(),
|
||||
ds_vector_before[i].first_seen, ds_vector_after[i].first_seen
|
||||
);
|
||||
ok(
|
||||
ds_vector_after[i].count_star - ds_vector_before[i].count_star == num_dummy_queries_executed,
|
||||
"Query `%s` should be executed %d times. Act:'%d'",
|
||||
ds_vector_after[i].digest_text.c_str(), num_dummy_queries_executed,
|
||||
ds_vector_after[i].count_star - ds_vector_before[i].count_star
|
||||
);
|
||||
ok(
|
||||
ds_vector_before[i].last_seen < ds_vector_after[i].last_seen &&
|
||||
ds_vector_before[i].sum_time < ds_vector_after[i].sum_time,
|
||||
"Last_seen and sum_time must have increased.\n"
|
||||
" Last_seen -> before:`%d` - after:`%d`.\n"
|
||||
" Sum_time -> before:`%d` - after:`%d`.",
|
||||
ds_vector_before[i].last_seen, ds_vector_after[i].last_seen,
|
||||
ds_vector_before[i].sum_time, ds_vector_after[i].sum_time
|
||||
);
|
||||
}
|
||||
|
||||
return exit_status();
|
||||
}
|
||||
Loading…
Reference in new issue