diff --git a/.gitignore b/.gitignore index 141845979..2e88e4698 100644 --- a/.gitignore +++ b/.gitignore @@ -157,6 +157,7 @@ deps/google-coredumper/google-coredumper/ test/.vagrant .DS_Store proxysql-tests.ini +test/sqlite_history_convert #heaptrack heaptrack.* diff --git a/test/Makefile b/test/Makefile new file mode 100644 index 000000000..e50df910d --- /dev/null +++ b/test/Makefile @@ -0,0 +1,51 @@ +DEPS_PATH=../deps + + +MARIADB_PATH=$(DEPS_PATH)/mariadb-client-library/mariadb_client +MARIADB_IDIR=$(MARIADB_PATH)/include + +JEMALLOC_PATH=$(DEPS_PATH)/jemalloc/jemalloc +JEMALLOC_IDIR=$(JEMALLOC_PATH)/include/jemalloc + +LIBCONFIG_PATH=$(DEPS_PATH)/libconfig/libconfig +LIBCONFIG_IDIR=-I$(LIBCONFIG_PATH)/lib + +#INJECTION_PATH=$(DEPS_PATH)/libinjection +#INJECTION_IDIR=$(INJECTION_PATH) + +RE2_PATH=$(DEPS_PATH)/re2/re2 +RE2_IDIR=$(RE2_PATH) + +PCRE_PATH=$(DEPS_PATH)/pcre/pcre + +SQLITE3_DIR=$(DEPS_PATH)/sqlite3/sqlite3 + +CLICKHOUSE_CPP_DIR=$(DEPS_PATH)/clickhouse-cpp/clickhouse-cpp + +LIBINJECTION_DIR=$(DEPS_PATH)/libinjection/libinjection +LIBINJECTION_IDIR=-I$(LIBINJECTION_DIR)/src + +LIBHTTPSERVER_DIR=$(DEPS_PATH)/libhttpserver/libhttpserver +LIBHTTPSERVER_IDIR=-I$(LIBHTTPSERVER_DIR)/src + +MICROHTTPD_DIR=$(DEPS_PATH)/libmicrohttpd/libmicrohttpd +MICROHTTPD_IDIR=-I$(MICROHTTPD_DIR) -I$(MICROHTTPD_DIR)/src/include + +CURL_DIR=$(DEPS_PATH)/curl/curl +CURL_IDIR=-I$(CURL_DIR)/include + +SSL_DIR=$(DEPS_PATH)/libssl/openssl/ +SSL_IDIR=$(SSL_DIR)/include + + +EV_DIR=$(DEPS_PATH)/libev/libev/ +EV_IDIR=$(EV_DIR) + +IDIR=../include + +IDIRS=-I$(IDIR) -I$(JEMALLOC_IDIR) -I$(MARIADB_IDIR) $(LIBCONFIG_IDIR) -I$(RE2_IDIR) -I$(SQLITE3_DIR) -I$(PCRE_PATH) -I/usr/local/include -I$(CLICKHOUSE_CPP_DIR) $(MICROHTTPD_IDIR) $(LIBHTTPSERVER_IDIR) $(LIBINJECTION_IDIR) $(CURL_IDIR) -I$(EV_DIR) -I$(SSL_IDIR) + + + +sqlite_history_convert: sqlite_history_convert.cpp + g++ -ggdb ../lib/SpookyV2.cpp ../lib/debug.cpp ../deps/sqlite3/sqlite3/sqlite3.o sqlite_history_convert.cpp ../lib/sqlite3db.cpp -o sqlite_history_convert $(IDIRS) -pthread -ldl diff --git a/test/sqlite_history_convert.cpp b/test/sqlite_history_convert.cpp new file mode 100644 index 000000000..aa86769dc --- /dev/null +++ b/test/sqlite_history_convert.cpp @@ -0,0 +1,116 @@ +#include +#include +#include +#include +#include +#include "sqlite3db.h" +#include +#include +#include +#include +#include +#include "SpookyV2.h" + +using namespace std; + +#define SAFE_SQLITE3_STEP2(_stmt) do {\ + do {\ + rc=sqlite3_step(_stmt);\ + if (rc==SQLITE_LOCKED || rc==SQLITE_BUSY) {\ + usleep(100);\ + }\ + } while (rc==SQLITE_LOCKED || rc==SQLITE_BUSY);\ +} while (0) + + +const char * const variable_names[] = { "Access_Denied_Max_Connections", "Access_Denied_Max_User_Connections", "Access_Denied_Wrong_Password", "Active_Transactions", "Backend_query_time_nsec", "Client_Connections_aborted", "Client_Connections_connected", "Client_Connections_created", "Client_Connections_hostgroup_locked", "Client_Connections_non_idle", "Com_autocommit", "Com_autocommit_filtered", "Com_backend_change_user", "Com_backend_init_db", "Com_backend_set_names", "Com_backend_stmt_close", "Com_backend_stmt_execute", "Com_backend_stmt_prepare", "Com_commit", "Com_commit_filtered", "Com_frontend_init_db", "Com_frontend_set_names", "Com_frontend_stmt_close", "Com_frontend_stmt_execute", "Com_frontend_stmt_prepare", "Com_frontend_use_db", "Com_rollback", "Com_rollback_filtered", "ConnPool_get_conn_failure", "ConnPool_get_conn_immediate", "ConnPool_get_conn_latency_awareness", "ConnPool_get_conn_success", "GTID_consistent_queries", "GTID_session_collected", "Mirror_concurrency", "Mirror_queue_length", "MyHGM_myconnpoll_destroy", "MyHGM_myconnpoll_get", "MyHGM_myconnpoll_get_ok", "MyHGM_myconnpoll_push", "MyHGM_myconnpoll_reset", "MySQL_Monitor_Workers", "MySQL_Monitor_Workers_Aux", "MySQL_Monitor_Workers_Started", "MySQL_Monitor_connect_check_ERR", "MySQL_Monitor_connect_check_OK", "MySQL_Monitor_ping_check_ERR", "MySQL_Monitor_ping_check_OK", "MySQL_Monitor_read_only_check_ERR", "MySQL_Monitor_read_only_check_OK", "MySQL_Monitor_replication_lag_check_ERR", "MySQL_Monitor_replication_lag_check_OK", "MySQL_Thread_Workers", "ProxySQL_Uptime", "Queries_backends_bytes_recv", "Queries_backends_bytes_sent", "Queries_frontends_bytes_recv", "Queries_frontends_bytes_sent", "Query_Processor_time_nsec", "Questions", "Selects_for_update__autocommit0", "Server_Connections_aborted", "Server_Connections_connected", "Server_Connections_created", "Server_Connections_delayed", "Servers_table_version", "Slow_queries", "automatic_detected_sql_injection", "aws_aurora_replicas_skipped_during_query", "backend_lagging_during_query", "backend_offline_during_query", "generated_error_packets", "hostgroup_locked_queries", "hostgroup_locked_set_cmds", "max_connect_timeouts", "mysql_backend_buffers_bytes", "mysql_frontend_buffers_bytes", "mysql_killed_backend_connections", "mysql_killed_backend_queries", "mysql_session_internal_bytes", "mysql_unexpected_frontend_com_quit", "mysql_unexpected_frontend_packets", "queries_with_max_lag_ms", "queries_with_max_lag_ms__delayed", "queries_with_max_lag_ms__total_wait_time_us", "whitelisted_sqli_fingerprint" }; + +void create_table(SQLite3DB& statsdb) { + statsdb.execute((char *)"DROP TABLE IF EXISTS history_mysql_status_variables_v2"); + int num_vars = sizeof(variable_names)/sizeof(const char *); + string query = "CREATE TABLE history_mysql_status_variables_v2 (timestamp INT NOT NULL PRIMARY KEY, "; + for (int i=0; i& varmap, SQLite3_result *res, uint64_t ts) { + string query = "INSERT INTO history_mysql_status_variables_v2 VALUES (?1,"; + int num_vars = sizeof(variable_names)/sizeof(const char *); + vector present; + for (int i=0; irows.begin() ; it != res->rows.end(); ++it) { + SQLite3_row *r=*it; + auto it2 = varmap.find(r->fields[0]); + if (it2 != varmap.end()) { + int id = it2->second; // ids are already offset by 2 + rc=sqlite3_bind_text(statement, id, r->fields[1] , -1, SQLITE_TRANSIENT); + assert(rc==SQLITE_OK); + present[id-2] = true; + } + } + for (int i=0; i varset; + unordered_map varmap; + for (int i=0; ifirst << " " << it->second << endl; + } + SQLite3_result * timestamps = new SQLite3_result(); + char * error; + int cols, affected_rows; + create_table(statsdb); + statsdb.execute_statement((char *)"SELECT DISTINCT timestamp FROM history_mysql_status_variables", &error, &cols, &affected_rows, ×tamps); + for (auto it = timestamps->rows.begin() ; it != timestamps->rows.end(); ++it) { + SQLite3_row *r=*it; + uint64_t ts = atoll(r->fields[0]); + //cout << r->fields[0] << endl; + SQLite3_result * vals_one_time = new SQLite3_result(); + string s = "SELECT variable_name, variable_value FROM history_mysql_status_variables WHERE timestamp = "; + s += to_string(ts); + statsdb.execute_statement(s.c_str(), &error, &cols, &affected_rows, &vals_one_time); + insert_row(statsdb,varmap, vals_one_time, ts); +/* + for (auto it2 = vals_one_time->rows.begin() ; it2 != vals_one_time->rows.end(); ++it2) { + SQLite3_row *r=*it2; + //cout << r->fields[1] << " " << r->fields[2] << endl; + } +*/ + } + return EXIT_SUCCESS; +}