From eac6e75dad4fccc1d6e7be2c226c78eba1d91ee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Mon, 17 May 2021 00:35:48 +0200 Subject: [PATCH] Adding a simulator for read_only test This commit introduces a first simulator for read_only monitoring ProxySQL itself will simulate hundreds of backends across hundreds of hostgroups, all configured in hundreds of clusters, and return read_only values for each of the simulated backend. The read_only value can be configured in table READONLY_STATUS available in the SQLite3Server module. If a backend isn't configured in table READONLY_STATUS , its default read_only value is 1. It is possible to simulate a lot of simultaneous failover running queries like the following: UPDATE READONLY_STATUS SET read_only=1; CREATE TABLE t1 AS SELECT hostname FROM READONLY_STATUS ORDER BY RANDOM() LIMIT 50; UPDATE READONLY_STATUS SET read_only=0 WHERE hostname IN (SELECT hostname FROM t1); DROP TABLE t1; --- Makefile | 15 ++++- Read_Only_Testing.md | 29 +++++++++ include/SQLite3_Server.h | 14 ++++- include/proxysql_admin.h | 4 ++ lib/MySQL_HostGroups_Manager.cpp | 11 ++-- lib/MySQL_Monitor.cpp | 8 +++ lib/ProxySQL_Admin.cpp | 45 +++++++++++++ src/SQLite3_Server.cpp | 104 +++++++++++++++++++++++++++---- 8 files changed, 211 insertions(+), 19 deletions(-) create mode 100644 Read_Only_Testing.md diff --git a/Makefile b/Makefile index 34827bc56..3115edae0 100644 --- a/Makefile +++ b/Makefile @@ -63,6 +63,9 @@ testgalera: build_deps_debug build_lib_testgalera build_src_testgalera .PHONY: testgrouprep testgrouprep: build_deps_debug build_lib_testgrouprep build_src_testgrouprep +.PHONY: testreadonly +testreadonly: build_deps_debug build_lib_testreadonly build_src_testreadonly + .PHONY: testall testall: build_deps_debug build_lib_testall build_src_testall @@ -117,13 +120,21 @@ build_src_testgrouprep: build_deps build_lib_testgrouprep build_lib_testgrouprep: build_deps_debug cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_GROUPREP" CC=${CC} CXX=${CXX} ${MAKE} +.PHONY: build_src_testreadonly +build_src_testreadonly: build_deps build_lib_testreadonly + cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_READONLY" CC=${CC} CXX=${CXX} ${MAKE} + +.PHONY: build_lib_testreadonly +build_lib_testreadonly: build_deps_debug + cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_READONLY" CC=${CC} CXX=${CXX} ${MAKE} + .PHONY: build_src_testall build_src_testall: build_deps build_lib_testall - cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_GALERA -DTEST_GROUPREP" CC=${CC} CXX=${CXX} ${MAKE} + cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_GALERA -DTEST_GROUPREP -DTEST_READONLY" CC=${CC} CXX=${CXX} ${MAKE} .PHONY: build_lib_testall build_lib_testall: build_deps_debug - cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_GALERA -DTEST_GROUPREP" CC=${CC} CXX=${CXX} ${MAKE} + cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_GALERA -DTEST_GROUPREP -DTEST_READONLY" CC=${CC} CXX=${CXX} ${MAKE} .PHONY: build_tap_test build_tap_test: build_src diff --git a/Read_Only_Testing.md b/Read_Only_Testing.md new file mode 100644 index 000000000..ad63237cb --- /dev/null +++ b/Read_Only_Testing.md @@ -0,0 +1,29 @@ + + +### Compiling + +To run `read_only` automated testing, ProxySQL needs to be compiled with `make testreadonly`. + + + +### shutdown mysqld + +When running automated testing, ProxySQL will listen on many IPs (30) an on port 3306. +You need to make sure that MySQL server is not running, or not listening on port 3306. + + +### Running proxysql + +`proxysql` needs to be executed with `--sqlite3-server` . +For example, to run it under `gdb`: `run -f -D . --sqlite3-server` + + +### Similate failover + +To simulate failover is enough to connect to sqlite3 server interface and update the `READONLY_STATUS` table. +Note that to connect it is necessary to use a user configured in `mysql_users` table. +To simulate a lot of failover at the same time, a query like the follow can be executed: + +``` + UPDATE READONLY_STATUS SET read_only=1; CREATE TABLE t1 AS SELECT hostname FROM READONLY_STATUS ORDER BY RANDOM() LIMIT 50; UPDATE READONLY_STATUS SET read_only=0 WHERE hostname IN (SELECT hostname FROM t1); DROP TABLE t1; +``` diff --git a/include/SQLite3_Server.h b/include/SQLite3_Server.h index 288d01156..28f342de1 100644 --- a/include/SQLite3_Server.h +++ b/include/SQLite3_Server.h @@ -45,7 +45,11 @@ class SQLite3_Server { #ifdef TEST_GROUPREP std::vector *tables_defs_grouprep; #endif // TEST_GROUPREP -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) +#ifdef TEST_READONLY + std::unordered_map readonly_map; + std::vector *tables_defs_readonly; +#endif // TEST_READONLY +#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) void insert_into_tables_defs(std::vector *, const char *table_name, const char *table_def); void drop_tables_defs(std::vector *tables_defs); void check_and_build_standard_tables(SQLite3DB *db, std::vector *tables_defs); @@ -73,6 +77,14 @@ class SQLite3_Server { void populate_grouprep_table(MySQL_Session *sess, int txs_behind = 0); void init_grouprep_ifaces_string(std::string& s); #endif // TEST_GROUPREP +#ifdef TEST_READONLY + pthread_mutex_t test_readonly_mutex; + void load_readonly_table(MySQL_Session *sess); + int readonly_test_value(char *p); + int readonly_map_size() { + return readonly_map.size(); + } +#endif // TEST_READONLY SQLite3_Server(); ~SQLite3_Server(); char **get_variables_list(); diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index b85ffae70..bb038acd2 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -424,6 +424,10 @@ class ProxySQL_Admin { void enable_grouprep_testing(); #endif // TEST_GROUPREP +#ifdef TEST_READONLY + void enable_readonly_testing(); +#endif // TEST_READONLY + unsigned int ProxySQL_Test___GenerateRandom_mysql_query_rules_fast_routing(unsigned int, bool); bool ProxySQL_Test___Verify_mysql_query_rules_fast_routing(int *ret1, int *ret2, int cnt, int dual); void ProxySQL_Test___MySQL_HostGroups_Manager_generate_many_clusters(); diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 2bfffb2c5..eba93a8da 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1880,11 +1880,12 @@ bool MySQL_HostGroups_Manager::commit() { proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_servers_incoming\n"); mydb->execute("DELETE FROM mysql_servers_incoming"); - proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_replication_hostgroups\n"); - mydb->execute("DELETE FROM mysql_replication_hostgroups"); - - generate_mysql_replication_hostgroups_table(); - + // replication + if (incoming_replication_hostgroups) { // this IF is extremely important, otherwise replication hostgroups may disappear + proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "DELETE FROM mysql_replication_hostgroups\n"); + mydb->execute("DELETE FROM mysql_replication_hostgroups"); + generate_mysql_replication_hostgroups_table(); + } // group replication if (incoming_group_replication_hostgroups) { diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index f70e1ae6b..e91fda825 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -1152,6 +1152,7 @@ void * monitor_read_only_thread(void *arg) { mmsd->t1=monotonic_time(); mmsd->interr=0; // reset the value +#ifndef TEST_READONLY if (mmsd->task_id == MON_INNODB_READ_ONLY) { mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT @@global.innodb_read_only read_only"); } else if (mmsd->task_id == MON_SUPER_READ_ONLY) { @@ -1163,6 +1164,13 @@ void * monitor_read_only_thread(void *arg) { } else { // default mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,"SELECT @@global.read_only read_only"); } +#else // TEST_READONLY + { + std::string s = "SELECT @@global.read_only read_only"; + s += " " + std::string(mmsd->hostname) + ":" + std::to_string(mmsd->port); + mmsd->async_exit_status=mysql_query_start(&mmsd->interr,mmsd->mysql,s.c_str()); + } +#endif // TEST_READONLY while (mmsd->async_exit_status) { mmsd->async_exit_status=wait_for_mysql(mmsd->mysql, mmsd->async_exit_status); unsigned long long now=monotonic_time(); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 10db17b0f..53ba13a9b 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -12460,6 +12460,51 @@ void ProxySQL_Admin::enable_grouprep_testing() { } #endif // TEST_GROUPREP +#ifdef TEST_READONLY +void ProxySQL_Admin::enable_readonly_testing() { + proxy_info("Admin is enabling Read Only Testing using SQLite3 Server and HGs from 4201 to 4800\n"); + mysql_servers_wrlock(); + string q; + q = "DELETE FROM mysql_servers WHERE hostgroup_id BETWEEN 4201 AND 4800"; + admindb->execute(q.c_str()); + //for (int i=1; i < 2; i++) { + for (int i=1; i < 4; i++) { + for (int j=2; j<100; j+=2) { + for (int k=1; k<5; k++) { + q = "INSERT INTO mysql_servers (hostgroup_id, hostname, use_ssl, comment) VALUES (" + std::to_string(4000+i*200+j) + ", '127.5."+ std::to_string(i) +"." + std::to_string(j*2+k) + "', 0, '')"; + admindb->execute(q.c_str()); + } + q = "INSERT INTO mysql_replication_hostgroups(writer_hostgroup, reader_hostgroup) VALUES (" + std::to_string(4000+i*200+j-1) + "," + std::to_string(4000+i*200+j) + ")"; + admindb->execute(q.c_str()); + } + } +/* + admindb->execute("INSERT INTO mysql_servers (hostgroup_id, hostname, use_ssl, comment) VALUES (3272, '127.2.1.1', 0, '')"); + admindb->execute("INSERT INTO mysql_servers (hostgroup_id, hostname, use_ssl, comment) VALUES (3273, '127.2.1.2', 0, '')"); + admindb->execute("INSERT INTO mysql_servers (hostgroup_id, hostname, use_ssl, comment) VALUES (3273, '127.2.1.3', 0, '')"); + admindb->execute("DELETE FROM mysql_group_replication_hostgroups"); + admindb->execute("INSERT INTO mysql_group_replication_hostgroups " + "(writer_hostgroup,backup_writer_hostgroup,reader_hostgroup,offline_hostgroup,active,max_writers," + "writer_is_also_reader,max_transactions_behind) VALUES (3272,3274,3273,3271,1,1,1,0);"); +*/ + load_mysql_servers_to_runtime(); + mysql_servers_wrunlock(); +/* + admindb->execute("UPDATE global_variables SET variable_value=5000 WHERE variable_name='mysql-monitor_groupreplication_healthcheck_interval'"); + admindb->execute("UPDATE global_variables SET variable_value=800 WHERE variable_name='mysql-monitor_groupreplication_healthcheck_timeout'"); + admindb->execute("UPDATE global_variables SET variable_value=3 WHERE variable_name='mysql-monitor_groupreplication_healthcheck_max_timeout_count'"); + admindb->execute("UPDATE global_variables SET variable_value=3 WHERE variable_name='mysql-monitor_groupreplication_max_transactions_behind_count'"); + load_mysql_variables_to_runtime(); + + admindb->execute("DELETE FROM mysql_users WHERE username='grouprep1'"); + admindb->execute("INSERT INTO mysql_users (username,password,default_hostgroup) VALUES ('grouprep1','pass1',3272)"); + init_users(); + + load_mysql_query_rules_to_runtime(); +*/ +} +#endif // TEST_READONLY + void ProxySQL_Admin::ProxySQL_Test___MySQL_HostGroups_Manager_generate_many_clusters() { mysql_servers_wrlock(); admindb->execute("DELETE FROM mysql_servers WHERE hostgroup_id BETWEEN 10001 AND 20000"); diff --git a/src/SQLite3_Server.cpp b/src/SQLite3_Server.cpp index 7723d2cac..ce6a29ee9 100644 --- a/src/SQLite3_Server.cpp +++ b/src/SQLite3_Server.cpp @@ -251,7 +251,7 @@ void SQLite3_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *p memcpy(query,(char *)pkt->ptr+sizeof(mysql_hdr)+1,query_length-1); query[query_length-1]=0; -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) +#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) if (sess->client_myds->proxy_addr.addr == NULL) { struct sockaddr addr; socklen_t addr_len=sizeof(struct sockaddr); @@ -281,7 +281,7 @@ void SQLite3_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *p sess->client_myds->proxy_addr.addr = strdup("unknown"); } } -#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP +#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY char *query_no_space=(char *)l_alloc(query_length); memcpy(query_no_space,query,query_length); @@ -356,13 +356,13 @@ void SQLite3_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *p if (query_no_space_length==SELECT_VERSION_COMMENT_LEN) { if (!strncasecmp(SELECT_VERSION_COMMENT, query_no_space, query_no_space_length)) { l_free(query_length,query); -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) +#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) char *a = (char *)"SELECT '(ProxySQL Automated Test Server) - %s'"; query = (char *)malloc(strlen(a)+strlen(sess->client_myds->proxy_addr.addr)); sprintf(query,a,sess->client_myds->proxy_addr.addr); #else query=l_strdup("SELECT '(ProxySQL SQLite3 Server)'"); -#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP +#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY query_length=strlen(query)+1; goto __run_query; } @@ -538,7 +538,7 @@ __end_show_commands: __run_query: if (run_query) { -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) +#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) if (strncasecmp("SELECT",query_no_space,6)==0) { #ifdef TEST_AURORA if (strstr(query_no_space,(char *)"REPLICA_HOST_STATUS")) { @@ -559,6 +559,24 @@ __run_query: if (testLag > 0) testLag--; } #endif // TEST_GROUPREP +#ifdef TEST_READONLY + if (strncasecmp("SELECT @@global.read_only read_only ",query_no_space, strlen("SELECT @@global.read_only read_only "))==0) { + if (strlen(query_no_space) > strlen("SELECT @@global.read_only read_only ")+5) { + pthread_mutex_lock(&GloSQLite3Server->test_readonly_mutex); + // the current test doesn't try to simulate failures, therefore it will return immediately + if (GloSQLite3Server->readonly_map_size() == 0) { + // probably never initialized + GloSQLite3Server->load_readonly_table(sess); + } + int rc = GloSQLite3Server->readonly_test_value(query_no_space+strlen("SELECT @@global.read_only read_only ")); + free(query); + char *a = (char *)"SELECT %d as read_only"; + query = (char *)malloc(strlen(a)+2); + sprintf(query,a,rc); + pthread_mutex_unlock(&GloSQLite3Server->test_readonly_mutex); + } + } +#endif // TEST_READONLY if (strstr(query_no_space,(char *)"Seconds_Behind_Master")) { free(query); char *a = (char *)"SELECT %d as Seconds_Behind_Master"; @@ -566,7 +584,7 @@ __run_query: sprintf(query,a,rand()%30+10); } } -#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP +#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY SQLite3_Session *sqlite_sess = (SQLite3_Session *)sess->thread->gen_args; sqlite_sess->sessdb->execute_statement(query, &error , &cols , &affected_rows , &resultset); #if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) @@ -626,6 +644,16 @@ __run_query: } sess->SQLite3_to_MySQL(resultset, error, affected_rows, &sess->client_myds->myprot, in_trans); delete resultset; +#ifdef TEST_READONLY + if (strncasecmp("SELECT",query_no_space,6)) { + if (strstr(query_no_space,(char *)"READONLY_STATUS")) { + // the table is writable + pthread_mutex_lock(&GloSQLite3Server->test_readonly_mutex); + GloSQLite3Server->load_readonly_table(sess); + pthread_mutex_unlock(&GloSQLite3Server->test_readonly_mutex); + } + } +#endif // TEST_READONLY } l_free(pkt->size-sizeof(mysql_hdr),query_no_space); // it is always freed here l_free(query_length,query); @@ -930,7 +958,7 @@ SQLite3_Server::SQLite3_Server() { variables.read_only=false; -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) +#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) string s = ""; #ifdef TEST_AURORA @@ -944,12 +972,17 @@ SQLite3_Server::SQLite3_Server() { #ifdef TEST_GROUPREP init_grouprep_ifaces_string(s); #endif // TEST_GROUPREP - +#ifdef TEST_READONLY + // for readonly test we listen on all IPs because we simulate a lot of clusters + if (!s.empty()) + s += ";"; + s += "0.0.0.0:3306"; +#endif //TEST_READONLY variables.mysql_ifaces=strdup(s.c_str()); #else variables.mysql_ifaces=strdup("127.0.0.1:6030"); -#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP +#endif // TEST_AURORA || TEST_GALERA || TEST_GROUPREP || TEST_READONLY }; @@ -1102,7 +1135,7 @@ void SQLite3_Server::populate_grouprep_table(MySQL_Session *sess, int txs_behind #endif // TEST_GALERA -#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) +#if defined(TEST_AURORA) || defined(TEST_GALERA) || defined(TEST_GROUPREP) || defined(TEST_READONLY) void SQLite3_Server::insert_into_tables_defs(std::vector *tables_defs, const char *table_name, const char *table_def) { table_def_t *td = new table_def_t; td->table_name=strdup(table_name); @@ -1132,7 +1165,7 @@ void SQLite3_Server::drop_tables_defs(std::vector *tables_defs) { delete td; } }; -#endif // TEST_AURORA || TEST_GALERA || defined(TEST_GROUPREP) +#endif // TEST_AURORA || TEST_GALERA || defined(TEST_GROUPREP) || defined(TEST_READONLY) void SQLite3_Server::wrlock() { pthread_rwlock_wrlock(&rwlock); @@ -1174,6 +1207,14 @@ bool SQLite3_Server::init() { check_and_build_standard_tables(sessdb, tables_defs_grouprep); GloAdmin->enable_grouprep_testing(); #endif // TEST_GALERA +#ifdef TEST_READONLY + tables_defs_readonly = new std::vector; + insert_into_tables_defs(tables_defs_readonly, + (const char *)"READONLY_STATUS", + (const char*)"CREATE TABLE READONLY_STATUS (hostname VARCHAR NOT NULL , port INT NOT NULL , read_only INT NOT NULL CHECK (read_only IN (0, 1)) DEFAULT 1 , PRIMARY KEY (hostname, port))"); + check_and_build_standard_tables(sessdb, tables_defs_readonly); + GloAdmin->enable_readonly_testing(); +#endif // TEST_READONLY child_func[0]=child_mysql; main_shutdown=0; @@ -1282,3 +1323,44 @@ void SQLite3_Server::send_MySQL_ERR(MySQL_Protocol *myprot, char *msg) { myprot->generate_pkt_ERR(true,NULL,NULL,1,1045,(char *)"28000",msg); myds->DSS=STATE_SLEEP; } + +#ifdef TEST_READONLY +void SQLite3_Server::load_readonly_table(MySQL_Session *sess) { + // this function needs to be called with lock on mutex readonly_mutex already acquired + readonly_map.clear(); + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + sessdb->execute_statement((char *)"SELECT * FROM READONLY_STATUS", &error , &cols , &affected_rows , &resultset); + if (resultset) { + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + std::string s = std::string(r->fields[0])+":"+std::string(r->fields[1]); + int ro = atoi(r->fields[2]); + bool b = ( ro ? true : false ); + readonly_map[s]=b; + } + } + delete resultset; + if (readonly_map.size()==0) { + GloAdmin->admindb->execute_statement((char *)"SELECT DISTINCT hostname, port FROM mysql_servers WHERE hostgroup_id BETWEEN 4202 AND 4700", &error , &cols , &affected_rows , &resultset); + for (std::vector::iterator it = resultset->rows.begin() ; it != resultset->rows.end(); ++it) { + SQLite3_row *r=*it; + std::string s = "INSERT INTO READONLY_STATUS VALUES ('" + std::string(r->fields[0]) + "'," + std::string(r->fields[1]) + ",1)"; + sessdb->execute(s.c_str()); + } + delete resultset; + } +} + +int SQLite3_Server::readonly_test_value(char *p) { + int rc = 1; // default read_only + std::string s = std::string(p); + std::unordered_map::iterator it = readonly_map.find(s); + if (it != readonly_map.end()) { + rc = it->second; + } + return rc; +} +#endif // TEST_READONLY