aherman
/
proxysql
mirror of https://github.com/sysown/proxysql
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'v2.0.6-aurora_discovery' into v2.0.7.4

Browse Source
pull/2228/head
René Cannaò 7 years ago
parent 234eb9ba27 a445493c01
commit 48ab04f5dc
4 changed files with 178 additions and 8 deletions
Show Stats Download Patch File Download Diff File

57
Aurora_Automated_Testing.md
Unescape View File

@ -0,0 +1,57 @@
### Compiling
To run AWS Aurora automated testing, ProxySQL needs to be compiled with `make testaurora`.
### Prepare /etc/hosts
Add the following in `/etc/hosts` to simulate 30 IPs that proxysql will use to emulate 3 clusters:
```
127.0.1.11 host.1.11.aws-test.com
127.0.1.12 host.1.12.aws-test.com
127.0.1.13 host.1.13.aws-test.com
127.0.1.14 host.1.14.aws-test.com
127.0.1.15 host.1.15.aws-test.com
127.0.1.16 host.1.16.aws-test.com
127.0.1.17 host.1.17.aws-test.com
127.0.1.18 host.1.18.aws-test.com
127.0.1.19 host.1.19.aws-test.com
127.0.1.20 host.1.20.aws-test.com
127.0.2.11 host.2.11.cluster2.aws.test
127.0.2.12 host.2.12.cluster2.aws.test
127.0.2.13 host.2.13.cluster2.aws.test
127.0.2.14 host.2.14.cluster2.aws.test
127.0.2.15 host.2.15.cluster2.aws.test
127.0.2.16 host.2.16.cluster2.aws.test
127.0.2.17 host.2.17.cluster2.aws.test
127.0.2.18 host.2.18.cluster2.aws.test
127.0.2.19 host.2.19.cluster2.aws.test
127.0.2.20 host.2.20.cluster2.aws.test
127.0.3.11 host.3.11.aws.3.test.com
127.0.3.12 host.3.12.aws.3.test.com
127.0.3.13 host.3.13.aws.3.test.com
127.0.3.14 host.3.14.aws.3.test.com
127.0.3.15 host.3.15.aws.3.test.com
127.0.3.16 host.3.16.aws.3.test.com
127.0.3.17 host.3.17.aws.3.test.com
127.0.3.18 host.3.18.aws.3.test.com
127.0.3.19 host.3.19.aws.3.test.com
127.0.3.20 host.3.20.aws.3.test.com
```
### shutdown mysqld
When running automated testing, ProxySQL will listen on many IPs (30) an on port 3306.
You need to make sure that MySQL server is not running, or not listening on port 3306.
### Running proxysql
`proxysql` needs to be executed with `--sqlite3-server` .
For example, to run it under `gdb`: `run -f -D . --sqlite3-server`

74
lib/MySQL_HostGroups_Manager.cpp
Unescape View File

@ -1197,6 +1197,7 @@ SQLite3_result * MySQL_HostGroups_Manager::execute_query(char *query, char **err
bool MySQL_HostGroups_Manager::commit() {
unsigned long long curtime1=monotonic_time();
wrlock();
// purge table
purge_mysql_servers_table();
@ -1592,9 +1593,15 @@ bool MySQL_HostGroups_Manager::commit() {
pthread_cond_broadcast(&status.servers_table_version_cond);
pthread_mutex_unlock(&status.servers_table_version_lock);
wrunlock();
unsigned long long curtime2=monotonic_time();
curtime1 = curtime1/1000;
curtime2 = curtime2/1000;
proxy_info("MySQL_HostGroups_Manager::commit() locked for %lluus\n", curtime2-curtime1);
if (GloMTH) {
GloMTH->signal_all_threads(1);
}
return true;
}
@ -5705,6 +5712,7 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_reader(int _whid, int _rhid
char *error=NULL;
int _writer_hostgroup = _whid;
int aurora_port = 3306;
int new_reader_weight = 0;
char *domain_name = strdup((char *)"");
{
pthread_mutex_lock(&AWS_Aurora_Info_mutex);
@ -5718,6 +5726,7 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_reader(int _whid, int _rhid
domain_name = strdup(info->domain_name);
}
aurora_port = info->aurora_port;
new_reader_weight = info->new_reader_weight;
}
pthread_mutex_unlock(&AWS_Aurora_Info_mutex);
}
@ -5787,6 +5796,71 @@ void MySQL_HostGroups_Manager::update_aws_aurora_set_reader(int _whid, int _rhid
wrunlock();
GloAdmin->mysql_servers_wrunlock();
free(query);
} else {
// we couldn't find the server
// autodiscovery algorithm here
char *full_hostname=(char *)malloc(strlen(_server_id)+strlen(domain_name)+1);
sprintf(full_hostname, "%s%s", _server_id, domain_name);
bool found = false;
GloAdmin->mysql_servers_wrlock();
unsigned int max_max_connections = 1000;
unsigned int max_use_ssl = 0;
wrlock();
for (int i=0; i<(int)MyHostGroups->len; i++) {
MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i);
if (_rhid!=(int)myhgc->hid) continue;
for (int j=0; j<(int)myhgc->mysrvs->cnt(); j++) {
MySrvC *mysrvc=(MySrvC *)myhgc->mysrvs->servers->index(j);
if (mysrvc->max_connections > max_max_connections) {
max_max_connections = mysrvc->max_connections;
}
if (mysrvc->use_ssl > max_use_ssl) {
max_use_ssl = mysrvc->use_ssl;
}
if (strcmp(mysrvc->address,full_hostname)==0 && mysrvc->port==aurora_port) {
found = true;
// we found the server, we just configure it online if it was offline
if (mysrvc->status == MYSQL_SERVER_STATUS_OFFLINE_HARD) {
mysrvc->status = MYSQL_SERVER_STATUS_ONLINE;
}
}
}
if (found == false) { // the server doesn't exist
MySrvC *mysrvc=new MySrvC(full_hostname, aurora_port, 0, new_reader_weight, MYSQL_SERVER_STATUS_ONLINE, 0, max_max_connections, 0, max_use_ssl, 0, (char *)""); // add new fields here if adding more columns in mysql_servers
proxy_info("Adding new discovered AWS Aurora node %s:%d with: hostgroup=%d, weight=%d, max_connections=%d\n" , full_hostname, aurora_port, _rhid , new_reader_weight, max_max_connections);
add(mysrvc,_rhid);
}
q=(char *)"DELETE FROM mysql_servers WHERE hostgroup_id IN (%d , %d)";
query = (char *)malloc(strlen(q)+64);
sprintf(query,q,_whid,_rhid);
mydb->execute(query);
generate_mysql_servers_table(&_whid);
generate_mysql_servers_table(&_rhid);
free(query);
}
wrunlock();
// it is now time to build a new structure in Monitor
pthread_mutex_lock(&AWS_Aurora_Info_mutex);
pthread_mutex_lock(&GloMyMon->aws_aurora_mutex);
{
char *error=NULL;
int cols=0;
int affected_rows=0;
SQLite3_result *resultset=NULL;
char *query=(char *)"SELECT writer_hostgroup, reader_hostgroup, hostname, port, MAX(use_ssl) use_ssl , max_lag_ms , check_interval_ms , check_timeout_ms FROM mysql_servers JOIN mysql_aws_aurora_hostgroups ON hostgroup_id=writer_hostgroup OR hostgroup_id=reader_hostgroup WHERE active=1 AND status NOT IN (2,3) GROUP BY hostname, port";
mydb->execute_statement(query, &error , &cols , &affected_rows , &resultset);
if (resultset) {
if (GloMyMon->AWS_Aurora_Hosts_resultset) {
delete GloMyMon->AWS_Aurora_Hosts_resultset;
}
GloMyMon->AWS_Aurora_Hosts_resultset=resultset;
GloMyMon->AWS_Aurora_Hosts_resultset_checksum=resultset->raw_checksum();
}
}
pthread_mutex_unlock(&GloMyMon->aws_aurora_mutex);
pthread_mutex_unlock(&AWS_Aurora_Info_mutex);
GloAdmin->mysql_servers_wrunlock();
free(full_hostname);
}
}
if (resultset) {

16
lib/ProxySQL_Admin.cpp
Unescape View File

@ -10301,7 +10301,13 @@ void ProxySQL_Admin::enable_aurora_testing() {
if (j==1) {
serverid = "host." + std::to_string(j) + "." + std::to_string(i+11) + ".aws-test.com";
} else {
serverid = "127.0." + std::to_string(j) + "." + std::to_string(i+11);
if (j==2) {
serverid = "host." + std::to_string(j) + "." + std::to_string(i+11) + ".cluster2.aws.test";
} else {
if (j==3) {
serverid = "host." + std::to_string(j) + "." + std::to_string(i+11) + ".aws.3.test.com";
}
}
}
string sessionid= "";
sessionid = "b80ef4b4-" + serverid + "-aa01";
@ -10316,8 +10322,8 @@ void ProxySQL_Admin::enable_aurora_testing() {
}
sqlite3_finalize(statement);
admindb->execute("INSERT INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, comment) VALUES (1271, 1272, 1, '.aws-test.com', 25, 120, 90, 1, 1, 'Automated Aurora Testing Cluster 1')");
admindb->execute("INSERT INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, comment) VALUES (1273, 1274, 1, 25, 120, 90, 0, 1, 'Automated Aurora Testing Cluster 2')");
admindb->execute("INSERT INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, comment) VALUES (1275, 1276, 1, 25, 120, 90, 0, 2, 'Automated Aurora Testing Cluster 3')");
admindb->execute("INSERT INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, comment) VALUES (1273, 1274, 1, '.cluster2.aws.test', 25, 120, 90, 0, 1, 'Automated Aurora Testing Cluster 2')");
admindb->execute("INSERT INTO mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, comment) VALUES (1275, 1276, 1, '.aws.3.test.com', 25, 120, 90, 0, 2, 'Automated Aurora Testing Cluster 3')");
admindb->execute("UPDATE mysql_aws_aurora_hostgroups SET active=1");
//admindb->execute("update mysql_servers set max_replication_lag=20");
load_mysql_servers_to_runtime();
@ -10334,5 +10340,9 @@ void ProxySQL_Admin::enable_aurora_testing() {
load_mysql_variables_to_runtime();
admindb->execute("INSERT INTO mysql_users (username,password,default_hostgroup) VALUES ('aurora1','pass1',1271), ('aurora2','pass2',1273), ('aurora3','pass3',1275)");
init_users();
admindb->execute("INSERT INTO mysql_query_rules (active, username, match_pattern, destination_hostgroup, apply) VALUES (1, 'aurora1', '^SELECT.*max_lag_ms', 1272, 1)");
admindb->execute("INSERT INTO mysql_query_rules (active, username, match_pattern, destination_hostgroup, apply) VALUES (1, 'aurora2', '^SELECT.*max_lag_ms', 1274, 1)");
admindb->execute("INSERT INTO mysql_query_rules (active, username, match_pattern, destination_hostgroup, apply) VALUES (1, 'aurora3', '^SELECT.*max_lag_ms', 1276, 1)");
load_mysql_query_rules_to_runtime();
}
#endif // TEST_AURORA

39
lib/SQLite3_Server.cpp
Unescape View File

@ -289,6 +289,29 @@ void SQLite3_Server_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *p
query_no_space[query_no_space_length]=0;
}
// fix bug #1047
if (
(!strncasecmp("BEGIN", query_no_space, strlen("BEGIN")))
||
(!strncasecmp("START TRANSACTION", query_no_space, strlen("START TRANSACTION")))
||
(!strncasecmp("COMMIT", query_no_space, strlen("COMMIT")))
||
(!strncasecmp("ROLLBACK", query_no_space, strlen("ROLLBACK")))
||
(!strncasecmp("SET character_set_results", query_no_space, strlen("SET character_set_results")))
||
(!strncasecmp("SET SQL_AUTO_IS_NULL", query_no_space, strlen("SET SQL_AUTO_IS_NULL")))
||
(!strncasecmp("SET NAMES", query_no_space, strlen("SET NAMES")))
||
(!strncasecmp("SET AUTOCOMMIT", query_no_space, strlen("SET AUTOCOMMIT")))
) {
GloSQLite3Server->send_MySQL_OK(&sess->client_myds->myprot, NULL);
run_query=false;
goto __run_query;
}
if (query_no_space_length==SELECT_VERSION_COMMENT_LEN) {
if (!strncasecmp(SELECT_VERSION_COMMENT, query_no_space, query_no_space_length)) {
l_free(query_length,query);
@ -773,7 +796,7 @@ SQLite3_Server::SQLite3_Server() {
pthread_mutex_init(&aurora_mutex,NULL);
unsigned int nas = time(NULL);
nas = nas % 3; // range
nas += 5; // min
nas += 4; // min
max_num_aurora_servers = 10; // hypothetical maximum number of nodes
for (unsigned int j=1; j<4; j++) {
cur_aurora_writer[j-1] = 0;
@ -891,13 +914,19 @@ void SQLite3_Server::populate_aws_aurora_table(MySQL_Session *sess) {
cur_aurora_writer[cluster_id] = rand() % num_aurora_servers[cluster_id];
proxy_info("Simulating a failover for AWS Aurora cluster %d , HGs (%d:%d)\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2);
}
if (rand() % 1000 == 0) {
if (num_aurora_servers[cluster_id] < max_num_aurora_servers) {
num_aurora_servers[cluster_id]++;
proxy_info("Simulating the add of a new server for AWS Aurora Cluster %d , HGs (%d:%d). Now adding server num %d\n", cluster_id, 1270 + cluster_id*2+1 , 1270 + cluster_id*2+2, num_aurora_servers[cluster_id]);
}
}
for (unsigned int i=0; i<num_aurora_servers[cluster_id]; i++) {
string serverid = "";
if (cluster_id==0) {
//if (cluster_id==0) {
serverid = "host." + std::to_string(cluster_id+1) + "." + std::to_string(i+11);
} else {
serverid = "127.0." + std::to_string(cluster_id+1) + "." + std::to_string(i+11);
}
//} else {
// serverid = "127.0." + std::to_string(cluster_id+1) + "." + std::to_string(i+11);
//}
string sessionid= "";
float lag_ms = 0;
if (i==cur_aurora_writer[cluster_id]) {

Write Preview
Loading…
Cancel
Save