From 133a690e8d6f52bb70daf3287a8850162b01977a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Fri, 13 Apr 2018 16:11:20 +0200 Subject: [PATCH 1/4] Reduce memory allocation --- include/MySQL_HostGroups_Manager.h | 2 +- include/MySQL_Protocol.h | 3 ++- include/mysql_connection.h | 1 + lib/MySQL_HostGroups_Manager.cpp | 4 ++-- lib/MySQL_Protocol.cpp | 8 +++++-- lib/MySQL_Session.cpp | 8 +++++-- lib/MySQL_Thread.cpp | 12 ++++++----- lib/mysql_connection.cpp | 34 ++++++++++++++++++++++++++---- 8 files changed, 55 insertions(+), 17 deletions(-) diff --git a/include/MySQL_HostGroups_Manager.h b/include/MySQL_HostGroups_Manager.h index 85ca5430c..47a19259e 100644 --- a/include/MySQL_HostGroups_Manager.h +++ b/include/MySQL_HostGroups_Manager.h @@ -498,7 +498,7 @@ class MySQL_HostGroups_Manager { SQLite3_result * SQL3_Connection_Pool(bool _reset); void push_MyConn_to_pool(MySQL_Connection *, bool _lock=true); - void push_MyConn_to_pool_array(MySQL_Connection **); + void push_MyConn_to_pool_array(MySQL_Connection **, unsigned int); void destroy_MyConn_from_pool(MySQL_Connection *, bool _lock=true); void replication_lag_action(int, char*, unsigned int, int); diff --git a/include/MySQL_Protocol.h b/include/MySQL_Protocol.h index 10d5950f1..d581ee8d5 100644 --- a/include/MySQL_Protocol.h +++ b/include/MySQL_Protocol.h @@ -21,7 +21,8 @@ class MySQL_ResultSet { unsigned long long resultset_size; PtrSizeArray PSarrayOUT; //PtrSizeArray *PSarrayOUT; - MySQL_ResultSet(MySQL_Protocol *_myprot, MYSQL_RES *_res, MYSQL *_my, MYSQL_STMT *_stmt=NULL); + MySQL_ResultSet(); + void init(MySQL_Protocol *_myprot, MYSQL_RES *_res, MYSQL *_my, MYSQL_STMT *_stmt=NULL); ~MySQL_ResultSet(); unsigned int add_row(MYSQL_ROW row); unsigned int add_row2(MYSQL_ROWS *row, unsigned char *offset); diff --git a/include/mysql_connection.h b/include/mysql_connection.h index 4f7573cd3..a407a0944 100644 --- a/include/mysql_connection.h +++ b/include/mysql_connection.h @@ -72,6 +72,7 @@ class MySQL_Connection { MYSQL_RES *mysql_result; MYSQL_ROW mysql_row; MySQL_ResultSet *MyRS; + MySQL_ResultSet *MyRS_reuse; MySrvC *parent; MySQL_Connection_userinfo *userinfo; MySQL_Data_Stream *myds; diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 14ab2bf33..224681b0f 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1851,12 +1851,12 @@ __exit_push_MyConn_to_pool: wrunlock(); } -void MySQL_HostGroups_Manager::push_MyConn_to_pool_array(MySQL_Connection **ca) { +void MySQL_HostGroups_Manager::push_MyConn_to_pool_array(MySQL_Connection **ca, unsigned int cnt) { unsigned int i=0; MySQL_Connection *c=NULL; c=ca[i]; wrlock(); - while (c) { + while (imysql, myconn->MyRS); if (myconn->MyRS) { // we also need to clear MyRS, so that the next staement will recreate it if needed - delete myconn->MyRS; + if (myconn->MyRS_reuse) { + delete myconn->MyRS_reuse; + } + myconn->MyRS_reuse = myconn->MyRS; myconn->MyRS=NULL; } NEXT_IMMEDIATE(PROCESSING_QUERY); @@ -4185,7 +4188,8 @@ void MySQL_Session::handler___client_DSS_QUERY_SENT___server_DSS_NOT_INITIALIZED void MySQL_Session::MySQL_Stmt_Result_to_MySQL_wire(MYSQL_STMT *stmt, MySQL_Connection *myconn) { MYSQL_RES *stmt_result=myconn->query.stmt_result; if (stmt_result) { - MySQL_ResultSet *MyRS=new MySQL_ResultSet(&client_myds->myprot, stmt_result, stmt->mysql, stmt); + MySQL_ResultSet *MyRS=new MySQL_ResultSet(); + MyRS->init(&client_myds->myprot, stmt_result, stmt->mysql, stmt); MyRS->get_resultset(client_myds->PSarrayOUT); //removed bool resultset_completed=MyRS->get_resultset(client_myds->PSarrayOUT); delete MyRS; diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index eb11f81e8..a43b110ff 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -2846,6 +2846,7 @@ __run_skip_1a: } GloQPro->update_query_processor_stats(); } + if (rc == -1 && errno == EINTR) // poll() timeout, try again continue; @@ -4770,15 +4771,16 @@ void MySQL_Thread::return_local_connections() { if (cached_connections->len==0) { return; } +/* MySQL_Connection **ca=(MySQL_Connection **)malloc(sizeof(MySQL_Connection *)*(cached_connections->len+1)); unsigned int i=0; +*/ +// ca[i]=NULL; + MyHGM->push_MyConn_to_pool_array((MySQL_Connection **)cached_connections->pdata, cached_connections->len); +// free(ca); while (cached_connections->len) { - ca[i]=(MySQL_Connection *)cached_connections->remove_index_fast(0); - i++; + cached_connections->remove_index_fast(0); } - ca[i]=NULL; - MyHGM->push_MyConn_to_pool_array(ca); - free(ca); } unsigned long long MySQL_Threads_Handler::get_ConnPool_get_conn_immediate() { diff --git a/lib/mysql_connection.cpp b/lib/mysql_connection.cpp index e566f10aa..c9e67b6fd 100644 --- a/lib/mysql_connection.cpp +++ b/lib/mysql_connection.cpp @@ -189,6 +189,7 @@ MySQL_Connection::MySQL_Connection() { largest_query_length=0; multiplex_delayed=false; MyRS=NULL; + MyRS_reuse=NULL; creation_time=0; processing_multi_statement=false; proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 4, "Creating new MySQL_Connection %p\n", this); @@ -216,6 +217,11 @@ MySQL_Connection::~MySQL_Connection() { } if (MyRS) { delete MyRS; + MyRS = NULL; + } + if (MyRS_reuse) { + delete MyRS_reuse; + MyRS_reuse = NULL; } if (query.stmt) { query.stmt=NULL; @@ -509,6 +515,7 @@ void MySQL_Connection::real_query_start() { } void MySQL_Connection::real_query_cont(short event) { + if (event == 0) return; proxy_debug(PROXY_DEBUG_MYSQL_PROTOCOL, 6,"event=%d\n", event); async_exit_status = mysql_real_query_cont(&interr ,mysql , mysql_status(event, true)); } @@ -877,7 +884,9 @@ handler_again: break; case ASYNC_NEXT_RESULT_CONT: - async_exit_status = mysql_next_result_cont(&interr, mysql, mysql_status(event, true)); + if (event) { + async_exit_status = mysql_next_result_cont(&interr, mysql, mysql_status(event, true)); + } if (async_exit_status) { next_event(ASYNC_NEXT_RESULT_CONT); } else { @@ -920,9 +929,23 @@ handler_again: NEXT_IMMEDIATE(ASYNC_QUERY_END); } else { if (myds->sess->mirror==false) { - MyRS=new MySQL_ResultSet(&myds->sess->client_myds->myprot, mysql_result, mysql); + if (MyRS_reuse == NULL) { + MyRS = new MySQL_ResultSet(); + MyRS->init(&myds->sess->client_myds->myprot, mysql_result, mysql); + } else { + MyRS = MyRS_reuse; + MyRS_reuse = NULL; + MyRS->init(&myds->sess->client_myds->myprot, mysql_result, mysql); + } } else { - MyRS=new MySQL_ResultSet(NULL, mysql_result, mysql); + if (MyRS_reuse == NULL) { + MyRS = new MySQL_ResultSet(); + MyRS->init(NULL, mysql_result, mysql); + } else { + MyRS = MyRS_reuse; + MyRS_reuse = NULL; + MyRS->init(NULL, mysql_result, mysql); + } } async_fetch_row_start=false; NEXT_IMMEDIATE(ASYNC_USE_RESULT_CONT); @@ -1489,7 +1512,10 @@ void MySQL_Connection::async_free_result() { } async_state_machine=ASYNC_IDLE; if (MyRS) { - delete MyRS; + if (MyRS_reuse) { + delete (MyRS_reuse); + } + MyRS_reuse = MyRS; MyRS=NULL; } } From bdca8ee6bbc6e03ca3ba2c349bedb61946dce38f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Fri, 13 Apr 2018 16:11:53 +0200 Subject: [PATCH 2/4] Adding PEM certificates in gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 567a87471..9094fcad0 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ oldcode/tests/connect_speed #binary src/proxysql +src/*pem binaries/*deb binaries/*rpm tools/eventslog_reader_sample From df651b45f3c47fd25642b7647ac39025601d9cbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 17 Apr 2018 09:44:29 +0200 Subject: [PATCH 3/4] Fix typo --- lib/MySQL_HostGroups_Manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 224681b0f..3a67df5dd 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -1121,7 +1121,7 @@ bool MySQL_HostGroups_Manager::commit() { //fprintf(stderr,"%lld\n", ptr); if (ptr==0) { if (GloMTH->variables.hostgroup_manager_verbose) { - proxy_info("Creating new server in HG %d : %s:%d , gtid_port=%d, weight=%d, status=%s\n", atoi(r->fields[0]), r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), atoi(r->fields[4]), (MySerStatus) atoi(r->fields[5])); + proxy_info("Creating new server in HG %d : %s:%d , gtid_port=%d, weight=%d, status=%d\n", atoi(r->fields[0]), r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), atoi(r->fields[4]), (MySerStatus) atoi(r->fields[5])); } MySrvC *mysrvc=new MySrvC(r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), atoi(r->fields[4]), (MySerStatus) atoi(r->fields[5]), atoi(r->fields[6]), atoi(r->fields[7]), atoi(r->fields[8]), atoi(r->fields[9]), atoi(r->fields[10]), r->fields[11]); // add new fields here if adding more columns in mysql_servers proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 5, "Adding new server %s:%d , weight=%d, status=%d, mem_ptr=%p into hostgroup=%d\n", r->fields[1], atoi(r->fields[2]), atoi(r->fields[3]), (MySerStatus) atoi(r->fields[4]), mysrvc, atoi(r->fields[0])); From 528a3477ed7d4064a6c1a0367c85f86725eb0a8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 17 Apr 2018 13:29:05 +0200 Subject: [PATCH 4/4] Allow hostname for GTID --- lib/MySQL_HostGroups_Manager.cpp | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 3a67df5dd..4947262a8 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -178,6 +178,7 @@ struct ev_io * new_connector(char *address, uint16_t gtid_port, uint16_t mysql_p close(s); return NULL; } +/* memset(&a, 0, sizeof(a)); a.sin_port = htons(gtid_port); a.sin_family = AF_INET; @@ -186,9 +187,27 @@ struct ev_io * new_connector(char *address, uint16_t gtid_port, uint16_t mysql_p close(s); return NULL; } +*/ ioctl_FIONBIO(s,1); - int status = connect(s, (struct sockaddr *) &a, sizeof(a)); + struct addrinfo hints; + struct addrinfo *res = NULL; + memset(&hints, 0, sizeof(hints)); + hints.ai_protocol= IPPROTO_TCP; + hints.ai_family= AF_UNSPEC; + hints.ai_socktype= SOCK_STREAM; + + char str_port[NI_MAXSERV+1]; + sprintf(str_port,"%d", gtid_port); + int gai_rc = getaddrinfo(address, str_port, &hints, &res); + if (gai_rc) { + freeaddrinfo(res); + //exit here + return NULL; + } + + //int status = connect(s, (struct sockaddr *) &a, sizeof(a)); + int status = connect(s, res->ai_addr, res->ai_addrlen); if ((status == 0) || ((status == -1) && (errno == EINPROGRESS))) { struct ev_io *c = (struct ev_io *)malloc(sizeof(struct ev_io)); if (c) {