aherman
/
proxysql
mirror of https://github.com/sysown/proxysql
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix 'connect_retries_on_failure' and 'connect_timeout' logic for 'fast_forward' sessions

Browse Source 
- Correct the behavior for 'connect_retries_on_failure' for
  'fast_forward' sessions to match regular sessions, reusing the
  same implementation based on 'MySQL_Data_Stream::max_connect_time'.
- Change 'connect_timeout' for 'fast_forward' sessions to be the
  highest of 'connect_timeout_server' and 'connect_timeout_server_max'.
- Add handling for 'COM_QUIT' packets for 'fast_forward' sessions which
  have not yet received a backend connection.
pull/4012/head
Javier Jaramago Fernández 4 years ago
parent f0799a43d8
commit 25cc58a50d
2 changed files with 64 additions and 46 deletions
Show Stats Download Patch File Download Diff File

21
lib/MySQL_HostGroups_Manager.cpp
Unescape View File

@ -875,6 +875,7 @@ void MySrvC::connect_error(int err_num, bool get_mutex) {
// NOTE: this function operates without any mutex
// although, it is not extremely important if any counter is lost
// as a single connection failure won't make a significant difference
proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Connect failed with code '%d'\n", err_num);
__sync_fetch_and_add(&connect_ERR,1);
__sync_fetch_and_add(&MyHGM->status.server_connections_aborted,1);
if (err_num >= 1048 && err_num <= 1052)
@ -921,7 +922,23 @@ void MySrvC::connect_error(int err_num, bool get_mutex) {
return;
}
// same time
int max_failures = ( mysql_thread___shun_on_failures > mysql_thread___connect_retries_on_failure ? mysql_thread___connect_retries_on_failure : mysql_thread___shun_on_failures) ;
/**
* @brief The expected configured retries set by 'mysql-connect_retries_on_failure' + '2' extra expected
* connection errors.
* @details This two extra connections errors are expected:
* 1. An initial connection error generated by the datastream and the connection when being created,
* this is, right after the session has requested a connection to the connection pool. This error takes
* places directly in the state machine from 'MySQL_Connection'. Because of this, we consider this
* additional error to be a consequence of the two states machines, and it's not considered for
* 'connect_retries'.
* 2. A second connection connection error, which is the initial connection error generated by 'MySQL_Session'
* when already in the 'CONNECTING_SERVER' state. This error is an 'extra error' to always consider, since
* it's not part of the retries specified by 'mysql_thread___connect_retries_on_failure', thus, we set the
* 'connect_retries' to be 'mysql_thread___connect_retries_on_failure + 1'.
*/
int connect_retries = mysql_thread___connect_retries_on_failure + 1;
int max_failures = mysql_thread___shun_on_failures > connect_retries ? connect_retries : mysql_thread___shun_on_failures;
if (__sync_add_and_fetch(&connect_ERR_at_time_last_detected_error,1) >= (unsigned int)max_failures) {
bool _shu=false;
if (get_mutex==true)
@ -3272,7 +3289,7 @@ MySQL_Connection * MySrvConnList::get_random_MyConn(MySQL_Session *sess, bool ff
unsigned int connections_to_free = 0;
if (conns_free >= 1) {
// connection cleanup is triggered when connectinos exceed 3/4 of the total
// connection cleanup is triggered when connections exceed 3/4 of the total
// allowed max connections, this cleanup ensures that at least *one connection*
// will be freed.
if (pct_max_connections <= (conns_free + conns_used)) {

89
lib/MySQL_Session.cpp
Unescape View File

@ -2757,28 +2757,30 @@ bool MySQL_Session::handler_again___status_CONNECTING_SERVER(int *_rc) {
mybe->server_myds->wait_until=thread->curtime+mysql_thread___connect_timeout_server*1000;
pause_until=0;
}
// If this is a 'fast_forward' session, we impose the 'connect_timeout' prior to actually getting the
// connection from the 'connection_pool'. This is used to ensure that we kill the session if
// 'CONNECTING_SERVER' isn't completed before this timeout expiring. For example, if 'max_connections'
// is reached for the target hostgroup.
if (session_fast_forward && mybe->server_myds->wait_until == 0) {
mybe->server_myds->wait_until=thread->curtime+mysql_thread___connect_timeout_server*1000;
}
if (mybe->server_myds->max_connect_time) {
if (thread->curtime >= mybe->server_myds->max_connect_time) {
if (mirror) {
PROXY_TRACE();
}
char buf[256];
sprintf(buf,"Max connect timeout reached while reaching hostgroup %d after %llums", current_hostgroup, (thread->curtime - CurrentQuery.start_time)/1000 );
string errmsg {};
const string session_info { session_fast_forward ? "for 'fast_forward' session " : "" };
const uint64_t query_time = (thread->curtime - CurrentQuery.start_time)/1000;
string_format(
"Max connect timeout reached while reaching hostgroup %d %safter %llums",
errmsg, current_hostgroup, session_info.c_str(), query_time
);
if (thread) {
thread->status_variables.stvar[st_var_max_connect_timeout_err]++;
}
client_myds->myprot.generate_pkt_ERR(true,NULL,NULL,1,9001,(char *)"HY000",buf, true);
client_myds->myprot.generate_pkt_ERR(true,NULL,NULL,1,9001,(char *)"HY000", errmsg.c_str(), true);
RequestEnd(mybe->server_myds);
std::string errmsg;
generate_status_one_hostgroup(current_hostgroup, errmsg);
proxy_error("%s . HG status: %s\n", buf, errmsg.c_str());
string hg_status {};
generate_status_one_hostgroup(current_hostgroup, hg_status);
proxy_error("%s . HG status: %s\n", errmsg.c_str(), hg_status.c_str());
while (previous_status.size()) {
previous_status.pop();
@ -2800,37 +2802,6 @@ bool MySQL_Session::handler_again___status_CONNECTING_SERVER(int *_rc) {
NEXT_IMMEDIATE_NEW(WAITING_CLIENT_DATA);
}
}
if (session_fast_forward && mybe->server_myds->wait_until && mybe->server_myds->wait_until <= thread->curtime) {
std::string errmsg {};
string_format(
"Connect timeout reached while reaching hostgroup %d for 'fast_forward' session",
errmsg, current_hostgroup
);
client_myds->myprot.generate_pkt_ERR(true,NULL,NULL,1,9001,(char *)"HY000", errmsg.c_str(), true);
std::string hosgroup_st {};
generate_status_one_hostgroup(current_hostgroup, hosgroup_st);
proxy_error("%s. HG status: %s\n", errmsg.c_str(), hosgroup_st.c_str());
RequestEnd(mybe->server_myds);
while (previous_status.size()) {
previous_status.pop();
}
if (mybe->server_myds->myconn) {
// See NOTE-3404.
mybe->server_myds->myconn->connect_cont(MYSQL_WAIT_TIMEOUT);
mybe->server_myds->destroy_MySQL_Connection_From_Pool(false);
if (mirror) {
PROXY_TRACE();
NEXT_IMMEDIATE_NEW(WAITING_CLIENT_DATA);
}
}
mybe->server_myds->wait_until=0;
NEXT_IMMEDIATE_NEW(WAITING_CLIENT_DATA);
}
if (mybe->server_myds->myconn==NULL) {
handler___client_DSS_QUERY_SENT___server_DSS_NOT_INITIALIZED__get_connection();
}
@ -3701,10 +3672,40 @@ __get_pkts_from_client:
}
proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Session=%p , client_myds=%p . Statuses: WAITING_CLIENT_DATA - STATE_SLEEP\n", this, client_myds);
if (session_fast_forward==true) { // if it is fast forward
// If this is a 'fast_forward' session that hasn't yet received a backend connection, we don't
// forward 'COM_QUIT' packets, since this will make the act of obtaining a connection pointless.
// Instead, we intercept the 'COM_QUIT' packet and end the 'MySQL_Session'.
unsigned char command = *(static_cast<unsigned char*>(pkt.ptr)+sizeof(mysql_hdr));
if (command == _MYSQL_COM_QUIT) {
proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Got COM_QUIT packet\n");
if (GloMyLogger) { GloMyLogger->log_audit_entry(PROXYSQL_MYSQL_AUTH_QUIT, this, NULL); }
l_free(pkt.size,pkt.ptr);
handler_ret = -1;
return handler_ret;
}
mybe=find_or_create_backend(current_hostgroup); // set a backend
mybe->server_myds->reinit_queues(); // reinitialize the queues in the myds . By default, they are not active
mybe->server_myds->PSarrayOUT->add(pkt.ptr, pkt.size); // move the first packet
previous_status.push(FAST_FORWARD); // next status will be FAST_FORWARD . Now we need a connection
// If this is a 'fast_forward' session, we impose the 'connect_timeout' prior to actually getting the
// connection from the 'connection_pool'. This is used to ensure that we kill the session if
// 'CONNECTING_SERVER' isn't completed before this timeout expiring. For example, if 'max_connections'
// is reached for the target hostgroup.
if (mybe->server_myds->max_connect_time == 0) {
uint64_t connect_timeout =
mysql_thread___connect_timeout_server < mysql_thread___connect_timeout_server_max ?
mysql_thread___connect_timeout_server_max : mysql_thread___connect_timeout_server;
mybe->server_myds->max_connect_time = thread->curtime + connect_timeout * 1000;
}
// Impose the same connection retrying policy as done for regular connections during
// 'MYSQL_CON_QUERY'.
mybe->server_myds->connect_retries_on_failure = mysql_thread___connect_retries_on_failure;
// 'CurrentQuery' isn't used for 'FAST_FORWARD' but we update it for using it as a session
// startup time for when a fast_forward session has attempted to obtain a connection.
CurrentQuery.start_time=thread->curtime;
{
//NEXT_IMMEDIATE(CONNECTING_SERVER); // we create a connection . next status will be FAST_FORWARD
// we can't use NEXT_IMMEDIATE() inside get_pkts_from_client()

Write Preview
Loading…
Cancel
Save