diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index 36e9cd72a..a3263f7b8 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -875,6 +875,7 @@ void MySrvC::connect_error(int err_num, bool get_mutex) { // NOTE: this function operates without any mutex // although, it is not extremely important if any counter is lost // as a single connection failure won't make a significant difference + proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Connect failed with code '%d'\n", err_num); __sync_fetch_and_add(&connect_ERR,1); __sync_fetch_and_add(&MyHGM->status.server_connections_aborted,1); if (err_num >= 1048 && err_num <= 1052) @@ -921,7 +922,23 @@ void MySrvC::connect_error(int err_num, bool get_mutex) { return; } // same time - int max_failures = ( mysql_thread___shun_on_failures > mysql_thread___connect_retries_on_failure ? mysql_thread___connect_retries_on_failure : mysql_thread___shun_on_failures) ; + /** + * @brief The expected configured retries set by 'mysql-connect_retries_on_failure' + '2' extra expected + * connection errors. + * @details This two extra connections errors are expected: + * 1. An initial connection error generated by the datastream and the connection when being created, + * this is, right after the session has requested a connection to the connection pool. This error takes + * places directly in the state machine from 'MySQL_Connection'. Because of this, we consider this + * additional error to be a consequence of the two states machines, and it's not considered for + * 'connect_retries'. + * 2. A second connection connection error, which is the initial connection error generated by 'MySQL_Session' + * when already in the 'CONNECTING_SERVER' state. This error is an 'extra error' to always consider, since + * it's not part of the retries specified by 'mysql_thread___connect_retries_on_failure', thus, we set the + * 'connect_retries' to be 'mysql_thread___connect_retries_on_failure + 1'. + */ + int connect_retries = mysql_thread___connect_retries_on_failure + 1; + int max_failures = mysql_thread___shun_on_failures > connect_retries ? connect_retries : mysql_thread___shun_on_failures; + if (__sync_add_and_fetch(&connect_ERR_at_time_last_detected_error,1) >= (unsigned int)max_failures) { bool _shu=false; if (get_mutex==true) @@ -3272,7 +3289,7 @@ MySQL_Connection * MySrvConnList::get_random_MyConn(MySQL_Session *sess, bool ff unsigned int connections_to_free = 0; if (conns_free >= 1) { - // connection cleanup is triggered when connectinos exceed 3/4 of the total + // connection cleanup is triggered when connections exceed 3/4 of the total // allowed max connections, this cleanup ensures that at least *one connection* // will be freed. if (pct_max_connections <= (conns_free + conns_used)) { diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index 966270e3a..22e004dcc 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -2757,28 +2757,30 @@ bool MySQL_Session::handler_again___status_CONNECTING_SERVER(int *_rc) { mybe->server_myds->wait_until=thread->curtime+mysql_thread___connect_timeout_server*1000; pause_until=0; } - // If this is a 'fast_forward' session, we impose the 'connect_timeout' prior to actually getting the - // connection from the 'connection_pool'. This is used to ensure that we kill the session if - // 'CONNECTING_SERVER' isn't completed before this timeout expiring. For example, if 'max_connections' - // is reached for the target hostgroup. - if (session_fast_forward && mybe->server_myds->wait_until == 0) { - mybe->server_myds->wait_until=thread->curtime+mysql_thread___connect_timeout_server*1000; - } if (mybe->server_myds->max_connect_time) { if (thread->curtime >= mybe->server_myds->max_connect_time) { if (mirror) { PROXY_TRACE(); } - char buf[256]; - sprintf(buf,"Max connect timeout reached while reaching hostgroup %d after %llums", current_hostgroup, (thread->curtime - CurrentQuery.start_time)/1000 ); + + string errmsg {}; + const string session_info { session_fast_forward ? "for 'fast_forward' session " : "" }; + const uint64_t query_time = (thread->curtime - CurrentQuery.start_time)/1000; + + string_format( + "Max connect timeout reached while reaching hostgroup %d %safter %llums", + errmsg, current_hostgroup, session_info.c_str(), query_time + ); + if (thread) { thread->status_variables.stvar[st_var_max_connect_timeout_err]++; } - client_myds->myprot.generate_pkt_ERR(true,NULL,NULL,1,9001,(char *)"HY000",buf, true); + client_myds->myprot.generate_pkt_ERR(true,NULL,NULL,1,9001,(char *)"HY000", errmsg.c_str(), true); RequestEnd(mybe->server_myds); - std::string errmsg; - generate_status_one_hostgroup(current_hostgroup, errmsg); - proxy_error("%s . HG status: %s\n", buf, errmsg.c_str()); + + string hg_status {}; + generate_status_one_hostgroup(current_hostgroup, hg_status); + proxy_error("%s . HG status: %s\n", errmsg.c_str(), hg_status.c_str()); while (previous_status.size()) { previous_status.pop(); @@ -2800,37 +2802,6 @@ bool MySQL_Session::handler_again___status_CONNECTING_SERVER(int *_rc) { NEXT_IMMEDIATE_NEW(WAITING_CLIENT_DATA); } } - if (session_fast_forward && mybe->server_myds->wait_until && mybe->server_myds->wait_until <= thread->curtime) { - std::string errmsg {}; - string_format( - "Connect timeout reached while reaching hostgroup %d for 'fast_forward' session", - errmsg, current_hostgroup - ); - client_myds->myprot.generate_pkt_ERR(true,NULL,NULL,1,9001,(char *)"HY000", errmsg.c_str(), true); - - std::string hosgroup_st {}; - generate_status_one_hostgroup(current_hostgroup, hosgroup_st); - proxy_error("%s. HG status: %s\n", errmsg.c_str(), hosgroup_st.c_str()); - - RequestEnd(mybe->server_myds); - while (previous_status.size()) { - previous_status.pop(); - } - - if (mybe->server_myds->myconn) { - // See NOTE-3404. - mybe->server_myds->myconn->connect_cont(MYSQL_WAIT_TIMEOUT); - mybe->server_myds->destroy_MySQL_Connection_From_Pool(false); - if (mirror) { - PROXY_TRACE(); - NEXT_IMMEDIATE_NEW(WAITING_CLIENT_DATA); - } - } - - mybe->server_myds->wait_until=0; - NEXT_IMMEDIATE_NEW(WAITING_CLIENT_DATA); - } - if (mybe->server_myds->myconn==NULL) { handler___client_DSS_QUERY_SENT___server_DSS_NOT_INITIALIZED__get_connection(); } @@ -3701,10 +3672,40 @@ __get_pkts_from_client: } proxy_debug(PROXY_DEBUG_MYSQL_CONNECTION, 5, "Session=%p , client_myds=%p . Statuses: WAITING_CLIENT_DATA - STATE_SLEEP\n", this, client_myds); if (session_fast_forward==true) { // if it is fast forward + // If this is a 'fast_forward' session that hasn't yet received a backend connection, we don't + // forward 'COM_QUIT' packets, since this will make the act of obtaining a connection pointless. + // Instead, we intercept the 'COM_QUIT' packet and end the 'MySQL_Session'. + unsigned char command = *(static_cast(pkt.ptr)+sizeof(mysql_hdr)); + if (command == _MYSQL_COM_QUIT) { + proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Got COM_QUIT packet\n"); + if (GloMyLogger) { GloMyLogger->log_audit_entry(PROXYSQL_MYSQL_AUTH_QUIT, this, NULL); } + l_free(pkt.size,pkt.ptr); + handler_ret = -1; + return handler_ret; + } + mybe=find_or_create_backend(current_hostgroup); // set a backend mybe->server_myds->reinit_queues(); // reinitialize the queues in the myds . By default, they are not active mybe->server_myds->PSarrayOUT->add(pkt.ptr, pkt.size); // move the first packet previous_status.push(FAST_FORWARD); // next status will be FAST_FORWARD . Now we need a connection + + // If this is a 'fast_forward' session, we impose the 'connect_timeout' prior to actually getting the + // connection from the 'connection_pool'. This is used to ensure that we kill the session if + // 'CONNECTING_SERVER' isn't completed before this timeout expiring. For example, if 'max_connections' + // is reached for the target hostgroup. + if (mybe->server_myds->max_connect_time == 0) { + uint64_t connect_timeout = + mysql_thread___connect_timeout_server < mysql_thread___connect_timeout_server_max ? + mysql_thread___connect_timeout_server_max : mysql_thread___connect_timeout_server; + mybe->server_myds->max_connect_time = thread->curtime + connect_timeout * 1000; + } + // Impose the same connection retrying policy as done for regular connections during + // 'MYSQL_CON_QUERY'. + mybe->server_myds->connect_retries_on_failure = mysql_thread___connect_retries_on_failure; + // 'CurrentQuery' isn't used for 'FAST_FORWARD' but we update it for using it as a session + // startup time for when a fast_forward session has attempted to obtain a connection. + CurrentQuery.start_time=thread->curtime; + { //NEXT_IMMEDIATE(CONNECTING_SERVER); // we create a connection . next status will be FAST_FORWARD // we can't use NEXT_IMMEDIATE() inside get_pkts_from_client()