diff --git a/lib/MySQL_HostGroups_Manager.cpp b/lib/MySQL_HostGroups_Manager.cpp index a3263f7b8..195b71fe7 100644 --- a/lib/MySQL_HostGroups_Manager.cpp +++ b/lib/MySQL_HostGroups_Manager.cpp @@ -3511,8 +3511,34 @@ void MySQL_HostGroups_Manager::destroy_MyConn_from_pool(MySQL_Connection *c, boo } } +inline double get_prometheus_counter_val( + std::map& counter_map, const std::string& endpoint_id +) { + const auto& counter_entry = counter_map.find(endpoint_id); + double current_val = 0; + + if (counter_entry != counter_map.end()) { + current_val = counter_entry->second->Value(); + } + + return current_val; +} + void MySQL_HostGroups_Manager::add(MySrvC *mysrvc, unsigned int _hid) { proxy_debug(PROXY_DEBUG_MYSQL_CONNPOOL, 7, "Adding MySrvC %p (%s:%d) for hostgroup %d\n", mysrvc, mysrvc->address, mysrvc->port, _hid); + + // Since metrics for servers are stored per-endpoint; the metrics for a particular endpoint can live longer than the + // 'MySrvC' itself. For example, a failover or a server config change could remove the server from a particular + // hostgroup, and a subsequent one bring it back to the original hostgroup. For this reason, everytime a 'mysrvc' is + // created and added to a particular hostgroup, we update the endpoint metrics for it. + std::string endpoint_id { std::to_string(_hid) + ":" + string { mysrvc->address } + ":" + std::to_string(mysrvc->port) }; + + mysrvc->bytes_recv = get_prometheus_counter_val(this->status.p_conn_pool_bytes_data_recv_map, endpoint_id); + mysrvc->bytes_sent = get_prometheus_counter_val(this->status.p_conn_pool_bytes_data_sent_map, endpoint_id); + mysrvc->connect_ERR = get_prometheus_counter_val(this->status.p_connection_pool_conn_err_map, endpoint_id); + mysrvc->connect_OK = get_prometheus_counter_val(this->status.p_connection_pool_conn_ok_map, endpoint_id); + mysrvc->queries_sent = get_prometheus_counter_val(this->status.p_connection_pool_queries_map, endpoint_id); + MyHGC *myhgc=MyHGC_lookup(_hid); myhgc->mysrvs->add(mysrvc); } diff --git a/test/tap/tests/test_prometheus_metrics-t.cpp b/test/tap/tests/test_prometheus_metrics-t.cpp index a895f80d2..763153c10 100644 --- a/test/tap/tests/test_prometheus_metrics-t.cpp +++ b/test/tap/tests/test_prometheus_metrics-t.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -28,6 +29,7 @@ using std::map; using std::vector; using std::pair; using std::string; +using std::tuple; std::vector split(const std::string& s, char delimiter) { std::vector tokens {}; @@ -41,6 +43,11 @@ std::vector split(const std::string& s, char delimiter) { return tokens; } +int mysql_query_d(MYSQL* mysql, const char* query) { + diag("Query: Issuing query '%s' to ('%s':%d)", query, mysql->host, mysql->port); + return mysql_query(mysql, query); +} + /** * @brief Extract the metrics values from the output of the admin command * 'SHOW PROMETHEUS METRICS'. @@ -70,6 +77,24 @@ std::map get_metric_values(std::string metrics_output) { return metrics_map; } +int get_cur_metrics(MYSQL* admin, map& metrics_vals) { + MYSQL_QUERY(admin, "SHOW PROMETHEUS METRICS\\G"); + MYSQL_RES* p_resulset = mysql_store_result(admin); + MYSQL_ROW data_row = mysql_fetch_row(p_resulset); + + std::string row_value {}; + if (data_row[0]) { + row_value = data_row[0]; + } else { + row_value = "NULL"; + } + + mysql_free_result(p_resulset); + metrics_vals = get_metric_values(row_value); + + return EXIT_SUCCESS; +} + /** * @brief Triggers the increment of 'auto_increment_delay_multiplex_metric'. * @param proxy Oppened MYSQL handler to ProxySQL. @@ -415,6 +440,125 @@ void check_message_count_parse_failure(const map& prev_metrics, } } +int get_target_metrics( + const map& metrics_map, const vector& metrics_ids, map& tg_metrics +) { + map metrics_vals {}; + + for (const string& metric_id : metrics_ids) { + const auto& metric_it = metrics_map.find(metric_id); + if (metric_it == metrics_map.end()) { + diag("%s: Unable to find target metric '%s'", __func__, metric_id.c_str()); + return EXIT_FAILURE; + } else { + metrics_vals.insert({metric_id, metric_it->second}); + } + } + + tg_metrics = metrics_vals; + + return EXIT_SUCCESS; +} + +bool rm_add_server_connpool_setup(MYSQL* proxy, MYSQL* admin, const CommandLine& cl) { + // Exercise some load on the hostgroup 0 + for (size_t i = 0; i < 10; i++) { + int rc = mysql_query_d(proxy, "/* hostgroup=0 */ SELECT 1"); + if (rc != EXIT_SUCCESS) { return EXIT_FAILURE; } + mysql_free_result(mysql_store_result(proxy)); + } + + // check metric value has been updated + return EXIT_SUCCESS; +} + +bool rm_add_server_connpool_counters(MYSQL* proxy, MYSQL* admin, const CommandLine& cl) { + // Delete server and add it again to hostgroup + diag("Removing current 'mysql_servers' for target hostgroup '0'"); + mysql_query_d(admin, "DELETE FROM mysql_servers WHERE hostgroup_id=0"); + mysql_query_d(admin, "LOAD MYSQL SERVERS TO RUNTIME"); + + diag("Recover original servers for target hostgroup '0'"); + mysql_query_d(admin, "LOAD MYSQL SERVERS FROM DISK"); + mysql_query_d(admin, "LOAD MYSQL SERVERS TO RUNTIME"); + + // Exercise some load on the hostgroup 0 + const char* query { "/* hostgroup=0,create_new_connection=1 */ SELECT 1" }; + int rc = mysql_query_d(proxy, query); + if (rc != EXIT_SUCCESS) { + diag("Failed to execute query '%s' with error '%s'", query, mysql_error(proxy)); + return false; + } + mysql_free_result(mysql_store_result(proxy)); + + return true; +} + +void check_server_data_recv(const map& prev_metrics, const map& after_metrics) { + // Endpoint we are going to target + const string endpoint_hg { "endpoint=\"127.0.0.1:13306\",hostgroup=\"0\"" }; + + // Metrics identifiers + const vector metrics_ids { + { "proxysql_connpool_data_bytes_total{" + endpoint_hg + ",traffic_flow=\"sent\"}" }, + { "proxysql_connpool_data_bytes_total{" + endpoint_hg + ",traffic_flow=\"recv\"}" }, + { "proxysql_connpool_conns_total{" + endpoint_hg + ",status=\"ok\"}" }, + // TODO: Not trivial to simulate connection error to the server + // { "proxysql_connpool_conns_total{" + inv_endpoint_hg + ",status=\"err\"}" }, + { "proxysql_connpool_conns_queries_total{" + endpoint_hg + "}" } + }; + + + // Get metrics prior to issue some traffic + diag("Obtaining metrics prior to issuing traffic to server"); + bool found_prev_metrics = false; + map prev_tg_metrics {}; + int prev_metrics_rc = get_target_metrics(prev_metrics, metrics_ids, prev_tg_metrics); + if (prev_metrics_rc == EXIT_SUCCESS) { + found_prev_metrics = true; + } else { + diag("Failed to find metrics prior to sending traffic to server"); + } + + diag("Obtaining metrics after to issuing traffic to server"); + bool found_after_metrics = false; + map after_tg_metrics {}; + int after_metrics_rc = get_target_metrics(after_metrics, metrics_ids, after_tg_metrics); + if (after_metrics_rc == EXIT_SUCCESS) { + found_after_metrics = true; + } else { + diag("Failed to find metrics after sending traffic to server"); + } + + ok(found_prev_metrics && found_after_metrics, "Metric was present in output from 'SHOW PROMETHEUS METRICS'"); + + // Check that all metrics increased from the previous values as expected + diag("Checking values have increased after the issued traffic"); + vector failed_metrics {}; + + for (const string& m_id : metrics_ids) { + const double pre_val = prev_tg_metrics[m_id]; + const double post_val = after_tg_metrics[m_id]; + + if (pre_val >= post_val) { + failed_metrics.push_back(m_id); + diag("Error: Metric '%s' failed to be incremented [%lf, %lf]", m_id.c_str(), pre_val, post_val); + } + } + + ok(failed_metrics.empty(), "All metric values were properly incremented after server rm/add from hostgroup"); +} + +using setup = function; +using metric_trigger = function; +using metric_check = function&, const map&)>; + +struct CHECK { + enum funcs { SETUP, TRIGGER, CHECKER, _END }; +}; + +bool placeholder_setup(MYSQL*, MYSQL*, const CommandLine&) { return true; } + /** * @brief Map of test identifier and pair functions holding the metrics tests: * - First function of the pair uses an open connection to ProxySQL and to ProxySQL Admin to perform @@ -422,92 +566,100 @@ void check_message_count_parse_failure(const map& prev_metrics, * - Second function performs the check to verify that the metric have been incremented properly. * This function should execute **one** 'ok(...)' inside when the values have been properly checked. */ -const vector, - function&, const map&)> - > ->> metric_tests { - { "proxysql_myhgm_auto_increment_multiplex_total", { trigger_auto_increment_delay_multiplex_metric, check_auto_increment_delay_multiplex_metric } }, - { "proxysql_access_denied_wrong_password_total", { trigger_access_denied_wrong_password_total, check_access_denied_wrong_password_total } }, - { "proxysql_com_rollback_total", { trigger_transaction_rollback_total, check_transaction_rollback_total } }, - { "proxysql_version_info", { get_proxysql_version_info, check_proxysql_version_info } }, +const vector>> metric_tests { + { + "proxysql_myhgm_auto_increment_multiplex_total", + { placeholder_setup, trigger_auto_increment_delay_multiplex_metric, check_auto_increment_delay_multiplex_metric } + }, + { + "proxysql_access_denied_wrong_password_total", + { placeholder_setup, trigger_access_denied_wrong_password_total, check_access_denied_wrong_password_total } + }, + { + "proxysql_com_rollback_total", + { placeholder_setup, trigger_transaction_rollback_total, check_transaction_rollback_total } }, + { + "proxysql_version_info", + { placeholder_setup, get_proxysql_version_info, check_proxysql_version_info } + }, + { + "rm_add_server_connpool_counters", + { placeholder_setup, rm_add_server_connpool_counters, { check_server_data_recv } }, + }, // Checks metric creation and initial value - { "proxysql_message_count_parse_failure_init", { trigger_message_count_parse_failure, check_message_count_parse_failure } }, + { + "proxysql_message_count_parse_failure_init", + { placeholder_setup, trigger_message_count_parse_failure, check_message_count_parse_failure } }, // Checks metric increment - { "proxysql_message_count_parse_failure_inc", { trigger_message_count_parse_failure, check_message_count_parse_failure } }, + { + "proxysql_message_count_parse_failure_inc", + { placeholder_setup, trigger_message_count_parse_failure, check_message_count_parse_failure } + }, }; +using std::map; + + int main(int argc, char** argv) { CommandLine cl; if (cl.getEnv()) { diag("Failed to get the required environmental variables."); - return -1; + return EXIT_FAILURE; } - plan(metric_tests.size() * 3); + plan(metric_tests.size() * 4); - for (const auto& metric_checker : metric_tests) { + for (const auto& metric_test : metric_tests) { // Initialize Admin connection MYSQL* proxysql_admin = mysql_init(NULL); if (!proxysql_admin) { fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql_admin)); - return -1; + return EXIT_FAILURE; } // Connnect to ProxySQL Admin if (!mysql_real_connect(proxysql_admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql_admin)); - return -1; + return EXIT_FAILURE; } - // Initialize ProxySQL connection MYSQL* proxysql = mysql_init(NULL); if (!proxysql) { fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql)); - return -1; + return EXIT_FAILURE; } // Connect to ProxySQL if (!mysql_real_connect(proxysql, cl.host, cl.username, cl.password, NULL, cl.port, NULL, 0)) { fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(proxysql)); - return exit_status(); + return EXIT_FAILURE; } // Log test start for metric - diag("Started test for metric '%s'", metric_checker.first.c_str()); + diag("Started test for metric '%s'", metric_test.first.c_str()); + + // Execute the action triggering the metric update + const auto& metric_setup = std::get(metric_test.second); + bool action_res = metric_setup(proxysql, proxysql_admin, cl); + ok(action_res, "Setup action to prepare the env was successful."); // Get the current metrics values - MYSQL_QUERY(proxysql_admin, "SHOW PROMETHEUS METRICS\\G"); - MYSQL_RES* p_resulset = mysql_store_result(proxysql_admin); - MYSQL_ROW data_row = mysql_fetch_row(p_resulset); - std::string row_value {}; - if (data_row[0]) { - row_value = data_row[0]; - } else { - row_value = "NULL"; - } - mysql_free_result(p_resulset); - const std::map prev_metrics { get_metric_values(row_value) }; + std::map prev_metrics {}; + int rc = get_cur_metrics(proxysql_admin, prev_metrics); + if (rc != EXIT_SUCCESS) { return EXIT_FAILURE; } // Execute the action triggering the metric update - bool action_res = metric_checker.second.first(proxysql, proxysql_admin, cl); - ok(action_res, "Action to update the metric was executed properly."); + const auto& metric_trigger = std::get(metric_test.second); + bool trigger_res = metric_trigger(proxysql, proxysql_admin, cl); + ok(trigger_res, "Action to update the metric was executed properly."); // Get the new updated metrics values - MYSQL_QUERY(proxysql_admin, "SHOW PROMETHEUS METRICS\\G"); - p_resulset = mysql_store_result(proxysql_admin); - data_row = mysql_fetch_row(p_resulset); - if (data_row[0]) { - row_value = data_row[0]; - } else { - row_value = "NULL"; - } - mysql_free_result(p_resulset); - const std::map after_metrics { get_metric_values(row_value) }; + std::map after_metrics {}; + rc = get_cur_metrics(proxysql_admin, after_metrics); + if (rc != EXIT_SUCCESS) { return EXIT_FAILURE; } // Check that the new metrics values matches the expected - metric_checker.second.second(prev_metrics, after_metrics); + const auto& metric_checker = std::get(metric_test.second); + metric_checker(prev_metrics, after_metrics); // Close the connections used for this test mysql_close(proxysql);