Merge pull request #4207 from sysown/v2.x-hostgroup_attributes_servers_defaults

Add new `servers_defaults` column to `mysql_hostgroup_attributes`
v2.x-test20230530
René Cannaò 3 years ago committed by GitHub
commit 182aef9ca6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -64,7 +64,7 @@ using json = nlohmann::json;
#define MYHGM_GEN_ADMIN_RUNTIME_SERVERS "SELECT hostgroup_id, hostname, port, gtid_port, CASE status WHEN 0 THEN \"ONLINE\" WHEN 1 THEN \"SHUNNED\" WHEN 2 THEN \"OFFLINE_SOFT\" WHEN 3 THEN \"OFFLINE_HARD\" WHEN 4 THEN \"SHUNNED\" END status, weight, compression, max_connections, max_replication_lag, use_ssl, max_latency_ms, comment FROM mysql_servers ORDER BY hostgroup_id, hostname, port"
#define MYHGM_MYSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')"
#define MYHGM_MYSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')"
typedef std::unordered_map<std::uint64_t, void *> umap_mysql_errors;
@ -145,17 +145,17 @@ class MySrvC { // MySQL Server Container
uint16_t port;
uint16_t gtid_port;
uint16_t flags;
unsigned int weight;
int64_t weight;
enum MySerStatus status;
unsigned int compression;
unsigned int max_connections;
int64_t max_connections;
unsigned int aws_aurora_current_lag_us;
unsigned int max_replication_lag;
unsigned int max_connections_used; // The maximum number of connections that has been opened
unsigned int connect_OK;
unsigned int connect_ERR;
unsigned int cur_replication_lag_count;
// note that these variables are in microsecond, while user defines max lantency in millisecond
// note that these variables are in microsecond, while user defines max latency in millisecond
unsigned int current_latency_us;
unsigned int max_latency_us;
time_t time_last_detected_error;
@ -166,18 +166,39 @@ class MySrvC { // MySQL Server Container
unsigned long long bytes_recv;
bool shunned_automatic;
bool shunned_and_kill_all_connections; // if a serious failure is detected, this will cause all connections to die even if the server is just shunned
bool use_ssl;
int32_t use_ssl;
char *comment;
MySrvConnList *ConnectionsUsed;
MySrvConnList *ConnectionsFree;
MySrvC(char *, uint16_t, uint16_t, unsigned int, enum MySerStatus, unsigned int, unsigned int _max_connections, unsigned int _max_replication_lag, unsigned int _use_ssl, unsigned int _max_latency_ms, char *_comment);
/**
* @brief Constructs a new MySQL Server Container.
* @details For 'server_defaults' parameters, if '-1' is supplied, they try to be obtained from
* 'servers_defaults' entry from 'mysql_hostgroup_attributes' when adding the server to it's target
* hostgroup(via 'MySQL_HostGroups_Manager::add'), if not found, value is set with 'mysql_servers'
* defaults.
* @param addr Address of the server, specified either by IP or hostname.
* @param port Server port.
* @param gitd_port If non-zero, enables GTID tracking for the server.
* @param _weight Server weight. 'server_defaults' param, check @details.
* @param _status Initial server status.
* @param _compression Enables compression for server connections.
* @param _max_connections Max server connections. 'server_defaults' param, check @details.
* @param _max_replication_lag If non-zero, enables replication lag checks.
* @param _use_ssl Enables SSL for server connections. 'servers_defaults' param, check @details.
* @param _max_latency_ms Max ping server latency. When exceeded, server gets excluded from conn-pool.
* @param _comment User defined comment.
*/
MySrvC(
char* addr, uint16_t port, uint16_t gitd_port, int64_t _weight, enum MySerStatus _status, unsigned int _compression,
int64_t _max_connections, unsigned int _max_replication_lag, int32_t _use_ssl, unsigned int _max_latency_ms,
char* _comment
);
~MySrvC();
void connect_error(int, bool get_mutex=true);
void shun_and_killall();
/**
* Update the maximum number of used connections
* @return
* the maximum number of used connections
* @brief Update the maximum number of used connections
* @return The maximum number of used connections
*/
unsigned int update_max_connections_used()
{
@ -222,6 +243,11 @@ class MyHGC { // MySQL Host Group Container
bool initialized; // this variable controls if attributes were ever configured or not. Used by reset_attributes()
json ignore_session_variables_json; // the JSON format of ignore_session_variables
} attributes;
struct {
int64_t weight;
int64_t max_connections;
int32_t use_ssl;
} servers_defaults;
void reset_attributes();
MyHGC(int);
~MyHGC();

@ -25,7 +25,11 @@
#include "ev.h"
#include <functional>
#include <mutex>
#include <type_traits>
using std::function;
#ifdef TEST_AURORA
static unsigned long long array_mysrvc_total = 0;
@ -727,7 +731,11 @@ void MySrvConnList::drop_all_connections() {
}
MySrvC::MySrvC(char *add, uint16_t p, uint16_t gp, unsigned int _weight, enum MySerStatus _status, unsigned int _compression /*, uint8_t _charset */, unsigned int _max_connections, unsigned int _max_replication_lag, unsigned int _use_ssl, unsigned int _max_latency_ms, char *_comment) {
MySrvC::MySrvC(
char* add, uint16_t p, uint16_t gp, int64_t _weight, enum MySerStatus _status, unsigned int _compression,
int64_t _max_connections, unsigned int _max_replication_lag, int32_t _use_ssl, unsigned int _max_latency_ms,
char* _comment
) {
address=strdup(add);
port=p;
gtid_port=gp;
@ -877,6 +885,10 @@ MyHGC::MyHGC(int _hid) {
new_connections_now = 0;
attributes.initialized = false;
reset_attributes();
// Uninitialized server defaults. Should later be initialized via 'mysql_hostgroup_attributes'.
servers_defaults.weight = -1;
servers_defaults.max_connections = -1;
servers_defaults.use_ssl = -1;
}
void MyHGC::reset_attributes() {
@ -2579,7 +2591,7 @@ SQLite3_result * MySQL_HostGroups_Manager::dump_table_mysql(const string& name)
} else if (name == "mysql_replication_hostgroups") {
query=(char *)"SELECT writer_hostgroup, reader_hostgroup, check_type, comment FROM mysql_replication_hostgroups";
} else if (name == "mysql_hostgroup_attributes") {
query=(char *)"SELECT hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex, connection_warming, throttle_connections_per_sec, ignore_session_variables, comment FROM mysql_hostgroup_attributes ORDER BY hostgroup_id";
query=(char *)"SELECT hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex, connection_warming, throttle_connections_per_sec, ignore_session_variables, servers_defaults, comment FROM mysql_hostgroup_attributes ORDER BY hostgroup_id";
} else if (name == "mysql_servers") {
query = (char *)MYHGM_GEN_ADMIN_RUNTIME_SERVERS;
} else {
@ -3428,6 +3440,32 @@ void MySQL_HostGroups_Manager::add(MySrvC *mysrvc, unsigned int _hid) {
mysrvc->queries_sent = get_prometheus_counter_val(this->status.p_connection_pool_queries_map, endpoint_id);
MyHGC *myhgc=MyHGC_lookup(_hid);
if (mysrvc->weight == -1) {
if (myhgc->servers_defaults.weight != -1) {
mysrvc->weight = myhgc->servers_defaults.weight;
} else {
// Same harcoded default as in 'CREATE TABLE mysql_servers ...'
mysrvc->weight = 1;
}
}
if (mysrvc->max_connections == -1) {
if (myhgc->servers_defaults.max_connections != -1) {
mysrvc->max_connections = myhgc->servers_defaults.max_connections;
} else {
// Same harcoded default as in 'CREATE TABLE mysql_servers ...'
mysrvc->max_connections = 1000;
}
}
if (mysrvc->use_ssl == -1) {
if (myhgc->servers_defaults.use_ssl != -1) {
mysrvc->use_ssl = myhgc->servers_defaults.use_ssl;
} else {
// Same harcoded default as in 'CREATE TABLE mysql_servers ...'
mysrvc->use_ssl = 0;
}
}
myhgc->mysrvs->add(mysrvc);
}
@ -6847,6 +6885,93 @@ bool AWS_Aurora_Info::update(int r, int _port, char *_end_addr, int maxl, int al
return ret;
}
/**
* @brief Helper function used to try to extract a value from the JSON field 'servers_defaults'.
*
* @param j JSON object constructed from 'servers_defaults' field.
* @param hid Hostgroup for which the 'servers_defaults' is defined in 'mysql_hostgroup_attributes'. Used for
* error logging.
* @param key The key for the value to be extracted.
* @param val_check A validation function, checks if the value is within a expected range.
*
* @return The value extracted from the supplied JSON. In case of error '-1', and error cause is logged.
*/
template <typename T, typename std::enable_if<std::is_integral<T>::value, bool>::type = true>
T j_get_srv_default_int_val(
const json& j, uint32_t hid, const string& key, const function<bool(T)>& val_check
) {
if (j.find(key) != j.end()) {
const json::value_t val_type = j[key].type();
const char* type_name = j[key].type_name();
if (val_type == json::value_t::number_integer || val_type == json::value_t::number_unsigned) {
T val = j[key].get<T>();
if (val_check(val)) {
return val;
} else {
proxy_error(
"Invalid value %ld supplied for 'mysql_hostgroup_attributes.servers_defaults.%s' for hostgroup %d."
" Value NOT UPDATED.\n",
static_cast<int64_t>(val), key.c_str(), hid
);
}
} else {
proxy_error(
"Invalid type '%s'(%hhu) supplied for 'mysql_hostgroup_attributes.servers_defaults.%s' for hostgroup %d."
" Value NOT UPDATED.\n",
type_name, val_type, key.c_str(), hid
);
}
}
return static_cast<T>(-1);
}
/**
* @brief Initializes the supplied 'MyHGC' with the specified 'servers_defaults'.
* @details Input verification is performed in the supplied 'server_defaults'. It's expected to be a valid
* JSON that may contain the following fields:
* - weight: Must be an unsigned integer >= 0.
* - max_connections: Must be an unsigned integer >= 0.
* - use_ssl: Must be a integer with either value 0 or 1.
*
* In case input verification fails for a field, supplied 'MyHGC' is NOT updated for that field. An error
* message is logged specifying the source of the error.
*
* @param servers_defaults String containing a JSON defined in 'mysql_hostgroup_attributes'.
* @param myhgc The 'MyHGC' of the target hostgroup of the supplied 'servers_defaults'.
*/
void init_myhgc_servers_defaults(char* servers_defaults, MyHGC* myhgc) {
uint32_t hid = myhgc->hid;
if (strcmp(servers_defaults, "") != 0) {
try {
nlohmann::json j = nlohmann::json::parse(servers_defaults);
const auto weight_check = [] (int64_t weight) -> bool { return weight >= 0; };
int64_t weight = j_get_srv_default_int_val<int64_t>(j, hid, "weight", weight_check);
myhgc->servers_defaults.weight = weight;
const auto max_conns_check = [] (int64_t max_conns) -> bool { return max_conns >= 0; };
int64_t max_conns = j_get_srv_default_int_val<int64_t>(j, hid, "max_connections", max_conns_check);
myhgc->servers_defaults.max_connections = max_conns;
const auto use_ssl_check = [] (int32_t use_ssl) -> bool { return use_ssl == 0 || use_ssl == 1; };
int32_t use_ssl = j_get_srv_default_int_val<int32_t>(j, hid, "use_ssl", use_ssl_check);
myhgc->servers_defaults.use_ssl = use_ssl;
} catch (const json::exception& e) {
proxy_error(
"JSON parsing for 'mysql_hostgroup_attributes.servers_defaults' for hostgroup %d failed with exception `%s`.\n",
hid, e.what()
);
}
}
}
void MySQL_HostGroups_Manager::generate_mysql_hostgroup_attributes_table() {
if (incoming_hostgroup_attributes==NULL) {
return;
@ -6857,8 +6982,8 @@ void MySQL_HostGroups_Manager::generate_mysql_hostgroup_attributes_table() {
const char * query=(const char *)"INSERT INTO mysql_hostgroup_attributes ( "
"hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, "
"init_connect, multiplex, connection_warming, throttle_connections_per_sec, "
"ignore_session_variables, comment) VALUES "
"(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)";
"ignore_session_variables, servers_defaults, comment) VALUES "
"(?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)";
//rc=(*proxy_sqlite3_prepare_v2)(mydb3, query, -1, &statement, 0);
rc = mydb->prepare_v2(query, &statement);
@ -6894,11 +7019,12 @@ void MySQL_HostGroups_Manager::generate_mysql_hostgroup_attributes_table() {
int connection_warming = atoi(r->fields[6]);
int throttle_connections_per_sec = atoi(r->fields[7]);
char * ignore_session_variables = r->fields[8];
char * comment = r->fields[9];
proxy_info("Loading MySQL Hostgroup Attributes info for (%d,%d,%d,%d,\"%s\",%d,%d,%d,\"%s\",\"%s\")\n",
char * servers_defaults = r->fields[9];
char * comment = r->fields[10];
proxy_info("Loading MySQL Hostgroup Attributes info for (%d,%d,%d,%d,\"%s\",%d,%d,%d,\"%s\",'%s',\"%s\")\n",
hid, max_num_online_servers, autocommit, free_connections_pct,
init_connect, multiplex, connection_warming, throttle_connections_per_sec,
ignore_session_variables, comment
ignore_session_variables, servers_defaults, comment
);
rc=(*proxy_sqlite3_bind_int64)(statement, 1, hid); ASSERT_SQLITE_OK(rc, mydb);
rc=(*proxy_sqlite3_bind_int64)(statement, 2, max_num_online_servers); ASSERT_SQLITE_OK(rc, mydb);
@ -6909,7 +7035,8 @@ void MySQL_HostGroups_Manager::generate_mysql_hostgroup_attributes_table() {
rc=(*proxy_sqlite3_bind_int64)(statement, 7, connection_warming); ASSERT_SQLITE_OK(rc, mydb);
rc=(*proxy_sqlite3_bind_int64)(statement, 8, throttle_connections_per_sec); ASSERT_SQLITE_OK(rc, mydb);
rc=(*proxy_sqlite3_bind_text)(statement, 9, ignore_session_variables, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb);
rc=(*proxy_sqlite3_bind_text)(statement, 10, comment, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb);
rc=(*proxy_sqlite3_bind_text)(statement, 10, servers_defaults, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb);
rc=(*proxy_sqlite3_bind_text)(statement, 11, comment, -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, mydb);
SAFE_SQLITE3_STEP2(statement);
rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, mydb);
rc=(*proxy_sqlite3_reset)(statement); ASSERT_SQLITE_OK(rc, mydb);
@ -6945,6 +7072,7 @@ void MySQL_HostGroups_Manager::generate_mysql_hostgroup_attributes_table() {
// TODO: assign the variables
}
}
init_myhgc_servers_defaults(servers_defaults, myhgc);
}
for (unsigned int i=0; i<MyHostGroups->len; i++) {
MyHGC *myhgc=(MyHGC *)MyHostGroups->index(i);

@ -592,10 +592,13 @@ MHD_Result http_handler(void *cls, struct MHD_Connection *connection, const char
#define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_0 "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')"
#define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_0
#define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_2 "CREATE TABLE mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')"
#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE runtime_mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')"
#define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_2
#define ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_HOSTGROUP_ATTRIBUTES "CREATE TABLE runtime_mysql_hostgroup_attributes (hostgroup_id INT NOT NULL PRIMARY KEY , max_num_online_servers INT CHECK (max_num_online_servers>=0 AND max_num_online_servers <= 1000000) NOT NULL DEFAULT 1000000 , autocommit INT CHECK (autocommit IN (-1, 0, 1)) NOT NULL DEFAULT -1 , free_connections_pct INT CHECK (free_connections_pct >= 0 AND free_connections_pct <= 100) NOT NULL DEFAULT 10 , init_connect VARCHAR NOT NULL DEFAULT '' , multiplex INT CHECK (multiplex IN (0, 1)) NOT NULL DEFAULT 1 , connection_warming INT CHECK (connection_warming IN (0, 1)) NOT NULL DEFAULT 0 , throttle_connections_per_sec INT CHECK (throttle_connections_per_sec >= 1 AND throttle_connections_per_sec <= 1000000) NOT NULL DEFAULT 1000000 , ignore_session_variables VARCHAR CHECK (JSON_VALID(ignore_session_variables) OR ignore_session_variables = '') NOT NULL DEFAULT '' , servers_defaults VARCHAR CHECK (JSON_VALID(servers_defaults) OR servers_defaults = '') NOT NULL DEFAULT '' , comment VARCHAR NOT NULL DEFAULT '')"
#define ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_2
// Cluster solution
@ -12235,7 +12238,7 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) {
StrQuery = "INSERT INTO ";
if (_runtime)
StrQuery += "runtime_";
StrQuery += "mysql_hostgroup_attributes (hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex, connection_warming, throttle_connections_per_sec, ignore_session_variables, comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)";
StrQuery += "mysql_hostgroup_attributes (hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex, connection_warming, throttle_connections_per_sec, ignore_session_variables, servers_defaults, comment) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)";
rc = admindb->prepare_v2(StrQuery.c_str(), &statement);
ASSERT_SQLITE_OK(rc, admindb);
//proxy_info("New mysql_aws_aurora_hostgroups table\n");
@ -12250,7 +12253,8 @@ void ProxySQL_Admin::save_mysql_servers_runtime_to_database(bool _runtime) {
rc=(*proxy_sqlite3_bind_int64)(statement, 7, atol(r->fields[6])); ASSERT_SQLITE_OK(rc, admindb); // connection_warming
rc=(*proxy_sqlite3_bind_int64)(statement, 8, atol(r->fields[7])); ASSERT_SQLITE_OK(rc, admindb); // throttle_connections_per_sec
rc=(*proxy_sqlite3_bind_text)(statement, 9, r->fields[8], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); // ignore_session_variables
rc=(*proxy_sqlite3_bind_text)(statement, 10, r->fields[9], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); // comment
rc=(*proxy_sqlite3_bind_text)(statement, 10, r->fields[9], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); // servers_defaults
rc=(*proxy_sqlite3_bind_text)(statement, 11, r->fields[10], -1, SQLITE_TRANSIENT); ASSERT_SQLITE_OK(rc, admindb); // comment
SAFE_SQLITE3_STEP2(statement);
rc=(*proxy_sqlite3_clear_bindings)(statement); ASSERT_SQLITE_OK(rc, admindb);
@ -13273,7 +13277,32 @@ void ProxySQL_Admin::disk_upgrade_mysql_servers() {
"SELECT writer_hostgroup, reader_hostgroup, active, aurora_port, domain_name, max_lag_ms, check_interval_ms, "
"check_timeout_ms, writer_is_also_reader, new_reader_weight, comment FROM mysql_aws_aurora_hostgroups_v208");
}
// upgrade mysql_hostgroup_attributes
rci=configdb->check_table_structure((char *)"mysql_hostgroup_attributes",(char *)ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES_V2_5_0);
if (rci) {
// upgrade is required
proxy_warning("Detected version pre-v2.5.2 of mysql_hostgroup_attributes\n");
proxy_warning("ONLINE UPGRADE of table mysql_hostgroup_attributes in progress\n");
// drop mysql_hostgroup_attributes table with suffix _v250
configdb->execute("DROP TABLE IF EXISTS mysql_hostgroup_attributes_v250");
// rename current table to add suffix _v250
configdb->execute("ALTER TABLE mysql_hostgroup_attributes RENAME TO mysql_hostgroup_attributes_v250");
// create new table
configdb->build_table((char *)"mysql_hostgroup_attributes",(char *)ADMIN_SQLITE_TABLE_MYSQL_HOSTGROUP_ATTRIBUTES,false);
// copy fields from old table
configdb->execute(
"INSERT INTO mysql_hostgroup_attributes ("
" hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex,"
" connection_warming, throttle_connections_per_sec, ignore_session_variables, comment"
") SELECT"
" hostgroup_id, max_num_online_servers, autocommit, free_connections_pct, init_connect, multiplex,"
" connection_warming, throttle_connections_per_sec, ignore_session_variables, comment"
" FROM mysql_hostgroup_attributes_v250"
);
}
configdb->execute("PRAGMA foreign_keys = ON");
}

@ -0,0 +1,180 @@
/**
* @file mysql_hostgroup_attributes-servers_defaults-t.cpp
* @brief Simple test doing several checks for 'servers_defaults':
* 1. Correct values can be inserted and check on 'mysql_hostgroup_attributes'.
* 2. Correct LOAD TO RUNTIME.
* 3. Correct SAVE TO DISK.
* 4. Runtime values reset when a single new value is set. This doesn't reflect the internal memory values.
* 5. Non-supported key fields in the JSON are allowed. This is by design.
* 6. Invalid values can be inserted, but is reported in the error log.
* 7. Invalid type values can be inserted, but is reported in the error log.
*/
#include <cstring>
#include <string>
#include <fstream>
#include <unistd.h>
#include <mysql.h>
#include <mysql/mysqld_error.h>
#include "json.hpp"
#include "tap.h"
#include "utils.h"
#include "command_line.h"
using nlohmann::json;
using std::string;
using std::fstream;
int update_and_check_servers_defaults(MYSQL* admin, const json& j_servers_defaults) {
const string INSERT_QUERY {
"INSERT INTO mysql_hostgroup_attributes (hostgroup_id, servers_defaults)"
" VALUES (0, '" + j_servers_defaults.dump() + "')"
};
// Since the value we are updating in a single field, we can just DELETE each time
MYSQL_QUERY_T(admin, "DELETE FROM mysql_hostgroup_attributes");
MYSQL_QUERY_T(admin, INSERT_QUERY.c_str());
MYSQL_QUERY_T(admin, "LOAD MYSQL SERVERS TO RUNTIME");
diag("Checking that RUNTIME value matches INSERTED");
MYSQL_QUERY_T(admin, "SELECT servers_defaults FROM runtime_mysql_hostgroup_attributes WHERE hostgroup_id=0");
const auto extract_json_result = [] (MYSQL* admin) {
json j_result {};
MYSQL_RES* myres = mysql_store_result(admin);
MYSQL_ROW myrow = mysql_fetch_row(myres);
if (myrow && myrow[0]) {
try {
j_result = json::parse(myrow[0]);
} catch (const std::exception& e) {
diag("ERROR: Failed to parse retrieved 'servers_defaults' - '%s'", e.what());
}
}
mysql_free_result(myres);
return j_result;
};
json j_runtime_servers_defaults = extract_json_result(admin);
ok(
j_servers_defaults == j_runtime_servers_defaults,
"INSERTED 'servers_defaults' should match RUNTIME value - Exp: `%s`, Act: `%s`",
j_servers_defaults.dump().c_str(), j_runtime_servers_defaults.dump().c_str()
);
MYSQL_QUERY_T(admin, "SAVE MYSQL SERVERS TO DISK");
diag("Checking that DISK value matches INSERTED");
MYSQL_QUERY_T(admin, "SELECT servers_defaults FROM disk.mysql_hostgroup_attributes WHERE hostgroup_id=0");
json j_disk_servers_defaults = extract_json_result(admin);
ok(
j_servers_defaults == j_disk_servers_defaults,
"INSERTED 'servers_defaults' should match RUNTIME value - Exp: `%s`, Act: `%s`",
j_servers_defaults.dump().c_str(), j_disk_servers_defaults.dump().c_str()
);
return EXIT_SUCCESS;
}
void check_matching_logline(fstream& f_log, string regex) {
// Minimal wait for the error log to be written
usleep(500 * 1000);
std::vector<line_match_t> matching_lines { get_matching_lines(f_log, regex) };
for (const line_match_t& line_match : matching_lines) {
diag(
"Found matching logline - pos: %ld, line: `%s`",
static_cast<int64_t>(std::get<LINE_MATCH_T::POS>(line_match)),
std::get<LINE_MATCH_T::LINE>(line_match).c_str()
);
}
ok(
matching_lines.size() == 1,
"Expected to find an invalid value logline matching regex - found_lines: %ld",
matching_lines.size()
);
// Set the file to the end of stream
f_log.seekg(0, std::ios::end);
}
int main(int, char**) {
plan(12);
CommandLine cl;
if (cl.getEnv()) {
diag("Failed to get the required environmental variables.");
return EXIT_FAILURE;
}
// Open the error log and fetch the final position
const string f_path { get_env("REGULAR_INFRA_DATADIR") + "/proxysql.log" };
fstream f_log {};
int of_err = open_file_and_seek_end(f_path, f_log);
if (of_err) {
diag("Failed to open ProxySQL log file. Aborting further testing...");
return EXIT_FAILURE;
}
MYSQL* admin = mysql_init(NULL);
if (!mysql_real_connect(admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) {
fprintf(stderr, "File %s, line %d, Error: %s\n", __FILE__, __LINE__, mysql_error(admin));
return EXIT_FAILURE;
}
// Cleanup
MYSQL_QUERY_T(admin, "DROP TABLE IF EXISTS mysql_hostgroup_attributes_0508");
MYSQL_QUERY_T(admin, "CREATE TABLE mysql_hostgroup_attributes_0508 AS SELECT * FROM disk.mysql_hostgroup_attributes");
MYSQL_QUERY_T(admin, "DELETE FROM mysql_hostgroup_attributes");
MYSQL_QUERY_T(admin, "LOAD MYSQL SERVERS TO RUNTIME");
// 1. Set multiple valid values, and check it's properly set.
json j_multiple_vals { { "weight", 100 }, { "max_connections", 10000 }, { "use_ssl", 1}, };
update_and_check_servers_defaults(admin, j_multiple_vals);
// 2. Set single valid value, and see others values have been reset.
json j_single_val { { "weight", 100 } };
update_and_check_servers_defaults(admin, j_single_val);
// 3. NOTE: Setting only valid keys isn't enforced right now. This is by design.
json j_inv_key { { "weiht", 100 } };
update_and_check_servers_defaults(admin, j_inv_key);
// 4. NOTE: Setting invalid value is allowed, error log should reflect the update error.
json j_inv_val { { "weight", -100 } };
update_and_check_servers_defaults(admin, j_inv_val);
const string inv_val_regex {
"\\[ERROR\\] Invalid value .+\\d supplied for 'mysql_hostgroup_attributes\\.servers_defaults\\.weight'"
" for hostgroup +\\d\\. Value NOT UPDATED\\."
};
check_matching_logline(f_log, inv_val_regex);
// 5. NOTE: Setting invalid type is allowed, error log should reflect the update error.
json j_inv_val_type { { "weight", "100" } };
update_and_check_servers_defaults(admin, j_inv_val_type);
const string inv_type_regex {
"\\[ERROR\\] Invalid type .*\\(\\d\\) supplied for 'mysql_hostgroup_attributes\\.servers_defaults\\.weight'"
" for hostgroup +\\d\\. Value NOT UPDATED\\."
};
check_matching_logline(f_log, inv_type_regex);
cleanup:
MYSQL_QUERY_T(admin, "DELETE FROM disk.mysql_hostgroup_attributes");
MYSQL_QUERY_T(admin, "INSERT INTO disk.mysql_hostgroup_attributes SELECT * FROM mysql_hostgroup_attributes_0508");
mysql_close(admin);
return exit_status();
}

@ -61,15 +61,22 @@ int main(int argc, char** argv) {
std::unordered_map<std::string,std::string> queries_and_checksums = {
{
"0x666CFBEEDB76EE9C",
"INSERT INTO mysql_hostgroup_attributes VALUES (19,1,1,10,'',1,1,10000,'','')"
"INSERT INTO mysql_hostgroup_attributes VALUES (19,1,1,10,'',1,1,10000,'','','')"
},
{
"0xE2FC2A5FEE8D18DC",
"INSERT INTO mysql_hostgroup_attributes VALUES (19,1,1,10,'',1,1,10000,'',''),(18,2,-1,20,'SET sql_mode=\"\"',0,0,100,'','hello world')",
"INSERT INTO mysql_hostgroup_attributes VALUES (19,1,1,10,'',1,1,10000,'','',''),(18,2,-1,20,'SET sql_mode=\"\"',0,0,100,'','','hello world')",
},
{
"0xFACE1C64FF1C373E",
"INSERT INTO mysql_hostgroup_attributes VALUES (19,1,1,10,'',1,1,10000,'',''),(18,2,-1,20,'SET sql_mode=\"\"',0,0,100,'','hello world'),(17,0,0,30,'SET long_query_time=0',1,0,123,'{\"session_variables\":[\"tmp_table_size\",\"join_buffer_size\"]}','filtering variables')"
"INSERT INTO mysql_hostgroup_attributes VALUES (19,1,1,10,'',1,1,10000,'','',''),(18,2,-1,20,'SET sql_mode=\"\"',0,0,100,'','','hello world'),(17,0,0,30,'SET long_query_time=0',1,0,123,'{\"session_variables\":[\"tmp_table_size\",\"join_buffer_size\"]}','','filtering variables')"
},
{
"0x161B2F2BB35BA05E",
"INSERT INTO mysql_hostgroup_attributes VALUES (19,1,1,10,'',1,1,10000,'','',''),"
"(18,2,-1,20,'SET sql_mode=\"\"',0,0,100,'','','hello world'),"
"(17,0,0,30,'SET long_query_time=0',1,0,123,'{\"session_variables\":[\"tmp_table_size\",\"join_buffer_size\"]}','','filtering variables'),"
"(20,3,-1,40,'SET sql_mode=\"\"',1,0,124,'{\"session_variables\":[\"tmp_table_size\",\"join_buffer_size\"]}','{\"weight\": 100, \"max_connections\": 1000}','servers defaults')"
},
};

Loading…
Cancel
Save