aherman
/
proxysql
mirror of https://github.com/sysown/proxysql
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adding support for more MySQL variables

Browse Source 
New variables supported:
- lc_messages
- lc_time_names
- long_query_time
- max_execution_time
- max_heap_table_size
- max_sort_length
- optimizer_prune_level
- optimizer_search_depth
- optimizer_switch
- sort_buffer_size
- sql_big_selects
- timestamp
- tmp_table_size

Note: sql_big_selects is still not fully supported when
combined with max_join_size.

In set testing TAP ,set large max_execution_time.
If max_execution_time is too small, the TAP test may fails because the
queries will timeout.
pull/3702/head
René Cannaò 4 years ago
parent 980b92813c
commit e983fc6827
7 changed files with 184 additions and 32 deletions
Show Stats Download Patch File Download Diff File

2
include/proxysql_structs.h
Unescape View File

@ -1131,7 +1131,7 @@ mysql_variable_st mysql_tracked_variables[] {
{ SQL_SQL_SAFE_UPDATES, SETTING_VARIABLE, true, false, false, true, (char *)"sql_safe_updates", NULL, (char *)"OFF" , false} ,
{ SQL_SQL_SELECT_LIMIT, SETTING_VARIABLE, false, false, true, false, (char *)"sql_select_limit", NULL, (char *)"DEFAULT" , false} ,
{ SQL_TIME_ZONE, SETTING_VARIABLE, true, false, false, false, (char *)"time_zone", NULL, (char *)"SYSTEM" , false} ,
{ SQL_TIMESTAMP, SETTING_VARIABLE, true, false, true, false, (char *)"timestamp", NULL, (char *)"" , false} ,
{ SQL_TIMESTAMP, SETTING_VARIABLE, false, false, true, false, (char *)"timestamp", NULL, (char *)"" , false} ,
{ SQL_TMP_TABLE_SIZE, SETTING_VARIABLE, false, false, true, false, (char *)"tmp_table_size", NULL, (char *)"" , false} ,
{ SQL_UNIQUE_CHECKS, SETTING_VARIABLE, true, false, false, true, (char *)"unique_checks", NULL, (char *)"" , false} ,

59
lib/MySQL_Session.cpp
Unescape View File

@ -69,6 +69,38 @@ static inline char is_normal_char(char c) {
return 0;
}
static const std::set<std::string> mysql_variables_boolean = {
"foreign_key_checks",
"innodb_strict_mode",
"innodb_table_locks",
"sql_auto_is_null",
"sql_big_selects",
"sql_log_bin",
"sql_safe_updates",
"unique_checks",
};
static const std::set<std::string> mysql_variables_numeric = {
"auto_increment_increment",
"auto_increment_offset",
"group_concat_max_len",
"innodb_lock_wait_timeout",
"join_buffer_size",
"lock_wait_timeout",
"long_query_time",
"max_execution_time",
"max_heap_table_size",
"max_join_size",
"max_sort_length",
"optimizer_prune_level",
"optimizer_search_depth",
"sort_buffer_size",
"sql_select_limit",
"timestamp",
"tmp_table_size",
"wsrep_sync_wait"
};
extern MARIADB_CHARSET_INFO * proxysql_find_charset_name(const char * const name);
extern MARIADB_CHARSET_INFO * proxysql_find_charset_collate_names(const char *csname, const char *collatename);
extern const MARIADB_CHARSET_INFO * proxysql_find_charset_nr(unsigned int nr);
@ -5597,6 +5629,9 @@ bool MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C
} else if (
(var == "default_storage_engine")
|| (var == "default_tmp_storage_engine")
|| (var == "lc_messages")
|| (var == "lc_time_names")
|| (var == "optimizer_switch")
) {
std::string value1 = *values;
std::size_t found_at = value1.find("@");
@ -5622,16 +5657,8 @@ bool MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C
}
}
}
// the following two blocks of code will be simplified later
} else if (
(var == "foreign_key_checks")
|| (var == "innodb_strict_mode")
|| (var == "innodb_table_locks")
|| (var == "sql_auto_is_null")
|| (var == "sql_log_bin")
|| (var == "sql_safe_updates")
|| (var == "unique_checks")
) {
//} else if (
} else if (mysql_variables_boolean.find(var) != mysql_variables_boolean.end()) {
int idx = SQL_NAME_LAST_HIGH_WM;
for (int i = 0 ; i < SQL_NAME_LAST_HIGH_WM ; i++) {
if (mysql_tracked_variables[i].is_bool) {
@ -5646,17 +5673,7 @@ bool MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C
return false;
}
}
} else if (
(var == "auto_increment_increment")
|| (var == "auto_increment_offset")
|| (var == "group_concat_max_len")
|| (var == "innodb_lock_wait_timeout")
|| (var == "join_buffer_size")
|| (var == "lock_wait_timeout")
|| (var == "max_join_size")
|| (var == "sql_select_limit")
|| (var == "wsrep_sync_wait")
) {
} else if (mysql_variables_numeric.find(var) != mysql_variables_numeric.end()) {
int idx = SQL_NAME_LAST_HIGH_WM;
for (int i = 0 ; i < SQL_NAME_LAST_HIGH_WM ; i++) {
if (mysql_tracked_variables[i].is_number) {

18
lib/MySQL_Variables.cpp
Unescape View File

@ -589,19 +589,35 @@ bool MySQL_Variables::parse_variable_number(MySQL_Session *sess, int idx, string
}
}
if (!only_digit_chars) {
if (strcasecmp(mysql_tracked_variables[idx].set_variable_name,(char *)"sql_select_limit")==0) { // only sql_select_limit allows value "default"
if (
(strcasecmp(mysql_tracked_variables[idx].set_variable_name,(char *)"sql_select_limit")==0) // sql_select_limit allows value "default"
||
(strcasecmp(mysql_tracked_variables[idx].set_variable_name,(char *)"max_join_size")==0) // max_join_size allows value "default"
) {
if (strcasecmp(v,"default")==0) {
only_digit_chars = true;
}
}
}
if (only_digit_chars) {
// see https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_max_join_size
if (value1 == "default") {
value1 = "18446744073709551615" ;
}
proxy_debug(PROXY_DEBUG_MYSQL_COM, 7, "Processing SET %s value %s\n", mysql_tracked_variables[idx].set_variable_name, value1.c_str());
uint32_t var_value_int=SpookyHash::Hash32(value1.c_str(),value1.length(),10);
if (mysql_variables.client_get_hash(sess, idx) != var_value_int) {
if (!mysql_variables.client_set_value(sess, idx, value1.c_str()))
return false;
proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Changing connection %s to %s\n", mysql_tracked_variables[idx].set_variable_name, value1.c_str());
if (idx == SQL_MAX_JOIN_SIZE) {
// see https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_max_join_size
if (value1 == "18446744073709551615") {
mysql_variables.client_set_value(sess, SQL_SQL_BIG_SELECTS, "ON");
} else {
mysql_variables.client_set_value(sess, SQL_SQL_BIG_SELECTS, "OFF");
}
}
}
//exit_after_SetParse = true;
} else {

4
lib/set_parser.cpp
Unescape View File

@ -37,7 +37,9 @@ std::map<std::string,std::vector<std::string>> SetParser::parse1() {
#define VAR_P1 "`?(@\\w+|\\w+)`?"
//#define VAR_VALUE "((?:[\\w/\\d:\\+\\-]|,)+)"
//#define VAR_VALUE "((?:CONCAT\\((?:(REPLACE|CONCAT)\\()+@@sql_mode,(?:(?:'|\\w|,| |\"|\\))+(?:\\)))|(?:[@\\w/\\d:\\+\\-]|,)+|(?:)))"
#define VAR_VALUE_P1 "((?:\\()*(?:SELECT)*(?: )*(?:CONCAT\\()*(?:(?:(?: )*REPLACE|IFNULL|CONCAT)\\()+(?: )*(?:NULL|@OLD_SQL_MODE|@@SQL_MODE),(?:(?:'|\\w|,| |\"|\\))+(?:\\))*)(?:\\))|(?:NULL)|(?:[@\\w/\\d:\\+\\-]|,)+|(?:(?:'{1}|\"{1})(?:)(?:'{1}|\"{1})))"
// added (?:[\\w]+=(?:on|off)|,)+ for optimizer_switch
#define VAR_VALUE_P1 "((?:\\()*(?:SELECT)*(?: )*(?:CONCAT\\()*(?:(?:(?: )*REPLACE|IFNULL|CONCAT)\\()+(?: )*(?:NULL|@OLD_SQL_MODE|@@SQL_MODE),(?:(?:'|\\w|,| |\"|\\))+(?:\\))*)(?:\\))|(?:NULL)|(?:[\\w]+=(?:on|off)|,)+|(?:[@\\w/\\d:\\+\\-]|,)+|(?:(?:'{1}|\"{1})(?:)(?:'{1}|\"{1})))"
const std::string pattern="(?:" NAMES SPACES QUOTES NAME_VALUE QUOTES "(?: +COLLATE +" QUOTES NAME_VALUE QUOTES "|)" "|" SESSION_P1 VAR_P1 SPACES "(?:|:)=" SPACES QUOTES VAR_VALUE_P1 QUOTES ") *,? *";
VALGRIND_DISABLE_ERROR_REPORTING;

85
test/tap/tests/generate_set_session_csv.cpp
Unescape View File

@ -66,7 +66,7 @@ class variable {
std::unordered_map<std::string, variable *> vars;
void add_value_j(std::string& j, std::string& s, variable *v) {
void add_value_j(std::string& j, const std::string& s, variable *v) {
if (v->is_bool == true) {
if (
(strcasestr(s.c_str(), "OFF") != NULL)
@ -115,6 +115,8 @@ int main() {
vars["sql_log_bin"]->add(bool_values);
vars["sql_safe_updates"] = new variable("sql_safe_updates", true, false, true);
vars["sql_safe_updates"]->add(bool_values);
vars["sql_big_selects"] = new variable("sql_safe_updates", true, false, true);
vars["sql_big_selects"]->add(bool_values);
// vars["wsrep_sync_wait"] = new variable("wsrep_sync_wait", true, false);
// vars["wsrep_sync_wait"]->add(bool_values);
vars["sql_auto_is_null"] = new variable("sql_auto_is_null", true, false, true);
@ -139,8 +141,8 @@ int main() {
vars["auto_increment_offset"] = new variable("auto_increment_offset", true, true, false);
vars["auto_increment_offset"]->add(int_values_small, 20);
vars["sql_select_limit"] = new variable("sql_select_limit", true, true, false);
vars["sql_select_limit"]->add(int_values_small, 50);
vars["sql_select_limit"]->add(int_values, 50);
vars["sql_select_limit"]->add(int_values_small, 80);
vars["sql_select_limit"]->add(int_values, 80);
vars["group_concat_max_len"] = new variable("group_concat_max_len", true, true, false);
vars["group_concat_max_len"]->add(int_values_small, 123);
vars["group_concat_max_len"]->add(int_values, 123);
@ -161,13 +163,67 @@ int main() {
vars["max_join_size"] = new variable("max_join_size", true, true, false);
vars["max_join_size"]->add(int_values, 1000);
vars["max_join_size"]->add("18446744073709551615");
vars["max_join_size"]->add("DEFAULT");
vars["tmp_table_size"] = new variable("tmp_table_size", true, true, false);
vars["tmp_table_size"]->add(int_values, 10050);
vars["tmp_table_size"]->add("18446744000000051615");
vars["max_heap_table_size"] = new variable("max_heap_table_size", true, true, false);
vars["max_heap_table_size"]->add(int_values, 20031);
vars["max_heap_table_size"]->add("8446744073709547520");
{
// join_buffer_size uses blocks of 1024 , so we need to round it
std::vector<std::string>& vals = vars["max_heap_table_size"]->values;
for (std::vector<std::string>::iterator it = vals.begin(); it != vals.end(); it++) {
unsigned long long a = std::stoll(*it);
a = a/1024;
a *= 1024;
*it = std::to_string(a);
}
}
vars["sort_buffer_size"] = new variable("sort_buffer_size", true, true, false);
vars["sort_buffer_size"]->add(int_values, 40123);
vars["sort_buffer_size"]->add("18446744073709551615");
vars["max_execution_time"] = new variable("max_execution_time", true, true, false);
vars["max_execution_time"]->add(int_values, 12930);
vars["max_execution_time"]->add("0");
vars["long_query_time"] = new variable("long_query_time", true, true, false);
vars["long_query_time"]->add(int_values, 2951);
vars["long_query_time"]->add(int_values_small, 498);
vars["long_query_time"]->add("0");
vars["max_sort_length"] = new variable("max_sort_length", true, true, false);
vars["max_sort_length"]->add(int_values, 1123);
vars["optimizer_prune_level"] = new variable("optimizer_prune_level", true, true, false);
vars["optimizer_prune_level"]->add({0, 1}, 0);
vars["optimizer_search_depth"] = new variable("optimizer_search_depth", true, true, false);
vars["optimizer_search_depth"]->add({0, 1, 12, 30, 62}, 0);
vars["timestamp"] = new variable("timestamp", true, true, false);
vars["timestamp"]->add("1640011196");
for (unsigned long long i = 0 ; i < 10 ; i++) {
unsigned long long ts = 1640011196 + i*3600*24;
vars["timestamp"]->add(std::to_string(ts));
}
vars["session_track_gtids"] = new variable("session_track_gtids", true, true, false);
vars["session_track_gtids"]->add("OWN_GTID");
// vars["session_track_gtids"]->add("OFF");
// vars["session_track_gtids"]->add("ALL_GTID");
vars["optimizer_switch"] = new variable("optimizer_switch", true, false, false);
vars["optimizer_switch"]->add(std::vector<std::string> {"'materialization=off'", "`materialization=on`", "\"materialization=off\""});
vars["optimizer_switch"]->add(std::vector<std::string> {"'index_merge_union=off'", "`index_merge_union=on`", "\"index_merge_union=off\""});
vars["optimizer_switch"]->add(std::vector<std::string> {"'semijoin=off'", "`semijoin=on`", "\"semijoin=off\""});
vars["lc_time_names"] = new variable("lc_time_names", true, false, false);
vars["lc_time_names"]->add(std::vector<std::string> {"en_US", "'en_US'", "`en_US`", "\"en_US\""});
vars["lc_time_names"]->add(std::vector<std::string> {"en_GB", "'en_GB'", "`en_GB`", "\"en_GB\""});
//vars["lc_time_names"]->add(std::vector<std::string> {"ja_JP", "'ja_JP'", "`ja_JP`", "\"ja_JP\""});
vars["lc_time_names"]->add(std::vector<std::string> {"pt_BR", "'pt_BR'", "`pt_BR`", "\"pt_BR\""});
vars["lc_messages"] = new variable("lc_messages", true, false, false);
vars["lc_messages"]->add(std::vector<std::string> {"it_IT", "'it_IT'", "`it_IT`", "\"it_IT\""});
vars["lc_messages"]->add(std::vector<std::string> {"es_ES", "'es_ES'", "`es_ES`", "\"es_ES\""});
vars["lc_messages"]->add(std::vector<std::string> {"fr_FR", "'fr_FR'", "`fr_FR`", "\"fr_FR\""});
vars["time_zone"] = new variable("time_zone", true, false, false);
vars["time_zone"]->add(std::vector<std::string> {"'+01:00'", "`+02:15`", "\"+03:30\""});
vars["time_zone"]->add(std::vector<std::string> {"'+04:45'", "`+05:00`", "\"+06:10\""});
@ -213,7 +269,7 @@ example:
i++;
}
}
for (int i=0; i<10000; i++) {
for (int i=0; i<20000; i++) {
int ne = rand()%4+1;
variable * va[ne];
for (int ine = 0; ine < ne; ) {
@ -242,8 +298,9 @@ example:
std::string s = v->values[r];
//std::cout << v->name << "=";
query += v->name + "=";
std::string s1 = "";
if (s[0] == '"') {
std::string s1 = std::string(s,0,s.length()-1);
s1 = std::string(s,0,s.length()-1);
//std::cout << "\\" << s1 << "\\\"";
query+= "\\" + s1 + "\\\"";
} else {
@ -251,6 +308,22 @@ example:
}
j += "\"" + v->name + "\":\"";
add_value_j(j,s,v);
if (v->name == "max_join_size") {
// see https://dev.mysql.com/doc/refman/5.7/en/server-system-variables.html#sysvar_max_join_size
std::string s2 = s;
if (s1.length() > 0) {
s2 = s1;
}
j+= "\", ";
j += "\"";
j += "sql_big_selects";
j += "\":\"";
if (strcasecmp(s2.c_str(),(char *)"default")==0 || s2 == "18446744073709551615") {
add_value_j(j,"ON",vars["sql_big_selects"]);
} else {
add_value_j(j,"OFF",vars["sql_big_selects"]);
}
}
if (ine != ne -1) {
query+= ", ";
j+= "\", ";

45
test/tap/tests/set_testing-240-t.cpp
Unescape View File

@ -243,6 +243,46 @@ void * my_conn_thread(void *arg) {
bool verified_special_sqlmode = false;
bool special_sqlmode = false;
bool parsing_optimizer_switch = false;
bool optimizer_switch_matches = false;
if (el.key() == "long_query_time") {
// we remove the decimals
std::string tsnd = mysql_vars["long_query_time"];
if (tsnd.find(".") != std::string::npos) {
tsnd = tsnd.substr(0, tsnd.find("."));
mysql_vars["long_query_time"]=tsnd;
}
}
if (el.key() == "timestamp") {
// we remove the decimals
std::string tsnd = mysql_vars["timestamp"];
if (tsnd.find(".") != std::string::npos) {
tsnd = tsnd.substr(0, tsnd.find("."));
mysql_vars["timestamp"]=tsnd;
}
}
if (el.key() == "max_join_size") {
if (el.value() == "DEFAULT") {
if (mysql_vars["max_join_size"] == "18446744073709551615") {
mysql_vars["max_join_size"] = "DEFAULT";
}
}
}
if (el.key() == "optimizer_switch") {
parsing_optimizer_switch = true;
std::string e_val { el.value() };
std::string k_val { k.value() };
std::string s_val { s.value() };
if (e_val == s_val) { // it matches in proxysql
if (strstr(k_val.c_str(), e_val.c_str()) != NULL) {
optimizer_switch_matches = true;
}
}
}
if (el.key() == "sql_mode") {
if (!el.value().is_string()) {
diag("Invalid value for 'sql_mode' found. Provided value should be of 'string' type");
@ -300,7 +340,8 @@ void * my_conn_thread(void *arg) {
(special_sqlmode == true && verified_special_sqlmode == false) ||
(k == mysql_vars.end()) ||
(s == proxysql_vars["conn"].end()) ||
(special_sqlmode == false &&
( (parsing_optimizer_switch == true) && (optimizer_switch_matches == false) ) ||
(special_sqlmode == false && parsing_optimizer_switch == false &&
(el.key() != "session_track_gtids" && (k.value() != el.value() || s.value() != el.value())) ||
(el.key() == "session_track_gtids" && !check_session_track_gtids(el.value(), s.value(), k.value()))
)
@ -362,7 +403,7 @@ int main(int argc, char *argv[]) {
fprintf(stderr, "Cannot read %s\n", fileName2.c_str());
return exit_status();
}
queries = 1500;
queries = 2500;
//queries = testCases.size();
plan(queries * num_threads);

3
test/tap/tests/set_testing.h
Unescape View File

@ -193,6 +193,9 @@ void queryVariables(MYSQL *mysql, json& j, std::string& paddress) {
query << ", 'default_storage_engine', 'default_tmp_storage_engine'";
query << ", 'innodb_lock_wait_timeout', 'innodb_strict_mode', 'innodb_table_locks'";
query << ", 'join_buffer_size', 'lock_wait_timeout'";
query << ", 'sort_buffer_size', 'optimizer_switch', 'optimizer_search_depth', 'optimizer_prune_level'";
query << ", 'max_execution_time', 'long_query_time', 'tmp_table_size', 'max_heap_table_size'";
query << ", 'lc_messages', 'lc_time_names', 'timestamp', 'max_sort_length', 'sql_big_selects'";
query << ")";
//fprintf(stderr, "TRACE : QUERY 3 : variables %s\n", query.str().c_str());
if (mysql_query(mysql, query.str().c_str())) {

Write Preview
Loading…
Cancel
Save