diff --git a/test/tap/tests/test_tokenizer-t.cpp b/test/tap/tests/test_tokenizer-t.cpp index a1ef73f88..3186c647d 100644 --- a/test/tap/tests/test_tokenizer-t.cpp +++ b/test/tap/tests/test_tokenizer-t.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -19,114 +20,258 @@ __thread bool mysql_thread___query_digests_replace_null = false; __thread bool mysql_thread___query_digests_no_digits = false; __thread int mysql_thread___query_digests_grouping_limit = 3; -const std::vector queries { - // floats - "select 1.1", - "select 1192.1102", - "select 99.1929", - // exponentials - "select 1.1e9", - "select 1.1e+9", - "select 1.1e-9", - // operators - "select 1 +1", - "select 1+ 1", - "select 1- 1", - "select 1 -1", - "select 1* 1", - "select 1 *1", - "select 1/ 1", - "select 1 /1", - "select 1% 1", - "select 1 %1", - // operators and commas - "select 1+ 1, 1 -1, 1 * 1 , 1/1 , 100 % 3", - "SELECT * FROM t t1, t t2 ,t t3,t t4 LIMIT 0", - // strings - "select * from t where t = \"foo\"", - "select \"1+ 1, 1 -1, 1 * 1 , 1/1 , 100 % 3\"", - // not modified - "select * fromt t", - // test query_digest reduction - "SELECT * FROM tablename WHERE id IN (1,2,3,4,5,6,7,8,9,10)", - "SELECT * FROM tablename WHERE id IN (1,2,3,4)", - // invalid request grouping - "SELECT * tablename where id IN (1,2,3,4,5,6,7,8, AND j in (1,2,3,4,5,6 and k=1", - // random insert queries - "INSERT INTO db.table(col1) VALUES ('val')", - "INSERT INTO db.table (col1) VALUES ('val')", - "INSERT INTO db.table( col1) VALUES ( 'val' )", - "INSERT INTO db.table( col1) VALUES ( 'val' )", - "INSERT INTO db.table ( col1 ) VALUES ( 'val' )", - "INSERT INTO db.table (col1, col2,col3,col4) VALUES ('val',2,3,'foo')", - "INSERT INTO db.table ( col1, col2,col3,col4 ) VALUES ('val',2,3,'foo')" -}; +using std::vector; +using std::pair; +using std::string; +using std::tuple; -const std::vector exp_results { +const vector> query_digest_pairs { + // initial '#' comments + { + "# random_comment \n select 1.1", + "select ?" + }, + { + "#random_comment \nselect 1.1", + "select ?" + }, + { + "#random_comment\nselect 1.1", + "select ?" + }, + // initial '--' comments + { + "-- random_comment \n select 1.1", + "select ?" + }, + { + "--random_comment \nselect 1.1", + "select ?" + }, + { + "--random_comment\nselect 1.1", + "select ?" + }, + // final/initial '#|--' comments + { + "# random_comment\n select 1.1 #final_comment\n ", + "select ?" + }, + // TODO: KnownIssue - 1 + // { + // "# random_comment\n select 1.1# final_comment \n", + // "select ?" + // // Actual: "select ?# final_comment" + // }, + { + "# random_comment\n select 1.1 #final_comment \n ", + "select ?" + }, + { + "-- random_comment\n select 1.1 --final_comment\n ", + "select ?" + }, + { + "-- random_comment\n select 1.1-- final_comment \n", + "select ?" + }, + { + "-- random_comment\n select 1.1 --final_comment \n ", + "select ?" + }, // floats - "select ?", - "select ?", - "select ?", + { "select 1.1", "select ?" }, + { "select 99.1929", "select ?" }, // exponentials - "select ?", - "select ?", - "select ?", + { "select 1.1e9", "select ?" }, + { "select 1.1e+9", "select ?" }, + { "select 1.1e-9", "select ?" }, + // TODO: KnownIssue - 2: Exponentials are case sensitive + // { "select 1.1E9", "select ?" }, + // { "select 1.1E+9", "select ?" }, + // { "select 1.1E-9", "select ?" }, // operators - "select ?+?", - "select ?+?", - "select ?-?", - "select ?-?", - "select ?*?", - "select ?*?", - "select ?/?", - "select ?/?", - "select ?%?", - "select ?%?", + { "select 1 +1", "select ?+?" }, + { "select 1+ 1", "select ?+?" }, + { "select 1- 1", "select ?-?" }, + { "select 1 -1", "select ?-?" }, + { "select 1* 1", "select ?*?" }, + { "select 1 *1", "select ?*?" }, + { "select 1/ 1", "select ?/?" }, + { "select 1 /1", "select ?/?" }, + { "select 1% 1", "select ?%?" }, + { "select 1 %1", "select ?%?" }, // operators and commas - "select ?+?,?-?,?*?,?/?,?%?", - "SELECT * FROM t t1,t t2,t t3,t t4 LIMIT ?", - // strings - "select * from t where t = ?", - "select ?", + { + "select 1+ 1, 1 -1, 1 * 1 , 1/1 , 100 % 3", + "select ?+?,?-?,?*?,?/?,?%?", + }, + { + "SELECT * FROM t t1, t t2 ,t t3,t t4 LIMIT 0", + "SELECT * FROM t t1,t t2,t t3,t t4 LIMIT ?" + }, + // mixing operators, commas and literals + { + "select 1+ 1,'1 -1', 1 * 1 , '1/1 ',100 % 3", + "select ?+?,?,?*?,?,?%?" + }, + { + "select 1+ 1 ,'1 -1' ,1 * 1 , '1 ' , 100 % 3", + "select ?+?,?,?*?,?,?%?" + }, + { + "select 1 + 1 , '1 - 1' , 1 * 1 , '1 ' , 100 % 3 ", + "select ?+?,?,?*?,?,?%?" + }, + // TODO: KnownIssue - 8: Operators not removed when extra space precedes the value + { + "select + 1", + "select +?" + }, + // strings - simple + { + "select * from t where t = \"foo\"", + "select * from t where t = ?", + }, + { + "select \"1+ 1, 1 -1, 1 * 1 , 1/1 , 100 % 3\"", + "select ?", + }, + // string - preceded by signs - outside parenthesis, not preceded by commas + { "select -\"1\"", "select -?", }, + { "select +\"1\",'foo'", "select +?,?", }, + // string - preceded by signs - inside parenthesis, or preceded by commas + { "select (-'89')", "select (?)", }, + { "select 10,-'89'", "select ?,?", }, + { "select 10, -'89' ", "select ?,?", }, + { "select 10, -'89 ',+'5'", "select ?,?,?", }, + // TODO: KnownIssue - 7: Spaces not removed after parenthesis when literal strings are preceded by '+|-' + { "select CONCAT( -'89'+'10')", "select CONCAT( ?+?)", }, + // ^ preserved space + { "select CONCAT( -'89'+'10')", "select CONCAT( ?+?)", }, + { "select CONCAT( -'89' + '10' )", "select CONCAT( ?+?)", }, + // TODO: KnownIssue - 8: Operators not removed when extra space precedes the literal (value) + { "select CONCAT(- '89')", "select CONCAT(-?)", }, + // ^ preserved operator + // not modified - "select * fromt t", + { "select * fromt t", "select * fromt t" }, // test query_digest reduction - "SELECT * FROM tablename WHERE id IN (?,?,?,...)", - "SELECT * FROM tablename WHERE id IN (?,?,?,...)", + { + "SELECT * FROM tablename WHERE id IN (1,2,3,4,5,6,7,8,9,10)", + "SELECT * FROM tablename WHERE id IN (?,?,?,...)" + }, + { + "SELECT * FROM tablename WHERE id IN (1,2,3,4)", + "SELECT * FROM tablename WHERE id IN (?,?,?,...)" + }, // invalid request grouping - "SELECT * tablename where id IN (?,?,?,... AND j in (?,?,?,... and k=?", + { + "SELECT * tablename where id IN (1,2,3,4,5,6,7,8, AND j in (1,2,3,4,5,6 and k=1", + "SELECT * tablename where id IN (?,?,?,...,AND j in (?,?,?,... and k=?" + }, // random insert queries - "INSERT INTO db.table(col1) VALUES (?)", - "INSERT INTO db.table (col1) VALUES (?)", - "INSERT INTO db.table( col1) VALUES ( ? )", - "INSERT INTO db.table( col1) VALUES ( ? )", - "INSERT INTO db.table ( col1 ) VALUES ( ? )", - "INSERT INTO db.table (col1,col2,col3,col4) VALUES (?,?,?,?)", - "INSERT INTO db.table ( col1,col2,col3,col4 ) VALUES (?,?,?,?)" + { + "INSERT INTO db.table(col1) VALUES ('val')", + "INSERT INTO db.table(col1) VALUES (?)" + }, + { + "INSERT INTO db.table (col1) VALUES ('val')", + "INSERT INTO db.table (col1) VALUES (?)" + }, + { + "INSERT INTO db.table( col1) VALUES ( 'val' )", + "INSERT INTO db.table( col1) VALUES (?)" + }, + { + "INSERT INTO db.table( col1) VALUES ( 'val' )", + "INSERT INTO db.table( col1) VALUES (?)" + }, + // TODO: KnownIssue - 6: When 'no-digits' is enabled, space before parenthesis closing brakcet is + // collapsed when an identifier name finish with a number. + // { + // "INSERT INTO db.table ( col1 ) VALUES ( 'val' )", + // "INSERT INTO db.table ( col1 ) VALUES (?)" + // }, + { + "INSERT INTO db.table (col1, col2,col3,col4) VALUES ('val',2,3,'foo')", + "INSERT INTO db.table (col1,col2,col3,col4) VALUES (?,?,?,...)" + }, + // TODO: KnownIssue - 6: When 'no-digits' is enabled, space before parenthesis closing brakcet is + // collapsed when an identifier name finish with a number. + // { + // "INSERT INTO db.table ( col1, col2,col3,col4 ) VALUES ('val',2,3,'foo')", + // "INSERT INTO db.table ( col1,col2,col3,col4 ) VALUES (?,?,?,...)" + // }, + { + "INSERT INTO db.table_25 (col1, col2,col3,col4) VALUES ('val',2,3,'foo')", + "INSERT INTO db.table_25 (col1,col2,col3,col4) VALUES (?,?,?,...)" + }, + { + "INSERT INTO db.table1_25 ( col_121,col2121 ,col12_3, col41203_ ) VALUES (?,?,?,...)", + "INSERT INTO db.table1_25 ( col_121,col2121,col12_3,col41203_ ) VALUES (?,?,?,...)" + }, + // TODO: KnownIssue - 5: Arithmetics operators breaks grouping + // { + // "INSERT INTO db.table ( col1, col2,col3,col4, col5 ) VALUES ('val',2,3,'foo', 5 + 10, 6 - 9)", + // "INSERT INTO db.table ( col1,col2,col3,col4,col5 ) VALUES (?,?,?,...)" + // // Actual: "INSERT INTO db.table ( col1,col2,col3,col4,col5 ) VALUES (?,?,?,... + -)" + // }, }; -const std::vector queries_grouping { +const vector> queries_digests_grouping { // test query_digest reduction - "SELECT * FROM tablename WHERE id IN (1,2, 3,4 ,5 ,6,7,8,9,10)", + { + "SELECT * FROM tablename WHERE id IN (1,2, 3,4 ,5 ,6,7,8,9,10)", + "SELECT * FROM tablename WHERE id IN (?,...)" + }, // invalid request grouping - "SELECT * tablename where id IN (1,2,3,4,5 , 6,7,8, AND j in (1, 2,3, 4 ,5,6,7,8,9 and k=1", - "SELECT (1.1, 1, 2, 13, 4.81, 12) FROM db.table", - "SELECT (1.1, 1.12934 , 21.32 , 91, 91, 12.93 ) FROM db.table2", - "SELECT (1.1, 1.12934 , 21.32 , 91.2 , 91, 12 ) FROM db.table7", - "SELECT (1.1, 1.12934, 21.32, 391,2381,28.493,1283 ) FROM db.table2", - "SELECT (1.1, 1.12934, 21.32 , 91, 91, 12.1 ) FROM db.table3" + { + "SELECT * tablename where id IN (1,2,3,4,5 , 6,7,8, AND j in (1, 2,3, 4 ,5,6,7,8,9 and k=1", + "SELECT * tablename where id IN (?,...,AND j in (?,... and k=?" + }, + // more random grouping + { + "SELECT (1.1, 1, 2, 13, 4.81, 12) FROM db.table", + "SELECT (?,...) FROM db.table" + }, + { + "SELECT (1.1, 1.12934 , 21.32 , 91, 91, 12.93 ) FROM db.table2", + "SELECT (?,...) FROM db.table2" + }, + { + "SELECT (1.1, 1.12934 , 21.32 , 91.2 , 91, 12 ) FROM db.table7", + "SELECT (?,...) FROM db.table7" + }, + { + "SELECT (1.1, 1.12934, 21.32, 391,2381,28.493,1283 ) FROM db.table2", + "SELECT (?,...) FROM db.table2" + }, + { + "SELECT (1.1, 1.12934, 21.32 , 91, 91, 12.1 ) FROM db.table3", + "SELECT (?,...) FROM db.table3" + } }; -const std::vector exp_queries_grouping { - // test query_digest reduction - "SELECT * FROM tablename WHERE id IN (?,...)", - // invalid request grouping - "SELECT * tablename where id IN (?,... AND j in (?,... and k=?", - "SELECT (?,...) FROM db.table", - "SELECT (?,...) FROM db.table2", - "SELECT (?,...) FROM db.table7", - "SELECT (?,...) FROM db.table2", - "SELECT (?,...) FROM db.table3" +const vector> null_queries_digests { + { + "select Null , '1*2/2',NULL,null , '1 ' , 100 % 3 ", + "select Null,?,NULL,null,?,?%?", + "select ?,?,?,?,?,?%?" + }, + // TODO: KnownIssue - 3: Grouping count isn't reset by NULL. + // { + // "INSERT INTO db.table VALUES ( Null , NULL, '',NULL, 'a', 'b', 'z',nuLL)", + // "INSERT INTO db.table VALUES (Null,NULL,?,NULL,?,?,?,nuLL)", + // "INSERT INTO db.table VALUES (?,?,?,...)" + // // Act: INSERT INTO db.table VALUES ( Null,NULL,?,NULL,?,?,...,nuLL) + // }, + { + "INSERT INTO db.table VALUES ( NULL, 'a', 'b', 'z', -4, nuLL)", + // TODO: KnownIssue - 4: Spaces preceding NULL values are not properly removed + "INSERT INTO db.table VALUES ( NULL,?,?,?,...,nuLL)", + "INSERT INTO db.table VALUES (?,?,?,...)" + }, }; std::string replace_str(const std::string& str, const std::string& match, const std::string& repl) { @@ -159,35 +304,145 @@ std::string increase_mark_num(const std::string query, uint32_t num) { return result; } +char is_digit_char(char c) { + if(c >= '0' && c <= '9') { + return 1; + } + return 0; +} + +vector extract_numbers(const string query) { + vector numbers {}; + string number {}; + + for (const char c : query) { + if (is_digit_char(c)) { + number += c; + } else { + if (!number.empty()) { + numbers.push_back(number); + number.clear(); + } + } + } + + return numbers; +} + +string replace_numbers(const string query, const char mark) { + vector numbers { extract_numbers(query) }; + std::sort( + numbers.begin(), numbers.end(), + [](const string& s1, const string& s2) -> bool { return s1.size() > s2.size(); } + ); + + string query_res { query }; + + for (const string& num : numbers) { + query_res = replace_str(query_res, num, string { mark }); + } + + return query_res; +} + int main(int argc, char** argv) { - if (queries.size() != exp_results.size()) { + if (query_digest_pairs.size() != query_digest_pairs.size()) { ok(0, "queries and exp_results sizes doesn't match"); return exit_status(); } + plan( + query_digest_pairs.size()*2 + queries_digests_grouping.size()*5 + null_queries_digests.size()*2 + ); + char buf[QUERY_DIGEST_BUF]; - for (size_t i = 0; i < queries.size(); i++) { - const auto& query = queries[i]; - const auto& exp_res = exp_results[i]; + const auto test_query_digests = [&](bool replace_digits) -> void { + mysql_thread___query_digests_no_digits=replace_digits; - char* c_res = mysql_query_digest_and_first_comment(const_cast(query.c_str()), query.length(), NULL, buf); - std::string result(c_res); + for (size_t i = 0; i < query_digest_pairs.size(); i++) { + const auto& query = query_digest_pairs[i].first; + const auto& query_str_rep = replace_str(query_digest_pairs[i].first, "\n", "\\n"); + std::string exp_res {}; - ok(result == exp_res, "Digest should be equal to exp result: '%s' == '%s'", result.c_str(), exp_res.c_str()); - } + if (replace_digits == false) { + exp_res = query_digest_pairs[i].second; + } else { + exp_res = replace_numbers(query_digest_pairs[i].second, '?'); + } + + char* c_res = mysql_query_digest_and_first_comment(const_cast(query.c_str()), query.length(), NULL, buf); + const std::string result(c_res); + std::string ok_msg {}; + + if (replace_digits == false) { + ok_msg = "Digest should be equal to exp result:\n * Query: `%s`,\n * Act: `%s`,\n * Exp: `%s`"; + } else { + ok_msg = "No-Digits digest should be equal to exp result:\n * Query: `%s`,\n * Act: `%s`,\n * Exp: `%s`"; + } + + ok( + result == exp_res, ok_msg.c_str(), + query_str_rep.c_str(), result.c_str(), exp_res.c_str() + ); + } + + mysql_thread___query_digests_no_digits=0; + }; + + /* Test queries without replacing digits */ + test_query_digests(false); + /* Test queries replacing digits */ + test_query_digests(true); + + const auto test_null_replacting = [&](bool replace_nulls) -> void { + mysql_thread___query_digests_replace_null=replace_nulls; + + for (size_t i = 0; i < null_queries_digests.size(); i++) { + const auto& query = std::get<0>(null_queries_digests[i]); + std::string exp_res {}; + + if (replace_nulls) { + exp_res = std::get<2>(null_queries_digests[i]); + } else { + exp_res = std::get<1>(null_queries_digests[i]); + } + + char* c_res = mysql_query_digest_and_first_comment(const_cast(query.c_str()), query.length(), NULL, buf); + std::string result(c_res); + + ok( + result == exp_res, + "Replaced NULL values digest should be equal to exp result:" + "\n * Query: `%s`,\n * Act: `%s`,\n * Exp: `%s`", + query.c_str(), result.c_str(), exp_res.c_str() + ); + } + + mysql_thread___query_digests_replace_null=0; + }; + + /* Test queries containing 'NULL', NOT replacing the 'NULL' values */ + test_null_replacting(false); + /* Test queries containing 'NULL', replacing the 'NULL' values */ + test_null_replacting(true); - for (size_t i = 0; i < queries_grouping.size(); i++) { + for (size_t i = 0; i < queries_digests_grouping.size(); i++) { for (int j = 1; j <= 5; j++) { mysql_thread___query_digests_grouping_limit = j; - const auto& query = queries_grouping[i]; - const auto& exp_res = increase_mark_num(exp_queries_grouping[i], j); + const auto& query = queries_digests_grouping[i].first; + const auto& exp_res = increase_mark_num(queries_digests_grouping[i].second, j); char* c_res = mysql_query_digest_and_first_comment(const_cast(query.c_str()), query.length(), NULL, buf); std::string result(c_res); - ok(result == exp_res, "Grouping digest should be equal to exp result: '%s' == '%s'", result.c_str(), exp_res.c_str()); + ok( + result == exp_res, + "Grouping digest should be equal to exp result:" + "\n * Query: `%s`,\n * Act: `%s`,\n * Exp: `%s`", + query.c_str(), result.c_str(), exp_res.c_str() + ); } }