Added unit test file for tokenizer functions

test/tap/tests/Makefile

@@ -75,13 +75,13 @@ all: tests
 
 .PHONY: clean
 clean:
-	rm -f basic-t set_character_set-t charset_unsigned_int-t select_config_file-t sqlite3-t galera_1_timeout_count galera_2_timeout_no_count aurora test_set_character_results-t test_ps_large_result-t set_testing-t test_firewall-t || true
+	rm -f basic-t set_character_set-t charset_unsigned_int-t select_config_file-t sqlite3-t galera_1_timeout_count galera_2_timeout_no_count aurora test_set_character_results-t test_ps_large_result-t set_testing-t test_firewall-t test_tokenizer-t|| true
 
 OPT=-O2
 
-SRC=basic-t.cpp set_character_set-t.cpp charset_unsigned_int-t.cpp select_config_file-t.cpp sqlite3-t.cpp  galera_1_timeout_count.cpp galera_2_timeout_no_count.cpp aurora.cpp test_set_character_results-t.cpp test_ps_large_result-t.cpp set_testing-t.cpp test_firewall-t.cpp
+SRC=basic-t.cpp set_character_set-t.cpp charset_unsigned_int-t.cpp select_config_file-t.cpp sqlite3-t.cpp  galera_1_timeout_count.cpp galera_2_timeout_no_count.cpp aurora.cpp test_set_character_results-t.cpp test_ps_large_result-t.cpp set_testing-t.cpp test_firewall-t.cpp test_tokenizer-t.cpp
 
-tests: basic-t set_character_set-t charset_unsigned_int-t select_config_file-t sqlite3-t test_set_character_results-t test_ps_large_result-t set_testing-t test_firewall-t
+tests: basic-t set_character_set-t charset_unsigned_int-t select_config_file-t sqlite3-t test_set_character_results-t test_ps_large_result-t set_testing-t test_firewall-t test_tokenizer-t
 testgalera: galera_1_timeout_count galera_2_timeout_no_count
 testaurora: aurora
 
@@ -120,3 +120,6 @@ aurora: aurora.cpp $(TAP_LIBDIR)/libtap.a
 
 set_testing-t: set_testing-t.cpp $(TAP_LIBDIR)/libtap.a
 	g++ set_testing-t.cpp -Wall $(INCLUDEDIRS) $(LDIRS) $(OPT) -std=c++11 $(MYLIBS) -ltap -pthread -O0 -ggdb -ldl -lssl -lcrypto -o set_testing-t
+
+test_tokenizer-t: test_tokenizer-t.cpp $(TAP_LIBDIR)/libtap.a
+	g++ test_tokenizer-t.cpp $(INCLUDEDIRS) $(LDIRS) $(OPT) -std=c++11 $(MYLIBS) -lproxysql -ltap -Wl,--no-as-needed -ldl -lpthread -o test_tokenizer-t

test/tap/tests/test_tokenizer-t.cpp

@@ -0,0 +1,98 @@
+#include <vector>
+#include <string>
+#include <cstring>
+
+#include "proxysql.h"
+
+#include "tap.h"
+#include "command_line.h"
+
+#include "utils.h"
+
+#include <ctype.h>
+
+__thread int mysql_thread___query_digests_max_query_length = 65000;
+__thread bool mysql_thread___query_digests_lowercase = false;
+__thread bool mysql_thread___query_digests_replace_null = false;
+__thread bool mysql_thread___query_digests_no_digits = false;
+
+const std::vector<std::string> queries {
+	// floats
+	"select 1.1",
+	"select 1192.1102",
+	"select 99.1929",
+	// exponentials
+	"select 1.1e9",
+	"select 1.1e+9",
+	"select 1.1e-9",
+	// operators
+	"select 1 +1",
+	"select 1+ 1",
+	"select 1- 1",
+	"select 1 -1",
+	"select 1* 1",
+	"select 1 *1",
+	"select 1/ 1",
+	"select 1 /1",
+	"select 1% 1",
+	"select 1 %1",
+	// operators and commas
+	"select 1+ 1, 1 -1, 1 * 1 , 1/1 , 100 % 3",
+	"SELECT * FROM t t1, t t2 ,t t3,t t4 LIMIT 0",
+	// strings
+	"select * from t where t = \"foo\"",
+	"select \"1+ 1, 1 -1, 1 * 1 , 1/1 , 100 % 3\"",
+	// not modified
+	"select * fromt t"
+};
+
+const std::vector<std::string> exp_results {
+	// floats
+	"select ?",
+	"select ?",
+	"select ?",
+	// exponentials
+	"select ?",
+	"select ?",
+	"select ?",
+	// operators
+	"select ?+?",
+	"select ?+?",
+	"select ?-?",
+	"select ?-?",
+	"select ?*?",
+	"select ?*?",
+	"select ?/?",
+	"select ?/?",
+	"select ?%?",
+	"select ?%?",
+	// operators and commas
+	"select ?+?,?-?,?*?,?/?,?%?",
+	"SELECT * FROM t t1,t t2,t t3,t t4 LIMIT ?",
+	// strings
+	"select * from t where t = ?",
+	"select ?",
+	// not modified
+	"select * fromt t"
+};
+
+int main(int argc, char** argv) {
+	if (queries.size() != exp_results.size()) {
+		ok(0, "queries and exp_results sizes doesn't match");
+		return exit_status();
+	}
+
+	char buf[QUERY_DIGEST_BUF];
+
+	for (size_t i = 0; i < queries.size(); i++) {
+		const auto& query = queries[i];
+		const auto& exp_res = exp_results[i];
+
+		char* c_res = mysql_query_digest_and_first_comment(const_cast<char*>(query.c_str()), query.length(), NULL, buf);
+		std::string result(c_res);
+
+		ok(result == exp_res, "result isn't equal to exp result: '%s' != '%s'", result.c_str(), exp_res.c_str());
+	}
+
+	return exit_status();
+}