Added 'AFL++' stability test for 'mysql_query_digests' impl

test/afl_digest_test/Makefile

@@ -0,0 +1,17 @@
+PROXYSQL_PATH=../..
+PROXYSQL_IDIR=$(PROXYSQL_PATH)/include
+
+CC = afl-g++-fast
+CFLAGS = -Wall -fpermissive -pthread
+OBJS = c_tokenizer.o
+
+all: afl_test
+
+afl_test: $(OBJS)
+	$(CC) $(CFLAGS) $(OBJS) -I$(PROXYSQL_IDIR) afl_mysql_query_digest.cpp -o afl_test
+
+c_tokenizer.o: c_tokenizer.c c_tokenizer.h
+	$(CC) $(CFLAGS) -c c_tokenizer.c
+
+clean:
+	rm -f *~ *.o afl_test

test/afl_digest_test/README.md

@@ -0,0 +1,51 @@
+## Description
+
+This folder provides a AFL++ stability test for fuzzy testing 'mysql_query_digest_and_first_comment_2'
+implementation.
+
+## Usage
+
+For compiling test it's enough to run the following commands in ProxySQL main WORKSPACE folder:
+
+```
+docker run -tid -v $(pwd):/src aflplusplus/aflplusplus
+docker exec -it $(CONTAINER_ID) /bin/bash
+cd /src/test/afl_digest_test/
+make
+```
+
+Then for launching an individual instance of `afl-fuzz` it's enough to run:
+
+```
+mkdir output
+afl-fuzz -M main-$HOSTNAME -i inputs/ -o output/ -- ./afl_test -d 1 -l 1 -n 1 -s 50 -g 0 -G 0
+```
+
+Where the options that can be specified for the fuzzing test are:
+
+```
+AFL fuzz testing for digest parsing
+
+USAGE: afl_test [OPTIONS]
+
+OPTIONS:
+
+-d, --replace-digits ARG          Query digest 'NoDigits'
+-G, --groups-grouping-limit ARG   Query digest 'GroupsGroupingLimit'
+-g, --grouping-limit ARG          Query digest 'GroupingLimit'
+-h, -help, --help, --usage        Display usage instructions.
+-l, --lowercase ARG               Query digest 'LowerCase'
+-n, --replace-null ARG            Query digest 'ReplaceNULL'
+-s, --digest-size ARG             Query digest 'MaxLength'
+
+```
+
+They can be optioned also by running `./afl_test -h`.
+
+## Parallel testing
+
+1\. For launching multiple instances of `afl-fuzz` the `launch_tests.sh` and `stop_tests.sh` scripts could be used.
+2\. For checking the overall progress of the parallel testing the following command can be used:
+```
+afl-whatsup -s output
+```

test/afl_digest_test/afl_mysql_query_digest.cpp

@@ -0,0 +1,180 @@
+#pragma GCC            optimize("O0")
+#define QUERY_DIGEST_BUF 128
+
+#include "c_tokenizer.h"
+#include "proxysql_utils.h"
+#include "ezOptionParser.hpp"
+
+#include <cstddef>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <signal.h>
+#include <string.h>
+#include <limits.h>
+
+#include <string>
+#include <iostream>
+
+__thread int mysql_thread___query_digests_max_query_length = 65000;
+__thread bool mysql_thread___query_digests_lowercase = false;
+__thread bool mysql_thread___query_digests_replace_null = true;
+__thread bool mysql_thread___query_digests_no_digits = false;
+__thread int mysql_thread___query_digests_grouping_limit = 3;
+__thread int mysql_thread___query_digests_groups_grouping_limit = 1;
+
+using std::string;
+using option_err = std::pair<int, string>;
+
+option_err check_and_set_option(ez::ezOptionParser& opts, string opt_id, int* config_option_val) {
+	option_err err_res { EXIT_SUCCESS, "" };
+
+	if (opts.isSet(opt_id)) {
+		string s_option_val {};
+		char* str_end = NULL;
+
+		opts.get(opt_id.c_str())->getString(s_option_val);
+		int option_val = std::strtoll(s_option_val.c_str(), &str_end, 10);
+
+		if (errno != ERANGE) {
+			*config_option_val = option_val;
+		} else {
+			string t_invalid_value_msg { "Invalid '%s' supplied" };
+			string invalid_value_msg {};
+			string_format(t_invalid_value_msg, invalid_value_msg, opt_id.c_str());
+
+			string t_err_msg { "File %s, line %d, Error: %s" };
+			string err_msg {};
+			string_format(t_err_msg, err_msg, __FILE__, __LINE__, invalid_value_msg.c_str());
+
+			err_res = { EXIT_FAILURE, err_msg };
+		}
+	}
+
+	return err_res;
+}
+
+void print_invalid_input_error(const option_err& err) {
+	std::cout << "InvalidInputError: " << err.second << "\n";
+}
+
+option_err parse_parameter_options(int argc, const char** argv) {
+	option_err err_res { EXIT_SUCCESS, "" };
+
+	ez::ezOptionParser opts {};
+	opts.overview = "AFL fuzz testing for digest parsing";
+	opts.syntax = "afl_test [OPTIONS]";
+	opts.footer = "\n\nHappy bug hunting :)";
+
+	opts.add(
+		(const char *)"", 0, 0, 0, (const char *)"Display usage instructions.",
+		(const char *)"-h", (const char *)"-help", (const char *)"--help", (const char *)"--usage"
+	);
+	opts.add(
+		(const char *)"", 1, 1, 0, (const char *)"Query digest 'MaxLength'",
+		(const char *)"-s", (const char *)"--digest-size"
+	);
+	opts.add(
+		(const char *)"", 1, 1, 0, (const char *)"Query digest 'LowerCase'",
+		(const char *)"-l", (const char *)"--lowercase"
+	);
+	opts.add(
+		(const char *)"", 1, 1, 0, (const char *)"Query digest 'ReplaceNULL'",
+		(const char *)"-n", (const char *)"--replace-null"
+	);
+	opts.add(
+		(const char *)"", 1, 1, 0, (const char *)"Query digest 'NoDigits'",
+		(const char *)"-d", (const char *)"--replace-digits"
+	);
+	opts.add(
+		(const char *)"", 1, 1, 0, (const char *)"Query digest 'GroupingLimit'",
+		(const char *)"-g", (const char *)"--grouping-limit"
+	);
+	opts.add(
+		(const char *)"", 1, 1, 0, (const char *)"Query digest 'GroupsGroupingLimit'",
+		(const char *)"-G", (const char *)"--groups-grouping-limit"
+	);
+
+	// parse the arguments
+	opts.parse(argc, argv);
+
+	// extract command line options
+	if (opts.isSet("-h")) {
+		std::string usage {};
+		opts.getUsage(usage);
+		std::cout << usage << std::endl;
+
+		exit(EXIT_SUCCESS);
+	}
+
+	int option_value = 0;
+
+	err_res = check_and_set_option(opts, "-s", &option_value);
+	if (err_res.first != EXIT_SUCCESS) {
+		return err_res;
+	} else {
+		mysql_thread___query_digests_max_query_length = option_value;
+	}
+	err_res = check_and_set_option(opts, "-l", &option_value);
+	if (err_res.first != EXIT_SUCCESS) {
+		return err_res;
+	} else {
+		mysql_thread___query_digests_lowercase = option_value;
+	}
+	err_res = check_and_set_option(opts, "-n", &option_value);
+	if (err_res.first != EXIT_SUCCESS) {
+		return err_res;
+	} else {
+		mysql_thread___query_digests_replace_null = option_value;
+	}
+	err_res = check_and_set_option(opts, "-d", &option_value);
+	if (err_res.first != EXIT_SUCCESS) {
+		return err_res;
+	} else {
+		mysql_thread___query_digests_no_digits = option_value;
+	}
+	err_res = check_and_set_option(opts, "-g", &option_value);
+	if (err_res.first != EXIT_SUCCESS) {
+		return err_res;
+	} else {
+		mysql_thread___query_digests_grouping_limit = option_value;
+	}
+	err_res = check_and_set_option(opts, "-G", &option_value);
+	if (err_res.first != EXIT_SUCCESS) {
+		return err_res;
+	} else {
+		mysql_thread___query_digests_groups_grouping_limit = option_value;
+	}
+
+	return err_res;
+}
+
+void process_digest_test(unsigned char* query, int len) {
+	char buf[QUERY_DIGEST_BUF];
+	char* first_comment = NULL;
+	mysql_query_digest_and_first_comment_2(query, len, &first_comment, ((len < QUERY_DIGEST_BUF) ? buf : NULL));
+}
+
+__AFL_FUZZ_INIT();
+
+int main(int argc, const char** argv) {
+	option_err opt_err = parse_parameter_options(argc, argv);
+	if (opt_err.first != EXIT_SUCCESS) {
+		std::cout << "InvalidSuppliedOption: " << opt_err.second << "\n";
+		return EXIT_FAILURE;
+	}
+
+#ifdef __AFL_HAVE_MANUAL_CONTROL
+	__AFL_INIT();
+#endif
+
+	unsigned char *buf = __AFL_FUZZ_TESTCASE_BUF; 
+	fflush(stdin);
+
+	while (__AFL_LOOP(10000)) {
+		int len = __AFL_FUZZ_TESTCASE_LEN;
+		process_digest_test(buf, len);
+	}
+
+	return 0;
+}

test/afl_digest_test/c_tokenizer.c

@@ -0,0 +1 @@
+../../lib/c_tokenizer.c

test/afl_digest_test/c_tokenizer.h

@@ -0,0 +1 @@
+../../include/c_tokenizer.h

test/afl_digest_test/inputs/sql_input_1.txt

@@ -0,0 +1 @@
+create table table_10_utf8_4 (\n`pk` int primary key,\n`col_bigint_undef_signed` bigint ,\n`col_bigint_undef_unsigned` bigint unsigned ,\n`col_bigint_key_signed` bigint ,\n`col_bigint_key_unsigned` bigint unsigned ,\n`col_float_undef_signed` float ,\n`col_float_undef_unsigned` float unsigned ,\n`col_float_key_signed` float ,\n`col_float_key_unsigned` float unsigned ,\n`col_double_undef_signed` double ,\n`col_double_undef_unsigned` double unsigned ,\n`col_double_key_signed` double ,\n`col_double_key_unsigned` double unsigned ,\n`col_decimal(40, 20)_undef_signed` decimal(40, 20) ,\n`col_decimal(40, 20)_undef_unsigned` decimal(40, 20) unsigned ,\n`col_decimal(40, 20)_key_signed` decimal(40, 20)  ,\n`col_decimal(40, 20)_key_unsigned` decimal(40, 20) unsigned ,\n`col_char(20)_undef_signed` char(20)  ,\n`col_char(20)_key_signed` char(20) ,\n`col_varchar(20)_undef_signed` varchar(20)  ,\n`col_varchar(20)_key_signed` varchar(20) ,\n`col_enum_undef_signed` enum('a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z') ,\n`col_enum_key_signed` enum('a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z') ,\nkey (`col_bigint_key_signed`),\nkey (`col_bigint_key_unsigned`),\nkey (`col_float_key_signed`),\nkey (`col_float_key_unsigned`),\nkey (`col_double_key_signed`),\nkey (`col_double_key_unsigned`),\nkey (`col_decimal(40, 20)_key_signed`),\nkey (`col_decimal(40, 20)_key_unsigned`),\nkey (`col_char(20)_key_signed`),\nkey (`col_varchar(20)_key_signed`),\nkey (`col_enum_key_signed`)\n) character set utf8 \npartition by hash(pk)\npartitions 4

test/afl_digest_test/inputs/sql_input_3.txt

@@ -0,0 +1 @@
+# WeirdFirstComment\nINSERT /* fst_comment */ INTO /*! random_comment */ db.table ( col1, col2,col3,col4, col5 ) VALUES         ('val',1, 2,3,'foo'), ('val2',3,NULL,4,'foo2'),         ('val2', 5,0x0239192,4,'foo2'), ('val2', 7,NULL,4,'foo2'),               (1.1E+9, 2.9E-9, 0x23914993, 928.2939123), ('val2',3,NULL,4,'foo2'), ('val2',3*3.293192493419231,NULL,4+2,'foo2'), ('val2', "9212312",NULL,92.1293123,"foo2"),                     ON DUPLICATE KEY UPDATE col1 = VALUES(col2) -- final_comment  \n

test/afl_digest_test/launch_tests.sh

@@ -0,0 +1,7 @@
+screen -d -m afl-fuzz -M main-$HOSTNAME -i inputs/ -o output/ -- ./afl_test -d 1 -l 1 -n 1 -s 50 -g 0 -G 0
+
+screen -d -m afl-fuzz -S variant-1 -i inputs/ -o output/ -- ./afl_test -d 1 -l 1 -n 1 -s 50 -g 1 -G 1
+screen -d -m afl-fuzz -S variant-2 -i inputs/ -o output/ -- ./afl_test -d 1 -l 1 -n 1 -s 100 -g 2 -G 2
+screen -d -m afl-fuzz -S variant-3 -i inputs/ -o output/ -- ./afl_test -d 1 -l 1 -n 1 -s 128 -g 3 -G 3
+screen -d -m afl-fuzz -S variant-4 -i inputs/ -o output/ -- ./afl_test -d 1 -l 1 -n 1 -s 300 -g 4 -G 4
+screen -d -m afl-fuzz -S variant-5 -i inputs/ -o output/ -- ./afl_test -d 1 -l 1 -n 1 -s 1000 -g 5 -G 5

test/afl_digest_test/stop_tests.sh

@@ -0,0 +1 @@
+screen -ls | grep '(Detached)' | awk '{print $1}' | xargs -I % -t screen -X -S % quit