Merge pull request #5487 from sysown/v3.0-5476

Query Processor rule management unit tests (Phase 2.4)

test/tap/test_helpers/test_init.cpp

@@ -33,6 +33,10 @@ extern PgSQL_Query_Cache *GloPgQC;
 extern MySQL_Query_Processor *GloMyQPro;
 extern PgSQL_Query_Processor *GloPgQPro;
 
+// GloMTH is declared extern in proxysql_utils.h.
+// GloPTH has no extern declaration in any header, so we add one here.
+extern PgSQL_Threads_Handler *GloPTH;
+
 // ============================================================================
 // Minimal initialization
 // ============================================================================
@@ -118,6 +122,32 @@ int test_init_query_processor() {
 		return 0;
 	}
 
+	// Query Processor constructors register Prometheus metrics and
+	// read variables from GloMTH/GloPTH. Ensure both are available.
+	if (GloVars.prometheus_registry == nullptr) {
+		GloVars.prometheus_registry = std::make_shared<prometheus::Registry>();
+	}
+	if (GloMTH == nullptr) {
+		GloMTH = new MySQL_Threads_Handler();
+	}
+	if (GloPTH == nullptr) {
+		GloPTH = new PgSQL_Threads_Handler();
+	}
+
+	// Trigger lazy initialization of VariablesPointers maps.
+	// The QP constructor calls get_variable_int() which requires
+	// these maps to be populated.
+	char **vl = GloMTH->get_variables_list();
+	if (vl) {
+		for (char **p = vl; *p != nullptr; ++p) free(*p);
+		free(vl);
+	}
+	vl = GloPTH->get_variables_list();
+	if (vl) {
+		for (char **p = vl; *p != nullptr; ++p) free(*p);
+		free(vl);
+	}
+
 	GloMyQPro = new MySQL_Query_Processor();
 	GloPgQPro = new PgSQL_Query_Processor();
 
@@ -133,4 +163,7 @@ void test_cleanup_query_processor() {
 		delete GloPgQPro;
 		GloPgQPro = nullptr;
 	}
+	// NOTE: We do NOT delete GloMTH/GloPTH here because other
+	// components may still reference them. Their cleanup relies
+	// on process exit.
 }

test/tap/tests/unit/Makefile

@@ -231,7 +231,7 @@ $(ODIR)/test_init.o: $(TEST_HELPERS_DIR)/test_init.cpp | $(ODIR)
 # Unit test targets
 # ===========================================================================
 
-UNIT_TESTS := smoke_test-t query_cache_unit-t
+UNIT_TESTS := smoke_test-t query_cache_unit-t query_processor_unit-t
 
 .PHONY: all
 all: $(UNIT_TESTS)
@@ -255,6 +255,11 @@ query_cache_unit-t: query_cache_unit-t.cpp $(TEST_HELPERS_OBJ) $(LIBPROXYSQLAR)
 		$(LIBPROXYSQLAR_FULL) $(STATIC_LIBS) $(MYLIBS) \
 		$(ALLOW_MULTI_DEF) -o $@
 
+query_processor_unit-t: query_processor_unit-t.cpp $(TEST_HELPERS_OBJ) $(LIBPROXYSQLAR)
+	$(CXX) $< $(TEST_HELPERS_OBJ) $(IDIRS) $(LDIRS) $(OPT) \
+		$(LIBPROXYSQLAR_FULL) $(STATIC_LIBS) $(MYLIBS) \
+		$(ALLOW_MULTI_DEF) -o $@
+
 
 # ===========================================================================
 # Clean

test/tap/tests/unit/query_processor_unit-t.cpp

@@ -0,0 +1,528 @@
+/**
+ * @file query_processor_unit-t.cpp
+ * @brief Unit tests for MySQL_Query_Processor and PgSQL_Query_Processor.
+ *
+ * Tests the query processor rule management in isolation without a
+ * running ProxySQL instance. Covers:
+ *   - Rule creation via new_query_rule() factory method
+ *   - Rule field storage and retrieval
+ *   - Rule insertion, sorting by rule_id, and commit
+ *   - Rule retrieval via get_current_query_rules() (SQLite3 result)
+ *   - Regex modifier parsing (CASELESS, GLOBAL)
+ *   - Active/inactive rule filtering
+ *   - PgSQL rule parity
+ *
+ * @note Full process_query() testing requires a MySQL_Session with
+ *       populated connection data (username, schema, client address),
+ *       which is beyond the scope of isolated unit tests. Those
+ *       scenarios are covered by the existing E2E TAP tests.
+ *
+ * @see Phase 2.4 of the Unit Testing Framework (GitHub issue #5476)
+ */
+
+#include "tap.h"
+#include "test_globals.h"
+#include "test_init.h"
+
+#include "proxysql.h"
+#include "MySQL_Query_Processor.h"
+#include "PgSQL_Query_Processor.h"
+
+#include <cstring>
+
+// Extern declarations for Glo* pointers (defined in test_globals.cpp)
+extern MySQL_Query_Processor *GloMyQPro;
+extern PgSQL_Query_Processor *GloPgQPro;
+
+// ============================================================================
+// Helpers
+// ============================================================================
+
+/**
+ * @brief Create a simple MySQL query rule with common defaults.
+ *
+ * Most fields default to -1/NULL/false (no match / no action).
+ * Only rule_id, active, and explicitly passed fields are set.
+ */
+static MySQL_Query_Processor_Rule_t *mysql_simple_rule(
+	int rule_id, bool active,
+	const char *match_pattern = nullptr,
+	int destination_hostgroup = -1,
+	bool apply = false,
+	const char *username = nullptr,
+	int flagIN = 0, int flagOUT = -1)
+{
+	return MySQL_Query_Processor::new_query_rule(
+		rule_id, active,
+		username,               // username
+		nullptr,                // schemaname
+		flagIN,                 // flagIN
+		nullptr,                // client_addr
+		nullptr,                // proxy_addr
+		-1,                     // proxy_port
+		nullptr,                // digest
+		nullptr,                // match_digest
+		match_pattern,          // match_pattern
+		false,                  // negate_match_pattern
+		nullptr,                // re_modifiers
+		flagOUT,                // flagOUT
+		nullptr,                // replace_pattern
+		destination_hostgroup,  // destination_hostgroup
+		-1,                     // cache_ttl
+		-1,                     // cache_empty_result
+		-1,                     // cache_timeout
+		-1,                     // reconnect
+		-1,                     // timeout
+		-1,                     // retries
+		-1,                     // delay
+		-1,                     // next_query_flagIN
+		-1,                     // mirror_flagOUT
+		-1,                     // mirror_hostgroup
+		nullptr,                // error_msg
+		nullptr,                // OK_msg
+		-1,                     // sticky_conn
+		-1,                     // multiplex
+		-1,                     // gtid_from_hostgroup
+		-1,                     // log
+		apply,                  // apply
+		nullptr,                // attributes
+		nullptr                 // comment
+	);
+}
+
+/**
+ * @brief Create a simple PgSQL query rule with common defaults.
+ */
+static PgSQL_Query_Processor_Rule_t *pgsql_simple_rule(
+	int rule_id, bool active,
+	const char *match_pattern = nullptr,
+	int destination_hostgroup = -1,
+	bool apply = false)
+{
+	return PgSQL_Query_Processor::new_query_rule(
+		rule_id, active,
+		nullptr, nullptr,       // username, schemaname
+		0,                      // flagIN
+		nullptr, nullptr, -1,   // client_addr, proxy_addr, proxy_port
+		nullptr,                // digest
+		nullptr,                // match_digest
+		match_pattern,          // match_pattern
+		false,                  // negate_match_pattern
+		nullptr,                // re_modifiers
+		-1,                     // flagOUT
+		nullptr,                // replace_pattern
+		destination_hostgroup,  // destination_hostgroup
+		-1, -1, -1,            // cache_ttl, cache_empty_result, cache_timeout
+		-1, -1, -1, -1,        // reconnect, timeout, retries, delay
+		-1, -1, -1,            // next_query_flagIN, mirror_flagOUT, mirror_hostgroup
+		nullptr, nullptr,       // error_msg, OK_msg
+		-1, -1,                 // sticky_conn, multiplex
+		-1,                     // log
+		apply,                  // apply
+		nullptr, nullptr        // attributes, comment
+	);
+}
+
+// ============================================================================
+// 1. Rule creation via new_query_rule()
+// ============================================================================
+
+/**
+ * @brief Test that new_query_rule() allocates and populates a rule.
+ */
+static void test_mysql_rule_creation() {
+	auto *rule = MySQL_Query_Processor::new_query_rule(
+		100,                    // rule_id
+		true,                   // active
+		"testuser",             // username
+		"testdb",               // schemaname
+		0,                      // flagIN
+		"192.168.1.%",          // client_addr
+		nullptr,                // proxy_addr
+		-1,                     // proxy_port
+		nullptr,                // digest
+		"^SELECT",              // match_digest
+		"SELECT.*FROM users",   // match_pattern
+		false,                  // negate_match_pattern
+		"CASELESS",             // re_modifiers
+		-1,                     // flagOUT
+		nullptr,                // replace_pattern
+		5,                      // destination_hostgroup
+		3000,                   // cache_ttl
+		1,                      // cache_empty_result
+		-1,                     // cache_timeout
+		-1,                     // reconnect
+		5000,                   // timeout
+		3,                      // retries
+		-1,                     // delay
+		-1,                     // next_query_flagIN
+		-1,                     // mirror_flagOUT
+		-1,                     // mirror_hostgroup
+		nullptr,                // error_msg
+		nullptr,                // OK_msg
+		1,                      // sticky_conn
+		0,                      // multiplex
+		-1,                     // gtid_from_hostgroup
+		1,                      // log
+		true,                   // apply
+		nullptr,                // attributes
+		"route reads to HG 5"  // comment
+	);
+
+	ok(rule != nullptr, "MySQL QP: new_query_rule() returns non-null");
+	ok(rule->rule_id == 100, "MySQL QP: rule_id is correct");
+	ok(rule->active == true, "MySQL QP: active is correct");
+	ok(rule->username != nullptr && strcmp(rule->username, "testuser") == 0,
+		"MySQL QP: username is stored correctly");
+	ok(rule->schemaname != nullptr && strcmp(rule->schemaname, "testdb") == 0,
+		"MySQL QP: schemaname is stored correctly");
+	ok(rule->match_digest != nullptr && strcmp(rule->match_digest, "^SELECT") == 0,
+		"MySQL QP: match_digest is stored correctly");
+	ok(rule->match_pattern != nullptr && strcmp(rule->match_pattern, "SELECT.*FROM users") == 0,
+		"MySQL QP: match_pattern is stored correctly");
+	ok(rule->destination_hostgroup == 5,
+		"MySQL QP: destination_hostgroup is correct");
+	ok(rule->cache_ttl == 3000,
+		"MySQL QP: cache_ttl is correct");
+	ok(rule->timeout == 5000,
+		"MySQL QP: timeout is correct");
+	ok(rule->retries == 3,
+		"MySQL QP: retries is correct");
+	ok(rule->sticky_conn == 1,
+		"MySQL QP: sticky_conn is correct");
+	ok(rule->multiplex == 0,
+		"MySQL QP: multiplex is correct");
+	ok(rule->apply == true,
+		"MySQL QP: apply flag is correct");
+	ok(rule->comment != nullptr && strcmp(rule->comment, "route reads to HG 5") == 0,
+		"MySQL QP: comment is stored correctly");
+	ok(rule->log == 1,
+		"MySQL QP: log is correct");
+	ok(rule->client_addr != nullptr && strcmp(rule->client_addr, "192.168.1.%") == 0,
+		"MySQL QP: client_addr is stored correctly");
+
+	// Verify re_modifiers parsed correctly
+	ok(rule->re_modifiers & QP_RE_MOD_CASELESS,
+		"MySQL QP: CASELESS re_modifier is set");
+
+	// This rule is not inserted into the QP, so we free it manually.
+	free(rule->username); free(rule->schemaname);
+	free(rule->match_digest); free(rule->match_pattern);
+	free(rule->client_addr); free(rule->comment);
+	free(rule);
+}
+
+// ============================================================================
+// 2. Rule insertion and retrieval via get_current_query_rules()
+// ============================================================================
+
+/**
+ * @brief Test inserting rules and retrieving them via SQL result set.
+ */
+static void test_mysql_insert_and_retrieve() {
+	// Create and insert rules
+	auto *r1 = mysql_simple_rule(10, true, "^SELECT", 1, true);
+	auto *r2 = mysql_simple_rule(20, true, "^INSERT", 2, true);
+	auto *r3 = mysql_simple_rule(30, false, "^DELETE", 3, true);  // inactive
+
+	GloMyQPro->insert((QP_rule_t *)r1);
+	GloMyQPro->insert((QP_rule_t *)r2);
+	GloMyQPro->insert((QP_rule_t *)r3);
+	GloMyQPro->sort();
+	GloMyQPro->commit();
+
+	SQLite3_result *result = GloMyQPro->get_current_query_rules();
+	ok(result != nullptr, "MySQL QP: get_current_query_rules() returns non-null");
+
+	if (result != nullptr) {
+		ok(result->rows_count == 3,
+			"MySQL QP: get_current_query_rules() returns 3 rules");
+		delete result;
+	} else {
+		ok(0, "MySQL QP: get_current_query_rules() returns 3 rules (skipped)");
+	}
+}
+
+/**
+ * @brief Test that rules are sorted by rule_id after sort().
+ */
+static void test_mysql_rule_sorting() {
+	// Reset rules
+	GloMyQPro->reset_all(true);
+
+	// Insert in reverse order
+	auto *r3 = mysql_simple_rule(300, true, nullptr, 3);
+	auto *r1 = mysql_simple_rule(100, true, nullptr, 1);
+	auto *r2 = mysql_simple_rule(200, true, nullptr, 2);
+
+	GloMyQPro->insert((QP_rule_t *)r3);
+	GloMyQPro->insert((QP_rule_t *)r1);
+	GloMyQPro->insert((QP_rule_t *)r2);
+	GloMyQPro->sort();
+	GloMyQPro->commit();
+
+	SQLite3_result *result = GloMyQPro->get_current_query_rules();
+	ok(result != nullptr && result->rows_count == 3,
+		"MySQL QP: 3 rules after insert in reverse order");
+
+	if (result != nullptr && result->rows_count == 3) {
+		// First row should be rule_id=100 (sorted ascending)
+		auto it = result->rows.begin();
+		ok(strcmp((*it)->fields[0], "100") == 0,
+			"MySQL QP: first rule after sort has rule_id=100");
+		++it;
+		ok(strcmp((*it)->fields[0], "200") == 0,
+			"MySQL QP: second rule after sort has rule_id=200");
+		++it;
+		ok(strcmp((*it)->fields[0], "300") == 0,
+			"MySQL QP: third rule after sort has rule_id=300");
+		delete result;
+	} else {
+		ok(0, "MySQL QP: first rule sorted (skipped)");
+		ok(0, "MySQL QP: second rule sorted (skipped)");
+		ok(0, "MySQL QP: third rule sorted (skipped)");
+		if (result) delete result;
+	}
+}
+
+// ============================================================================
+// 3. Regex modifier parsing
+// ============================================================================
+
+/**
+ * @brief Test re_modifiers parsing for CASELESS, GLOBAL, and combined.
+ */
+static void test_regex_modifiers() {
+	auto *r1 = mysql_simple_rule(1, true);
+	ok((r1->re_modifiers & QP_RE_MOD_CASELESS) == 0,
+		"MySQL QP: no modifiers when re_modifiers is null");
+	free(r1);
+
+	// Create rule with CASELESS
+	auto *r2 = MySQL_Query_Processor::new_query_rule(
+		2, true, nullptr, nullptr, 0, nullptr, nullptr, -1,
+		nullptr, nullptr, "test", false, "CASELESS",
+		-1, nullptr, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+		nullptr, nullptr, -1, -1, -1, -1, false, nullptr, nullptr);
+	ok((r2->re_modifiers & QP_RE_MOD_CASELESS) != 0,
+		"MySQL QP: CASELESS modifier parsed");
+	ok((r2->re_modifiers & QP_RE_MOD_GLOBAL) == 0,
+		"MySQL QP: GLOBAL not set when only CASELESS specified");
+	free(r2->match_pattern);
+	free(r2);
+
+	// Create rule with CASELESS,GLOBAL
+	auto *r3 = MySQL_Query_Processor::new_query_rule(
+		3, true, nullptr, nullptr, 0, nullptr, nullptr, -1,
+		nullptr, nullptr, "test", false, "CASELESS,GLOBAL",
+		-1, nullptr, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
+		nullptr, nullptr, -1, -1, -1, -1, false, nullptr, nullptr);
+	ok((r3->re_modifiers & QP_RE_MOD_CASELESS) != 0,
+		"MySQL QP: CASELESS set in combined modifiers");
+	ok((r3->re_modifiers & QP_RE_MOD_GLOBAL) != 0,
+		"MySQL QP: GLOBAL set in combined modifiers");
+	free(r3->match_pattern);
+	free(r3);
+}
+
+// ============================================================================
+// 4. Rule with all match fields populated
+// ============================================================================
+
+/**
+ * @brief Test rule with error_msg, OK_msg, and replace_pattern.
+ */
+static void test_mysql_rule_special_fields() {
+	auto *rule = MySQL_Query_Processor::new_query_rule(
+		50, true,
+		nullptr, nullptr,       // username, schemaname
+		0,                      // flagIN
+		nullptr, nullptr, -1,   // client_addr, proxy_addr, proxy_port
+		nullptr,                // digest
+		nullptr,                // match_digest
+		"^BLOCKED",             // match_pattern
+		false,                  // negate_match_pattern
+		nullptr,                // re_modifiers
+		-1,                     // flagOUT
+		"REWRITTEN",            // replace_pattern
+		-1,                     // destination_hostgroup
+		-1, -1, -1,            // cache_ttl, cache_empty_result, cache_timeout
+		-1, -1, -1, -1,        // reconnect, timeout, retries, delay
+		-1, -1, -1,            // next_query_flagIN, mirror_flagOUT, mirror_hostgroup
+		"Access denied",        // error_msg
+		"Query OK",             // OK_msg
+		-1, -1, -1, -1,        // sticky_conn, multiplex, gtid_from_hostgroup, log
+		true, nullptr, nullptr  // apply, attributes, comment
+	);
+
+	ok(rule->error_msg != nullptr && strcmp(rule->error_msg, "Access denied") == 0,
+		"MySQL QP: error_msg stored correctly");
+	ok(rule->OK_msg != nullptr && strcmp(rule->OK_msg, "Query OK") == 0,
+		"MySQL QP: OK_msg stored correctly");
+	ok(rule->replace_pattern != nullptr && strcmp(rule->replace_pattern, "REWRITTEN") == 0,
+		"MySQL QP: replace_pattern stored correctly");
+
+	free(rule->match_pattern); free(rule->replace_pattern);
+	free(rule->error_msg); free(rule->OK_msg);
+	free(rule);
+}
+
+// ============================================================================
+// 5. flagIN/flagOUT chaining
+// ============================================================================
+
+/**
+ * @brief Test rule creation with flagIN/flagOUT for chain matching.
+ */
+static void test_mysql_flag_chaining() {
+	GloMyQPro->reset_all(true);
+
+	// Rule 1: flagIN=0, flagOUT=1 (passes to next stage)
+	auto *r1 = mysql_simple_rule(10, true, nullptr, -1, false,
+		nullptr, 0, 1);
+	// Rule 2: flagIN=1, applies with destination
+	auto *r2 = mysql_simple_rule(20, true, nullptr, 5, true,
+		nullptr, 1, -1);
+
+	GloMyQPro->insert((QP_rule_t *)r1);
+	GloMyQPro->insert((QP_rule_t *)r2);
+	GloMyQPro->sort();
+	GloMyQPro->commit();
+
+	SQLite3_result *result = GloMyQPro->get_current_query_rules();
+	ok(result != nullptr && result->rows_count == 2,
+		"MySQL QP: 2 chained rules inserted correctly");
+	if (result) delete result;
+}
+
+// ============================================================================
+// 6. Rule with username filter
+// ============================================================================
+
+/**
+ * @brief Test rule creation with username filter.
+ */
+static void test_mysql_rule_with_username() {
+	GloMyQPro->reset_all(true);
+
+	auto *r1 = mysql_simple_rule(10, true, "^SELECT", 1, true, "admin");
+	auto *r2 = mysql_simple_rule(20, true, "^SELECT", 2, true, "readonly");
+
+	GloMyQPro->insert((QP_rule_t *)r1);
+	GloMyQPro->insert((QP_rule_t *)r2);
+	GloMyQPro->sort();
+	GloMyQPro->commit();
+
+	SQLite3_result *result = GloMyQPro->get_current_query_rules();
+	ok(result != nullptr && result->rows_count == 2,
+		"MySQL QP: 2 rules with username filters inserted");
+	if (result) delete result;
+}
+
+// ============================================================================
+// 7. Reset all rules
+// ============================================================================
+
+/**
+ * @brief Test reset_all() clears all rules.
+ */
+static void test_mysql_reset_all() {
+	GloMyQPro->reset_all(true);
+
+	GloMyQPro->insert((QP_rule_t *)mysql_simple_rule(10, true));
+	GloMyQPro->insert((QP_rule_t *)mysql_simple_rule(20, true));
+	GloMyQPro->commit();
+
+	GloMyQPro->reset_all(true);
+
+	SQLite3_result *result = GloMyQPro->get_current_query_rules();
+	ok(result != nullptr && result->rows_count == 0,
+		"MySQL QP: no rules after reset_all()");
+	if (result) delete result;
+}
+
+// ============================================================================
+// 8. Stats commands counters
+// ============================================================================
+
+/**
+ * @brief Test get_stats_commands_counters() returns a valid result.
+ */
+static void test_mysql_stats_counters() {
+	SQLite3_result *result = GloMyQPro->get_stats_commands_counters();
+	ok(result != nullptr,
+		"MySQL QP: get_stats_commands_counters() returns non-null");
+	if (result != nullptr) {
+		ok(result->columns > 0,
+			"MySQL QP: stats counters result has columns");
+		delete result;
+	} else {
+		ok(0, "MySQL QP: stats counters result has columns (skipped)");
+	}
+}
+
+// ============================================================================
+// 9. PgSQL Query Processor: Basic operations
+// ============================================================================
+
+/**
+ * @brief Test PgSQL rule creation and insertion.
+ */
+static void test_pgsql_rule_creation_and_insert() {
+	auto *rule = pgsql_simple_rule(10, true, "^SELECT", 1, true);
+	ok(rule != nullptr, "PgSQL QP: new_query_rule() returns non-null");
+	ok(rule->rule_id == 10, "PgSQL QP: rule_id is correct");
+	ok(rule->destination_hostgroup == 1,
+		"PgSQL QP: destination_hostgroup is correct");
+
+	GloPgQPro->insert((QP_rule_t *)rule);
+	GloPgQPro->sort();
+	GloPgQPro->commit();
+
+	SQLite3_result *result = GloPgQPro->get_current_query_rules();
+	ok(result != nullptr && result->rows_count >= 1,
+		"PgSQL QP: rule appears in get_current_query_rules()");
+	if (result) delete result;
+}
+
+/**
+ * @brief Test PgSQL reset and stats.
+ */
+static void test_pgsql_reset_and_stats() {
+	GloPgQPro->reset_all(true);
+	SQLite3_result *result = GloPgQPro->get_current_query_rules();
+	ok(result != nullptr && result->rows_count == 0,
+		"PgSQL QP: no rules after reset_all()");
+	if (result) delete result;
+}
+
+// ============================================================================
+// Main
+// ============================================================================
+
+int main() {
+	plan(42);
+
+	test_init_minimal();
+	test_init_query_processor();
+
+	// MySQL tests
+	test_mysql_rule_creation();              // 18 tests
+	test_mysql_insert_and_retrieve();        // 2 tests
+	test_mysql_rule_sorting();               // 4 tests
+	test_regex_modifiers();                  // 5 tests
+	test_mysql_rule_special_fields();        // 3 tests
+	test_mysql_flag_chaining();              // 1 test
+	test_mysql_rule_with_username();         // 1 test
+	test_mysql_reset_all();                  // 1 test
+	test_mysql_stats_counters();             // 2 tests
+
+	// PgSQL tests
+	test_pgsql_rule_creation_and_insert();   // 4 tests
+	test_pgsql_reset_and_stats();            // 1 test
+
+	test_cleanup_query_processor();
+	test_cleanup_minimal();
+
+	return exit_status();
+}