diff --git a/test/tap/NOISE_TESTING.md b/test/tap/NOISE_TESTING.md new file mode 100644 index 000000000..b4c9e3faa --- /dev/null +++ b/test/tap/NOISE_TESTING.md @@ -0,0 +1,81 @@ +# ProxySQL TAP Test Noise Injection Framework + +The Noise Injection framework (Approach 2) is designed to increase the complexity and realism of functional TAP tests by introducing concurrent background activity. This helps identify race conditions, deadlocks, and stability issues that might not surface in single-threaded functional tests. + +## Overview + +When enabled, a TAP test can spawn one or more background "noise" tools. These tools run independently of the test logic, generating load against various ProxySQL interfaces (MySQL, PostgreSQL, Admin, Stats). + +- **Global Toggle:** Controlled by an environment variable. +- **Automatic Cleanup:** All spawned tools are automatically killed when the test finishes via `exit_status()`. +- **Isolation:** Noise tools run in their own process groups with I/O redirected to `/dev/null` to avoid polluting TAP output. + +## Configuration + +### Environment Variable +The framework is globally controlled by the `TAP_USE_NOISE` environment variable. + +| Value | Effect | +| :--- | :--- | +| `1` or `true` | Enables noise injection. | +| `0` or `false` (default) | Disables noise injection. `spawn_noise()` becomes a no-op. | + +### Path Resolution +Noise tools are typically located in `test/tap/noise/`. When calling `spawn_noise`, you can provide the relative path to these scripts or absolute paths to system binaries. + +## Standard Noise Tools + +Initial noise scripts are provided in `test/tap/noise/`: + +1. **`noise_stats_poller.py`**: + * **Action**: Periodically queries `stats_mysql_query_digest` and `stats_mysql_connection_pool`. + * **Arguments**: `--host`, `--port`, `--user`, `--password`, `--interval`. +2. **`noise_admin_pinger.sh`**: + * **Action**: Executes `SELECT 1` against the Admin interface. + * **Arguments**: `[host] [port] [user] [pass] [interval]`. +3. **`noise_pgsql_poller.sh`**: + * **Action**: Generates simple PostgreSQL traffic using `psql`. + * **Arguments**: `[host] [port] [user] [pass] [interval]`. + +## Usage in C++ TAP Tests + +Include `utils.h` and `noise_utils.h`. + +### External Tools +Use `spawn_noise` to run scripts or binaries in a separate process. +```cpp +spawn_noise(cl, "../noise/noise_stats_poller.py", {"--interval", "0.1"}); +``` + +### Internal Threads +Use `spawn_internal_noise` to run built-in C++ functions in background threads within the same process. This is **highly recommended for debugging with GDB**, as stopping the test process will also pause the noise. + +```cpp +#include "noise_utils.h" + +// ... inside main ... +spawn_internal_noise(cl, internal_noise_admin_pinger); +``` + +#### Standard Internal Noise Functions: +- `internal_noise_admin_pinger`: Executes `SELECT 1` against Admin every 500ms. +- `internal_noise_stats_poller`: Polls various `stats_*` tables every 200ms. +- `internal_noise_prometheus_poller`: Fetches Prometheus metrics via both MySQL and PostgreSQL protocol every 1000ms. +- `internal_noise_random_stats_poller`: Shuffles and queries a set of MySQL and PostgreSQL stats tables (e.g., `stats_mysql_query_digest`, `stats_pgsql_processlist`) every 500ms. + +## Internal Safety Mechanisms + +1. **Process Group Isolation**: `spawn_noise` calls `setpgid(0, 0)` in the child. This ensures that signals like `SIGINT` (Ctrl+C) sent to the test runner are not automatically forwarded to the noise tools, allowing the `utils` library to manage their shutdown sequence explicitly. +2. **Double-Hook Cleanup**: + * **Primary**: `exit_status()` calls `stop_noise_tools()`. + * **Fallback**: An `atexit()` handler is registered during the first `spawn_noise` call to catch unexpected (but clean) exits. +3. **Graceful Termination**: The framework sends `SIGTERM` first, waits 100ms for the process to reap, and follows up with `SIGKILL` if the process is still alive. + +## Testing the Framework + +A dedicated verification test is provided: +```bash +# From test/tap/tests +TAP_USE_NOISE=1 ./test_noise_injection-t +``` +This test spawns a bash sub-process, verifies it is alive via its PID, and then verifies it is successfully killed by the cleanup logic. diff --git a/test/tap/tap/Makefile b/test/tap/tap/Makefile index 75bd9ec5c..c391c508a 100644 --- a/test/tap/tap/Makefile +++ b/test/tap/tap/Makefile @@ -59,17 +59,20 @@ utils_mysql8.o: utils.cpp cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a libcurl.s tap.o: tap.cpp cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a libcurl.so -lssl -lcrypto libcpp_dotenv.so $(CXX) -fPIC -c tap.cpp $(IDIRS) $(OPT) +noise_utils.o: noise_utils.cpp noise_utils.h utils.h command_line.h + $(CXX) -fPIC -c noise_utils.cpp $(IDIRS) -I$(MARIADB_IDIR) -I$(POSTGRESQL_IDIR) $(OPT) + mcp_client.o: mcp_client.cpp mcp_client.h libcurl.so $(CXX) -fPIC -c mcp_client.cpp $(IDIRS) $(OPT) -libtap_mariadb.a: tap.o command_line.o utils_mariadb.o mcp_client.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a - ar rcs libtap_mariadb.a tap.o command_line.o utils_mariadb.o mcp_client.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo +libtap_mariadb.a: tap.o command_line.o utils_mariadb.o noise_utils.o mcp_client.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a + ar rcs libtap_mariadb.a tap.o command_line.o utils_mariadb.o noise_utils.o mcp_client.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo -libtap_mysql57.a: tap.o command_line.o utils_mysql57.o mcp_client.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a - ar rcs libtap_mysql57.a tap.o command_line.o utils_mysql57.o mcp_client.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo +libtap_mysql57.a: tap.o command_line.o utils_mysql57.o noise_utils.o mcp_client.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a + ar rcs libtap_mysql57.a tap.o command_line.o utils_mysql57.o noise_utils.o mcp_client.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo -libtap_mysql8.a: tap.o command_line.o utils_mysql8.o mcp_client.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a - ar rcs libtap_mysql8.a tap.o command_line.o utils_mysql8.o mcp_client.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo +libtap_mysql8.a: tap.o command_line.o utils_mysql8.o noise_utils.o mcp_client.o cpp-dotenv/static/cpp-dotenv/libcpp_dotenv.a + ar rcs libtap_mysql8.a tap.o command_line.o utils_mysql8.o noise_utils.o mcp_client.o $(SQLITE3_LDIR)/sqlite3.o $(PROXYSQL_LDIR)/obj/sha256crypt.oo libtap.so: libtap_mariadb.a cpp-dotenv/dynamic/cpp-dotenv/libcpp_dotenv.so libre2.so $(CXX) -shared -o libtap.so -Wl,--whole-archive libtap_mariadb.a -Wl,--no-whole-archive $(LWGCOV) diff --git a/test/tap/tap/noise_utils.cpp b/test/tap/tap/noise_utils.cpp new file mode 100644 index 000000000..a94954490 --- /dev/null +++ b/test/tap/tap/noise_utils.cpp @@ -0,0 +1,176 @@ +#include +#include +#include +#include +#include "noise_utils.h" +#include "utils.h" +#include "tap.h" +#include "mysql.h" +#include "libpq-fe.h" + +static std::vector internal_noise_threads; +static std::atomic stop_internal_noise{false}; + +// Helper for PostgreSQL noise +static void pg_noise_query(PGconn* conn, const char* query) { + PGresult* res = PQexec(conn, query); + if (res) PQclear(res); +} + +void spawn_internal_noise(const CommandLine& cl, internal_noise_func_t func) { + if (!cl.use_noise) { + return; + } + + stop_internal_noise = false; + internal_noise_threads.emplace_back(func, std::ref(cl), std::ref(stop_internal_noise)); + diag("Spawned internal noise thread"); +} + +void stop_internal_noise_threads() { + stop_internal_noise = true; + for (auto& t : internal_noise_threads) { + if (t.joinable()) { + t.join(); + } + } + internal_noise_threads.clear(); +} + +// --- Standard Internal Noise Functions Implementation --- + +void internal_noise_admin_pinger(const CommandLine& cl, std::atomic& stop) { + MYSQL* admin = mysql_init(NULL); + if (!admin) return; + + if (!mysql_real_connect(admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { + mysql_close(admin); + return; + } + + while (!stop) { + if (mysql_query(admin, "SELECT 1")) { + // Silently ignore errors in noise thread + } else { + MYSQL_RES* res = mysql_store_result(admin); + if (res) mysql_free_result(res); + } + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + + mysql_close(admin); +} + +void internal_noise_stats_poller(const CommandLine& cl, std::atomic& stop) { + MYSQL* admin = mysql_init(NULL); + if (!admin) return; + + if (!mysql_real_connect(admin, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0)) { + mysql_close(admin); + return; + } + + while (!stop) { + const char* queries[] = { + "SELECT * FROM stats_mysql_query_digest", + "SELECT * FROM stats_mysql_connection_pool", + "SELECT * FROM stats_mysql_processlist" + }; + + for (const char* q : queries) { + if (stop) break; + if (mysql_query(admin, q)) { + // Ignore + } else { + MYSQL_RES* res = mysql_store_result(admin); + if (res) mysql_free_result(res); + } + } + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + } + + mysql_close(admin); +} + +void internal_noise_prometheus_poller(const CommandLine& cl, std::atomic& stop) { + MYSQL* admin_my = mysql_init(NULL); + PGconn* admin_pg = NULL; + + if (admin_my) { + mysql_real_connect(admin_my, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0); + } + + std::string conninfo = "host=" + std::string(cl.host) + " port=" + std::to_string(cl.pgsql_admin_port) + + " user=" + std::string(cl.admin_username) + " password=" + std::string(cl.admin_password) + + " dbname=stats connect_timeout=2"; + admin_pg = PQconnectdb(conninfo.c_str()); + + while (!stop) { + if (admin_my && mysql_ping(admin_my) == 0) { + if (mysql_query(admin_my, "SELECT * FROM stats_prometheus_metrics") == 0) { + MYSQL_RES* res = mysql_store_result(admin_my); + if (res) mysql_free_result(res); + } + } + + if (admin_pg && PQstatus(admin_pg) == CONNECTION_OK) { + pg_noise_query(admin_pg, "SELECT * FROM stats_prometheus_metrics"); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1000)); + } + + if (admin_my) mysql_close(admin_my); + if (admin_pg) PQfinish(admin_pg); +} + +void internal_noise_random_stats_poller(const CommandLine& cl, std::atomic& stop) { + MYSQL* admin_my = mysql_init(NULL); + PGconn* admin_pg = NULL; + + if (admin_my) { + mysql_real_connect(admin_my, cl.host, cl.admin_username, cl.admin_password, NULL, cl.admin_port, NULL, 0); + } + + std::string conninfo = "host=" + std::string(cl.host) + " port=" + std::to_string(cl.pgsql_admin_port) + + " user=" + std::string(cl.admin_username) + " password=" + std::string(cl.admin_password) + + " dbname=stats connect_timeout=2"; + admin_pg = PQconnectdb(conninfo.c_str()); + + std::vector my_tables = { + "stats_mysql_query_digest", "stats_mysql_connection_pool", "stats_mysql_processlist", + "stats_mysql_global", "stats_mysql_user_stats", "stats_mysql_query_rules", "stats_mysql_commands_counters" + }; + std::vector pg_tables = { + "stats_pgsql_query_digest", "stats_pgsql_connection_pool", "stats_pgsql_processlist", + "stats_pgsql_global", "stats_pgsql_commands_counters" + }; + + std::random_device rd; + std::mt19937 g(rd()); + + while (!stop) { + std::shuffle(my_tables.begin(), my_tables.end(), g); + std::shuffle(pg_tables.begin(), pg_tables.end(), g); + + for (size_t i = 0; i < 3; ++i) { + if (stop) break; + if (admin_my && mysql_ping(admin_my) == 0) { + std::string q = "SELECT * FROM " + my_tables[i % my_tables.size()] + " LIMIT 10"; + if (mysql_query(admin_my, q.c_str()) == 0) { + MYSQL_RES* res = mysql_store_result(admin_my); + if (res) mysql_free_result(res); + } + } + if (admin_pg && PQstatus(admin_pg) == CONNECTION_OK) { + std::string q = "SELECT * FROM " + pg_tables[i % pg_tables.size()] + " LIMIT 10"; + pg_noise_query(admin_pg, q.c_str()); + } + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + + if (admin_my) mysql_close(admin_my); + if (admin_pg) PQfinish(admin_pg); +} diff --git a/test/tap/tap/noise_utils.h b/test/tap/tap/noise_utils.h new file mode 100644 index 000000000..eb1ec8298 --- /dev/null +++ b/test/tap/tap/noise_utils.h @@ -0,0 +1,52 @@ +#ifndef NOISE_UTILS_H +#define NOISE_UTILS_H + +#include +#include +#include +#include +#include +#include "command_line.h" + +/** + * @brief Type for internal noise functions. + * @param cl CommandLine configuration. + * @param stop Atomic boolean to signal the thread to exit. + */ +typedef std::function&)> internal_noise_func_t; + +/** + * @brief Spawns an internal noise function in a separate thread. + * @param cl The CommandLine object containing configuration. + * @param func The function to execute in the background. + */ +void spawn_internal_noise(const CommandLine& cl, internal_noise_func_t func); + +/** + * @brief Stops all internal noise threads. + */ +void stop_internal_noise_threads(); + +// --- Standard Internal Noise Functions --- + +/** + * @brief Periodically executes 'SELECT 1' against the ProxySQL Admin interface. + */ +void internal_noise_admin_pinger(const CommandLine& cl, std::atomic& stop); + +/** + * @brief Periodically polls stats_mysql_query_digest. + */ +void internal_noise_stats_poller(const CommandLine& cl, std::atomic& stop); + +/** + * @brief Periodically fetches Prometheus metrics via MySQL and PostgreSQL protocols. + */ +void internal_noise_prometheus_poller(const CommandLine& cl, std::atomic& stop); + +/** + * @brief Periodically queries random stats tables via MySQL and PostgreSQL protocols. + */ +void internal_noise_random_stats_poller(const CommandLine& cl, std::atomic& stop); + +#endif // #ifndef NOISE_UTILS_H diff --git a/test/tap/tap/utils.cpp b/test/tap/tap/utils.cpp index 8633f18b6..247743d8d 100644 --- a/test/tap/tap/utils.cpp +++ b/test/tap/tap/utils.cpp @@ -22,6 +22,7 @@ #include "mysql.h" #include "utils.h" #include "tap.h" +#include "noise_utils.h" using std::pair; using std::map; @@ -2441,6 +2442,7 @@ static std::vector background_noise_pids; static bool atexit_noise_registered = false; extern "C" void stop_noise_tools() { + stop_internal_noise_threads(); for (pid_t pid : background_noise_pids) { kill(pid, SIGTERM); // Small wait and reap diff --git a/test/tap/tests/test_noise_injection-t.cpp b/test/tap/tests/test_noise_injection-t.cpp index b879e9cb6..270699a36 100644 --- a/test/tap/tests/test_noise_injection-t.cpp +++ b/test/tap/tests/test_noise_injection-t.cpp @@ -7,6 +7,7 @@ #include "tap.h" #include "command_line.h" #include "utils.h" +#include "noise_utils.h" int main(int argc, char** argv) { CommandLine cl; @@ -15,50 +16,43 @@ int main(int argc, char** argv) { return 1; } - // Force noise enabled for this test if environment variable is not set - // but CommandLine::getEnv() already read it. - // To properly test, we should run this with TAP_USE_NOISE=1 - if (!cl.use_noise) { skip_all("TAP_USE_NOISE is not enabled. Skip noise injection test."); } - plan(3); - - // Use a simple script that just sleeps or a standard one - // We'll use our new stats poller but with a long interval or just 'sleep 100' - spawn_noise(cl, "/bin/sleep", {"100"}); + plan(5); - // We can't easily get the PID from here as it's hidden in utils.cpp - // but we can check if a sleep 100 process exists. - // However, multiple might exist. - - // Better way: use a specific noise tool that writes its PID to a file + // --- External Noise Test --- std::string pid_file = "/tmp/proxysql_noise_test.pid"; std::string cmd = "echo $$ > " + pid_file + " && exec sleep 100"; spawn_noise(cl, "/bin/bash", {"-c", cmd}); sleep(1); // Give it time to start - ok(access(pid_file.c_str(), F_OK) == 0, "Noise process started and created PID file"); + ok(access(pid_file.c_str(), F_OK) == 0, "External noise process started and created PID file"); - // Read PID from file FILE* f = fopen(pid_file.c_str(), "r"); pid_t pid = 0; if (f) { if (fscanf(f, "%d", &pid) != 1) pid = 0; fclose(f); } - diag("Noise process PID: %d", pid); + diag("External noise process PID: %d", pid); + + ok(pid > 0 && kill(pid, 0) == 0, "External noise process is alive"); - // Verify it is alive - ok(pid > 0 && kill(pid, 0) == 0, "Noise process is alive"); + // --- Internal Noise Test --- + spawn_internal_noise(cl, internal_noise_admin_pinger); + // There isn't an easy way to verify the thread is running from outside + // but we can verify it doesn't crash the test and that stop works. + ok(1, "Internal noise thread spawned without crash"); - // We can manually call stop_noise_tools() to verify it works + // --- Cleanup Verification --- stop_noise_tools(); sleep(1); // Give it time to be killed - ok(pid > 0 && kill(pid, 0) != 0, "Noise process was killed"); + ok(pid > 0 && kill(pid, 0) != 0, "External noise process was killed"); + ok(1, "Internal noise tools stopped (implied by join finishing)"); if (access(pid_file.c_str(), F_OK) == 0) { unlink(pid_file.c_str());