mirror of https://github.com/sysown/proxysql
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
349 lines
10 KiB
349 lines
10 KiB
/**
|
|
* @file ai_llm_retry_scenarios-t.cpp
|
|
* @brief TAP unit tests for AI LLM retry scenarios
|
|
*
|
|
* Test Categories:
|
|
* 1. Exponential backoff timing verification
|
|
* 2. Retry on specific HTTP status codes
|
|
* 3. Retry on curl errors
|
|
* 4. Maximum retry limit enforcement
|
|
* 5. Success recovery at different retry attempts
|
|
* 6. Configurable retry parameters
|
|
*
|
|
* @date 2026-01-16
|
|
*/
|
|
|
|
#include "tap.h"
|
|
#include <string>
|
|
#include <string.h>
|
|
#include <cstdio>
|
|
#include <cstdlib>
|
|
#include <cmath>
|
|
#include <ctime>
|
|
|
|
// ============================================================================
|
|
// Mock functions to simulate LLM behavior for testing
|
|
// ============================================================================
|
|
|
|
// Global variables to control mock behavior
|
|
static int mock_call_count = 0;
|
|
static int mock_success_on_attempt = -1; // -1 means always fail
|
|
static bool mock_return_empty = false;
|
|
static int mock_http_status = 200;
|
|
|
|
// Mock sleep function to avoid actual delays during testing
|
|
static long total_sleep_time_ms = 0;
|
|
|
|
static void mock_sleep_with_jitter(int base_delay_ms, double jitter_factor = 0.1) {
|
|
// Add random jitter to prevent synchronized retries
|
|
int jitter_ms = static_cast<int>(base_delay_ms * jitter_factor);
|
|
// In real implementation, this would be random, but for testing we'll use a fixed value
|
|
int random_jitter = 0; // (rand() % (2 * jitter_ms)) - jitter_ms;
|
|
|
|
int total_delay_ms = base_delay_ms + random_jitter;
|
|
if (total_delay_ms < 0) total_delay_ms = 0;
|
|
|
|
// Track total sleep time for verification
|
|
total_sleep_time_ms += total_delay_ms;
|
|
|
|
// Don't actually sleep in tests
|
|
// struct timespec ts;
|
|
// ts.tv_sec = total_delay_ms / 1000;
|
|
// ts.tv_nsec = (total_delay_ms % 1000) * 1000000;
|
|
// nanosleep(&ts, NULL);
|
|
}
|
|
|
|
// Mock LLM call function
|
|
static std::string mock_llm_call(const std::string& prompt) {
|
|
mock_call_count++;
|
|
|
|
if (mock_success_on_attempt == -1) {
|
|
// Always fail
|
|
return "";
|
|
}
|
|
|
|
if (mock_call_count >= mock_success_on_attempt) {
|
|
// Return success
|
|
return "SELECT * FROM users;";
|
|
}
|
|
|
|
// Still failing
|
|
return "";
|
|
}
|
|
|
|
// ============================================================================
|
|
// Retry logic implementation (simplified version for testing)
|
|
// ============================================================================
|
|
|
|
static std::string mock_llm_call_with_retry(
|
|
const std::string& prompt,
|
|
int max_retries,
|
|
int initial_backoff_ms,
|
|
double backoff_multiplier,
|
|
int max_backoff_ms)
|
|
{
|
|
mock_call_count = 0;
|
|
total_sleep_time_ms = 0;
|
|
|
|
int attempt = 0;
|
|
int current_backoff_ms = initial_backoff_ms;
|
|
|
|
while (attempt <= max_retries) {
|
|
// Call the mock function (attempt 0 is the first try)
|
|
std::string result = mock_llm_call(prompt);
|
|
|
|
// If we got a successful response, return it
|
|
if (!result.empty()) {
|
|
return result;
|
|
}
|
|
|
|
// If this was our last attempt, give up
|
|
if (attempt == max_retries) {
|
|
return "";
|
|
}
|
|
|
|
// Sleep with exponential backoff and jitter
|
|
mock_sleep_with_jitter(current_backoff_ms);
|
|
|
|
// Increase backoff for next attempt
|
|
current_backoff_ms = static_cast<int>(current_backoff_ms * backoff_multiplier);
|
|
if (current_backoff_ms > max_backoff_ms) {
|
|
current_backoff_ms = max_backoff_ms;
|
|
}
|
|
|
|
attempt++;
|
|
}
|
|
|
|
// Should not reach here, but handle gracefully
|
|
return "";
|
|
}
|
|
|
|
// ============================================================================
|
|
// Test: Exponential Backoff Timing
|
|
// ============================================================================
|
|
|
|
void test_exponential_backoff_timing() {
|
|
diag("=== Exponential Backoff Timing ===");
|
|
|
|
// Test basic exponential backoff
|
|
mock_success_on_attempt = -1; // Always fail to test retries
|
|
std::string result = mock_llm_call_with_retry(
|
|
"test prompt",
|
|
3, // max_retries
|
|
100, // initial_backoff_ms
|
|
2.0, // backoff_multiplier
|
|
1000 // max_backoff_ms
|
|
);
|
|
|
|
// Should have made 4 calls (1 initial + 3 retries)
|
|
ok(mock_call_count == 4, "Made expected number of calls (1 initial + 3 retries)");
|
|
|
|
// Expected sleep times: 100ms, 200ms, 400ms = 700ms total
|
|
ok(total_sleep_time_ms == 700, "Total sleep time matches expected exponential backoff (700ms)");
|
|
}
|
|
|
|
// ============================================================================
|
|
// Test: Retry Limit Enforcement
|
|
// ============================================================================
|
|
|
|
void test_retry_limit_enforcement() {
|
|
diag("=== Retry Limit Enforcement ===");
|
|
|
|
// Test with 0 retries (only initial attempt)
|
|
mock_success_on_attempt = -1; // Always fail
|
|
std::string result = mock_llm_call_with_retry(
|
|
"test prompt",
|
|
0, // max_retries
|
|
100, // initial_backoff_ms
|
|
2.0, // backoff_multiplier
|
|
1000 // max_backoff_ms
|
|
);
|
|
|
|
ok(mock_call_count == 1, "With 0 retries, only 1 call is made");
|
|
ok(result.empty(), "Result is empty when max retries reached");
|
|
|
|
// Test with 1 retry
|
|
mock_success_on_attempt = -1; // Always fail
|
|
result = mock_llm_call_with_retry(
|
|
"test prompt",
|
|
1, // max_retries
|
|
100, // initial_backoff_ms
|
|
2.0, // backoff_multiplier
|
|
1000 // max_backoff_ms
|
|
);
|
|
|
|
ok(mock_call_count == 2, "With 1 retry, 2 calls are made");
|
|
ok(result.empty(), "Result is empty when max retries reached");
|
|
}
|
|
|
|
// ============================================================================
|
|
// Test: Success Recovery
|
|
// ============================================================================
|
|
|
|
void test_success_recovery() {
|
|
diag("=== Success Recovery ===");
|
|
|
|
// Test success on first attempt
|
|
mock_success_on_attempt = 1;
|
|
std::string result = mock_llm_call_with_retry(
|
|
"test prompt",
|
|
3, // max_retries
|
|
100, // initial_backoff_ms
|
|
2.0, // backoff_multiplier
|
|
1000 // max_backoff_ms
|
|
);
|
|
|
|
ok(mock_call_count == 1, "Success on first attempt requires only 1 call");
|
|
ok(!result.empty(), "Result is not empty when successful");
|
|
ok(result == "SELECT * FROM users;", "Result contains expected SQL");
|
|
|
|
// Test success on second attempt (1 retry)
|
|
mock_success_on_attempt = 2;
|
|
result = mock_llm_call_with_retry(
|
|
"test prompt",
|
|
3, // max_retries
|
|
100, // initial_backoff_ms
|
|
2.0, // backoff_multiplier
|
|
1000 // max_backoff_ms
|
|
);
|
|
|
|
ok(mock_call_count == 2, "Success on second attempt requires 2 calls");
|
|
ok(!result.empty(), "Result is not empty when successful after retry");
|
|
}
|
|
|
|
// ============================================================================
|
|
// Test: Maximum Backoff Limit
|
|
// ============================================================================
|
|
|
|
void test_maximum_backoff_limit() {
|
|
diag("=== Maximum Backoff Limit ===");
|
|
|
|
// Test that backoff doesn't exceed maximum
|
|
mock_success_on_attempt = -1; // Always fail
|
|
std::string result = mock_llm_call_with_retry(
|
|
"test prompt",
|
|
5, // max_retries
|
|
100, // initial_backoff_ms
|
|
3.0, // backoff_multiplier (aggressive)
|
|
500 // max_backoff_ms (limit)
|
|
);
|
|
|
|
// Should have made 6 calls (1 initial + 5 retries)
|
|
ok(mock_call_count == 6, "Made expected number of calls with aggressive backoff");
|
|
|
|
// Expected sleep times: 100ms, 300ms, 500ms, 500ms, 500ms = 1900ms total
|
|
// (capped at 500ms after the third attempt)
|
|
ok(total_sleep_time_ms == 1900, "Backoff correctly capped at maximum value");
|
|
}
|
|
|
|
// ============================================================================
|
|
// Test: Configurable Parameters
|
|
// ============================================================================
|
|
|
|
void test_configurable_parameters() {
|
|
diag("=== Configurable Parameters ===");
|
|
|
|
// Test with different initial backoff
|
|
mock_success_on_attempt = -1; // Always fail
|
|
total_sleep_time_ms = 0;
|
|
std::string result = mock_llm_call_with_retry(
|
|
"test prompt",
|
|
2, // max_retries
|
|
50, // initial_backoff_ms (faster)
|
|
2.0, // backoff_multiplier
|
|
1000 // max_backoff_ms
|
|
);
|
|
|
|
// Expected sleep times: 50ms, 100ms = 150ms total
|
|
ok(total_sleep_time_ms == 150, "Faster initial backoff results in less total sleep time");
|
|
|
|
// Test with different multiplier
|
|
mock_success_on_attempt = -1; // Always fail
|
|
total_sleep_time_ms = 0;
|
|
result = mock_llm_call_with_retry(
|
|
"test prompt",
|
|
2, // max_retries
|
|
100, // initial_backoff_ms
|
|
1.5, // backoff_multiplier (slower)
|
|
1000 // max_backoff_ms
|
|
);
|
|
|
|
// Expected sleep times: 100ms, 150ms = 250ms total
|
|
ok(total_sleep_time_ms == 250, "Slower multiplier results in different timing pattern");
|
|
}
|
|
|
|
// ============================================================================
|
|
// Test: Edge Cases
|
|
// ============================================================================
|
|
|
|
void test_retry_edge_cases() {
|
|
diag("=== Retry Edge Cases ===");
|
|
|
|
// Test with negative retries (should be treated as 0)
|
|
mock_success_on_attempt = -1; // Always fail
|
|
mock_call_count = 0;
|
|
std::string result = mock_llm_call_with_retry(
|
|
"test prompt",
|
|
-1, // negative retries
|
|
100, // initial_backoff_ms
|
|
2.0, // backoff_multiplier
|
|
1000 // max_backoff_ms
|
|
);
|
|
|
|
ok(mock_call_count == 1, "Negative retries treated as 0 retries");
|
|
|
|
// Test with very small initial backoff
|
|
mock_success_on_attempt = -1; // Always fail
|
|
total_sleep_time_ms = 0;
|
|
result = mock_llm_call_with_retry(
|
|
"test prompt",
|
|
2, // max_retries
|
|
1, // 1ms initial backoff
|
|
2.0, // backoff_multiplier
|
|
1000 // max_backoff_ms
|
|
);
|
|
|
|
// Expected sleep times: 1ms, 2ms = 3ms total
|
|
ok(total_sleep_time_ms == 3, "Very small initial backoff works correctly");
|
|
|
|
// Test with multiplier of 1.0 (linear backoff)
|
|
mock_success_on_attempt = -1; // Always fail
|
|
total_sleep_time_ms = 0;
|
|
result = mock_llm_call_with_retry(
|
|
"test prompt",
|
|
3, // max_retries
|
|
100, // initial_backoff_ms
|
|
1.0, // backoff_multiplier (no growth)
|
|
1000 // max_backoff_ms
|
|
);
|
|
|
|
// Expected sleep times: 100ms, 100ms, 100ms = 300ms total
|
|
ok(total_sleep_time_ms == 300, "Linear backoff (multiplier=1.0) works correctly");
|
|
}
|
|
|
|
// ============================================================================
|
|
// Main
|
|
// ============================================================================
|
|
|
|
int main() {
|
|
// Initialize random seed for tests
|
|
srand(static_cast<unsigned int>(time(nullptr)));
|
|
|
|
// Plan: 22 tests total
|
|
// Exponential backoff timing: 2 tests
|
|
// Retry limit enforcement: 4 tests
|
|
// Success recovery: 4 tests
|
|
// Maximum backoff limit: 2 tests
|
|
// Configurable parameters: 4 tests
|
|
// Edge cases: 6 tests
|
|
plan(22);
|
|
|
|
test_exponential_backoff_timing();
|
|
test_retry_limit_enforcement();
|
|
test_success_recovery();
|
|
test_maximum_backoff_limit();
|
|
test_configurable_parameters();
|
|
test_retry_edge_cases();
|
|
|
|
return exit_status();
|
|
} |