mirror of https://github.com/sysown/proxysql
Enhance AI features with improved validation, memory safety, error handling, and performance monitoring
- Rename validate_provider_name to validate_provider_format for clarity - Add null checks and error handling for all strdup() operations - Enhance error messages with more context and HTTP status codes - Implement performance monitoring with timing metrics for LLM calls and cache operations - Add comprehensive test coverage for edge cases, retry scenarios, and performance - Extend status variables to track performance metrics - Update MySQL session to report timing information to AI managerpull/5310/head
Rene Cannao
3 months ago
parent
3032dffed4
commit
ae4200dbc0
@ -0,0 +1,303 @@
|
||||
/**
|
||||
* @file ai_error_handling_edge_cases-t.cpp
|
||||
* @brief TAP unit tests for AI error handling edge cases
|
||||
*
|
||||
* Test Categories:
|
||||
* 1. API key validation edge cases (special characters, boundary lengths)
|
||||
* 2. URL validation edge cases (IPv6, unusual ports, malformed patterns)
|
||||
* 3. Timeout scenarios simulation
|
||||
* 4. Connection failure handling
|
||||
* 5. Rate limiting error responses
|
||||
* 6. Invalid LLM response formats
|
||||
*
|
||||
* @date 2026-01-16
|
||||
*/
|
||||
|
||||
#include "tap.h"
|
||||
#include <string.h>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
|
||||
// ============================================================================
|
||||
// Standalone validation functions (matching AI_Features_Manager.cpp logic)
|
||||
// ============================================================================
|
||||
|
||||
static bool validate_url_format(const char* url) {
|
||||
if (!url || strlen(url) == 0) {
|
||||
return true; // Empty URL is valid (will use defaults)
|
||||
}
|
||||
|
||||
// Check for protocol prefix (http://, https://)
|
||||
const char* http_prefix = "http://";
|
||||
const char* https_prefix = "https://";
|
||||
|
||||
bool has_protocol = (strncmp(url, http_prefix, strlen(http_prefix)) == 0 ||
|
||||
strncmp(url, https_prefix, strlen(https_prefix)) == 0);
|
||||
|
||||
if (!has_protocol) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check for host part (at least something after ://)
|
||||
const char* host_start = strstr(url, "://");
|
||||
if (!host_start || strlen(host_start + 3) == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool validate_api_key_format(const char* key, const char* provider_name) {
|
||||
if (!key || strlen(key) == 0) {
|
||||
return true; // Empty key is valid for local endpoints
|
||||
}
|
||||
|
||||
size_t len = strlen(key);
|
||||
|
||||
// Check for whitespace
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
if (key[i] == ' ' || key[i] == '\t' || key[i] == '\n' || key[i] == '\r') {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Check minimum length (most API keys are at least 20 chars)
|
||||
if (len < 10) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check for incomplete OpenAI key format
|
||||
if (strncmp(key, "sk-", 3) == 0 && len < 20) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check for incomplete Anthropic key format
|
||||
if (strncmp(key, "sk-ant-", 7) == 0 && len < 25) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool validate_numeric_range(const char* value, int min_val, int max_val, const char* var_name) {
|
||||
if (!value || strlen(value) == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
int int_val = atoi(value);
|
||||
|
||||
if (int_val < min_val || int_val > max_val) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool validate_provider_format(const char* provider) {
|
||||
if (!provider || strlen(provider) == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const char* valid_formats[] = {"openai", "anthropic", NULL};
|
||||
for (int i = 0; valid_formats[i]; i++) {
|
||||
if (strcmp(provider, valid_formats[i]) == 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test: API Key Validation Edge Cases
|
||||
// ============================================================================
|
||||
|
||||
void test_api_key_edge_cases() {
|
||||
diag("=== API Key Validation Edge Cases ===");
|
||||
|
||||
// Test very short keys
|
||||
ok(!validate_api_key_format("a", "openai"),
|
||||
"Very short key (1 char) rejected");
|
||||
ok(!validate_api_key_format("sk", "openai"),
|
||||
"Very short OpenAI-like key (2 chars) rejected");
|
||||
ok(!validate_api_key_format("sk-ant", "anthropic"),
|
||||
"Very short Anthropic-like key (6 chars) rejected");
|
||||
|
||||
// Test keys with special characters
|
||||
ok(validate_api_key_format("sk-abc123!@#$%^&*()", "openai"),
|
||||
"API key with special characters accepted");
|
||||
ok(validate_api_key_format("sk-ant-xyz789_+-=[]{}|;':\",./<>?", "anthropic"),
|
||||
"Anthropic key with special characters accepted");
|
||||
|
||||
// Test keys with exactly minimum valid lengths
|
||||
ok(validate_api_key_format("sk-abcdefghij", "openai"),
|
||||
"OpenAI key with exactly 10 chars accepted");
|
||||
ok(validate_api_key_format("sk-ant-abcdefghijklmnop", "anthropic"),
|
||||
"Anthropic key with exactly 25 chars accepted");
|
||||
|
||||
// Test keys with whitespace at boundaries (should be rejected)
|
||||
ok(!validate_api_key_format(" sk-abcdefghij", "openai"),
|
||||
"API key with leading space rejected");
|
||||
ok(!validate_api_key_format("sk-abcdefghij ", "openai"),
|
||||
"API key with trailing space rejected");
|
||||
ok(!validate_api_key_format("sk-abc def-ghij", "openai"),
|
||||
"API key with internal space rejected");
|
||||
ok(!validate_api_key_format("sk-abcdefghij\t", "openai"),
|
||||
"API key with tab rejected");
|
||||
ok(!validate_api_key_format("sk-abcdefghij\n", "openai"),
|
||||
"API key with newline rejected");
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test: URL Validation Edge Cases
|
||||
// ============================================================================
|
||||
|
||||
void test_url_edge_cases() {
|
||||
diag("=== URL Validation Edge Cases ===");
|
||||
|
||||
// Test IPv6 URLs
|
||||
ok(validate_url_format("http://[2001:db8::1]:8080/v1/chat/completions"),
|
||||
"IPv6 URL with port accepted");
|
||||
ok(validate_url_format("https://[::1]/v1/chat/completions"),
|
||||
"IPv6 localhost URL accepted");
|
||||
|
||||
// Test unusual ports
|
||||
ok(validate_url_format("http://localhost:1/v1/chat/completions"),
|
||||
"URL with port 1 accepted");
|
||||
ok(validate_url_format("http://localhost:65535/v1/chat/completions"),
|
||||
"URL with port 65535 accepted");
|
||||
|
||||
// Test URLs with paths and query parameters
|
||||
ok(validate_url_format("https://api.openai.com/v1/chat/completions?timeout=30"),
|
||||
"URL with query parameters accepted");
|
||||
ok(validate_url_format("http://localhost:11434/v1/chat/completions/model/llama3"),
|
||||
"URL with additional path segments accepted");
|
||||
|
||||
// Test malformed URLs that should be rejected
|
||||
ok(!validate_url_format("http://"),
|
||||
"URL with only protocol rejected");
|
||||
ok(!validate_url_format("http://:8080"),
|
||||
"URL with port but no host rejected");
|
||||
ok(!validate_url_format("localhost:8080/v1/chat/completions"),
|
||||
"URL without protocol rejected");
|
||||
ok(!validate_url_format("ftp://localhost/v1/chat/completions"),
|
||||
"FTP URL rejected (only HTTP/HTTPS supported)");
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test: Numeric Range Edge Cases
|
||||
// ============================================================================
|
||||
|
||||
void test_numeric_range_edge_cases() {
|
||||
diag("=== Numeric Range Edge Cases ===");
|
||||
|
||||
// Test boundary values
|
||||
ok(validate_numeric_range("0", 0, 100, "test_var"),
|
||||
"Minimum boundary value accepted");
|
||||
ok(validate_numeric_range("100", 0, 100, "test_var"),
|
||||
"Maximum boundary value accepted");
|
||||
ok(!validate_numeric_range("-1", 0, 100, "test_var"),
|
||||
"Value below minimum rejected");
|
||||
ok(!validate_numeric_range("101", 0, 100, "test_var"),
|
||||
"Value above maximum rejected");
|
||||
|
||||
// Test string values that are valid numbers
|
||||
ok(validate_numeric_range("50", 0, 100, "test_var"),
|
||||
"Valid number string accepted");
|
||||
ok(!validate_numeric_range("abc", 0, 100, "test_var"),
|
||||
"Non-numeric string rejected");
|
||||
ok(!validate_numeric_range("50abc", 0, 100, "test_var"),
|
||||
"String starting with number rejected");
|
||||
ok(!validate_numeric_range("", 0, 100, "test_var"),
|
||||
"Empty string rejected");
|
||||
|
||||
// Test negative numbers
|
||||
ok(validate_numeric_range("-50", -100, 0, "test_var"),
|
||||
"Negative number within range accepted");
|
||||
ok(!validate_numeric_range("-150", -100, 0, "test_var"),
|
||||
"Negative number below range rejected");
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test: Provider Format Edge Cases
|
||||
// ============================================================================
|
||||
|
||||
void test_provider_format_edge_cases() {
|
||||
diag("=== Provider Format Edge Cases ===");
|
||||
|
||||
// Test case sensitivity
|
||||
ok(!validate_provider_format("OpenAI"),
|
||||
"Uppercase 'OpenAI' rejected (case sensitive)");
|
||||
ok(!validate_provider_format("OPENAI"),
|
||||
"Uppercase 'OPENAI' rejected (case sensitive)");
|
||||
ok(!validate_provider_format("Anthropic"),
|
||||
"Uppercase 'Anthropic' rejected (case sensitive)");
|
||||
ok(!validate_provider_format("ANTHROPIC"),
|
||||
"Uppercase 'ANTHROPIC' rejected (case sensitive)");
|
||||
|
||||
// Test provider names with whitespace
|
||||
ok(!validate_provider_format(" openai"),
|
||||
"Provider with leading space rejected");
|
||||
ok(!validate_provider_format("openai "),
|
||||
"Provider with trailing space rejected");
|
||||
ok(!validate_provider_format(" openai "),
|
||||
"Provider with leading and trailing spaces rejected");
|
||||
ok(!validate_provider_format("open ai"),
|
||||
"Provider with internal space rejected");
|
||||
|
||||
// Test empty and NULL cases
|
||||
ok(!validate_provider_format(""),
|
||||
"Empty provider format rejected");
|
||||
ok(!validate_provider_format(NULL),
|
||||
"NULL provider format rejected");
|
||||
|
||||
// Test similar but invalid provider names
|
||||
ok(!validate_provider_format("openai2"),
|
||||
"Similar but invalid provider 'openai2' rejected");
|
||||
ok(!validate_provider_format("anthropic2"),
|
||||
"Similar but invalid provider 'anthropic2' rejected");
|
||||
ok(!validate_provider_format("ollama"),
|
||||
"Provider 'ollama' rejected (use 'openai' format instead)");
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test: Edge Cases and Boundary Conditions
|
||||
// ============================================================================
|
||||
|
||||
void test_general_edge_cases() {
|
||||
diag("=== General Edge Cases ===");
|
||||
|
||||
// Test extremely long strings
|
||||
char* long_string = (char*)malloc(10000);
|
||||
memset(long_string, 'a', 9999);
|
||||
long_string[9999] = '\0';
|
||||
ok(validate_api_key_format(long_string, "openai"),
|
||||
"Extremely long API key accepted");
|
||||
free(long_string);
|
||||
|
||||
// Test strings with special Unicode characters (if supported)
|
||||
// Note: This is a basic test - actual Unicode support depends on system
|
||||
ok(validate_api_key_format("sk-testkey123", "openai"),
|
||||
"Standard ASCII key accepted");
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Main
|
||||
// ============================================================================
|
||||
|
||||
int main() {
|
||||
// Plan: 35 tests total
|
||||
// API key edge cases: 10 tests
|
||||
// URL edge cases: 9 tests
|
||||
// Numeric range edge cases: 8 tests
|
||||
// Provider format edge cases: 8 tests
|
||||
plan(35);
|
||||
|
||||
test_api_key_edge_cases();
|
||||
test_url_edge_cases();
|
||||
test_numeric_range_edge_cases();
|
||||
test_provider_format_edge_cases();
|
||||
test_general_edge_cases();
|
||||
|
||||
return exit_status();
|
||||
}
|
||||
@ -0,0 +1,348 @@
|
||||
/**
|
||||
* @file ai_llm_retry_scenarios-t.cpp
|
||||
* @brief TAP unit tests for AI LLM retry scenarios
|
||||
*
|
||||
* Test Categories:
|
||||
* 1. Exponential backoff timing verification
|
||||
* 2. Retry on specific HTTP status codes
|
||||
* 3. Retry on curl errors
|
||||
* 4. Maximum retry limit enforcement
|
||||
* 5. Success recovery at different retry attempts
|
||||
* 6. Configurable retry parameters
|
||||
*
|
||||
* @date 2026-01-16
|
||||
*/
|
||||
|
||||
#include "tap.h"
|
||||
#include <string.h>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cmath>
|
||||
#include <ctime>
|
||||
|
||||
// ============================================================================
|
||||
// Mock functions to simulate LLM behavior for testing
|
||||
// ============================================================================
|
||||
|
||||
// Global variables to control mock behavior
|
||||
static int mock_call_count = 0;
|
||||
static int mock_success_on_attempt = -1; // -1 means always fail
|
||||
static bool mock_return_empty = false;
|
||||
static int mock_http_status = 200;
|
||||
|
||||
// Mock sleep function to avoid actual delays during testing
|
||||
static long total_sleep_time_ms = 0;
|
||||
|
||||
static void mock_sleep_with_jitter(int base_delay_ms, double jitter_factor = 0.1) {
|
||||
// Add random jitter to prevent synchronized retries
|
||||
int jitter_ms = static_cast<int>(base_delay_ms * jitter_factor);
|
||||
// In real implementation, this would be random, but for testing we'll use a fixed value
|
||||
int random_jitter = 0; // (rand() % (2 * jitter_ms)) - jitter_ms;
|
||||
|
||||
int total_delay_ms = base_delay_ms + random_jitter;
|
||||
if (total_delay_ms < 0) total_delay_ms = 0;
|
||||
|
||||
// Track total sleep time for verification
|
||||
total_sleep_time_ms += total_delay_ms;
|
||||
|
||||
// Don't actually sleep in tests
|
||||
// struct timespec ts;
|
||||
// ts.tv_sec = total_delay_ms / 1000;
|
||||
// ts.tv_nsec = (total_delay_ms % 1000) * 1000000;
|
||||
// nanosleep(&ts, NULL);
|
||||
}
|
||||
|
||||
// Mock LLM call function
|
||||
static std::string mock_llm_call(const std::string& prompt) {
|
||||
mock_call_count++;
|
||||
|
||||
if (mock_success_on_attempt == -1) {
|
||||
// Always fail
|
||||
return "";
|
||||
}
|
||||
|
||||
if (mock_call_count >= mock_success_on_attempt) {
|
||||
// Return success
|
||||
return "SELECT * FROM users;";
|
||||
}
|
||||
|
||||
// Still failing
|
||||
return "";
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Retry logic implementation (simplified version for testing)
|
||||
// ============================================================================
|
||||
|
||||
static std::string mock_llm_call_with_retry(
|
||||
const std::string& prompt,
|
||||
int max_retries,
|
||||
int initial_backoff_ms,
|
||||
double backoff_multiplier,
|
||||
int max_backoff_ms)
|
||||
{
|
||||
mock_call_count = 0;
|
||||
total_sleep_time_ms = 0;
|
||||
|
||||
int attempt = 0;
|
||||
int current_backoff_ms = initial_backoff_ms;
|
||||
|
||||
while (attempt <= max_retries) {
|
||||
// Call the mock function (attempt 0 is the first try)
|
||||
std::string result = mock_llm_call(prompt);
|
||||
|
||||
// If we got a successful response, return it
|
||||
if (!result.empty()) {
|
||||
return result;
|
||||
}
|
||||
|
||||
// If this was our last attempt, give up
|
||||
if (attempt == max_retries) {
|
||||
return "";
|
||||
}
|
||||
|
||||
// Sleep with exponential backoff and jitter
|
||||
mock_sleep_with_jitter(current_backoff_ms);
|
||||
|
||||
// Increase backoff for next attempt
|
||||
current_backoff_ms = static_cast<int>(current_backoff_ms * backoff_multiplier);
|
||||
if (current_backoff_ms > max_backoff_ms) {
|
||||
current_backoff_ms = max_backoff_ms;
|
||||
}
|
||||
|
||||
attempt++;
|
||||
}
|
||||
|
||||
// Should not reach here, but handle gracefully
|
||||
return "";
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test: Exponential Backoff Timing
|
||||
// ============================================================================
|
||||
|
||||
void test_exponential_backoff_timing() {
|
||||
diag("=== Exponential Backoff Timing ===");
|
||||
|
||||
// Test basic exponential backoff
|
||||
mock_success_on_attempt = -1; // Always fail to test retries
|
||||
std::string result = mock_llm_call_with_retry(
|
||||
"test prompt",
|
||||
3, // max_retries
|
||||
100, // initial_backoff_ms
|
||||
2.0, // backoff_multiplier
|
||||
1000 // max_backoff_ms
|
||||
);
|
||||
|
||||
// Should have made 4 calls (1 initial + 3 retries)
|
||||
ok(mock_call_count == 4, "Made expected number of calls (1 initial + 3 retries)");
|
||||
|
||||
// Expected sleep times: 100ms, 200ms, 400ms = 700ms total
|
||||
ok(total_sleep_time_ms == 700, "Total sleep time matches expected exponential backoff (700ms)");
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test: Retry Limit Enforcement
|
||||
// ============================================================================
|
||||
|
||||
void test_retry_limit_enforcement() {
|
||||
diag("=== Retry Limit Enforcement ===");
|
||||
|
||||
// Test with 0 retries (only initial attempt)
|
||||
mock_success_on_attempt = -1; // Always fail
|
||||
std::string result = mock_llm_call_with_retry(
|
||||
"test prompt",
|
||||
0, // max_retries
|
||||
100, // initial_backoff_ms
|
||||
2.0, // backoff_multiplier
|
||||
1000 // max_backoff_ms
|
||||
);
|
||||
|
||||
ok(mock_call_count == 1, "With 0 retries, only 1 call is made");
|
||||
ok(result.empty(), "Result is empty when max retries reached");
|
||||
|
||||
// Test with 1 retry
|
||||
mock_success_on_attempt = -1; // Always fail
|
||||
result = mock_llm_call_with_retry(
|
||||
"test prompt",
|
||||
1, // max_retries
|
||||
100, // initial_backoff_ms
|
||||
2.0, // backoff_multiplier
|
||||
1000 // max_backoff_ms
|
||||
);
|
||||
|
||||
ok(mock_call_count == 2, "With 1 retry, 2 calls are made");
|
||||
ok(result.empty(), "Result is empty when max retries reached");
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test: Success Recovery
|
||||
// ============================================================================
|
||||
|
||||
void test_success_recovery() {
|
||||
diag("=== Success Recovery ===");
|
||||
|
||||
// Test success on first attempt
|
||||
mock_success_on_attempt = 1;
|
||||
std::string result = mock_llm_call_with_retry(
|
||||
"test prompt",
|
||||
3, // max_retries
|
||||
100, // initial_backoff_ms
|
||||
2.0, // backoff_multiplier
|
||||
1000 // max_backoff_ms
|
||||
);
|
||||
|
||||
ok(mock_call_count == 1, "Success on first attempt requires only 1 call");
|
||||
ok(!result.empty(), "Result is not empty when successful");
|
||||
ok(result == "SELECT * FROM users;", "Result contains expected SQL");
|
||||
|
||||
// Test success on second attempt (1 retry)
|
||||
mock_success_on_attempt = 2;
|
||||
result = mock_llm_call_with_retry(
|
||||
"test prompt",
|
||||
3, // max_retries
|
||||
100, // initial_backoff_ms
|
||||
2.0, // backoff_multiplier
|
||||
1000 // max_backoff_ms
|
||||
);
|
||||
|
||||
ok(mock_call_count == 2, "Success on second attempt requires 2 calls");
|
||||
ok(!result.empty(), "Result is not empty when successful after retry");
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test: Maximum Backoff Limit
|
||||
// ============================================================================
|
||||
|
||||
void test_maximum_backoff_limit() {
|
||||
diag("=== Maximum Backoff Limit ===");
|
||||
|
||||
// Test that backoff doesn't exceed maximum
|
||||
mock_success_on_attempt = -1; // Always fail
|
||||
std::string result = mock_llm_call_with_retry(
|
||||
"test prompt",
|
||||
5, // max_retries
|
||||
100, // initial_backoff_ms
|
||||
3.0, // backoff_multiplier (aggressive)
|
||||
500 // max_backoff_ms (limit)
|
||||
);
|
||||
|
||||
// Should have made 6 calls (1 initial + 5 retries)
|
||||
ok(mock_call_count == 6, "Made expected number of calls with aggressive backoff");
|
||||
|
||||
// Expected sleep times: 100ms, 300ms, 500ms, 500ms, 500ms = 1900ms total
|
||||
// (capped at 500ms after the third attempt)
|
||||
ok(total_sleep_time_ms == 1900, "Backoff correctly capped at maximum value");
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test: Configurable Parameters
|
||||
// ============================================================================
|
||||
|
||||
void test_configurable_parameters() {
|
||||
diag("=== Configurable Parameters ===");
|
||||
|
||||
// Test with different initial backoff
|
||||
mock_success_on_attempt = -1; // Always fail
|
||||
total_sleep_time_ms = 0;
|
||||
std::string result = mock_llm_call_with_retry(
|
||||
"test prompt",
|
||||
2, // max_retries
|
||||
50, // initial_backoff_ms (faster)
|
||||
2.0, // backoff_multiplier
|
||||
1000 // max_backoff_ms
|
||||
);
|
||||
|
||||
// Expected sleep times: 50ms, 100ms = 150ms total
|
||||
ok(total_sleep_time_ms == 150, "Faster initial backoff results in less total sleep time");
|
||||
|
||||
// Test with different multiplier
|
||||
mock_success_on_attempt = -1; // Always fail
|
||||
total_sleep_time_ms = 0;
|
||||
result = mock_llm_call_with_retry(
|
||||
"test prompt",
|
||||
2, // max_retries
|
||||
100, // initial_backoff_ms
|
||||
1.5, // backoff_multiplier (slower)
|
||||
1000 // max_backoff_ms
|
||||
);
|
||||
|
||||
// Expected sleep times: 100ms, 150ms = 250ms total
|
||||
ok(total_sleep_time_ms == 250, "Slower multiplier results in different timing pattern");
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test: Edge Cases
|
||||
// ============================================================================
|
||||
|
||||
void test_retry_edge_cases() {
|
||||
diag("=== Retry Edge Cases ===");
|
||||
|
||||
// Test with negative retries (should be treated as 0)
|
||||
mock_success_on_attempt = -1; // Always fail
|
||||
mock_call_count = 0;
|
||||
std::string result = mock_llm_call_with_retry(
|
||||
"test prompt",
|
||||
-1, // negative retries
|
||||
100, // initial_backoff_ms
|
||||
2.0, // backoff_multiplier
|
||||
1000 // max_backoff_ms
|
||||
);
|
||||
|
||||
ok(mock_call_count == 1, "Negative retries treated as 0 retries");
|
||||
|
||||
// Test with very small initial backoff
|
||||
mock_success_on_attempt = -1; // Always fail
|
||||
total_sleep_time_ms = 0;
|
||||
result = mock_llm_call_with_retry(
|
||||
"test prompt",
|
||||
2, // max_retries
|
||||
1, // 1ms initial backoff
|
||||
2.0, // backoff_multiplier
|
||||
1000 // max_backoff_ms
|
||||
);
|
||||
|
||||
// Expected sleep times: 1ms, 2ms = 3ms total
|
||||
ok(total_sleep_time_ms == 3, "Very small initial backoff works correctly");
|
||||
|
||||
// Test with multiplier of 1.0 (linear backoff)
|
||||
mock_success_on_attempt = -1; // Always fail
|
||||
total_sleep_time_ms = 0;
|
||||
result = mock_llm_call_with_retry(
|
||||
"test prompt",
|
||||
3, // max_retries
|
||||
100, // initial_backoff_ms
|
||||
1.0, // backoff_multiplier (no growth)
|
||||
1000 // max_backoff_ms
|
||||
);
|
||||
|
||||
// Expected sleep times: 100ms, 100ms, 100ms = 300ms total
|
||||
ok(total_sleep_time_ms == 300, "Linear backoff (multiplier=1.0) works correctly");
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Main
|
||||
// ============================================================================
|
||||
|
||||
int main() {
|
||||
// Initialize random seed for tests
|
||||
srand(static_cast<unsigned int>(time(nullptr)));
|
||||
|
||||
// Plan: 22 tests total
|
||||
// Exponential backoff timing: 2 tests
|
||||
// Retry limit enforcement: 4 tests
|
||||
// Success recovery: 4 tests
|
||||
// Maximum backoff limit: 2 tests
|
||||
// Configurable parameters: 4 tests
|
||||
// Edge cases: 6 tests
|
||||
plan(22);
|
||||
|
||||
test_exponential_backoff_timing();
|
||||
test_retry_limit_enforcement();
|
||||
test_success_recovery();
|
||||
test_maximum_backoff_limit();
|
||||
test_configurable_parameters();
|
||||
test_retry_edge_cases();
|
||||
|
||||
return exit_status();
|
||||
}
|
||||
@ -0,0 +1,407 @@
|
||||
/**
|
||||
* @file vector_db_performance-t.cpp
|
||||
* @brief TAP unit tests for vector database performance
|
||||
*
|
||||
* Test Categories:
|
||||
* 1. Embedding generation timing for various text lengths
|
||||
* 2. KNN similarity search performance with different dataset sizes
|
||||
* 3. Cache hit vs miss performance comparison
|
||||
* 4. Concurrent access performance and thread safety
|
||||
* 5. Memory usage monitoring during vector operations
|
||||
* 6. Large dataset handling (1K+, 10K+ entries)
|
||||
*
|
||||
* @date 2026-01-16
|
||||
*/
|
||||
|
||||
#include "tap.h"
|
||||
#include <string.h>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <vector>
|
||||
#include <chrono>
|
||||
#include <thread>
|
||||
#include <algorithm>
|
||||
|
||||
// ============================================================================
|
||||
// Mock structures and functions to simulate vector database operations
|
||||
// ============================================================================
|
||||
|
||||
// Mock embedding generation (simulates GenAI embedding)
|
||||
static std::vector<float> mock_generate_embedding(const std::string& text) {
|
||||
// Simulate time taken for embedding generation based on text length
|
||||
// In real implementation, this would call GloGATH->embed_documents()
|
||||
|
||||
// Simple mock: create a fixed-size embedding with values based on text
|
||||
std::vector<float> embedding(1536, 0.0f); // Standard embedding size
|
||||
|
||||
// Fill with pseudo-random values based on text content
|
||||
unsigned int hash = 0;
|
||||
for (char c : text) {
|
||||
hash = hash * 31 + static_cast<unsigned char>(c);
|
||||
}
|
||||
|
||||
// Use hash to generate deterministic but varied embedding values
|
||||
for (size_t i = 0; i < embedding.size() && i < sizeof(hash); i++) {
|
||||
embedding[i] = static_cast<float>((hash >> (i * 8)) & 0xFF) / 255.0f;
|
||||
}
|
||||
|
||||
return embedding;
|
||||
}
|
||||
|
||||
// Mock cache entry structure
|
||||
struct MockCacheEntry {
|
||||
std::string natural_language;
|
||||
std::string generated_sql;
|
||||
std::vector<float> embedding;
|
||||
long long timestamp;
|
||||
};
|
||||
|
||||
// Mock vector database
|
||||
class MockVectorDB {
|
||||
private:
|
||||
std::vector<MockCacheEntry> entries;
|
||||
size_t max_entries;
|
||||
|
||||
public:
|
||||
MockVectorDB(size_t max_size = 10000) : max_entries(max_size) {}
|
||||
|
||||
// Simulate cache storage with timing
|
||||
long long store_entry(const std::string& query, const std::string& sql) {
|
||||
auto start = std::chrono::high_resolution_clock::now();
|
||||
|
||||
// Generate embedding
|
||||
std::vector<float> embedding = mock_generate_embedding(query);
|
||||
|
||||
// Check if we need to evict old entries
|
||||
if (entries.size() >= max_entries) {
|
||||
// Remove oldest entry (simple FIFO)
|
||||
entries.erase(entries.begin());
|
||||
}
|
||||
|
||||
// Add new entry
|
||||
MockCacheEntry entry;
|
||||
entry.natural_language = query;
|
||||
entry.generated_sql = sql;
|
||||
entry.embedding = embedding;
|
||||
entry.timestamp = std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
std::chrono::system_clock::now().time_since_epoch()).count();
|
||||
|
||||
entries.push_back(entry);
|
||||
|
||||
auto end = std::chrono::high_resolution_clock::now();
|
||||
auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
|
||||
return duration.count();
|
||||
}
|
||||
|
||||
// Simulate cache lookup with timing
|
||||
std::pair<long long, std::string> lookup_entry(const std::string& query, float similarity_threshold = 0.85f) {
|
||||
auto start = std::chrono::high_resolution_clock::now();
|
||||
|
||||
// Generate embedding for query
|
||||
std::vector<float> query_embedding = mock_generate_embedding(query);
|
||||
|
||||
// Find best match using cosine similarity
|
||||
float best_similarity = -1.0f;
|
||||
std::string best_sql = "";
|
||||
|
||||
for (const auto& entry : entries) {
|
||||
float similarity = cosine_similarity(query_embedding, entry.embedding);
|
||||
if (similarity > best_similarity && similarity >= similarity_threshold) {
|
||||
best_similarity = similarity;
|
||||
best_sql = entry.generated_sql;
|
||||
}
|
||||
}
|
||||
|
||||
auto end = std::chrono::high_resolution_clock::now();
|
||||
auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
|
||||
return std::make_pair(duration.count(), best_sql);
|
||||
}
|
||||
|
||||
// Calculate cosine similarity between two vectors
|
||||
float cosine_similarity(const std::vector<float>& a, const std::vector<float>& b) {
|
||||
if (a.size() != b.size() || a.empty()) return 0.0f;
|
||||
|
||||
float dot_product = 0.0f;
|
||||
float norm_a = 0.0f;
|
||||
float norm_b = 0.0f;
|
||||
|
||||
for (size_t i = 0; i < a.size(); i++) {
|
||||
dot_product += a[i] * b[i];
|
||||
norm_a += a[i] * a[i];
|
||||
norm_b += b[i] * b[i];
|
||||
}
|
||||
|
||||
if (norm_a == 0.0f || norm_b == 0.0f) return 0.0f;
|
||||
|
||||
return dot_product / (sqrt(norm_a) * sqrt(norm_b));
|
||||
}
|
||||
|
||||
size_t size() const { return entries.size(); }
|
||||
void clear() { entries.clear(); }
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Test: Embedding Generation Timing
|
||||
// ============================================================================
|
||||
|
||||
void test_embedding_timing() {
|
||||
diag("=== Embedding Generation Timing ===");
|
||||
|
||||
// Test with different text lengths
|
||||
std::vector<std::string> test_texts = {
|
||||
"Short query",
|
||||
"A medium length query with more words to process",
|
||||
"A very long query that contains many words and should take more time to process because it has significantly more text content that needs to be analyzed and converted into embeddings for vector database operations",
|
||||
std::string(1000, 'A') // Very long text
|
||||
};
|
||||
|
||||
std::vector<long long> timings;
|
||||
|
||||
for (const auto& text : test_texts) {
|
||||
auto start = std::chrono::high_resolution_clock::now();
|
||||
auto embedding = mock_generate_embedding(text);
|
||||
auto end = std::chrono::high_resolution_clock::now();
|
||||
|
||||
auto duration = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
|
||||
timings.push_back(duration.count());
|
||||
|
||||
ok(embedding.size() == 1536, "Embedding has correct size for text length %zu", text.length());
|
||||
}
|
||||
|
||||
// Verify that longer texts take more time (roughly)
|
||||
ok(timings[0] <= timings[1], "Medium text takes longer than short text");
|
||||
ok(timings[1] <= timings[2], "Long text takes longer than medium text");
|
||||
|
||||
diag("Embedding times (microseconds): Short=%lld, Medium=%lld, Long=%lld, VeryLong=%lld",
|
||||
timings[0], timings[1], timings[2], timings[3]);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test: KNN Search Performance
|
||||
// ============================================================================
|
||||
|
||||
void test_knn_search_performance() {
|
||||
diag("=== KNN Search Performance ===");
|
||||
|
||||
MockVectorDB db;
|
||||
|
||||
// Populate database with test entries
|
||||
const size_t small_dataset = 100;
|
||||
const size_t medium_dataset = 1000;
|
||||
const size_t large_dataset = 10000;
|
||||
|
||||
// Test with small dataset
|
||||
for (size_t i = 0; i < small_dataset; i++) {
|
||||
std::string query = "Test query " + std::to_string(i);
|
||||
std::string sql = "SELECT * FROM table WHERE id = " + std::to_string(i);
|
||||
db.store_entry(query, sql);
|
||||
}
|
||||
|
||||
// Test search performance
|
||||
auto result = db.lookup_entry("Test query 50");
|
||||
ok(result.second == "SELECT * FROM table WHERE id = 50" || result.second.empty(),
|
||||
"Search finds correct entry or no match in small dataset");
|
||||
|
||||
diag("Small dataset (%zu entries) search time: %lld microseconds", small_dataset, result.first);
|
||||
|
||||
// Clear and test with medium dataset
|
||||
db.clear();
|
||||
for (size_t i = 0; i < medium_dataset; i++) {
|
||||
std::string query = "Test query " + std::to_string(i);
|
||||
std::string sql = "SELECT * FROM table WHERE id = " + std::to_string(i);
|
||||
db.store_entry(query, sql);
|
||||
}
|
||||
|
||||
result = db.lookup_entry("Test query 500");
|
||||
ok(result.second == "SELECT * FROM table WHERE id = 500" || result.second.empty(),
|
||||
"Search finds correct entry or no match in medium dataset");
|
||||
|
||||
diag("Medium dataset (%zu entries) search time: %lld microseconds", medium_dataset, result.first);
|
||||
|
||||
// Test with query that won't match exactly (tests full search)
|
||||
result = db.lookup_entry("Completely different query");
|
||||
ok(result.second.empty(), "No match found for completely different query");
|
||||
|
||||
diag("Non-matching query search time: %lld microseconds", result.first);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test: Cache Hit vs Miss Performance
|
||||
// ============================================================================
|
||||
|
||||
void test_cache_hit_miss_performance() {
|
||||
diag("=== Cache Hit vs Miss Performance ===");
|
||||
|
||||
MockVectorDB db;
|
||||
|
||||
// Add some entries
|
||||
db.store_entry("Show me all users", "SELECT * FROM users;");
|
||||
db.store_entry("Count the orders", "SELECT COUNT(*) FROM orders;");
|
||||
|
||||
// Test cache hit
|
||||
auto hit_result = db.lookup_entry("Show me all users");
|
||||
ok(!hit_result.second.empty(), "Cache hit returns result");
|
||||
|
||||
// Test cache miss
|
||||
auto miss_result = db.lookup_entry("List all products");
|
||||
ok(miss_result.second.empty(), "Cache miss returns empty result");
|
||||
|
||||
// Verify hit is faster than miss (should be roughly similar in mock, but let's check)
|
||||
diag("Cache hit time: %lld microseconds, Cache miss time: %lld microseconds",
|
||||
hit_result.first, miss_result.first);
|
||||
|
||||
// Both should be reasonable times
|
||||
ok(hit_result.first < 100000, "Cache hit time is reasonable (< 100ms)");
|
||||
ok(miss_result.first < 100000, "Cache miss time is reasonable (< 100ms)");
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test: Memory Usage Monitoring
|
||||
// ============================================================================
|
||||
|
||||
void test_memory_usage() {
|
||||
diag("=== Memory Usage Monitoring ===");
|
||||
|
||||
// This is a conceptual test - in real implementation, we would monitor actual memory usage
|
||||
// For now, we'll test that the database doesn't grow unreasonably
|
||||
|
||||
MockVectorDB db(1000); // Limit to 1000 entries
|
||||
|
||||
// Add many entries
|
||||
for (size_t i = 0; i < 500; i++) {
|
||||
std::string query = "Query " + std::to_string(i);
|
||||
std::string sql = "SELECT * FROM table WHERE id = " + std::to_string(i);
|
||||
db.store_entry(query, sql);
|
||||
}
|
||||
|
||||
ok(db.size() == 500, "Database has expected number of entries (500)");
|
||||
|
||||
// Add more entries to test size limit
|
||||
for (size_t i = 500; i < 1200; i++) {
|
||||
std::string query = "Query " + std::to_string(i);
|
||||
std::string sql = "SELECT * FROM table WHERE id = " + std::to_string(i);
|
||||
db.store_entry(query, sql);
|
||||
}
|
||||
|
||||
// Should be capped at 1000 entries due to limit
|
||||
ok(db.size() <= 1000, "Database size respects maximum limit");
|
||||
|
||||
diag("Database size after adding 1200 entries: %zu", db.size());
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test: Large Dataset Handling
|
||||
// ============================================================================
|
||||
|
||||
void test_large_dataset_handling() {
|
||||
diag("=== Large Dataset Handling ===");
|
||||
|
||||
MockVectorDB db;
|
||||
|
||||
// Test handling of large dataset (10K entries)
|
||||
const size_t large_size = 10000;
|
||||
|
||||
auto start_insert = std::chrono::high_resolution_clock::now();
|
||||
|
||||
// Insert large number of entries
|
||||
for (size_t i = 0; i < large_size; i++) {
|
||||
std::string query = "Large dataset query " + std::to_string(i);
|
||||
std::string sql = "SELECT * FROM large_table WHERE id = " + std::to_string(i);
|
||||
|
||||
// Every 1000 entries, report progress
|
||||
if (i % 1000 == 0 && i > 0) {
|
||||
diag("Inserted %zu entries...", i);
|
||||
}
|
||||
|
||||
db.store_entry(query, sql);
|
||||
}
|
||||
|
||||
auto end_insert = std::chrono::high_resolution_clock::now();
|
||||
auto insert_duration = std::chrono::duration_cast<std::chrono::milliseconds>(end_insert - start_insert);
|
||||
|
||||
ok(db.size() == large_size, "Large dataset (%zu entries) inserted successfully", large_size);
|
||||
diag("Time to insert %zu entries: %lld ms", large_size, insert_duration.count());
|
||||
|
||||
// Test search performance in large dataset
|
||||
auto search_result = db.lookup_entry("Large dataset query 5000");
|
||||
ok(search_result.second == "SELECT * FROM large_table WHERE id = 5000" || search_result.second.empty(),
|
||||
"Search works in large dataset");
|
||||
|
||||
diag("Search time in %zu entry dataset: %lld microseconds", large_size, search_result.first);
|
||||
|
||||
// Performance should be reasonable even with large dataset
|
||||
ok(search_result.first < 500000, "Search time reasonable in large dataset (< 500ms)");
|
||||
ok(insert_duration.count() < 30000, "Insert time reasonable for large dataset (< 30s)");
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Test: Concurrent Access Performance
|
||||
// ============================================================================
|
||||
|
||||
void test_concurrent_access() {
|
||||
diag("=== Concurrent Access Performance ===");
|
||||
|
||||
// This is a simplified test - in real implementation, we would test actual thread safety
|
||||
MockVectorDB db;
|
||||
|
||||
// Populate with some data
|
||||
for (size_t i = 0; i < 100; i++) {
|
||||
std::string query = "Concurrent test " + std::to_string(i);
|
||||
std::string sql = "SELECT * FROM concurrent_table WHERE id = " + std::to_string(i);
|
||||
db.store_entry(query, sql);
|
||||
}
|
||||
|
||||
// Simulate concurrent access by running multiple operations
|
||||
const int num_operations = 10;
|
||||
std::vector<long long> timings;
|
||||
|
||||
auto start = std::chrono::high_resolution_clock::now();
|
||||
|
||||
for (int i = 0; i < num_operations; i++) {
|
||||
auto result = db.lookup_entry("Concurrent test " + std::to_string(i * 2));
|
||||
timings.push_back(result.first);
|
||||
}
|
||||
|
||||
auto end = std::chrono::high_resolution_clock::now();
|
||||
auto total_duration = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
|
||||
|
||||
// All operations should complete successfully
|
||||
ok(timings.size() == static_cast<size_t>(num_operations), "All concurrent operations completed");
|
||||
|
||||
// Calculate average time
|
||||
long long total_time = 0;
|
||||
for (long long time : timings) {
|
||||
total_time += time;
|
||||
}
|
||||
long long avg_time = total_time / num_operations;
|
||||
|
||||
diag("Average time per concurrent operation: %lld microseconds", avg_time);
|
||||
diag("Total time for %d operations: %lld microseconds", num_operations, total_duration.count());
|
||||
|
||||
// Operations should be reasonably fast
|
||||
ok(avg_time < 50000, "Average concurrent operation time reasonable (< 50ms)");
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Main
|
||||
// ============================================================================
|
||||
|
||||
int main() {
|
||||
// Plan: 25 tests total
|
||||
// Embedding timing: 5 tests
|
||||
// KNN search performance: 4 tests
|
||||
// Cache hit vs miss: 3 tests
|
||||
// Memory usage: 3 tests
|
||||
// Large dataset handling: 5 tests
|
||||
// Concurrent access: 5 tests
|
||||
plan(25);
|
||||
|
||||
test_embedding_timing();
|
||||
test_knn_search_performance();
|
||||
test_cache_hit_miss_performance();
|
||||
test_memory_usage();
|
||||
test_large_dataset_handling();
|
||||
test_concurrent_access();
|
||||
|
||||
return exit_status();
|
||||
}
|
||||
Loading…
Reference in new issue