From e9a6dd0b3ee39991f9b60f6137039d77a345d4c0 Mon Sep 17 00:00:00 2001 From: Rene Cannao Date: Sun, 11 Jan 2026 15:05:15 +0000 Subject: [PATCH] Add comprehensive MCP testing suite in scripts/mcp/ Created a complete testing suite for the MCP module with MySQL connection pool and exploration tools. Files added: - README.md: Comprehensive testing documentation - setup_test_db.sh: Docker-based test MySQL database setup - Start/stop/status/connect commands - Creates sample schema (customers, orders, products, order_items) - Includes views and stored procedures for testing - configure_mcp.sh: ProxySQL MCP module configuration - Configures MySQL connection parameters - Enables/disables MCP server - Shows current configuration status - test_mcp_tools.sh: Main MCP tools test suite - Tests all 15 MCP tools (list_schemas, list_tables, etc.) - Includes catalog tests (upsert, get, search, delete) - Reports pass/fail statistics - stress_test.sh: Concurrent connection stress testing - Configurable number of concurrent requests - Response time measurement - Success rate calculation - test_catalog.sh: Catalog/LLM memory specific tests - 12 catalog operation tests - FTS search testing - CRUD verification All scripts are executable and include: - Command-line argument parsing - Colored output for readability - Error handling and validation - Usage/help documentation - Environment variable support --- scripts/mcp/README.md | 155 +++++++++ scripts/mcp/configure_mcp.sh | 301 +++++++++++++++++ scripts/mcp/setup_test_db.sh | 401 +++++++++++++++++++++++ scripts/mcp/stress_test.sh | 286 ++++++++++++++++ scripts/mcp/test_catalog.sh | 438 +++++++++++++++++++++++++ scripts/mcp/test_mcp_tools.sh | 598 ++++++++++++++++++++++++++++++++++ 6 files changed, 2179 insertions(+) create mode 100644 scripts/mcp/README.md create mode 100755 scripts/mcp/configure_mcp.sh create mode 100755 scripts/mcp/setup_test_db.sh create mode 100755 scripts/mcp/stress_test.sh create mode 100755 scripts/mcp/test_catalog.sh create mode 100755 scripts/mcp/test_mcp_tools.sh diff --git a/scripts/mcp/README.md b/scripts/mcp/README.md new file mode 100644 index 000000000..e1776ded8 --- /dev/null +++ b/scripts/mcp/README.md @@ -0,0 +1,155 @@ +# MCP Module Testing Suite + +This directory contains scripts to test the ProxySQL MCP (Model Context Protocol) module with MySQL connection pool and exploration tools. + +## Prerequisites + +- ProxySQL must be installed and built with MCP support +- MySQL server (either running or Docker capability) +- `mysql` client installed +- `curl` installed for HTTP testing +- `jq` installed for JSON parsing (optional but recommended) + +## Quick Start + +```bash +# 1. Start a test MySQL server (Docker) +./setup_test_db.sh start + +# 2. Configure ProxySQL MCP module +./configure_mcp.sh + +# 3. Run all MCP tool tests +./test_mcp_tools.sh + +# 4. Run stress test (optional) +./stress_test.sh + +# 5. Stop test MySQL server (Docker) +./setup_test_db.sh stop +``` + +## Scripts + +| Script | Purpose | +|--------|---------| +| `setup_test_db.sh` | Create/start a test MySQL database with sample data | +| `configure_mcp.sh` | Configure ProxySQL MCP module variables | +| `test_mcp_tools.sh` | Test all MCP tools via HTTPS/JSON-RPC | +| `stress_test.sh` | Concurrent connection stress test | +| `test_catalog.sh` | Test catalog (LLM memory) functionality | + +## Manual Testing + +### Test via curl + +```bash +# Test list_schemas +curl -k https://127.0.0.1:6071/query -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "jsonrpc": "2.0", + "method": "tools/call", + "params": {"name": "list_schemas", "arguments": {}}, + "id": 1 + }' + +# Test list_tables +curl -k https://127.0.0.1:6071/query -X POST \ + -H "Content-Type: application/json" \ + -d '{ + "jsonrpc": "2.0", + "method": "tools/call", + "params": {"name": "list_tables", "arguments": {"schema": "testdb"}}, + "id": 1 + }' +``` + +### Test via mysql admin + +```sql +-- Connect to ProxySQL admin +mysql -h 127.0.0.1 -P 6032 -u admin -padmin + +-- Check MCP configuration +SHOW VARIABLES LIKE 'mcp-%'; + +-- Check connection pool status +SELECT * FROM stats_mcp_connections; +``` + +## Expected Results + +### Successful Connection Pool Initialization + +ProxySQL log should show: +``` +MySQL_Tool_Handler: Connected to 127.0.0.1:3307 +MySQL_Tool_Handler: Connection pool initialized with 1 connection(s) +MySQL Tool Handler initialized for schema 'testdb' +``` + +### Successful Tool Response + +```json +{ + "jsonrpc": "2.0", + "result": [ + {"name": "testdb", "table_count": 2}, + {"name": "mysql", "table_count": 0} + ], + "id": 1 +} +``` + +## Troubleshooting + +### MCP server not starting + +Check ProxySQL logs: +```bash +tail -f proxysql.log | grep -i mcp +``` + +### Connection pool failing + +Verify MySQL is accessible: +```bash +mysql -h 127.0.0.1 -P 3307 -u root -ptest testdb -e "SELECT 1" +``` + +### Certificate errors + +The tests use `-k` to skip SSL verification. For production: +```bash +export MCP_CERT=/path/to/cert.pem +export MCP_KEY=/path/to/key.pem +``` + +## MCP Tools Reference + +| Tool | Description | +|------|-------------| +| `list_schemas` | List available databases | +| `list_tables` | List tables in a schema | +| `describe_table` | Get table schema (columns, keys, indexes) | +| `sample_rows` | Sample rows from a table | +| `sample_distinct` | Sample distinct values from a column | +| `run_sql_readonly` | Execute read-only SQL with guardrails | +| `explain_sql` | Get query execution plan | +| `catalog_upsert` | Store entry in LLM catalog | +| `catalog_get` | Retrieve entry from LLM catalog | +| `catalog_search` | Search LLM catalog | + +## Default Configuration + +| Variable | Default | Description | +|----------|---------|-------------| +| `mcp-enabled` | false | Enable MCP server | +| `mcp-port` | 6071 | HTTPS port for MCP | +| `mcp-mysql_hosts` | 127.0.0.1 | MySQL server host(s) | +| `mcp-mysql_ports` | 3306 | MySQL server port(s) | +| `mcp-mysql_user` | (empty) | MySQL username | +| `mcp-mysql_password` | (empty) | MySQL password | +| `mcp-mysql_schema` | (empty) | Default schema | +| `mcp-catalog_path` | /var/lib/proxysql/mcp_catalog.db | Catalog database path | diff --git a/scripts/mcp/configure_mcp.sh b/scripts/mcp/configure_mcp.sh new file mode 100755 index 000000000..23b99eeeb --- /dev/null +++ b/scripts/mcp/configure_mcp.sh @@ -0,0 +1,301 @@ +#!/bin/bash +# +# configure_mcp.sh - Configure ProxySQL MCP module +# +# Usage: +# ./configure_mcp.sh [options] +# +# Options: +# -h, --host HOST MySQL host (default: 127.0.0.1) +# -P, --port PORT MySQL port (default: 3307) +# -u, --user USER MySQL user (default: root) +# -p, --password PASS MySQL password (default: test123) +# -d, --database DB MySQL database (default: testdb) +# --mcp-port PORT MCP server port (default: 6071) +# --enable Enable MCP server +# --disable Disable MCP server +# --status Show current MCP configuration +# + +set -e + +# Default configuration +MYSQL_HOST="127.0.0.1" +MYSQL_PORT="3307" +MYSQL_USER="root" +MYSQL_PASSWORD="test123" +MYSQL_DATABASE="testdb" +MCP_PORT="6071" +MCP_ENABLED="false" + +# ProxySQL admin configuration +PROXYSQL_ADMIN_HOST="${PROXYSQL_ADMIN_HOST:-127.0.0.1}" +PROXYSQL_ADMIN_PORT="${PROXYSQL_ADMIN_PORT:-6032}" +PROXYSQL_ADMIN_USER="${PROXYSQL_ADMIN_USER:-admin}" +PROXYSQL_ADMIN_PASSWORD="${PROXYSQL_ADMIN_PASSWORD:-admin}" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +log_step() { + echo -e "${BLUE}[STEP]${NC} $1" +} + +# Execute MySQL command via ProxySQL admin +exec_admin() { + mysql -h "${PROXYSQL_ADMIN_HOST}" -P "${PROXYSQL_ADMIN_PORT}" \ + -u "${PROXYSQL_ADMIN_USER}" -p"${PROXYSQL_ADMIN_PASSWORD}" \ + -e "$1" 2>/dev/null +} + +# Check if ProxySQL admin is accessible +check_proxysql_admin() { + log_step "Checking ProxySQL admin connection..." + if exec_admin "SELECT 1" >/dev/null 2>&1; then + log_info "Connected to ProxySQL admin at ${PROXYSQL_ADMIN_HOST}:${PROXYSQL_ADMIN_PORT}" + return 0 + else + log_error "Cannot connect to ProxySQL admin at ${PROXYSQL_ADMIN_HOST}:${PROXYSQL_ADMIN_PORT}" + log_error "Please ensure ProxySQL is running" + return 1 + fi +} + +# Check if MySQL is accessible +check_mysql_connection() { + log_step "Checking MySQL connection..." + if mysql -h "${MYSQL_HOST}" -P "${MYSQL_PORT}" \ + -u "${MYSQL_USER}" -p"${MYSQL_PASSWORD}" \ + -e "SELECT 1" >/dev/null 2>&1; then + log_info "Connected to MySQL at ${MYSQL_HOST}:${MYSQL_PORT}" + return 0 + else + log_error "Cannot connect to MySQL at ${MYSQL_HOST}:${MYSQL_PORT}" + log_error "Please ensure MySQL is running and credentials are correct" + return 1 + fi +} + +# Configure MCP variables +configure_mcp() { + local enable="$1" + + log_step "Configuring MCP variables..." + + # Set MySQL connection configuration + cat </dev/null 2>&1; then + log_info "MCP variables loaded to RUNTIME" + else + log_error "Failed to load MCP variables to RUNTIME" + return 1 + fi +} + +# Show current MCP configuration +show_status() { + log_step "Current MCP configuration:" + echo "" + exec_admin "SHOW VARIABLES LIKE 'mcp-%';" | column -t + echo "" +} + +# Test MCP server connectivity +test_mcp_server() { + log_step "Testing MCP server connectivity..." + + # Wait a moment for server to start + sleep 2 + + # Test ping endpoint + local response + response=$(curl -k -s -X POST "https://${PROXYSQL_ADMIN_HOST}:${MCP_PORT}/config" \ + -H "Content-Type: application/json" \ + -d '{"jsonrpc":"2.0","method":"ping","id":1}' 2>/dev/null || echo "") + + if [ -n "$response" ]; then + log_info "MCP server is responding" + echo " Response: $response" + else + log_warn "MCP server not responding (may still be starting)" + fi +} + +# Parse command line arguments +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + -h|--host) + MYSQL_HOST="$2" + shift 2 + ;; + -P|--port) + MYSQL_PORT="$2" + shift 2 + ;; + -u|--user) + MYSQL_USER="$2" + shift 2 + ;; + -p|--password) + MYSQL_PASSWORD="$2" + shift 2 + ;; + -d|--database) + MYSQL_DATABASE="$2" + shift 2 + ;; + --mcp-port) + MCP_PORT="$2" + shift 2 + ;; + --enable) + MCP_ENABLED="true" + shift + ;; + --disable) + MCP_ENABLED="false" + shift + ;; + --status) + show_status + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use --help for usage information" + exit 1 + ;; + esac + done +} + +# Show usage +show_usage() { + cat < /dev/null; then + log_error "Docker is not installed or not in PATH" + log_info "Please install Docker or use an existing MySQL server" + exit 1 + fi +} + +# Start test MySQL container +start_mysql() { + log_info "Starting test MySQL container..." + + # Check if container already exists + if docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then + log_warn "Container '${CONTAINER_NAME}' already exists" + read -p "Remove and recreate? (y/N): " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + docker rm -f "${CONTAINER_NAME}" > /dev/null 2>&1 || true + else + log_info "Starting existing container..." + docker start "${CONTAINER_NAME}" + return 0 + fi + fi + + # Create and start container + docker run -d \ + --name "${CONTAINER_NAME}" \ + -p "${MYSQL_PORT}:3306" \ + -e MYSQL_ROOT_PASSWORD="${MYSQL_ROOT_PASSWORD}" \ + -e MYSQL_DATABASE="${MYSQL_DATABASE}" \ + -v "${SCRIPT_DIR}/init_testdb.sql:/docker-entrypoint-initdb.d/01-init.sql:ro" \ + mysql:${MYSQL_VERSION} \ + --default-authentication-plugin=mysql_native_password + + log_info "Waiting for MySQL to be ready..." + for i in {1..30}; do + if docker exec "${CONTAINER_NAME}" mysqladmin ping -h localhost --silent 2>/dev/null; then + log_info "MySQL is ready!" + break + fi + sleep 1 + done + + # Run initialization script if not via volume + if [ ! -f "${SCRIPT_DIR}/init_testdb.sql" ]; then + log_info "Creating test schema and data..." + sleep 5 # Give MySQL extra time to fully start + docker exec -i "${CONTAINER_NAME}" mysql -uroot -p"${MYSQL_ROOT_PASSWORD}" "${MYSQL_DATABASE}" <<'EOSQL' +CREATE DATABASE IF NOT EXISTS testdb; +USE testdb; + +CREATE TABLE IF NOT EXISTS customers ( + id INT PRIMARY KEY AUTO_INCREMENT, + name VARCHAR(100), + email VARCHAR(100), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + INDEX idx_email (email) +); + +CREATE TABLE IF NOT EXISTS orders ( + id INT PRIMARY KEY AUTO_INCREMENT, + customer_id INT NOT NULL, + order_date DATE, + total DECIMAL(10,2), + status VARCHAR(20), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (customer_id) REFERENCES customers(id), + INDEX idx_customer (customer_id), + INDEX idx_status (status) +); + +CREATE TABLE IF NOT EXISTS products ( + id INT PRIMARY KEY AUTO_INCREMENT, + name VARCHAR(200), + category VARCHAR(50), + price DECIMAL(10,2), + stock INT DEFAULT 0, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + INDEX idx_category (category) +); + +CREATE TABLE IF NOT EXISTS order_items ( + id INT PRIMARY KEY AUTO_INCREMENT, + order_id INT NOT NULL, + product_id INT NOT NULL, + quantity INT DEFAULT 1, + price DECIMAL(10,2), + FOREIGN KEY (order_id) REFERENCES orders(id), + FOREIGN KEY (product_id) REFERENCES products(id) +); + +-- Insert sample customers +INSERT INTO customers (name, email) VALUES + ('Alice Johnson', 'alice@example.com'), + ('Bob Smith', 'bob@example.com'), + ('Charlie Brown', 'charlie@example.com'), + ('Diana Prince', 'diana@example.com'), + ('Eve Davis', 'eve@example.com'); + +-- Insert sample products +INSERT INTO products (name, category, price, stock) VALUES + ('Laptop', 'Electronics', 999.99, 50), + ('Mouse', 'Electronics', 29.99, 200), + ('Keyboard', 'Electronics', 79.99, 150), + ('Desk Chair', 'Furniture', 199.99, 75), + ('Coffee Mug', 'Kitchen', 12.99, 500); + +-- Insert sample orders +INSERT INTO orders (customer_id, order_date, total, status) VALUES + (1, '2024-01-15', 1029.98, 'completed'), + (2, '2024-01-16', 79.99, 'shipped'), + (1, '2024-01-17', 212.98, 'pending'), + (3, '2024-01-18', 199.99, 'completed'), + (4, '2024-01-19', 1099.98, 'shipped'); + +-- Insert sample order items +INSERT INTO order_items (order_id, product_id, quantity, price) VALUES + (1, 1, 1, 999.99), + (1, 2, 1, 29.99), + (2, 3, 1, 79.99), + (3, 1, 1, 999.99), + (3, 3, 1, 79.99), + (3, 5, 3, 38.97), + (4, 4, 1, 199.99), + (5, 1, 1, 999.99), + (5, 4, 1, 199.99); + +-- Create a view +CREATE OR REPLACE VIEW customer_orders AS +SELECT + c.id AS customer_id, + c.name AS customer_name, + COUNT(o.id) AS order_count, + SUM(o.total) AS total_spent +FROM customers c +LEFT JOIN orders o ON c.id = o.customer_id +GROUP BY c.id, c.name; + +-- Create a stored procedure +DELIMITER // +CREATE PROCEDURE get_customer_stats(IN customer_id INT) +BEGIN + SELECT + c.name, + COUNT(o.id) AS order_count, + COALESCE(SUM(o.total), 0) AS total_spent + FROM customers c + LEFT JOIN orders o ON c.id = o.customer_id + WHERE c.id = customer_id; +END // +DELIMITER ; +EOSQL + fi + + log_info "Test MySQL database is ready!" + log_info " Host: 127.0.0.1" + log_info " Port: ${MYSQL_PORT}" + log_info " User: root" + log_info " Password: ${MYSQL_ROOT_PASSWORD}" + log_info " Database: ${MYSQL_DATABASE}" +} + +# Stop and remove test MySQL container +stop_mysql() { + log_info "Stopping test MySQL container..." + if docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then + docker stop "${CONTAINER_NAME}" + log_info "Container stopped" + else + log_warn "Container '${CONTAINER_NAME}' is not running" + fi + + if docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then + read -p "Remove container '${CONTAINER_NAME}'? (y/N): " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + docker rm "${CONTAINER_NAME}" + log_info "Container removed" + fi + fi +} + +# Check status of test MySQL +status_mysql() { + log_info "Checking test MySQL status..." + + if docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then + echo -e "${GREEN}●${NC} Container '${CONTAINER_NAME}' is ${GREEN}running${NC}" + + # Show connection details + echo "" + echo "Connection Details:" + echo " Host: 127.0.0.1" + echo " Port: ${MYSQL_PORT}" + echo " User: root" + echo " Password: ${MYSQL_ROOT_PASSWORD}" + echo " Database: ${MYSQL_DATABASE}" + + # Test connection + if docker exec "${CONTAINER_NAME}" mysqladmin ping -h localhost --silent 2>/dev/null; then + echo -e " Status: ${GREEN}Accepting connections${NC}" + else + echo -e " Status: ${RED}Not responding${NC}" + fi + + # Show database info + echo "" + echo "Database Info:" + docker exec "${CONTAINER_NAME}" mysql -uroot -p"${MYSQL_ROOT_PASSWORD}" -e " + SELECT + table_name AS 'Table', + table_rows AS 'Rows', + ROUND((data_length + index_length) / 1024, 2) AS 'Size (KB)' + FROM information_schema.tables + WHERE table_schema = '${MYSQL_DATABASE}' + ORDER BY table_name; + " 2>/dev/null | column -t + elif docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then + echo -e "${YELLOW}○${NC} Container '${CONTAINER_NAME}' exists but is ${YELLOW}stopped${NC}" + echo "Start with: $0 start" + else + echo -e "${RED}✗${NC} Container '${CONTAINER_NAME}' does not exist" + echo "Create with: $0 start" + fi +} + +# Connect to test MySQL +connect_mysql() { + log_info "Connecting to test MySQL..." + if ! docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then + log_error "Container '${CONTAINER_NAME}' is not running" + exit 1 + fi + + docker exec -it "${CONTAINER_NAME}" mysql -uroot -p"${MYSQL_ROOT_PASSWORD}" "${MYSQL_DATABASE}" +} + +# Create initialization SQL file +create_init_sql() { + cat > "${SCRIPT_DIR}/init_testdb.sql" <<'EOSQL' +-- Test Database Schema for MCP Testing + +CREATE DATABASE IF NOT EXISTS testdb; +USE testdb; + +CREATE TABLE IF NOT EXISTS customers ( + id INT PRIMARY KEY AUTO_INCREMENT, + name VARCHAR(100), + email VARCHAR(100), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + INDEX idx_email (email) +); + +CREATE TABLE IF NOT EXISTS orders ( + id INT PRIMARY KEY AUTO_INCREMENT, + customer_id INT NOT NULL, + order_date DATE, + total DECIMAL(10,2), + status VARCHAR(20), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (customer_id) REFERENCES customers(id), + INDEX idx_customer (customer_id), + INDEX idx_status (status) +); + +CREATE TABLE IF NOT EXISTS products ( + id INT PRIMARY KEY AUTO_INCREMENT, + name VARCHAR(200), + category VARCHAR(50), + price DECIMAL(10,2), + stock INT DEFAULT 0, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + INDEX idx_category (category) +); + +CREATE TABLE IF NOT EXISTS order_items ( + id INT PRIMARY KEY AUTO_INCREMENT, + order_id INT NOT NULL, + product_id INT NOT NULL, + quantity INT DEFAULT 1, + price DECIMAL(10,2), + FOREIGN KEY (order_id) REFERENCES orders(id), + FOREIGN KEY (product_id) REFERENCES products(id) +); + +-- Insert sample customers +INSERT INTO customers (name, email) VALUES + ('Alice Johnson', 'alice@example.com'), + ('Bob Smith', 'bob@example.com'), + ('Charlie Brown', 'charlie@example.com'), + ('Diana Prince', 'diana@example.com'), + ('Eve Davis', 'eve@example.com'); + +-- Insert sample products +INSERT INTO products (name, category, price, stock) VALUES + ('Laptop', 'Electronics', 999.99, 50), + ('Mouse', 'Electronics', 29.99, 200), + ('Keyboard', 'Electronics', 79.99, 150), + ('Desk Chair', 'Furniture', 199.99, 75), + ('Coffee Mug', 'Kitchen', 12.99, 500); + +-- Insert sample orders +INSERT INTO orders (customer_id, order_date, total, status) VALUES + (1, '2024-01-15', 1029.98, 'completed'), + (2, '2024-01-16', 79.99, 'shipped'), + (1, '2024-01-17', 212.98, 'pending'), + (3, '2024-01-18', 199.99, 'completed'), + (4, '2024-01-19', 1099.98, 'shipped'); + +-- Insert sample order items +INSERT INTO order_items (order_id, product_id, quantity, price) VALUES + (1, 1, 1, 999.99), + (1, 2, 1, 29.99), + (2, 3, 1, 79.99), + (3, 1, 1, 999.99), + (3, 3, 1, 79.99), + (3, 5, 3, 38.97), + (4, 4, 1, 199.99), + (5, 1, 1, 999.99), + (5, 4, 1, 199.99); +EOSQL + + log_info "Created ${SCRIPT_DIR}/init_testdb.sql" +} + +# Main script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +case "${1:-start}" in + start) + check_docker + start_mysql + ;; + stop) + check_docker + stop_mysql + ;; + status) + check_docker + status_mysql + ;; + connect) + check_docker + connect_mysql + ;; + create-sql) + create_init_sql + ;; + *) + echo "Usage: $0 {start|stop|status|connect|create-sql}" + echo "" + echo "Commands:" + echo " start - Start test MySQL container" + echo " stop - Stop test MySQL container" + echo " status - Check status of test MySQL" + echo " connect - Connect to test MySQL shell" + echo " create-sql - Create init_testdb.sql file" + exit 1 + ;; +esac diff --git a/scripts/mcp/stress_test.sh b/scripts/mcp/stress_test.sh new file mode 100755 index 000000000..a04459681 --- /dev/null +++ b/scripts/mcp/stress_test.sh @@ -0,0 +1,286 @@ +#!/bin/bash +# +# stress_test.sh - Concurrent connection stress test for MCP tools +# +# Usage: +# ./stress_test.sh [options] +# +# Options: +# -n, --num-requests N Number of concurrent requests (default: 10) +# -t, --tool NAME Tool to test (default: sample_rows) +# -d, --delay SEC Delay between requests in ms (default: 0) +# -v, --verbose Show individual responses +# -h, --help Show help +# + +set -e + +# Configuration +MCP_HOST="${MCP_HOST:-127.0.0.1}" +MCP_PORT="${MCP_PORT:-6071}" +MCP_URL="https://${MCP_HOST}:${MCP_PORT}/query" + +# Test options +NUM_REQUESTS="${NUM_REQUESTS:-10}" +TOOL_NAME="${TOOL_NAME:-sample_rows}" +DELAY_MS="${DELAY_MS:-0}" +VERBOSE=false + +# Statistics +TOTAL_REQUESTS=0 +SUCCESSFUL_REQUESTS=0 +FAILED_REQUESTS=0 +TOTAL_TIME=0 + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +log_info() { + echo -e "${GREEN}[INFO]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +log_warn() { + echo -e "${YELLOW}[WARN]${NC} $1" +} + +# Execute MCP request +mcp_request() { + local id="$1" + + local payload + payload=$(cat </dev/null) + + local end_time + end_time=$(date +%s%N) + + local duration + duration=$(( (end_time - start_time) / 1000000 )) # Convert to milliseconds + + local body + body=$(echo "$response" | head -n -1) + + local code + code=$(echo "$response" | tail -n 1) + + echo "${body}|${duration}|${code}" +} + +# Run concurrent requests +run_stress_test() { + log_info "Running stress test with ${NUM_REQUESTS} concurrent requests..." + log_info "Tool: ${TOOL_NAME}" + log_info "Target: ${MCP_URL}" + echo "" + + # Create temp directory for results + local tmpdir + tmpdir=$(mktemp -d) + trap "rm -rf ${tmpdir}" EXIT + + local pids=() + + # Launch requests in background + for i in $(seq 1 "${NUM_REQUESTS}"); do + ( + if [ -n "${DELAY_MS}" ] && [ "${DELAY_MS}" -gt 0 ]; then + sleep $(( (RANDOM % ${DELAY_MS}) / 1000 )).$(( (RANDOM % 1000) )) + fi + + local result + result=$(mcp_request "${i}") + + local body + local duration + local code + + body=$(echo "${result}" | cut -d'|' -f1) + duration=$(echo "${result}" | cut -d'|' -f2) + code=$(echo "${result}" | cut -d'|' -f3) + + echo "${body}" > "${tmpdir}/response_${i}.json" + echo "${duration}" > "${tmpdir}/duration_${i}.txt" + echo "${code}" > "${tmpdir}/code_${i}.txt" + ) & + pids+=($!) + done + + # Wait for all requests to complete + local start_time + start_time=$(date +%s) + + for pid in "${pids[@]}"; do + wait ${pid} || true + done + + local end_time + end_time=$(date +%s) + + local total_wall_time + total_wall_time=$((end_time - start_time)) + + # Collect results + for i in $(seq 1 "${NUM_REQUESTS}"); do + TOTAL_REQUESTS=$((TOTAL_REQUESTS + 1)) + + local code + code=$(cat "${tmpdir}/code_${i}.txt" 2>/dev/null || echo "000") + + if [ "${code}" = "200" ]; then + SUCCESSFUL_REQUESTS=$((SUCCESSFUL_REQUESTS + 1)) + else + FAILED_REQUESTS=$((FAILED_REQUESTS + 1)) + fi + + local duration + duration=$(cat "${tmpdir}/duration_${i}.txt" 2>/dev/null || echo "0") + TOTAL_TIME=$((TOTAL_TIME + duration)) + + if [ "${VERBOSE}" = "true" ]; then + local body + body=$(cat "${tmpdir}/response_${i}.json" 2>/dev/null || echo "{}") + echo "Request ${i}: [${code}] ${duration}ms" + if [ "${code}" != "200" ]; then + echo " Response: ${body}" + fi + fi + done + + # Calculate statistics + local avg_time + if [ ${TOTAL_REQUESTS} -gt 0 ]; then + avg_time=$((TOTAL_TIME / TOTAL_REQUESTS)) + else + avg_time=0 + fi + + local requests_per_second + if [ ${total_wall_time} -gt 0 ]; then + requests_per_second=$(awk "BEGIN {printf \"%.2f\", ${NUM_REQUESTS} / ${total_wall_time}}") + else + requests_per_second="N/A" + fi + + # Print summary + echo "" + echo "======================================" + echo "Stress Test Results" + echo "======================================" + echo "Concurrent requests: ${NUM_REQUESTS}" + echo "Total wall time: ${total_wall_time}s" + echo "" + echo "Total requests: ${TOTAL_REQUESTS}" + echo -e "Successful: ${GREEN}${SUCCESSFUL_REQUESTS}${NC}" + echo -e "Failed: ${RED}${FAILED_REQUESTS}${NC}" + echo "" + echo "Average response time: ${avg_time}ms" + echo "Requests/second: ${requests_per_second}" + echo "" + + # Calculate success rate + if [ ${TOTAL_REQUESTS} -gt 0 ]; then + local success_rate + success_rate=$(awk "BEGIN {printf \"%.1f\", (${SUCCESSFUL_REQUESTS} * 100) / ${TOTAL_REQUESTS}}") + echo "Success rate: ${success_rate}%" + echo "" + + if [ ${FAILED_REQUESTS} -eq 0 ]; then + log_info "All requests succeeded!" + return 0 + else + log_error "Some requests failed!" + return 1 + fi + else + log_error "No requests were completed!" + return 1 + fi +} + +# Parse command line arguments +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + -n|--num-requests) + NUM_REQUESTS="$2" + shift 2 + ;; + -t|--tool) + TOOL_NAME="$2" + shift 2 + ;; + -d|--delay) + DELAY_MS="$2" + shift 2 + ;; + -v|--verbose) + VERBOSE=true + shift + ;; + -h|--help) + cat </dev/null) + + echo "${response}" +} + +# Test catalog operations +test_catalog() { + local test_id="$1" + local operation="$2" + local payload="$3" + local expected="$4" + + log_test "${test_id}: ${operation}" + + local response + response=$(mcp_request "${payload}") + + if [ "${VERBOSE}" = "true" ]; then + echo "Payload: ${payload}" + echo "Response: ${response}" + fi + + if echo "${response}" | grep -q "${expected}"; then + log_info "✓ ${test_id}" + return 0 + else + log_error "✗ ${test_id}" + if [ "${VERBOSE}" = "true" ]; then + echo "Expected to find: ${expected}" + fi + return 1 + fi +} + +# Main test flow +run_catalog_tests() { + echo "======================================" + echo "Catalog (LLM Memory) Test Suite" + echo "======================================" + echo "" + echo "Testing catalog operations for LLM memory persistence" + echo "" + + local passed=0 + local failed=0 + + # Test 1: Upsert a table schema entry + local payload1 + payload1='{ + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "name": "catalog_upsert", + "arguments": { + "kind": "table", + "key": "testdb.customers", + "document": "{\"table\": \"customers\", \"columns\": [{\"name\": \"id\", \"type\": \"INT\"}, {\"name\": \"name\", \"type\": \"VARCHAR\"}], \"row_count\": 5}", + "tags": "schema,testdb", + "links": "testdb.orders:customer_id" + } + }, + "id": 1 +}' + + if test_catalog "CAT001" "Upsert table schema" "${payload1}" '"success"[[:space:]]*:[[:space:]]*true'; then + passed=$((passed + 1)) + else + failed=$((failed + 1)) + fi + + # Test 2: Upsert a domain knowledge entry + local payload2 + payload2='{ + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "name": "catalog_upsert", + "arguments": { + "kind": "domain", + "key": "customer_management", + "document": "{\"description\": \"Customer management domain\", \"entities\": [\"customers\", \"orders\", \"products\"], \"relationships\": [\"customer has many orders\", \"order belongs to customer\"]}", + "tags": "domain,business", + "links": "" + } + }, + "id": 2 +}' + + if test_catalog "CAT002" "Upsert domain knowledge" "${payload2}" '"success"[[:space:]]*:[[:space:]]*true'; then + passed=$((passed + 1)) + else + failed=$((failed + 1)) + fi + + # Test 3: Get the upserted table entry + local payload3 + payload3='{ + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "name": "catalog_get", + "arguments": { + "kind": "table", + "key": "testdb.customers" + } + }, + "id": 3 +}' + + if test_catalog "CAT003" "Get table entry" "${payload3}" '"columns"'; then + passed=$((passed + 1)) + else + failed=$((failed + 1)) + fi + + # Test 4: Get the upserted domain entry + local payload4 + payload4='{ + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "name": "catalog_get", + "arguments": { + "kind": "domain", + "key": "customer_management" + } + }, + "id": 4 +}' + + if test_catalog "CAT004" "Get domain entry" "${payload4}" '"entities"'; then + passed=$((passed + 1)) + else + failed=$((failed + 1)) + fi + + # Test 5: Search for table entries + local payload5 + payload5='{ + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "name": "catalog_search", + "arguments": { + "query": "customers", + "limit": 10 + } + }, + "id": 5 +}' + + if test_catalog "CAT005" "Search catalog" "${payload5}" '"results"'; then + passed=$((passed + 1)) + else + failed=$((failed + 1)) + fi + + # Test 6: List entries by kind + local payload6 + payload6='{ + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "name": "catalog_list", + "arguments": { + "kind": "table", + "limit": 10 + } + }, + "id": 6 +}' + + if test_catalog "CAT006" "List by kind" "${payload6}" '"results"'; then + passed=$((passed + 1)) + else + failed=$((failed + 1)) + fi + + # Test 7: Update existing entry + local payload7 + payload7='{ + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "name": "catalog_upsert", + "arguments": { + "kind": "table", + "key": "testdb.customers", + "document": "{\"table\": \"customers\", \"columns\": [{\"name\": \"id\", \"type\": \"INT\"}, {\"name\": \"name\", \"type\": \"VARCHAR\"}, {\"name\": \"email\", \"type\": \"VARCHAR\"}], \"row_count\": 5, \"updated\": true}", + "tags": "schema,testdb,updated", + "links": "testdb.orders:customer_id" + } + }, + "id": 7 +}' + + if test_catalog "CAT007" "Update existing entry" "${payload7}" '"success"[[:space:]]*:[[:space:]]*true'; then + passed=$((passed + 1)) + else + failed=$((failed + 1)) + fi + + # Test 8: Verify update + local payload8 + payload8='{ + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "name": "catalog_get", + "arguments": { + "kind": "table", + "key": "testdb.customers" + } + }, + "id": 8 +}' + + if test_catalog "CAT008" "Verify update" "${payload8}" '"updated"[[:space:]]*:[[:space:]]*true'; then + passed=$((passed + 1)) + else + failed=$((failed + 1)) + fi + + # Test 9: Test FTS search with special characters + local payload9 + payload9='{ + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "name": "catalog_search", + "arguments": { + "query": "customer*", + "limit": 10 + } + }, + "id": 9 +}' + + if test_catalog "CAT009" "FTS search with wildcard" "${payload9}" '"results"'; then + passed=$((passed + 1)) + else + failed=$((failed + 1)) + fi + + # Test 10: Delete entry + local payload10 + payload10='{ + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "name": "catalog_delete", + "arguments": { + "kind": "table", + "key": "testdb.customers" + } + }, + "id": 10 +}' + + if test_catalog "CAT010" "Delete entry" "${payload10}" '"success"[[:space:]]*:[[:space:]]*true'; then + passed=$((passed + 1)) + else + failed=$((failed + 1)) + fi + + # Test 11: Verify deletion + local payload11 + payload11='{ + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "name": "catalog_get", + "arguments": { + "kind": "table", + "key": "testdb.customers" + } + }, + "id": 11 +}' + + # This should return an error since we deleted it + log_test "CAT011: Verify deletion (should fail)" + local response11 + response11=$(mcp_request "${payload11}") + + if echo "${response11}" | grep -q '"error"'; then + log_info "✓ CAT011" + passed=$((passed + 1)) + else + log_error "✗ CAT011" + failed=$((failed + 1)) + fi + + # Test 12: Cleanup - delete domain entry + local payload12 + payload12='{ + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "name": "catalog_delete", + "arguments": { + "kind": "domain", + "key": "customer_management" + } + }, + "id": 12 +}' + + if test_catalog "CAT012" "Cleanup domain entry" "${payload12}" '"success"[[:space:]]*:[[:space:]]*true'; then + passed=$((passed + 1)) + else + failed=$((failed + 1)) + fi + + # Print summary + echo "" + echo "======================================" + echo "Test Summary" + echo "======================================" + echo "Total tests: $((passed + failed))" + echo -e "Passed: ${GREEN}${passed}${NC}" + echo -e "Failed: ${RED}${failed}${NC}" + echo "" + + if [ ${failed} -gt 0 ]; then + log_error "Some tests failed!" + return 1 + else + log_info "All catalog tests passed!" + return 0 + fi +} + +# Parse command line arguments +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + -v|--verbose) + VERBOSE=true + shift + ;; + -h|--help) + cat </dev/null) + + local body=$(echo "$response" | head -n -1) + local code=$(echo "$response" | tail -n 1) + + if [ "${VERBOSE}" = "true" ]; then + echo "Request: ${payload}" + echo "Response (${code}): ${body}" + fi + + echo "${body}" + return 0 +} + +# Check if MCP server is accessible +check_mcp_server() { + log_test "Checking MCP server accessibility..." + + local response + response=$(mcp_request "${MCP_CONFIG_URL}" '{"jsonrpc":"2.0","method":"ping","id":1}') + + if echo "${response}" | grep -q "result"; then + log_info "MCP server is accessible" + return 0 + else + log_error "MCP server is not accessible" + log_error "Response: ${response}" + return 1 + fi +} + +# Assert that JSON contains expected value +assert_json_contains() { + local response="$1" + local field="$2" + local expected="$3" + + if echo "${response}" | grep -q "\"${field}\"[[:space:]]*:[[:space:]]*${expected}"; then + return 0 + fi + + # Try with jq if available + if command -v jq &> /dev/null; then + local actual + actual=$(echo "${response}" | jq -r "${field}" 2>/dev/null) + if [ "${actual}" = "${expected}" ]; then + return 0 + fi + fi + + return 1 +} + +# Assert that JSON array contains expected value +assert_json_array_contains() { + local response="$1" + local field="$2" + local expected="$3" + + if echo "${response}" | grep -q "${expected}"; then + return 0 + fi + + return 1 +} + +# Test a tool +test_tool() { + local tool_name="$1" + local arguments="$2" + local expected_field="$3" + local expected_value="$4" + + TOTAL_TESTS=$((TOTAL_TESTS + 1)) + + log_test "Testing tool: ${tool_name}" + + local payload + payload=$(cat <