diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 000000000..36943df34 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,74 @@ +### Scripts description + +This is a set of example scripts to show the capabilities of the RESTAPI interface and how to interface with it. + +### Prepare ProxySQL + +1. Launch ProxySQL: + +``` +./src/proxysql -M --sqlite3-server --idle-threads -f -c $PATH/restapi_examples/datadir/proxysql.cnf -D $PATH/restapi_examples/datadir +``` + +2. Configure ProxySQL: + +``` +cd $RESTAPI_EXAMPLES_DIR +./proxysql_config.sh +``` + +3. Install requirements + +``` +cd $RESTAPI_EXAMPLES_DIR/requirements +./install_requirements.sh +``` + +### Query the endpoints + +1. Flush Query Cache: `curl -i -X GET http://localhost:6070/sync/flush_query_cache` +2. Change host status: + - Assuming local ProxySQL: + ``` + curl -i -X POST -d '{ "hostgroup_id": "", "hostname": "sbtest1", "port": 3306, "status": "OFFLINE_HARD" }' http://localhost:6070/sync/change_host_status + ``` + - Specifying server: + ``` + curl -i -X POST -d '{ "admin_host": "127.0.0.1", "admin_port": "6032", "admin_user": "radmin", "admin_pass": "radmin", "hostgroup_id": "0", "hostname": "sbtest1", "port": 3306, "status": "ONLINE_HARD" }' http://localhost:6070/sync/change_host_status + ``` +2. Add or replace MySQL user: + - Assuming local ProxySQL: + ``` + curl -i -X POST -d '{ "user": "sbtest1", "pass": "sbtest1" }' http://localhost:6070/sync/add_mysql_user + ``` + - Add user and load to runtime (Assuming local instance): + ``` + curl -i -X POST -d '{ "user": "sbtest1", "pass": "sbtest1", "to_runtime": 1 }' http://localhost:6070/sync/add_mysql_user + ``` + - Specifying server: + ``` + curl -i -X POST -d '{ "admin_host": "127.0.0.1", "admin_port": "6032", "admin_user": "radmin", "admin_pass": "radmin", "user": "sbtest1", "pass": "sbtest1" }' http://localhost:6070/sync/add_mysql_user + ``` +3. Kill idle backend connections: + - Assuming local ProxySQL: + ``` + curl -i -X POST -d '{ "timeout": 10 }' http://localhost:6070/sync/kill_idle_backend_conns + ``` + - Specifying server: + ``` + curl -i -X POST -d '{ "admin_host": "127.0.0.1", "admin_port": 6032, "admin_user": "radmin", "admin_pass": "radmin", "timeout": 10 }' http://localhost:6070/sync/kill_idle_backend_conns + ``` +4. Scrap tables from 'stats' schema: + - Assuming local ProxySQL: + ``` + curl -i -X POST -d '{ "table": "stats_mysql_users" }' http://localhost:6070/sync/scrap_stats + ``` + - Specifying server: + ``` + curl -i -X POST -d '{ "admin_host": "127.0.0.1", "admin_port": 6032, "admin_user": "radmin", "admin_pass": "radmin", "table": "stats_mysql_users" }' http://localhost:6070/sync/scrap_stats + ``` + +### Scripts doc + +- All scripts allows to perform the target operations on a local or remote ProxySQL instance. +- Notice how the unique 'GET' request is for 'QUERY CACHE' flushing, since it doesn't require any parameters. diff --git a/scripts/add_mysql_user.sh b/scripts/add_mysql_user.sh new file mode 100755 index 000000000..78c3fb164 --- /dev/null +++ b/scripts/add_mysql_user.sh @@ -0,0 +1,68 @@ +#!/bin/sh + +# Add a MySQL user: +# +# - Optional params (with default values): '{ "admin_host": "127.0.0.1", "admin_port": 6032, "admin_user": "radmin", "admin_pass": "radmin" }' +# - Mandatory params: '{ "user": "username", "pass": "password" }' + +if [ ! "$1" ]; then + echo { \"err\": \"Missing required argument specifying username/password\" } + exit 0 +fi + +# Script mandatory parameters +user=$(echo $1 | jq '.user') +pass=$(echo $1 | jq '.pass') + +# Script optional parameters +admin_host=$(echo $1 | jq -r '.admin_host') +admin_port=$(echo $1 | jq -r '.admin_port') +admin_user=$(echo $1 | jq -r '.admin_user') +admin_pass=$(echo $1 | jq -r '.admin_pass') +to_runtime=$(echo $1 | jq -r '.to_runtime') + +if [ $user == "null" ]; then + echo { \"err_code\": 1, \"res\": \"Missing required argument username\" } + exit 0 +fi +if [ $pass == "null" ]; then + echo { \"err_code\": 1, \"res\": \"Missing required argument password\" } + exit 0 +fi + +# Optional parameters +if [ $admin_host == "null" ]; then + admin_host="127.0.0.1" +fi +if [ $admin_port == "null" ]; then + admin_port=6032 +fi +if [ $admin_user == "null" ]; then + admin_user="radmin" +fi +if [ $admin_pass == "null" ]; then + admin_pass="radmin" +fi +if [ $to_runtime == "null" ]; then + to_runtime=0 +fi + +cmd_output=$(mysql -h$admin_host -P$admin_port -u$admin_user -p$admin_pass -e \ + "INSERT OR REPLACE INTO mysql_users (username,password) VALUES ($user, $pass)" 2> $(pwd)/add_mysql_err.log) + +if [ $? -eq 0 ]; then + if [ $to_runtime -eq 1 ]; then + cmd_output=$(mysql -h$admin_host -P$admin_port -u$admin_user -p$admin_pass -e \ + "LOAD MYSQL USERS TO RUNTIME" 2> $(pwd)/add_mysql_err.log) + fi + + if [ $? -eq 0 ]; then + echo { \"err_code\": 0, \"res\": \"$cmd_output\" } + else + echo { \"err_code\": 1, \"res\": \"$(cat $(pwd)/add_mysql_err.log)\" } + fi +else + echo { \"err_code\": 1, \"res\": \"$(cat $(pwd)/add_mysql_err.log)\" } +fi + +rm $(pwd)/add_mysql_err.log diff --git a/scripts/change_host_status.sh b/scripts/change_host_status.sh new file mode 100755 index 000000000..0a86625d6 --- /dev/null +++ b/scripts/change_host_status.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +# Change a particular host status: +# +# - Optional params (with default values): '{ "admin_host": "127.0.0.1", "admin_port": 6032, "admin_user": "radmin", "admin_pass": "radmin" }' +# - Mandatory params: '{ "hostgroup_id": N, "hostname": "N.N.N.N", "port": N, "status": "ONLINE|OFFLINE_HARD" }' + +if [ ! "$1" ]; then + echo { \"err\": \"Missing required argument specifying hostgroup_id/hostname/port/status\" } + exit 0 +fi + +# Script mandatory parameters +hostgroup_id=$(echo $1 | jq '.hostgroup_id') +hostname=$(echo $1 | jq '.hostname') +port=$(echo $1 | jq '.port') +status=$(echo $1 | jq '.status') + +# Script optional parameters +admin_host=$(echo $1 | jq -r '.admin_host') +admin_port=$(echo $1 | jq -r '.admin_port') +admin_user=$(echo $1 | jq -r '.admin_user') +admin_pass=$(echo $1 | jq -r '.admin_pass') + +if [ $hostgroup_id == "null" ]; then + echo { \"err_code\": 1, \"res\": \"Missing required argument \'hostgroup_id\'\" } + exit 0 +fi +if [ $hostname == "null" ]; then + echo { \"err_code\": 1, \"res\": \"Missing required argument \'hostname\'\" } + exit 0 +fi +if [ $port == "null" ]; then + echo { \"err_code\": 1, \"res\": \"Missing required argument \'port\'\" } + exit 0 +fi +if [ $status == "null" ]; then + echo { \"err_code\": 1, \"res\": \"Missing required argument \'status\'\" } + exit 0 +fi + +# Optional parameters +if [ $admin_host == "null" ]; then + admin_host="127.0.0.1" +fi +if [ $admin_port == "null" ]; then + admin_port=6032 +fi +if [ $admin_user == "null" ]; then + admin_user="radmin" +fi +if [ $admin_pass == "null" ]; then + admin_pass="radmin" +fi + +cmd_output=$(mysql -h$admin_host -P$admin_port -u$admin_user -p$admin_pass -e \ + "UPDATE mysql_servers SET status=$status WHERE hostgroup_id=$hostgroup_id AND hostname=$hostname AND port=$port" 2> $(pwd)/change_host_st_err.log) + +if [ $? -eq 0 ]; then + echo { \"err_code\": 0, \"res\": \"$cmd_output\" } +else + echo { \"err_code\": 1, \"res\": \"$(cat $(pwd)/change_host_st_err.log)\" } +fi + +rm $(pwd)/change_host_st_err.log diff --git a/scripts/datadir/proxysql.cnf b/scripts/datadir/proxysql.cnf new file mode 100644 index 000000000..dfc7335c2 --- /dev/null +++ b/scripts/datadir/proxysql.cnf @@ -0,0 +1,33 @@ +admin_variables= +{ + admin_credentials="admin:admin;radmin:radmin" + mysql_ifaces="0.0.0.0:6032" + hash_passwords=false +} + +mysql_variables= +{ + threads=2 + max_connections=2048 + default_query_delay=0 + default_query_timeout=36000000 + have_compress=true + poll_timeout=2000 + interfaces="0.0.0.0:6033" + default_schema="information_schema" + stacksize=1048576 + server_version="5.5.30" + connect_timeout_server=3000 + monitor_username="monitor" + monitor_password="monitor" + monitor_history=600000 + monitor_connect_interval=60000 + monitor_ping_interval=10000 + monitor_read_only_interval=1500 + monitor_read_only_timeout=500 + ping_interval_server_msec=120000 + ping_timeout_server=500 + commands_stats=true + sessions_sort=true + connect_retries_on_failure=10 +} diff --git a/scripts/flush_query_cache.sh b/scripts/flush_query_cache.sh new file mode 100755 index 000000000..6d28d1777 --- /dev/null +++ b/scripts/flush_query_cache.sh @@ -0,0 +1,42 @@ +#!/usr/bin/sh + +# Flush Query cache from a ProxySQL instance: +# +# - Optional params (with default values): '{ "admin_host": "127.0.0.1", "admin_port": 6032, "admin_user": "radmin", "admin_pass": "radmin" }' + +if [ ! "$1" ]; then + j_arg="{}" +else + j_arg=$1 +fi + +# Script optional parameters +admin_host=$(echo $j_arg | jq -r '.admin_host') +admin_port=$(echo $j_arg | jq -r '.admin_port') +admin_user=$(echo $j_arg | jq -r '.admin_user') +admin_pass=$(echo $j_arg | jq -r '.admin_pass') + +# Optional parameters +if [ $admin_host == "null" ]; then + admin_host="127.0.0.1" +fi +if [ $admin_port == "null" ]; then + admin_port=6032 +fi +if [ $admin_user == "null" ]; then + admin_user="radmin" +fi +if [ $admin_pass == "null" ]; then + admin_pass="radmin" +fi + +cmd_output=$(mysql -h$admin_host -P$admin_port -u$admin_user -p$admin_pass -e \ + "PROXYSQL FLUSH QUERY CACHE" 2> $(pwd)/flush_cache_err.log) + +if [ $? -eq 0 ]; then + echo { \"err_code\": 0, \"res\": \"$cmd_output\" } +else + echo { \"err_code\": 1, \"res\": \"$(cat $(pwd)/flush_cache_err.log)\" } +fi + +rm $(pwd)/flush_cache_err.log diff --git a/scripts/kill_idle_backend_conns.py b/scripts/kill_idle_backend_conns.py new file mode 100755 index 000000000..54dcb0a16 --- /dev/null +++ b/scripts/kill_idle_backend_conns.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +""" +Kills all ProxySQL idle backend connections from a particular instance. + +- Optional params (with default values): '{ "admin_host": "127.0.0.1", "admin_port": 6032, "admin_user": "radmin", "admin_pass": "radmin" }' +- Mandatory params: '{ "timeout": N }' +""" + +import json +import jsonschema +import sys +import time + +import pymysql.cursors + +schema = { + "type": "object", + "properties": { + "admin_host": {"type": "string"}, + "admin_port": {"type": "number"}, + "admin_user": {"type": "string"}, + "admin_pass": {"type": "string"}, + "timeout": {"type": "number"} + }, + "required": ["timeout"] +} + +def validate_params(): + """Validates the JSON encoded received parameters.""" + res = {} + + if len(sys.argv) != 2: + res = {"err_code": 1, "res": "Invalid number of parameters"} + else: + try: + j_arg = json.loads(sys.argv[1]) + jsonschema.validate(instance=j_arg, schema=schema) + + res = {"err_code": 0, "res": ""} + except jsonschema.exceptions.ValidationError as err: + res = {"err_code": 1, "res": "Params validation failed: `" + str(err) + "`"} + except json.decoder.JSONDecodeError as err: + res = {"err_code": 1, "res": "Invalid supplied JSON: `" + str(err) + "`"} + + return res + +if __name__ == "__main__": + p_res = validate_params() + + if p_res["err_code"] != 0: + print(json.dumps(p_res)) + exit(0) + + params = json.loads(sys.argv[1]) + + if params.get('admin_host') is None: + params['admin_host'] = "127.0.0.1" + if params.get('admin_port') is None: + params['admin_port'] = 6032 + if params.get('admin_user') is None: + params['admin_user'] = "radmin" + if params.get('admin_pass') is None: + params['admin_pass'] = "radmin" + + try: + proxy_admin_conn = pymysql.connect( + host=params['admin_host'], + user=params['admin_user'], + password=params['admin_pass'], + port=int(params['admin_port']), + cursorclass=pymysql.cursors.DictCursor, + defer_connect=True + ) + + proxy_admin_conn.client_flag |= pymysql.constants.CLIENT.MULTI_STATEMENTS + proxy_admin_conn.connect() + proxy_admin_conn.autocommit(True) + except Exception as err: + print(json.dumps({"err_code": 1, "res": "Connection attempt failed: `" + str(err) + "`"})) + exit(0) + + with proxy_admin_conn: + with proxy_admin_conn.cursor() as cursor: + # Backup the current 'free_connections_pct' + prev_free_conns_pct = "" + + s_var_query = "SELECT variable_value FROM global_variables WHERE variable_name='mysql-free_connections_pct'" + cursor.execute(s_var_query) + my_varval = cursor.fetchall() + + if len(my_varval) != 1: + print(json.dumps({"err_code": 1, "res": "Invalid resulset received for query `" + s_var_query + "`"})) + exit(0) + else: + prev_free_conns_pct = my_varval[0]['variable_value'] + + # Set the 'free_connections_pct' to '0', to performa purge of the idle backend connections + update_query = "UPDATE global_variables SET variable_value=%s WHERE variable_name='mysql-free_connections_pct'" + cursor.execute(update_query, "0") + cursor.execute("LOAD MYSQL VARIABLES TO RUNTIME") + + # Loop with a timeout until ProxySQL has cleaned all the backend connections + free_conns = -1 + waited = 0 + timeout = int(params['timeout']) + + while free_conns != 0 and waited < timeout: + s_free_query = "SELECT SUM(ConnFree) FROM stats.stats_mysql_connection_pool" + + cursor.execute(s_free_query) + my_rows = cursor.fetchall() + + if len(my_rows) != 1: + print(json.dumps({"err_code": 1, "res": "Invalid resulset received for query `" + s_free_query + "`"})) + exit(0) + else: + free_conns = int(my_rows[0]['SUM(ConnFree)']) + + time.sleep(1) + + # Recover previous 'mysql-free_connections_pct' + cursor.execute(update_query, prev_free_conns_pct) + cursor.execute("LOAD MYSQL VARIABLES TO RUNTIME") + + if waited >= timeout: + print(json.dumps({"err_code": 1, "res": "Operation timedout"})) + else: + print(json.dumps({"err_code": 0, "res": "Success!"})) diff --git a/scripts/export_users.py b/scripts/legacy/export_users.py similarity index 100% rename from scripts/export_users.py rename to scripts/legacy/export_users.py diff --git a/scripts/metrics.py b/scripts/legacy/metrics.py similarity index 100% rename from scripts/metrics.py rename to scripts/legacy/metrics.py diff --git a/scripts/proxysql_config.sh b/scripts/proxysql_config.sh new file mode 100755 index 000000000..16b66374d --- /dev/null +++ b/scripts/proxysql_config.sh @@ -0,0 +1,24 @@ +# Configure a local ProxySQL instance for enabling the example RESTAPI scripts in this folder + +# Enable restapi +mysql -h127.0.0.1 -P6032 -uradmin -pradmin -e "SET admin-restapi_enabled='true'" +mysql -h127.0.0.1 -P6032 -uradmin -pradmin -e "SET admin-restapi_port=6070" +mysql -h127.0.0.1 -P6032 -uradmin -pradmin -e "LOAD ADMIN VARIABLES TO RUNTIME" + +# Clenaup current routes +mysql -h127.0.0.1 -P6032 -uradmin -pradmin -e "DELETE FROM restapi_routes" + +# Add new routes +mysql -h127.0.0.1 -P6032 -uradmin -pradmin -e \ + "INSERT INTO restapi_routes (active, timeout_ms, method, uri, script, comment) VALUES (1,3000,'GET','flush_query_cache','$(pwd)/flush_query_cache.sh','Flush the query cache')" +mysql -h127.0.0.1 -P6032 -uradmin -pradmin -e \ + "INSERT INTO restapi_routes (active, timeout_ms, method, uri, script, comment) VALUES (1,3000,'POST','change_host_status','$(pwd)/change_host_status.sh','Change the specified host status')" +mysql -h127.0.0.1 -P6032 -uradmin -pradmin -e \ + "INSERT INTO restapi_routes (active, timeout_ms, method, uri, script, comment) VALUES (1,3000,'POST','add_mysql_user','$(pwd)/add_mysql_user.sh','Adds a new MySQL user')" +mysql -h127.0.0.1 -P6032 -uradmin -pradmin -e \ + "INSERT INTO restapi_routes (active, timeout_ms, method, uri, script, comment) VALUES (1,20000,'POST','kill_idle_backend_conns','$(pwd)/kill_idle_backend_conns.py','Kills all idle backend connections')" +mysql -h127.0.0.1 -P6032 -uradmin -pradmin -e \ + "INSERT INTO restapi_routes (active, timeout_ms, method, uri, script, comment) VALUES (1,3000,'POST','scrap_stats','$(pwd)/stats_scrapper.py','Allow stats table scrapping')" + +# Load the RESTAPI to runtime +mysql -h127.0.0.1 -P6032 -uradmin -pradmin -e "LOAD RESTAPI TO RUNTIME" diff --git a/scripts/requirements/install_requirements.sh b/scripts/requirements/install_requirements.sh new file mode 100755 index 000000000..ebc7f7b57 --- /dev/null +++ b/scripts/requirements/install_requirements.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +pip install -r requirements.txt +sudo apt-get install -y jq diff --git a/scripts/requirements/requirements.txt b/scripts/requirements/requirements.txt new file mode 100644 index 000000000..b22f5dca4 --- /dev/null +++ b/scripts/requirements/requirements.txt @@ -0,0 +1,2 @@ +jsonschema +pymysql diff --git a/scripts/stats_scrapper.py b/scripts/stats_scrapper.py new file mode 100755 index 000000000..aa40de782 --- /dev/null +++ b/scripts/stats_scrapper.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +""" +Retuns the content of a particular table from Admin 'stats' schema. + +- Optional params (with default values): '{ "admin_host": "127.0.0.1", "admin_port": 6032, "admin_user": "radmin", "admin_pass": "radmin" }' +- Mandatory params: '{ "table": "tablename" }' +""" + +import json +import jsonschema +import sys + +import pymysql.cursors + +schema = { + "type": "object", + "properties": { + "admin_host": {"type": "string"}, + "admin_port": {"type": "number"}, + "admin_user": {"type": "string"}, + "admin_pass": {"type": "string"}, + "table": {"type": "string"}, + }, + "required": ["table"] +} + +def validate_params(): + """Validates the JSON encoded received parameters.""" + res = {} + + if len(sys.argv) != 2: + res = {"err_code": 1, "res": "Invalid number of parameters"} + else: + try: + j_arg = json.loads(sys.argv[1]) + jsonschema.validate(instance=j_arg, schema=schema) + + res = {"err_code": 0, "res": ""} + except jsonschema.exceptions.ValidationError as err: + res = {"err_code": 1, "res": "Params validation failed: `" + str(err) + "`"} + except json.decoder.JSONDecodeError as err: + res = {"err_code": 1, "res": "Invalid supplied JSON: `" + str(err) + "`"} + + return res + +if __name__ == "__main__": + p_res = validate_params() + + if p_res["err_code"] != 0: + print(json.dumps(p_res)) + exit(0) + + params = json.loads(sys.argv[1]) + + if params.get('admin_host') is None: + params['admin_host'] = "127.0.0.1" + if params.get('admin_port') is None: + params['admin_port'] = 6032 + if params.get('admin_user') is None: + params['admin_user'] = "radmin" + if params.get('admin_pass') is None: + params['admin_pass'] = "radmin" + + try: + proxy_admin_conn = pymysql.connect( + host=params['admin_host'], + user=params['admin_user'], + password=params['admin_pass'], + port=int(params['admin_port']), + cursorclass=pymysql.cursors.DictCursor, + defer_connect=True + ) + + proxy_admin_conn.client_flag |= pymysql.constants.CLIENT.MULTI_STATEMENTS + proxy_admin_conn.connect() + proxy_admin_conn.autocommit(True) + except Exception as err: + print(json.dumps({"err_code": 1, "res": "Connection attempt failed: `" + str(err) + "`"})) + exit(0) + + s_query = "SELECT * FROM stats.%s" + + with proxy_admin_conn: + with proxy_admin_conn.cursor() as cursor: + cursor.execute(s_query, params['table']) + rows = cursor.fetchall() + + print(json.dumps({"err_code": 0, "res": rows}))