From 801b42cc3c535cd0fb482d2275186df62d2908c5 Mon Sep 17 00:00:00 2001 From: Valentin Rakush Date: Sat, 21 Dec 2019 16:04:25 +0000 Subject: [PATCH 1/6] Implement call to external script and return stdout in response --- lib/ProxySQL_RESTAPI_Server.cpp | 44 +++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/lib/ProxySQL_RESTAPI_Server.cpp b/lib/ProxySQL_RESTAPI_Server.cpp index 30b278074..70b638901 100644 --- a/lib/ProxySQL_RESTAPI_Server.cpp +++ b/lib/ProxySQL_RESTAPI_Server.cpp @@ -15,12 +15,45 @@ using namespace httpserver; class hello_world_resource : public http_resource { public: - const std::shared_ptr render_GET(const http_request&) { - return std::shared_ptr(new string_response("GET: Hello, World!")); + const std::shared_ptr render_GET(const http_request& req) { + return std::shared_ptr(new string_response("GET: Hello, World!\n")); } - const std::shared_ptr render(const http_request&) { - return std::shared_ptr(new string_response("OTHER: Hello, World!")); + const std::shared_ptr render_POST(const http_request& req) { + //TODO : validate json correctness in the req + + int pipefd[2]; + if (pipe(pipefd) == -1) { + return std::shared_ptr(new string_response("{\"error\":\"Cannot create pipe.\"}")); + } + + pid_t pid; + if ((pid=fork()) == -1) { + return std::shared_ptr(new string_response("{\"error\":\"Cannot fork.\"}")); + } + + char buf[1024]; + if (pid == 0) { + dup2(pipefd[1], STDOUT_FILENO); + close(pipefd[0]); + close(pipefd[1]); + char* args[] = {"a", (char*)req.get_content().data(), NULL}; + if (execve("/home/val/workspace/script.py", args, NULL) == -1) { + return std::shared_ptr(new string_response("{\"error\":\"Error calling execve().\"}")); + } + exit(EXIT_SUCCESS); + } + else { + close(pipefd[1]); + int nbytes = read(pipefd[0], buf, sizeof(buf) - 1); + if (nbytes == -1) { + return std::shared_ptr(new string_response("{\"error\":\"Error reading pipe.\"}")); + } + //TODO : validate json correctness in the buf + close(pipefd[0]); + wait(NULL); + } + return std::shared_ptr(new string_response(buf)); } }; @@ -34,13 +67,14 @@ ProxySQL_RESTAPI_Server::ProxySQL_RESTAPI_Server(int p) { // for now, this is COMPLETELY DISABLED // just adding a POC - return; +// return; //ws = NULL; port = p; ws = new webserver(create_webserver(p)); //hello_world_resource hwr; hr = new hello_world_resource(); + //ws->register_resource("/hello", &hwr); ws->register_resource("/hello", hr); if (pthread_create(&thread_id, NULL, restapi_server_thread, ws) !=0 ) { From f0687f368811062cf264271a703ce45188a636ee Mon Sep 17 00:00:00 2001 From: Valentin Rakush Date: Tue, 24 Dec 2019 20:45:26 +0000 Subject: [PATCH 2/6] Implement restapi functionality - restapi_routes table in admin module - calling python script (json validation, error handling) - scripts: metrics, export_users - tests adjusted --- include/proxysql_admin.h | 2 + include/proxysql_config.h | 2 + include/proxysql_restapi.h | 44 +++++++ lib/Makefile | 2 +- lib/ProxySQL_Admin.cpp | 124 +++++++++++++++++- lib/ProxySQL_Config.cpp | 103 +++++++++++++++ lib/ProxySQL_RESTAPI_Server.cpp | 60 ++++++++- lib/ProxySQL_Restapi.cpp | 107 +++++++++++++++ scripts/export_users.py | 27 ++++ scripts/metrics.py | 16 +++ src/main.cpp | 2 + .../proxysql_reference_select_config_file.cnf | 11 ++ test/tap/tests/select_config_file-t.cpp | 11 ++ 13 files changed, 503 insertions(+), 8 deletions(-) create mode 100644 include/proxysql_restapi.h create mode 100644 lib/ProxySQL_Restapi.cpp create mode 100755 scripts/export_users.py create mode 100755 scripts/metrics.py diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 14f5c5956..2fa79bcdc 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -10,6 +10,7 @@ typedef struct { uint32_t hash; uint32_t key; } t_symstruct; class ProxySQL_Config; +class ProxySQL_Restapi; class Scheduler_Row { public: @@ -271,6 +272,7 @@ class ProxySQL_Admin { void stats___mysql_gtid_executed(); ProxySQL_Config& proxysql_config(); + ProxySQL_Restapi& proxysql_restapi(); void flush_error_log(); bool GenericRefreshStatistics(const char *query_no_space, unsigned int query_no_space_length, bool admin); diff --git a/include/proxysql_config.h b/include/proxysql_config.h index 59e32d277..bc095b76c 100644 --- a/include/proxysql_config.h +++ b/include/proxysql_config.h @@ -17,6 +17,7 @@ public: int Read_MySQL_Query_Rules_from_configfile(); int Read_MySQL_Servers_from_configfile(); int Read_Scheduler_from_configfile(); + int Read_Restapi_from_configfile(); int Read_ProxySQL_Servers_from_configfile(); void addField(std::string& data, const char* name, const char* value, const char* dq="\""); @@ -25,6 +26,7 @@ public: int Write_MySQL_Query_Rules_to_configfile(std::string& data); int Write_MySQL_Servers_to_configfile(std::string& data); int Write_Scheduler_to_configfile(std::string& data); + int Write_Restapi_to_configfile(std::string& data); int Write_ProxySQL_Servers_to_configfile(std::string& data); }; diff --git a/include/proxysql_restapi.h b/include/proxysql_restapi.h new file mode 100644 index 000000000..ebe780217 --- /dev/null +++ b/include/proxysql_restapi.h @@ -0,0 +1,44 @@ +#ifndef __PROXYSQL_RESTAPI_H__ +#define __PROXYSQL_RESTAPI_H__ + +#include "proxy_defines.h" +#include "proxysql.h" +#include "cpp.h" +#include + +class SQLite3DB; + +class Restapi_Row { +public: + unsigned int id; + bool is_active; + unsigned int interval_ms; + std::string uri; + std::string script; + std::string comment; + unsigned int version; + Restapi_Row(unsigned int _id, bool _is_active, unsigned int _in, const std::string& _uri, const std::string& _script, const std::string& _comment); +}; + +class ProxySQL_Restapi { + SQLite3DB* admindb; +public: + ProxySQL_Restapi(SQLite3DB* db); + virtual ~ProxySQL_Restapi(); + + unsigned int last_version; + unsigned int version; +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_t rwlock; +#else + rwlock_t rwlock; +#endif + std::vector Restapi_Rows; + void update_table(SQLite3_result *result); + void load_restapi_to_runtime(); + void save_restapi_runtime_to_database(bool); + void flush_restapi__from_memory_to_disk(); + void flush_restapi__from_disk_to_memory(); +}; + +#endif // #ifndef __PROXYSQL_RESTAPI_H__ diff --git a/lib/Makefile b/lib/Makefile index c1c366d9c..6aef3a705 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -95,7 +95,7 @@ default: libproxysql.a _OBJ = c_tokenizer.o OBJ = $(patsubst %,$(ODIR)/%,$(_OBJ)) -_OBJ_CXX = ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Query_Cache.oo SpookyV2.oo MySQL_Authentication.oo gen_utils.oo sqlite3db.oo mysql_connection.oo MySQL_HostGroups_Manager.oo mysql_data_stream.oo MySQL_Thread.oo MySQL_Session.oo MySQL_Protocol.oo mysql_backend.oo Query_Processor.oo ProxySQL_Admin.oo ProxySQL_Config.oo MySQL_Monitor.oo MySQL_Logger.oo thread.oo MySQL_PreparedStatement.oo ProxySQL_Cluster.oo ClickHouse_Authentication.oo ClickHouse_Server.oo ProxySQL_Statistics.oo Chart_bundle_js.oo ProxySQL_HTTP_Server.oo ProxySQL_RESTAPI_Server.oo font-awesome.min.css.oo main-bundle.min.css.oo set_parser.oo +_OBJ_CXX = ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Query_Cache.oo SpookyV2.oo MySQL_Authentication.oo gen_utils.oo sqlite3db.oo mysql_connection.oo MySQL_HostGroups_Manager.oo mysql_data_stream.oo MySQL_Thread.oo MySQL_Session.oo MySQL_Protocol.oo mysql_backend.oo Query_Processor.oo ProxySQL_Admin.oo ProxySQL_Config.oo ProxySQL_Restapi.oo MySQL_Monitor.oo MySQL_Logger.oo thread.oo MySQL_PreparedStatement.oo ProxySQL_Cluster.oo ClickHouse_Authentication.oo ClickHouse_Server.oo ProxySQL_Statistics.oo Chart_bundle_js.oo ProxySQL_HTTP_Server.oo ProxySQL_RESTAPI_Server.oo font-awesome.min.css.oo main-bundle.min.css.oo set_parser.oo OBJ_CXX = $(patsubst %,$(ODIR)/%,$(_OBJ_CXX)) HEADERS = ../include/*.h ../include/*.hpp diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 303ce52e6..495e4237e 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -6,6 +6,7 @@ #include "re2/regexp.h" #include "proxysql.h" #include "proxysql_config.h" +#include "proxysql_restapi.h" #include "cpp.h" #include "MySQL_Data_Stream.h" @@ -362,6 +363,10 @@ static int http_handler(void *cls, struct MHD_Connection *connection, const char #define ADMIN_SQLITE_TABLE_MYSQL_COLLATIONS "CREATE TABLE mysql_collations (Id INTEGER NOT NULL PRIMARY KEY , Collation VARCHAR NOT NULL , Charset VARCHAR NOT NULL , `Default` VARCHAR NOT NULL)" +#define ADMIN_SQLITE_TABLE_RESTAPI_ROUTES "CREATE TABLE restapi_routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>-100 AND interval_ms<=100000000) NOT NULL , uri VARCHAR NOT NULL, script VARCHAR NOT NULL, comment VARCHAR NO NULL DEFAULT '')" + +#define ADMIN_SQLITE_TABLE_RUNTIME_RESTAPI_ROUTES "CREATE TABLE runtime_restapi_routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>-100 AND interval_ms<=100000000) NOT NULL , uri VARCHAR NOT NULL, script VARCHAR NOT NULL, comment VARCHAR NO NULL DEFAULT '')" + #define ADMIN_SQLITE_TABLE_SCHEDULER "CREATE TABLE scheduler (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , comment VARCHAR NOT NULL DEFAULT '')" #define ADMIN_SQLITE_TABLE_SCHEDULER_V1_2_0 "CREATE TABLE scheduler (id INTEGER NOT NULL , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , PRIMARY KEY(id))" @@ -564,6 +569,11 @@ ProxySQL_Config& ProxySQL_Admin::proxysql_config() { return instance; } +ProxySQL_Restapi& ProxySQL_Admin::proxysql_restapi() { + static ProxySQL_Restapi instance = ProxySQL_Restapi(admindb); + return instance; +} + int ProxySQL_Admin::FlushDigestTableToDisk(SQLite3DB *_db) { int r = 0; if (!GloQPro) return 0; @@ -1296,6 +1306,101 @@ bool admin_handler_command_load_or_save(char *query_no_space, unsigned int query } #endif /* DEBUG */ + if ((query_no_space_length>13) && ( (!strncasecmp("SAVE RESTAPI ", query_no_space, 13)) || (!strncasecmp("LOAD RESTAPI ", query_no_space, 13))) ) { + + if ( + (query_no_space_length==strlen("LOAD RESTAPI TO MEMORY") && !strncasecmp("LOAD RESTAPI TO MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD RESTAPI TO MEM") && !strncasecmp("LOAD RESTAPI TO MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD RESTAPI FROM DISK") && !strncasecmp("LOAD RESTAPI FROM DISK",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->proxysql_restapi().flush_restapi__from_disk_to_memory(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading restapi to to MEMORY\n"); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); + return false; + } + + if ( + (query_no_space_length==strlen("SAVE RESTAPI FROM MEMORY") && !strncasecmp("SAVE RESTAPI FROM MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE RESTAPI FROM MEM") && !strncasecmp("SAVE RESTAPI FROM MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE RESTAPI TO DISK") && !strncasecmp("SAVE RESTAPI TO DISK",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->proxysql_restapi().flush_restapi__from_memory_to_disk(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saving restapi to DISK\n"); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); + return false; + } + + if ( + (query_no_space_length==strlen("LOAD RESTAPI FROM MEMORY") && !strncasecmp("LOAD RESTAPI FROM MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD RESTAPI FROM MEM") && !strncasecmp("LOAD RESTAPI FROM MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD RESTAPI TO RUNTIME") && !strncasecmp("LOAD RESTAPI TO RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD RESTAPI TO RUN") && !strncasecmp("LOAD RESTAPI TO RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->proxysql_restapi().load_restapi_to_runtime(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded restapito RUNTIME\n"); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); + return false; + } + + if ( + (query_no_space_length==strlen("LOAD RESTAPI FROM CONFIG") && !strncasecmp("LOAD RESTAPI FROM CONFIG",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + if (GloVars.configfile_open) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); + if (GloVars.confFile->OpenFile(NULL)==true) { + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + int rows=0; + rows=SPA->proxysql_config().Read_Restapi_from_configfile(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded restapi from CONFIG\n"); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, rows); + GloVars.confFile->CloseFile(); + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); + char *s=(char *)"Unable to open or parse config file %s"; + char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); + sprintf(m,s,GloVars.config_file); + SPA->send_MySQL_ERR(&sess->client_myds->myprot, m); + free(m); + } + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); + SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Config file unknown"); + } + return false; + } + + if ( + (query_no_space_length==strlen("SAVE RESTAPI TO MEMORY") && !strncasecmp("SAVE RESTAPI TO MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE RESTAPI TO MEM") && !strncasecmp("SAVE RESTAPI TO MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE RESTAPI FROM RUNTIME") && !strncasecmp("SAVE RESTAPI FROM RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE RESTAPI FROM RUN") && !strncasecmp("SAVE RESTAPI FROM RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->save_scheduler_runtime_to_database(false); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved scheduler from RUNTIME\n"); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); + return false; + } + } + if ((query_no_space_length>15) && ( (!strncasecmp("SAVE SCHEDULER ", query_no_space, 15)) || (!strncasecmp("LOAD SCHEDULER ", query_no_space, 15))) ) { if ( @@ -2516,6 +2621,7 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign bool dump_global_variables=false; bool runtime_scheduler=false; + bool runtime_restapi_routes=false; bool runtime_mysql_users=false; bool runtime_mysql_firewall=false; bool runtime_mysql_ldap_mapping=false; @@ -2653,6 +2759,9 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign if (strstr(query_no_space,"runtime_scheduler")) { runtime_scheduler=true; refresh=true; } + if (strstr(query_no_space,"runtime_restapi_routes")) { + runtime_restapi_routes=true; refresh=true; + } if (strstr(query_no_space,"runtime_proxysql_servers")) { runtime_proxysql_servers=true; refresh=true; } @@ -2775,6 +2884,9 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign if (runtime_scheduler) { save_scheduler_runtime_to_database(true); } + if (runtime_restapi_routes) { + proxysql_restapi().save_restapi_runtime_to_database(true); + } if (runtime_checksums_values) { dump_checksums_values_table(); } @@ -3808,6 +3920,7 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { rc = pa->proxysql_config().Write_MySQL_Query_Rules_to_configfile(data); rc = pa->proxysql_config().Write_MySQL_Servers_to_configfile(data); rc = pa->proxysql_config().Write_Scheduler_to_configfile(data); + rc = pa->proxysql_config().Write_Restapi_to_configfile(data); rc = pa->proxysql_config().Write_ProxySQL_Servers_to_configfile(data); if (rc) { std::stringstream ss; @@ -3847,6 +3960,7 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { rc = pa->proxysql_config().Write_MySQL_Query_Rules_to_configfile(data); rc = pa->proxysql_config().Write_MySQL_Servers_to_configfile(data); rc = pa->proxysql_config().Write_Scheduler_to_configfile(data); + rc = pa->proxysql_config().Write_Restapi_to_configfile(data); rc = pa->proxysql_config().Write_ProxySQL_Servers_to_configfile(data); if (rc) { std::stringstream ss; @@ -4782,6 +4896,8 @@ bool ProxySQL_Admin::init() { insert_into_tables_defs(tables_defs_admin,"runtime_mysql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_FIREWALL_WHITELIST_RULES); insert_into_tables_defs(tables_defs_admin,"mysql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS); insert_into_tables_defs(tables_defs_admin,"runtime_mysql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS); + insert_into_tables_defs(tables_defs_admin, "restapi_routes", ADMIN_SQLITE_TABLE_RESTAPI_ROUTES); + insert_into_tables_defs(tables_defs_admin, "runtime_restapi_routes", ADMIN_SQLITE_TABLE_RUNTIME_RESTAPI_ROUTES); #ifdef DEBUG insert_into_tables_defs(tables_defs_admin,"debug_levels", ADMIN_SQLITE_TABLE_DEBUG_LEVELS); #endif /* DEBUG */ @@ -4808,6 +4924,7 @@ bool ProxySQL_Admin::init() { insert_into_tables_defs(tables_defs_config,"mysql_firewall_whitelist_users", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_USERS); insert_into_tables_defs(tables_defs_config,"mysql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_RULES); insert_into_tables_defs(tables_defs_config,"mysql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS); + insert_into_tables_defs(tables_defs_config, "restapi_routes", ADMIN_SQLITE_TABLE_RESTAPI_ROUTES); #ifdef DEBUG insert_into_tables_defs(tables_defs_config,"debug_levels", ADMIN_SQLITE_TABLE_DEBUG_LEVELS); #endif /* DEBUG */ @@ -4890,6 +5007,7 @@ bool ProxySQL_Admin::init() { // workaround for issue #708 statsdb->execute("INSERT OR IGNORE INTO global_variables VALUES('mysql-max_allowed_packet',4194304)"); + #ifdef DEBUG if (GloVars.global.gdbg==false && GloVars.__cmd_proxysql_gdbg) { proxy_debug(PROXY_DEBUG_ADMIN, 4, "Enabling GloVars.global.gdbg because GloVars.__cmd_proxysql_gdbg==%d\n", GloVars.__cmd_proxysql_gdbg); @@ -4906,6 +5024,7 @@ bool ProxySQL_Admin::init() { proxysql_config().Read_Global_Variables_from_configfile("admin"); proxysql_config().Read_Global_Variables_from_configfile("mysql"); proxysql_config().Read_Scheduler_from_configfile(); + proxysql_config().Read_Restapi_from_configfile(); proxysql_config().Read_ProxySQL_Servers_from_configfile(); __insert_or_replace_disktable_select_maintable(); } @@ -4942,7 +5061,7 @@ bool ProxySQL_Admin::init() { #ifdef DEBUG std::cerr << "Admin initialized in "; #endif - return true; +return true; }; @@ -8103,6 +8222,7 @@ void ProxySQL_Admin::__insert_or_replace_maintable_select_disktable() { admindb->execute("INSERT OR REPLACE INTO main.mysql_firewall_whitelist_sqli_fingerprints SELECT * FROM disk.mysql_firewall_whitelist_sqli_fingerprints"); admindb->execute("INSERT OR REPLACE INTO main.global_variables SELECT * FROM disk.global_variables"); admindb->execute("INSERT OR REPLACE INTO main.scheduler SELECT * FROM disk.scheduler"); + admindb->execute("INSERT OR REPLACE INTO main.restapi_routes SELECT * FROM disk.restapi_routes"); admindb->execute("INSERT OR REPLACE INTO main.proxysql_servers SELECT * FROM disk.proxysql_servers"); #ifdef DEBUG admindb->execute("INSERT OR REPLACE INTO main.debug_levels SELECT * FROM disk.debug_levels"); @@ -10398,6 +10518,8 @@ void ProxySQL_Admin::disk_upgrade_mysql_users() { configdb->execute("PRAGMA foreign_keys = ON"); } +Restapi_Row::Restapi_Row(unsigned int _id, bool _is_active, unsigned int _in, const std::string& _uri, const std::string& _script, const std::string& _comment) : + id(_id), is_active(_is_active), interval_ms(_in), uri(_uri), script(_script), comment(_comment) {} Scheduler_Row::Scheduler_Row(unsigned int _id, bool _is_active, unsigned int _in, char *_f, char *a1, char *a2, char *a3, char *a4, char *a5, char *_comment) { int i; diff --git a/lib/ProxySQL_Config.cpp b/lib/ProxySQL_Config.cpp index b7e40b4b5..5f9118d25 100644 --- a/lib/ProxySQL_Config.cpp +++ b/lib/ProxySQL_Config.cpp @@ -338,6 +338,109 @@ int ProxySQL_Config::Read_Scheduler_from_configfile() { return rows; } +int ProxySQL_Config::Write_Restapi_to_configfile(std::string& data) { + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* sqlite_resultset = NULL; + + char *query=(char *)"SELECT * FROM restapi_routes"; + admindb->execute_statement(query, &error, &cols, &affected_rows, &sqlite_resultset); + if (error) { + proxy_error("Error on read from restapi_router: %s\n", error); + return -1; + } else { + if (sqlite_resultset) { + data += "restapi:\n(\n"; + bool isNext = false; + for (auto r : sqlite_resultset->rows) { + if (isNext) + data += ",\n"; + data += "\t{\n"; + addField(data, "id", r->fields[0], ""); + addField(data, "active", r->fields[1], ""); + addField(data, "interval_ms", r->fields[2], ""); + addField(data, "uri", r->fields[3]); + addField(data, "script", r->fields[4]); + addField(data, "comment", r->fields[5]); + + data += "\t}"; + isNext = true; + } + data += "\n)\n"; + } + } + + if (sqlite_resultset) + delete sqlite_resultset; + + return 0; +} + +int ProxySQL_Config::Read_Restapi_from_configfile() { + const Setting& root = GloVars.confFile->cfg.getRoot(); + if (root.exists("restapi")==false) return 0; + const Setting &routes = root["restapi"]; + int count = routes.getLength(); + //fprintf(stderr, "Found %d users\n",count); + int i; + int rows=0; + admindb->execute("PRAGMA foreign_keys = OFF"); + char *q=(char *)"INSERT OR REPLACE INTO restapi (id, active, interval_ms, uri, script, comment) VALUES (%d, %d, %d, '%s', '%s', '%s')"; + for (i=0; i< count; i++) { + const Setting &route = routes[i]; + int id; + int active=1; + // variable for parsing interval_ms + int interval_ms=0; + + + std::string uri; + std::string script; + std::string comment=""; + + // validate arguments + if (route.lookupValue("id", id)==false) { + proxy_error("Admin: detected a restapi route in config file without a mandatory id\n"); + continue; + } + route.lookupValue("active", active); + route.lookupValue("interval_ms", interval_ms); + if (route.lookupValue("uri", uri)==false) { + proxy_error("Admin: detected a restapi route in config file without a mandatory uri\n"); + continue; + } + if (route.lookupValue("script", script)==false) { + proxy_error("Admin: detected a restapi route in config file without a mandatory script\n"); + continue; + } + route.lookupValue("comment", comment); + + int query_len=0; + query_len+=strlen(q) + + strlen(std::to_string(id).c_str()) + + strlen(std::to_string(active).c_str()) + + strlen(std::to_string(interval_ms).c_str()) + + strlen(uri.c_str()) + + strlen(script.c_str()) + + strlen(comment.c_str()) + + 40; + char *query=(char *)malloc(query_len); + sprintf(query, q, + id, active, + interval_ms, + uri.c_str(), + script.c_str(), + comment.c_str() + ); + admindb->execute(query); + free(query); + rows++; + } + admindb->execute("PRAGMA foreign_keys = ON"); + return rows; +} + int ProxySQL_Config::Write_MySQL_Query_Rules_to_configfile(std::string& data) { char* error = NULL; int cols = 0; diff --git a/lib/ProxySQL_RESTAPI_Server.cpp b/lib/ProxySQL_RESTAPI_Server.cpp index 70b638901..2fdd6b399 100644 --- a/lib/ProxySQL_RESTAPI_Server.cpp +++ b/lib/ProxySQL_RESTAPI_Server.cpp @@ -2,6 +2,9 @@ #include "cpp.h" #include "httpserver.hpp" +#include + + #include "ProxySQL_RESTAPI_Server.hpp" using namespace httpserver; @@ -13,6 +16,8 @@ using namespace httpserver; #endif /* DEBUG */ #define PROXYSQL_RESTAPI_SERVER_VERSION "2.0.1121" DEB +extern ProxySQL_Admin *GloAdmin; + class hello_world_resource : public http_resource { public: const std::shared_ptr render_GET(const http_request& req) { @@ -32,14 +37,46 @@ public: return std::shared_ptr(new string_response("{\"error\":\"Cannot fork.\"}")); } - char buf[1024]; + // validate json correctness + try { + nlohmann::json valid=nlohmann::json::parse(req.get_content()); + } + catch(nlohmann::json::exception& e) { + std::stringstream ss; + ss << "{\"type\":\"in\", \"error\":\"" << e.what() << "\"}"; + return std::shared_ptr(new string_response(ss.str())); + } + + + char buf[1024] = {0}; if (pid == 0) { dup2(pipefd[1], STDOUT_FILENO); close(pipefd[0]); close(pipefd[1]); - char* args[] = {"a", (char*)req.get_content().data(), NULL}; - if (execve("/home/val/workspace/script.py", args, NULL) == -1) { - return std::shared_ptr(new string_response("{\"error\":\"Error calling execve().\"}")); + + SQLite3_result *resultset=NULL; + int affected_rows; + int cols; + char *error=NULL; + std::stringstream ss; + ss << "SELECT * FROM restapi_routes WHERE uri='" << req.get_path_piece(1) << "'"; + bool rc=GloAdmin->admindb->execute_statement(ss.str().c_str(), &error, &cols, &affected_rows, &resultset); + if (!rc) { + proxy_error("Cannot query script for given path [%s]\n", req.get_path_piece(1)); + } + if (!resultset || resultset->rows_count == 0) { + std::stringstream ss; + ss << "{\"error\":\"The script for route [" << req.get_path() << "] was not found.\"}"; + return std::shared_ptr(new string_response(ss.str())); + } + std::string script = resultset->rows[0]->fields[4]; + char* const args[] = {const_cast("a"), const_cast(req.get_content().c_str()), NULL}; + if (execve(resultset->rows[0]->fields[4], args, NULL) == -1) { + char path_buffer[PATH_MAX]; + char* cwd = getcwd(path_buffer, sizeof(path_buffer)-1); + std::stringstream ss; + ss << "{\"error\":\"Error calling execve().\", \"cwd\":\"" << cwd << "\", \"file\":\"" << resultset->rows[0]->fields[4] << "\"}"; + return std::shared_ptr(new string_response(ss.str())); } exit(EXIT_SUCCESS); } @@ -49,7 +86,18 @@ public: if (nbytes == -1) { return std::shared_ptr(new string_response("{\"error\":\"Error reading pipe.\"}")); } - //TODO : validate json correctness in the buf + + // validate json correctness + try { + nlohmann::json j=nlohmann::json::parse(buf); + } + catch(nlohmann::json::exception& e) { + std::stringstream ss; + ss << "{\"type\":\"out\", \"error\":\"" << e.what() << "\"}"; + proxy_error("Error parsing script output. %s\n", buf); + return std::shared_ptr(new string_response(ss.str())); + } + close(pipefd[0]); wait(NULL); } @@ -76,7 +124,7 @@ ProxySQL_RESTAPI_Server::ProxySQL_RESTAPI_Server(int p) { hr = new hello_world_resource(); //ws->register_resource("/hello", &hwr); - ws->register_resource("/hello", hr); + ws->register_resource("/hello", hr, true); if (pthread_create(&thread_id, NULL, restapi_server_thread, ws) !=0 ) { perror("Thread creation"); exit(EXIT_FAILURE); diff --git a/lib/ProxySQL_Restapi.cpp b/lib/ProxySQL_Restapi.cpp new file mode 100644 index 000000000..22e194d3c --- /dev/null +++ b/lib/ProxySQL_Restapi.cpp @@ -0,0 +1,107 @@ +#include "proxysql_restapi.h" +#include "proxysql.h" +#include "proxysql_atomic.h" +#include "cpp.h" + +#include + +ProxySQL_Restapi::ProxySQL_Restapi(SQLite3DB* db) { + assert(db); + + admindb = db; +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_init(&rwlock,NULL); +#else + spinlock_rwlock_init(&rwlock); +#endif + version=0; +} + + +ProxySQL_Restapi::~ProxySQL_Restapi() {} + +void ProxySQL_Restapi::update_table(SQLite3_result *resultset) { +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_wrlock(&rwlock); +#else + spin_wrlock(&rwlock); +#endif + // delete all current rows + Restapi_Rows.clear(); + for (auto r : resultset->rows) { + unsigned int id = strtoul(r->fields[0], NULL, 10); + bool is_active=false; + if (atoi(r->fields[1])) { + is_active=true; + } + unsigned int interval_ms=strtoul(r->fields[2], NULL, 10); + Restapi_Rows.push_back({id, is_active, interval_ms, r->fields[3], r->fields[4], r->fields[5]}); + } + + // increase version + __sync_fetch_and_add(&version,1); + // unlock +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_unlock(&rwlock); +#else + spin_wrunlock(&rwlock); +#endif +} + +void ProxySQL_Restapi::flush_restapi__from_disk_to_memory() { + admindb->wrlock(); + admindb->execute("DELETE FROM main.restapi_routes"); + admindb->execute("INSERT INTO main.restapi_routes SELECT * FROM disk.restapi_routes"); + admindb->wrunlock(); +} + +void ProxySQL_Restapi::flush_restapi__from_memory_to_disk() { + admindb->wrlock(); + admindb->execute("DELETE FROM disk.restapi_routes"); + admindb->execute("INSERT INTO disk.restapi_routes SELECT * FROM main.restapi_routes"); + admindb->wrunlock(); +} + +void ProxySQL_Restapi::save_restapi_runtime_to_database(bool _runtime) { + const char *query = _runtime ? "DELETE FROM main.runtime_restapi_routes" : "DELETE FROM main.restapi_routes"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + admindb->execute(query); + // read lock the scheduler +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_rdlock(&rwlock); +#else + spin_rdlock(&rwlock); +#endif + const char* table = _runtime ? " runtime_restapi_routes " : " restapi_routes "; + for (auto r : Restapi_Rows) { + std::stringstream ss; + ss << "INSERT INTO " << table << " VALUES(" << r.id << "," << r.is_active << "," + << r.interval_ms << ",'" << r.uri << "','" << r.script << "','" << r.comment << "')"; + + admindb->execute(ss.str().c_str()); + } + // unlock the scheduler +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_unlock(&rwlock); +#else + spin_rdunlock(&rwlock); +#endif +} + +void ProxySQL_Restapi::load_restapi_to_runtime() { + char *error=NULL; + int cols=0; + int affected_rows=0; + SQLite3_result *resultset=NULL; + char *query=(char *)"SELECT * FROM restapi_routes"; + admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + if (error) { + proxy_error("Error on %s : %s\n", query, error); + } else { + update_table(resultset); + } + if (resultset) delete resultset; + resultset=NULL; +} + + diff --git a/scripts/export_users.py b/scripts/export_users.py new file mode 100755 index 000000000..8d3e1e796 --- /dev/null +++ b/scripts/export_users.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# +#./scripts/export_users.py '{"db":{"user":"root", "password":"a", "port":"3306", "host":"127.0.0.1"},"admin":{"user":"admin","password":"admin","port":"6032","host":"127.0.0.1"}}' +# + +import sys +import subprocess +import json +from MySQLdb import _mysql + +if len(sys.argv) > 1: + params=json.loads(sys.argv[1]) + db_mysql=_mysql.connect(host=params['db']['host'],user=params['db']['user'],passwd=params['db']['password'],port=int(params['db']['port'])) + db_mysql.query('SELECT user, authentication_string from mysql.user') + records=db_mysql.store_result().fetch_row(maxrows=0) + db_proxy_admin=_mysql.connect(host=params['admin']['host'],user=params['admin']['user'],passwd=params['admin']['password'],port=int(params['admin']['port'])) + for row in records: + db_proxy_admin.query('INSERT OR REPLACE INTO mysql_users (username, password) values ("'+str(row[0])+'","'+str(row[1])+'")') + + result='{"num_records":"'+str(len(records))+'"}' + try: + subprocess.check_output(['mysql', '-u'+params['admin']['user'], '-p'+params['admin']['password'], '-h'+params['admin']['host'], '-P'+params['admin']['port'], '-e', 'load mysql users to runtime'],stderr= subprocess.STDOUT) + except subprocess.CalledProcessError as e: + result='"Error calling mysql: ' + e.output.replace("'", "") + '"' + + print('{"params":'+str(sys.argv[1])+', "result":'+result+'}') + diff --git a/scripts/metrics.py b/scripts/metrics.py new file mode 100755 index 000000000..6ea4c421a --- /dev/null +++ b/scripts/metrics.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python + +import sys +import subprocess +import json + + +if len(sys.argv) > 1: + params=json.loads(sys.argv[1]) + out='' + try: + out=subprocess.check_output(['mysql', '-u'+params['user'], '-p'+params['password'], '-h'+params['host'], '-P'+params['port'], '-e', 'select * from stats.stats_memory_metrics'],stderr= subprocess.STDOUT) + except subprocess.CalledProcessError as e: + out="Error calling mysql: " + e.output.replace("'", "") + print('{"params":'+sys.argv[1].encode('string-escape')+', "result":"'+out.encode('string-escape')+'"}') + diff --git a/src/main.cpp b/src/main.cpp index 3c047d9f5..6009f8894 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -17,6 +17,7 @@ #include "query_processor.h" #include "MySQL_Authentication.hpp" #include "MySQL_LDAP_Authentication.hpp" +#include "proxysql_restapi.h" #include @@ -1261,6 +1262,7 @@ void ProxySQL_Main_init_phase3___start_all() { GloAdmin->init_mysql_servers(); GloAdmin->init_proxysql_servers(); GloAdmin->load_scheduler_to_runtime(); + GloAdmin->proxysql_restapi().load_restapi_to_runtime(); #ifdef DEBUG std::cerr << "Main phase3 : GloAdmin initialized in "; #endif diff --git a/test/tap/tests/proxysql_reference_select_config_file.cnf b/test/tap/tests/proxysql_reference_select_config_file.cnf index cc2f68f79..8f5e0214f 100644 --- a/test/tap/tests/proxysql_reference_select_config_file.cnf +++ b/test/tap/tests/proxysql_reference_select_config_file.cnf @@ -358,6 +358,17 @@ scheduler: comment="comment" } ) +restapi: +( + { + id=1 + active=1 + interval_ms=1000 + uri="test" + script="/script.py" + comment="comment" + } +) proxysql_servers: ( { diff --git a/test/tap/tests/select_config_file-t.cpp b/test/tap/tests/select_config_file-t.cpp index 64aa84aed..1d953bfc7 100644 --- a/test/tap/tests/select_config_file-t.cpp +++ b/test/tap/tests/select_config_file-t.cpp @@ -62,11 +62,14 @@ int main(int argc, char** argv) { MYSQL_QUERY(mysql, "delete from mysql_galera_hostgroups"); MYSQL_QUERY(mysql, "delete from mysql_aws_aurora_hostgroups"); MYSQL_QUERY(mysql, "delete from scheduler"); + MYSQL_QUERY(mysql, "delete from restapi_routes"); MYSQL_QUERY(mysql, "delete from proxysql_servers"); MYSQL_QUERY(mysql, "insert into proxysql_servers (hostname, port, weight, comment) values ('hostname', 3333, 12, 'comment');"); MYSQL_QUERY(mysql, "insert into scheduler (id, active, interval_ms, filename, arg1, arg2, arg3, arg4, arg5, comment) values " " (1,1,1000,'filename','a1','a2','a3','a4','a5','comment');"); + MYSQL_QUERY(mysql, "insert into restapi_routes (id, active, interval_ms, uri, script, comment) values " + " (1,1,1000,'test','/script.py','comment');"); MYSQL_QUERY(mysql, "insert into mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, aurora_port, " " domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, " " add_lag_ms, min_lag_ms, lag_num_checks, comment) " @@ -107,6 +110,13 @@ int main(int argc, char** argv) { std::string str = strStream.str(); //str holds the content of the file ok(!str.compare(resultset), "Files are equal"); +#if 0 + std::ofstream f; + f.open("out.txt", std::ios::out); + f << resultset; + f.close(); +#endif + } MYSQL_QUERY(mysql, "load mysql variables from disk"); @@ -114,6 +124,7 @@ int main(int argc, char** argv) { MYSQL_QUERY(mysql, "load mysql users from disk"); MYSQL_QUERY(mysql, "load mysql servers from disk"); MYSQL_QUERY(mysql, "load scheduler from disk"); + MYSQL_QUERY(mysql, "load restapi from disk"); MYSQL_QUERY(mysql, "load proxysql servers from disk"); mysql_close(mysql); From 66400a35c7780858d66c7f8235a3542ebf907511 Mon Sep 17 00:00:00 2001 From: Valentin Rakush Date: Thu, 26 Dec 2019 19:49:11 +0000 Subject: [PATCH 3/6] Implementation of the GET and POST methods --- include/ProxySQL_RESTAPI_Server.hpp | 5 +- lib/ProxySQL_RESTAPI_Server.cpp | 211 +++++++++++++++++++--------- scripts/metrics.py | 1 - 3 files changed, 146 insertions(+), 71 deletions(-) diff --git a/include/ProxySQL_RESTAPI_Server.hpp b/include/ProxySQL_RESTAPI_Server.hpp index 8ce7aedd6..fc0847e89 100644 --- a/include/ProxySQL_RESTAPI_Server.hpp +++ b/include/ProxySQL_RESTAPI_Server.hpp @@ -8,11 +8,10 @@ class ProxySQL_RESTAPI_Server { private: - //httpserver::webserver *ws; - httpserver::webserver * ws; + std::unique_ptr ws; int port; pthread_t thread_id; - httpserver::http_resource *hr; + std::unique_ptr endpoint; public: ProxySQL_RESTAPI_Server(int p); ~ProxySQL_RESTAPI_Server(); diff --git a/lib/ProxySQL_RESTAPI_Server.cpp b/lib/ProxySQL_RESTAPI_Server.cpp index 2fdd6b399..24136dfb6 100644 --- a/lib/ProxySQL_RESTAPI_Server.cpp +++ b/lib/ProxySQL_RESTAPI_Server.cpp @@ -18,91 +18,177 @@ using namespace httpserver; extern ProxySQL_Admin *GloAdmin; -class hello_world_resource : public http_resource { -public: - const std::shared_ptr render_GET(const http_request& req) { - return std::shared_ptr(new string_response("GET: Hello, World!\n")); - } - - const std::shared_ptr render_POST(const http_request& req) { - //TODO : validate json correctness in the req - - int pipefd[2]; - if (pipe(pipefd) == -1) { - return std::shared_ptr(new string_response("{\"error\":\"Cannot create pipe.\"}")); - } +class sync_resource : public http_resource { +private: + const std::shared_ptr find_script(const http_request& req, std::string& script, int &interval_ms) { + SQLite3_result *resultset=NULL; + int affected_rows; + int cols; + char *error=NULL; + std::stringstream ss; + ss << "SELECT * FROM restapi_routes WHERE uri='" << req.get_path_piece(1) << "' and active=1"; + bool rc=GloAdmin->admindb->execute_statement(ss.str().c_str(), &error, &cols, &affected_rows, &resultset); + if (!rc) { + proxy_error("Cannot query script for given path [%s]\n", req.get_path_piece(1)); + std::stringstream ss; + if (error) { + ss << "{\"error\":\"The script for route [" << req.get_path() << "] was not found. Error: " << error << "Error: \"}"; + proxy_error("Path %s, error %s\n", req.get_path().c_str(), error); + } + else { + ss << "{\"error\":\"The script for route [" << req.get_path() << "] was not found.\"}"; + proxy_error("Path %s\n", req.get_path().c_str()); + } + return std::shared_ptr(new string_response(ss.str())); + } + if (resultset && resultset->rows_count != 1) { + std::stringstream ss; + ss << "{\"error\":\"The script for route [" << req.get_path() << "] was not found. count = " << resultset->rows_count << "\" }"; + proxy_error("Script for route %s was not found\n", req.get_path().c_str()); + return std::shared_ptr(new string_response(ss.str())); + } + script = resultset->rows[0]->fields[4]; + interval_ms = atoi(resultset->rows[0]->fields[2]); + if (resultset) {delete resultset; resultset=NULL;} + return std::shared_ptr(nullptr); + } - pid_t pid; - if ((pid=fork()) == -1) { - return std::shared_ptr(new string_response("{\"error\":\"Cannot fork.\"}")); + const std::shared_ptr process_request(const http_request& req, const std::string& _params) { + std::string params = req.get_content(); + if (params.empty()) + params = _params; + if (params.empty()) { + proxy_error("Empty parameters\n"); + return std::shared_ptr(new string_response("{\"error\":\"Empty parameters\"}")); } - // validate json correctness try { - nlohmann::json valid=nlohmann::json::parse(req.get_content()); + nlohmann::json valid=nlohmann::json::parse(params); } catch(nlohmann::json::exception& e) { std::stringstream ss; ss << "{\"type\":\"in\", \"error\":\"" << e.what() << "\"}"; + proxy_error("Error parsing input json parameters. %s\n", ss.str().c_str()); return std::shared_ptr(new string_response(ss.str())); } + std::string script; + int interval_ms; + auto result=find_script(req, script, interval_ms); + if (nullptr!=result) + return result; + + int pipefd[2]; + if (pipe(pipefd) == -1) { + proxy_error("Cannot create pipe\n"); + return std::shared_ptr(new string_response("{\"error\":\"Cannot create pipe.\"}")); + } + + pid_t pid; + if ((pid=fork()) == -1) { + proxy_error("Cannot fork\n"); + return std::shared_ptr(new string_response("{\"error\":\"Cannot fork.\"}")); + } - char buf[1024] = {0}; + char buf[65536] = {0}; if (pid == 0) { dup2(pipefd[1], STDOUT_FILENO); close(pipefd[0]); close(pipefd[1]); - SQLite3_result *resultset=NULL; - int affected_rows; - int cols; - char *error=NULL; - std::stringstream ss; - ss << "SELECT * FROM restapi_routes WHERE uri='" << req.get_path_piece(1) << "'"; - bool rc=GloAdmin->admindb->execute_statement(ss.str().c_str(), &error, &cols, &affected_rows, &resultset); - if (!rc) { - proxy_error("Cannot query script for given path [%s]\n", req.get_path_piece(1)); - } - if (!resultset || resultset->rows_count == 0) { - std::stringstream ss; - ss << "{\"error\":\"The script for route [" << req.get_path() << "] was not found.\"}"; - return std::shared_ptr(new string_response(ss.str())); - } - std::string script = resultset->rows[0]->fields[4]; - char* const args[] = {const_cast("a"), const_cast(req.get_content().c_str()), NULL}; - if (execve(resultset->rows[0]->fields[4], args, NULL) == -1) { + char* const args[] = {const_cast(script.c_str()), const_cast(params.c_str()), NULL}; + if (execve(script.c_str(), args, NULL) == -1) { char path_buffer[PATH_MAX]; char* cwd = getcwd(path_buffer, sizeof(path_buffer)-1); std::stringstream ss; - ss << "{\"error\":\"Error calling execve().\", \"cwd\":\"" << cwd << "\", \"file\":\"" << resultset->rows[0]->fields[4] << "\"}"; + ss << "{\"error\":\"Error calling execve().\", \"cwd\":\"" << cwd << "\", \"file\":\"" << script << "\"}"; + proxy_error("%s\n", ss.str().c_str()); return std::shared_ptr(new string_response(ss.str())); } exit(EXIT_SUCCESS); } else { close(pipefd[1]); - int nbytes = read(pipefd[0], buf, sizeof(buf) - 1); - if (nbytes == -1) { - return std::shared_ptr(new string_response("{\"error\":\"Error reading pipe.\"}")); - } - // validate json correctness - try { - nlohmann::json j=nlohmann::json::parse(buf); + fd_set set; + + FD_ZERO(&set); + FD_SET(pipefd[0], &set); + + struct timeval timeout; + timeout.tv_sec=interval_ms/1000; + timeout.tv_usec=(interval_ms%1000)*1000; + + int rv = select(pipefd[0]+1,&set,NULL,NULL,&timeout); + if (rv == -1) { + proxy_error("Error calling select for path %s\n", req.get_path().c_str()); + std::stringstream ss; + ss << "{\"error\":\"Error calling select().\", \"path\":\"" << req.get_path() << "\"}"; + return std::shared_ptr(new string_response(ss.str())); } - catch(nlohmann::json::exception& e) { + else if (rv == 0) { + proxy_error("Timeout reading script output %s\n", script.c_str()); std::stringstream ss; - ss << "{\"type\":\"out\", \"error\":\"" << e.what() << "\"}"; - proxy_error("Error parsing script output. %s\n", buf); + ss << "{\"error\":\"Timeout reading script output. Script file: " << script << "\"}"; return std::shared_ptr(new string_response(ss.str())); } - + else { + int nbytes = read(pipefd[0], buf, sizeof(buf) - 1); + if (nbytes == -1) { + proxy_error("Error reading pipe\n"); + return std::shared_ptr(new string_response("{\"error\":\"Error reading pipe.\"}")); + } + + // validate json correctness + try { + nlohmann::json j=nlohmann::json::parse(buf); + } + catch(nlohmann::json::exception& e) { + std::stringstream ss; + ss << "{\"type\":\"out\", \"error\":\"" << e.what() << "\"}"; + proxy_error("Error parsing script output. %s\n", buf); + return std::shared_ptr(new string_response(ss.str())); + } + } close(pipefd[0]); - wait(NULL); + + int status; + waitpid(pid, &status, 0); } return std::shared_ptr(new string_response(buf)); } + +public: + const std::shared_ptr render(const http_request& req) { + proxy_info("Render generic request with method %s for uri %s\n", req.get_method().c_str(), req.get_path().c_str()); + std::stringstream ss; + ss << "{\"error\":\"HTTP method " << req.get_method().c_str() << " is not implemented\"}"; + return std::shared_ptr(new string_response(ss.str().c_str())); + } + + const std::shared_ptr render_GET(const http_request& req) { + auto args = req.get_args(); + + size_t last = 0; + std::stringstream params; + params << "{"; + for (auto arg : args) { + params << "\"" << arg.first << "\":\"" << arg.second << "\""; + if (last < args.size()-1) { + params << ","; + last++; + } + } + params << "}"; + + return process_request(req, params.str()); + } + + const std::shared_ptr render_POST(const http_request& req) { + std::string params=req.get_content(); + return process_request(req, params); + } + }; void * restapi_server_thread(void *arg) { @@ -112,25 +198,16 @@ void * restapi_server_thread(void *arg) { } ProxySQL_RESTAPI_Server::ProxySQL_RESTAPI_Server(int p) { + ws = std::unique_ptr(new webserver(create_webserver(p))); + auto sr = new sync_resource(); + + endpoint = std::unique_ptr(sr); - // for now, this is COMPLETELY DISABLED - // just adding a POC -// return; - - //ws = NULL; - port = p; - ws = new webserver(create_webserver(p)); - //hello_world_resource hwr; - hr = new hello_world_resource(); - - //ws->register_resource("/hello", &hwr); - ws->register_resource("/hello", hr, true); - if (pthread_create(&thread_id, NULL, restapi_server_thread, ws) !=0 ) { - perror("Thread creation"); - exit(EXIT_FAILURE); + ws->register_resource("/sync", endpoint.get(), true); + if (pthread_create(&thread_id, NULL, restapi_server_thread, ws.get()) !=0 ) { + perror("Thread creation"); + exit(EXIT_FAILURE); } - //webserver ws2 = create_webserver(8080); - //webserws = create_webserver(8080); } ProxySQL_RESTAPI_Server::~ProxySQL_RESTAPI_Server() { diff --git a/scripts/metrics.py b/scripts/metrics.py index 6ea4c421a..49fc50ce2 100755 --- a/scripts/metrics.py +++ b/scripts/metrics.py @@ -4,7 +4,6 @@ import sys import subprocess import json - if len(sys.argv) > 1: params=json.loads(sys.argv[1]) out='' From 4a5d986aa9bd9c234fb45e0ad44a4cbe3b96ad1e Mon Sep 17 00:00:00 2001 From: Valentin Rakush Date: Fri, 27 Dec 2019 09:07:59 +0000 Subject: [PATCH 4/6] Use HTTP method for configuration --- lib/ProxySQL_Admin.cpp | 4 ++-- lib/ProxySQL_RESTAPI_Server.cpp | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 495e4237e..2c0056409 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -363,9 +363,9 @@ static int http_handler(void *cls, struct MHD_Connection *connection, const char #define ADMIN_SQLITE_TABLE_MYSQL_COLLATIONS "CREATE TABLE mysql_collations (Id INTEGER NOT NULL PRIMARY KEY , Collation VARCHAR NOT NULL , Charset VARCHAR NOT NULL , `Default` VARCHAR NOT NULL)" -#define ADMIN_SQLITE_TABLE_RESTAPI_ROUTES "CREATE TABLE restapi_routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>-100 AND interval_ms<=100000000) NOT NULL , uri VARCHAR NOT NULL, script VARCHAR NOT NULL, comment VARCHAR NO NULL DEFAULT '')" +#define ADMIN_SQLITE_TABLE_RESTAPI_ROUTES "CREATE TABLE restapi_routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>-100 AND interval_ms<=100000000) NOT NULL , method VARCHAR NOT NULL CHECK (UPPER(method) IN ('GET','POST')), uri VARCHAR NOT NULL, script VARCHAR NOT NULL, comment VARCHAR NO NULL DEFAULT '')" -#define ADMIN_SQLITE_TABLE_RUNTIME_RESTAPI_ROUTES "CREATE TABLE runtime_restapi_routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>-100 AND interval_ms<=100000000) NOT NULL , uri VARCHAR NOT NULL, script VARCHAR NOT NULL, comment VARCHAR NO NULL DEFAULT '')" +#define ADMIN_SQLITE_TABLE_RUNTIME_RESTAPI_ROUTES "CREATE TABLE runtime_restapi_routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>-100 AND interval_ms<=100000000) NOT NULL , method VARCHAR NOT NULL CHECK (UPPER(method) IN ('GET','POST')) , uri VARCHAR NOT NULL, script VARCHAR NOT NULL, comment VARCHAR NO NULL DEFAULT '')" #define ADMIN_SQLITE_TABLE_SCHEDULER "CREATE TABLE scheduler (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , comment VARCHAR NOT NULL DEFAULT '')" diff --git a/lib/ProxySQL_RESTAPI_Server.cpp b/lib/ProxySQL_RESTAPI_Server.cpp index 24136dfb6..c4f63a72e 100644 --- a/lib/ProxySQL_RESTAPI_Server.cpp +++ b/lib/ProxySQL_RESTAPI_Server.cpp @@ -26,28 +26,28 @@ private: int cols; char *error=NULL; std::stringstream ss; - ss << "SELECT * FROM restapi_routes WHERE uri='" << req.get_path_piece(1) << "' and active=1"; + ss << "SELECT * FROM restapi_routes WHERE uri='" << req.get_path_piece(1) << "' and method='" << req.get_method() << "' and active=1"; bool rc=GloAdmin->admindb->execute_statement(ss.str().c_str(), &error, &cols, &affected_rows, &resultset); if (!rc) { - proxy_error("Cannot query script for given path [%s]\n", req.get_path_piece(1)); + proxy_error("Cannot query script for given method [%s] and uri [%s]\n", req.get_method().c_str(), req.get_path_piece(1).c_str()); std::stringstream ss; if (error) { - ss << "{\"error\":\"The script for route [" << req.get_path() << "] was not found. Error: " << error << "Error: \"}"; + ss << "{\"error\":\"The script for method [" << req.get_method() << "] and route [" << req.get_path() << "] was not found. Error: " << error << "Error: \"}"; proxy_error("Path %s, error %s\n", req.get_path().c_str(), error); } else { - ss << "{\"error\":\"The script for route [" << req.get_path() << "] was not found.\"}"; + ss << "{\"error\":\"The script for method [" << req.get_method() << "] and route [" << req.get_path() << "] was not found.\"}"; proxy_error("Path %s\n", req.get_path().c_str()); } return std::shared_ptr(new string_response(ss.str())); } if (resultset && resultset->rows_count != 1) { std::stringstream ss; - ss << "{\"error\":\"The script for route [" << req.get_path() << "] was not found. count = " << resultset->rows_count << "\" }"; - proxy_error("Script for route %s was not found\n", req.get_path().c_str()); + ss << "{\"error\":\"The script for method [" << req.get_method() << "] and route [" << req.get_path() << "] was not found. Rows count returned [" << resultset->rows_count << "]\" }"; + proxy_error("Script for method [%s] and route [%s] was not found\n", req.get_method().c_str(), req.get_path().c_str()); return std::shared_ptr(new string_response(ss.str())); } - script = resultset->rows[0]->fields[4]; + script = resultset->rows[0]->fields[5]; interval_ms = atoi(resultset->rows[0]->fields[2]); if (resultset) {delete resultset; resultset=NULL;} return std::shared_ptr(nullptr); From 30351dfd0f49459b99d42fc33785ab37a5388dd7 Mon Sep 17 00:00:00 2001 From: Valentin Rakush Date: Sun, 12 Jan 2020 20:44:05 +0000 Subject: [PATCH 5/6] Fix issue with method field --- include/proxysql_restapi.h | 5 +++-- lib/ProxySQL_Admin.cpp | 3 --- lib/ProxySQL_Config.cpp | 17 ++++++++++++----- lib/ProxySQL_Restapi.cpp | 18 ++++++++---------- .../proxysql_reference_select_config_file.cnf | 3 ++- test/tap/tests/select_config_file-t.cpp | 4 ++-- 6 files changed, 27 insertions(+), 23 deletions(-) diff --git a/include/proxysql_restapi.h b/include/proxysql_restapi.h index ebe780217..9660a1264 100644 --- a/include/proxysql_restapi.h +++ b/include/proxysql_restapi.h @@ -13,11 +13,12 @@ public: unsigned int id; bool is_active; unsigned int interval_ms; + std::string method; std::string uri; std::string script; std::string comment; unsigned int version; - Restapi_Row(unsigned int _id, bool _is_active, unsigned int _in, const std::string& _uri, const std::string& _script, const std::string& _comment); + Restapi_Row(unsigned int _id, bool _is_active, unsigned int _in, const std::string& _method, const std::string& _uri, const std::string& _script, const std::string& _comment); }; class ProxySQL_Restapi { @@ -34,7 +35,7 @@ public: rwlock_t rwlock; #endif std::vector Restapi_Rows; - void update_table(SQLite3_result *result); + void update_restapi_table(SQLite3_result *result); void load_restapi_to_runtime(); void save_restapi_runtime_to_database(bool); void flush_restapi__from_memory_to_disk(); diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 2c0056409..b02dc6d0b 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -10518,9 +10518,6 @@ void ProxySQL_Admin::disk_upgrade_mysql_users() { configdb->execute("PRAGMA foreign_keys = ON"); } -Restapi_Row::Restapi_Row(unsigned int _id, bool _is_active, unsigned int _in, const std::string& _uri, const std::string& _script, const std::string& _comment) : - id(_id), is_active(_is_active), interval_ms(_in), uri(_uri), script(_script), comment(_comment) {} - Scheduler_Row::Scheduler_Row(unsigned int _id, bool _is_active, unsigned int _in, char *_f, char *a1, char *a2, char *a3, char *a4, char *a5, char *_comment) { int i; id=_id; diff --git a/lib/ProxySQL_Config.cpp b/lib/ProxySQL_Config.cpp index 5f9118d25..aeca63411 100644 --- a/lib/ProxySQL_Config.cpp +++ b/lib/ProxySQL_Config.cpp @@ -360,9 +360,10 @@ int ProxySQL_Config::Write_Restapi_to_configfile(std::string& data) { addField(data, "id", r->fields[0], ""); addField(data, "active", r->fields[1], ""); addField(data, "interval_ms", r->fields[2], ""); - addField(data, "uri", r->fields[3]); - addField(data, "script", r->fields[4]); - addField(data, "comment", r->fields[5]); + addField(data, "method", r->fields[3], ""); + addField(data, "uri", r->fields[4]); + addField(data, "script", r->fields[5]); + addField(data, "comment", r->fields[6]); data += "\t}"; isNext = true; @@ -386,7 +387,7 @@ int ProxySQL_Config::Read_Restapi_from_configfile() { int i; int rows=0; admindb->execute("PRAGMA foreign_keys = OFF"); - char *q=(char *)"INSERT OR REPLACE INTO restapi (id, active, interval_ms, uri, script, comment) VALUES (%d, %d, %d, '%s', '%s', '%s')"; + char *q=(char *)"INSERT OR REPLACE INTO restapi VALUES (%d, %d, %d, '%s', '%s', '%s', '%s')"; for (i=0; i< count; i++) { const Setting &route = routes[i]; int id; @@ -394,7 +395,7 @@ int ProxySQL_Config::Read_Restapi_from_configfile() { // variable for parsing interval_ms int interval_ms=0; - + std::string method; std::string uri; std::string script; std::string comment=""; @@ -406,6 +407,10 @@ int ProxySQL_Config::Read_Restapi_from_configfile() { } route.lookupValue("active", active); route.lookupValue("interval_ms", interval_ms); + if (route.lookupValue("method", method)==false) { + proxy_error("Admin: detected a restapi route in config file without a mandatory method\n"); + continue; + } if (route.lookupValue("uri", uri)==false) { proxy_error("Admin: detected a restapi route in config file without a mandatory uri\n"); continue; @@ -421,6 +426,7 @@ int ProxySQL_Config::Read_Restapi_from_configfile() { strlen(std::to_string(id).c_str()) + strlen(std::to_string(active).c_str()) + strlen(std::to_string(interval_ms).c_str()) + + strlen(method.c_str()) + strlen(uri.c_str()) + strlen(script.c_str()) + strlen(comment.c_str()) + @@ -429,6 +435,7 @@ int ProxySQL_Config::Read_Restapi_from_configfile() { sprintf(query, q, id, active, interval_ms, + method.c_str(), uri.c_str(), script.c_str(), comment.c_str() diff --git a/lib/ProxySQL_Restapi.cpp b/lib/ProxySQL_Restapi.cpp index 22e194d3c..ae536f2f9 100644 --- a/lib/ProxySQL_Restapi.cpp +++ b/lib/ProxySQL_Restapi.cpp @@ -5,6 +5,9 @@ #include +Restapi_Row::Restapi_Row(unsigned int _id, bool _is_active, unsigned int _in, const std::string& _method, const std::string& _uri, const std::string& _script, const std::string& _comment) : + id(_id), is_active(_is_active), interval_ms(_in), method(_method), uri(_uri), script(_script), comment(_comment) {} + ProxySQL_Restapi::ProxySQL_Restapi(SQLite3DB* db) { assert(db); @@ -20,7 +23,7 @@ ProxySQL_Restapi::ProxySQL_Restapi(SQLite3DB* db) { ProxySQL_Restapi::~ProxySQL_Restapi() {} -void ProxySQL_Restapi::update_table(SQLite3_result *resultset) { +void ProxySQL_Restapi::update_restapi_table(SQLite3_result *resultset) { #ifdef PA_PTHREAD_MUTEX pthread_rwlock_wrlock(&rwlock); #else @@ -35,7 +38,7 @@ void ProxySQL_Restapi::update_table(SQLite3_result *resultset) { is_active=true; } unsigned int interval_ms=strtoul(r->fields[2], NULL, 10); - Restapi_Rows.push_back({id, is_active, interval_ms, r->fields[3], r->fields[4], r->fields[5]}); + Restapi_Rows.push_back({id, is_active, interval_ms, r->fields[3], r->fields[4], r->fields[5], r->fields[6]}); } // increase version @@ -76,7 +79,7 @@ void ProxySQL_Restapi::save_restapi_runtime_to_database(bool _runtime) { for (auto r : Restapi_Rows) { std::stringstream ss; ss << "INSERT INTO " << table << " VALUES(" << r.id << "," << r.is_active << "," - << r.interval_ms << ",'" << r.uri << "','" << r.script << "','" << r.comment << "')"; + << r.interval_ms << ",'" << r.method << "','" << r.uri << "','" << r.script << "','" << r.comment << "')"; admindb->execute(ss.str().c_str()); } @@ -90,18 +93,13 @@ void ProxySQL_Restapi::save_restapi_runtime_to_database(bool _runtime) { void ProxySQL_Restapi::load_restapi_to_runtime() { char *error=NULL; - int cols=0; - int affected_rows=0; - SQLite3_result *resultset=NULL; char *query=(char *)"SELECT * FROM restapi_routes"; - admindb->execute_statement(query, &error , &cols , &affected_rows , &resultset); + std::unique_ptr resultset = std::unique_ptr(admindb->execute_statement(query, &error)); if (error) { proxy_error("Error on %s : %s\n", query, error); } else { - update_table(resultset); + update_restapi_table(resultset.get()); } - if (resultset) delete resultset; - resultset=NULL; } diff --git a/test/tap/tests/proxysql_reference_select_config_file.cnf b/test/tap/tests/proxysql_reference_select_config_file.cnf index 8f5e0214f..71f3bc75b 100644 --- a/test/tap/tests/proxysql_reference_select_config_file.cnf +++ b/test/tap/tests/proxysql_reference_select_config_file.cnf @@ -364,8 +364,9 @@ restapi: id=1 active=1 interval_ms=1000 + method=GET uri="test" - script="/script.py" + script="./scripts/script.py" comment="comment" } ) diff --git a/test/tap/tests/select_config_file-t.cpp b/test/tap/tests/select_config_file-t.cpp index 1d953bfc7..5c6459106 100644 --- a/test/tap/tests/select_config_file-t.cpp +++ b/test/tap/tests/select_config_file-t.cpp @@ -68,8 +68,8 @@ int main(int argc, char** argv) { MYSQL_QUERY(mysql, "insert into proxysql_servers (hostname, port, weight, comment) values ('hostname', 3333, 12, 'comment');"); MYSQL_QUERY(mysql, "insert into scheduler (id, active, interval_ms, filename, arg1, arg2, arg3, arg4, arg5, comment) values " " (1,1,1000,'filename','a1','a2','a3','a4','a5','comment');"); - MYSQL_QUERY(mysql, "insert into restapi_routes (id, active, interval_ms, uri, script, comment) values " - " (1,1,1000,'test','/script.py','comment');"); + MYSQL_QUERY(mysql, "insert into restapi_routes values " + " (1,1,1000,'GET', 'test','./scripts/script.py','comment');"); MYSQL_QUERY(mysql, "insert into mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, aurora_port, " " domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, " " add_lag_ms, min_lag_ms, lag_num_checks, comment) " From 8c1d4ef8f30079dfd6af6a8a729a93d253a20880 Mon Sep 17 00:00:00 2001 From: Valentin Rakush Date: Mon, 13 Jan 2020 06:54:10 +0000 Subject: [PATCH 6/6] Fix problem with runtime table. Add CORS and content type headers --- lib/ProxySQL_Admin.cpp | 4 +- lib/ProxySQL_RESTAPI_Server.cpp | 84 ++++++++++++++++++++++++--------- lib/ProxySQL_Restapi.cpp | 1 + 3 files changed, 65 insertions(+), 24 deletions(-) diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index b02dc6d0b..f76ec125c 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -363,9 +363,9 @@ static int http_handler(void *cls, struct MHD_Connection *connection, const char #define ADMIN_SQLITE_TABLE_MYSQL_COLLATIONS "CREATE TABLE mysql_collations (Id INTEGER NOT NULL PRIMARY KEY , Collation VARCHAR NOT NULL , Charset VARCHAR NOT NULL , `Default` VARCHAR NOT NULL)" -#define ADMIN_SQLITE_TABLE_RESTAPI_ROUTES "CREATE TABLE restapi_routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>-100 AND interval_ms<=100000000) NOT NULL , method VARCHAR NOT NULL CHECK (UPPER(method) IN ('GET','POST')), uri VARCHAR NOT NULL, script VARCHAR NOT NULL, comment VARCHAR NO NULL DEFAULT '')" +#define ADMIN_SQLITE_TABLE_RESTAPI_ROUTES "CREATE TABLE restapi_routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , method VARCHAR NOT NULL CHECK (UPPER(method) IN ('GET','POST')) , uri VARCHAR NOT NULL , script VARCHAR NOT NULL , comment VARCHAR NOT NULL DEFAULT '')" -#define ADMIN_SQLITE_TABLE_RUNTIME_RESTAPI_ROUTES "CREATE TABLE runtime_restapi_routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>-100 AND interval_ms<=100000000) NOT NULL , method VARCHAR NOT NULL CHECK (UPPER(method) IN ('GET','POST')) , uri VARCHAR NOT NULL, script VARCHAR NOT NULL, comment VARCHAR NO NULL DEFAULT '')" +#define ADMIN_SQLITE_TABLE_RUNTIME_RESTAPI_ROUTES "CREATE TABLE runtime_restapi_routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , method VARCHAR NOT NULL CHECK (UPPER(method) IN ('GET','POST')) , uri VARCHAR NOT NULL , script VARCHAR NOT NULL , comment VARCHAR NOT NULL DEFAULT '')" #define ADMIN_SQLITE_TABLE_SCHEDULER "CREATE TABLE scheduler (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , comment VARCHAR NOT NULL DEFAULT '')" diff --git a/lib/ProxySQL_RESTAPI_Server.cpp b/lib/ProxySQL_RESTAPI_Server.cpp index c4f63a72e..c61bf6bb1 100644 --- a/lib/ProxySQL_RESTAPI_Server.cpp +++ b/lib/ProxySQL_RESTAPI_Server.cpp @@ -20,15 +20,17 @@ extern ProxySQL_Admin *GloAdmin; class sync_resource : public http_resource { private: + void add_headers(std::shared_ptr &response) { + response->with_header("Content-Type", "application/json"); + response->with_header("Access-Control-Allow-Origin", "*"); + } + const std::shared_ptr find_script(const http_request& req, std::string& script, int &interval_ms) { - SQLite3_result *resultset=NULL; - int affected_rows; - int cols; char *error=NULL; std::stringstream ss; - ss << "SELECT * FROM restapi_routes WHERE uri='" << req.get_path_piece(1) << "' and method='" << req.get_method() << "' and active=1"; - bool rc=GloAdmin->admindb->execute_statement(ss.str().c_str(), &error, &cols, &affected_rows, &resultset); - if (!rc) { + ss << "SELECT * FROM runtime_restapi_routes WHERE uri='" << req.get_path_piece(1) << "' and method='" << req.get_method() << "' and active=1"; + std::unique_ptr resultset = std::unique_ptr(GloAdmin->admindb->execute_statement(ss.str().c_str(), &error)); + if (!resultset) { proxy_error("Cannot query script for given method [%s] and uri [%s]\n", req.get_method().c_str(), req.get_path_piece(1).c_str()); std::stringstream ss; if (error) { @@ -39,17 +41,20 @@ private: ss << "{\"error\":\"The script for method [" << req.get_method() << "] and route [" << req.get_path() << "] was not found.\"}"; proxy_error("Path %s\n", req.get_path().c_str()); } - return std::shared_ptr(new string_response(ss.str())); + auto response = std::shared_ptr(new string_response(ss.str())); + add_headers(response); + return response; } if (resultset && resultset->rows_count != 1) { std::stringstream ss; ss << "{\"error\":\"The script for method [" << req.get_method() << "] and route [" << req.get_path() << "] was not found. Rows count returned [" << resultset->rows_count << "]\" }"; proxy_error("Script for method [%s] and route [%s] was not found\n", req.get_method().c_str(), req.get_path().c_str()); - return std::shared_ptr(new string_response(ss.str())); + auto response = std::shared_ptr(new string_response(ss.str())); + add_headers(response); + return response; } script = resultset->rows[0]->fields[5]; interval_ms = atoi(resultset->rows[0]->fields[2]); - if (resultset) {delete resultset; resultset=NULL;} return std::shared_ptr(nullptr); } @@ -59,7 +64,10 @@ private: params = _params; if (params.empty()) { proxy_error("Empty parameters\n"); - return std::shared_ptr(new string_response("{\"error\":\"Empty parameters\"}")); + + auto response = std::shared_ptr(new string_response("{\"error\":\"Empty parameters\"}")); + add_headers(response); + return response; } try { @@ -69,25 +77,36 @@ private: std::stringstream ss; ss << "{\"type\":\"in\", \"error\":\"" << e.what() << "\"}"; proxy_error("Error parsing input json parameters. %s\n", ss.str().c_str()); - return std::shared_ptr(new string_response(ss.str())); + + auto response = std::shared_ptr(new string_response(ss.str())); + add_headers(response); + return response; } std::string script; int interval_ms; auto result=find_script(req, script, interval_ms); + + // result == nullpts means that script was found and we can execute it. continue. if (nullptr!=result) return result; int pipefd[2]; - if (pipe(pipefd) == -1) { + if (pipe(pipefd) == -1) { proxy_error("Cannot create pipe\n"); - return std::shared_ptr(new string_response("{\"error\":\"Cannot create pipe.\"}")); - } + + auto response = std::shared_ptr(new string_response("{\"error\":\"Cannot create pipe.\"}")); + add_headers(response); + return response; + } pid_t pid; if ((pid=fork()) == -1) { proxy_error("Cannot fork\n"); - return std::shared_ptr(new string_response("{\"error\":\"Cannot fork.\"}")); + + auto response = std::shared_ptr(new string_response("{\"error\":\"Cannot fork.\"}")); + add_headers(response); + return response; } char buf[65536] = {0}; @@ -103,7 +122,10 @@ private: std::stringstream ss; ss << "{\"error\":\"Error calling execve().\", \"cwd\":\"" << cwd << "\", \"file\":\"" << script << "\"}"; proxy_error("%s\n", ss.str().c_str()); - return std::shared_ptr(new string_response(ss.str())); + + auto response = std::shared_ptr(new string_response(ss.str())); + add_headers(response); + return response; } exit(EXIT_SUCCESS); } @@ -124,19 +146,28 @@ private: proxy_error("Error calling select for path %s\n", req.get_path().c_str()); std::stringstream ss; ss << "{\"error\":\"Error calling select().\", \"path\":\"" << req.get_path() << "\"}"; - return std::shared_ptr(new string_response(ss.str())); + + auto response = std::shared_ptr(new string_response(ss.str())); + add_headers(response); + return response; } else if (rv == 0) { proxy_error("Timeout reading script output %s\n", script.c_str()); std::stringstream ss; ss << "{\"error\":\"Timeout reading script output. Script file: " << script << "\"}"; - return std::shared_ptr(new string_response(ss.str())); + + auto response = std::shared_ptr(new string_response(ss.str())); + add_headers(response); + return response; } else { int nbytes = read(pipefd[0], buf, sizeof(buf) - 1); if (nbytes == -1) { proxy_error("Error reading pipe\n"); - return std::shared_ptr(new string_response("{\"error\":\"Error reading pipe.\"}")); + + auto response = std::shared_ptr(new string_response("{\"error\":\"Error reading pipe.\"}")); + add_headers(response); + return response; } // validate json correctness @@ -147,7 +178,10 @@ private: std::stringstream ss; ss << "{\"type\":\"out\", \"error\":\"" << e.what() << "\"}"; proxy_error("Error parsing script output. %s\n", buf); - return std::shared_ptr(new string_response(ss.str())); + + auto response = std::shared_ptr(new string_response(ss.str())); + add_headers(response); + return response; } } close(pipefd[0]); @@ -155,7 +189,9 @@ private: int status; waitpid(pid, &status, 0); } - return std::shared_ptr(new string_response(buf)); + auto response = std::shared_ptr(new string_response(buf)); + add_headers(response); + return response; } public: @@ -163,7 +199,11 @@ public: proxy_info("Render generic request with method %s for uri %s\n", req.get_method().c_str(), req.get_path().c_str()); std::stringstream ss; ss << "{\"error\":\"HTTP method " << req.get_method().c_str() << " is not implemented\"}"; - return std::shared_ptr(new string_response(ss.str().c_str())); + + auto response = std::shared_ptr(new string_response(ss.str().c_str())); + response->with_header("Content-Type", "application/json"); + response->with_header("Access-Control-Allow-Origin", "*"); + return response; } const std::shared_ptr render_GET(const http_request& req) { diff --git a/lib/ProxySQL_Restapi.cpp b/lib/ProxySQL_Restapi.cpp index ae536f2f9..2c851c377 100644 --- a/lib/ProxySQL_Restapi.cpp +++ b/lib/ProxySQL_Restapi.cpp @@ -99,6 +99,7 @@ void ProxySQL_Restapi::load_restapi_to_runtime() { proxy_error("Error on %s : %s\n", query, error); } else { update_restapi_table(resultset.get()); + save_restapi_runtime_to_database(true); } }