diff --git a/include/ProxySQL_RESTAPI_Server.hpp b/include/ProxySQL_RESTAPI_Server.hpp index 8ce7aedd6..fc0847e89 100644 --- a/include/ProxySQL_RESTAPI_Server.hpp +++ b/include/ProxySQL_RESTAPI_Server.hpp @@ -8,11 +8,10 @@ class ProxySQL_RESTAPI_Server { private: - //httpserver::webserver *ws; - httpserver::webserver * ws; + std::unique_ptr ws; int port; pthread_t thread_id; - httpserver::http_resource *hr; + std::unique_ptr endpoint; public: ProxySQL_RESTAPI_Server(int p); ~ProxySQL_RESTAPI_Server(); diff --git a/include/proxysql_admin.h b/include/proxysql_admin.h index 14f5c5956..2fa79bcdc 100644 --- a/include/proxysql_admin.h +++ b/include/proxysql_admin.h @@ -10,6 +10,7 @@ typedef struct { uint32_t hash; uint32_t key; } t_symstruct; class ProxySQL_Config; +class ProxySQL_Restapi; class Scheduler_Row { public: @@ -271,6 +272,7 @@ class ProxySQL_Admin { void stats___mysql_gtid_executed(); ProxySQL_Config& proxysql_config(); + ProxySQL_Restapi& proxysql_restapi(); void flush_error_log(); bool GenericRefreshStatistics(const char *query_no_space, unsigned int query_no_space_length, bool admin); diff --git a/include/proxysql_config.h b/include/proxysql_config.h index 59e32d277..bc095b76c 100644 --- a/include/proxysql_config.h +++ b/include/proxysql_config.h @@ -17,6 +17,7 @@ public: int Read_MySQL_Query_Rules_from_configfile(); int Read_MySQL_Servers_from_configfile(); int Read_Scheduler_from_configfile(); + int Read_Restapi_from_configfile(); int Read_ProxySQL_Servers_from_configfile(); void addField(std::string& data, const char* name, const char* value, const char* dq="\""); @@ -25,6 +26,7 @@ public: int Write_MySQL_Query_Rules_to_configfile(std::string& data); int Write_MySQL_Servers_to_configfile(std::string& data); int Write_Scheduler_to_configfile(std::string& data); + int Write_Restapi_to_configfile(std::string& data); int Write_ProxySQL_Servers_to_configfile(std::string& data); }; diff --git a/include/proxysql_restapi.h b/include/proxysql_restapi.h new file mode 100644 index 000000000..9660a1264 --- /dev/null +++ b/include/proxysql_restapi.h @@ -0,0 +1,45 @@ +#ifndef __PROXYSQL_RESTAPI_H__ +#define __PROXYSQL_RESTAPI_H__ + +#include "proxy_defines.h" +#include "proxysql.h" +#include "cpp.h" +#include + +class SQLite3DB; + +class Restapi_Row { +public: + unsigned int id; + bool is_active; + unsigned int interval_ms; + std::string method; + std::string uri; + std::string script; + std::string comment; + unsigned int version; + Restapi_Row(unsigned int _id, bool _is_active, unsigned int _in, const std::string& _method, const std::string& _uri, const std::string& _script, const std::string& _comment); +}; + +class ProxySQL_Restapi { + SQLite3DB* admindb; +public: + ProxySQL_Restapi(SQLite3DB* db); + virtual ~ProxySQL_Restapi(); + + unsigned int last_version; + unsigned int version; +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_t rwlock; +#else + rwlock_t rwlock; +#endif + std::vector Restapi_Rows; + void update_restapi_table(SQLite3_result *result); + void load_restapi_to_runtime(); + void save_restapi_runtime_to_database(bool); + void flush_restapi__from_memory_to_disk(); + void flush_restapi__from_disk_to_memory(); +}; + +#endif // #ifndef __PROXYSQL_RESTAPI_H__ diff --git a/lib/Makefile b/lib/Makefile index c1c366d9c..6aef3a705 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -95,7 +95,7 @@ default: libproxysql.a _OBJ = c_tokenizer.o OBJ = $(patsubst %,$(ODIR)/%,$(_OBJ)) -_OBJ_CXX = ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Query_Cache.oo SpookyV2.oo MySQL_Authentication.oo gen_utils.oo sqlite3db.oo mysql_connection.oo MySQL_HostGroups_Manager.oo mysql_data_stream.oo MySQL_Thread.oo MySQL_Session.oo MySQL_Protocol.oo mysql_backend.oo Query_Processor.oo ProxySQL_Admin.oo ProxySQL_Config.oo MySQL_Monitor.oo MySQL_Logger.oo thread.oo MySQL_PreparedStatement.oo ProxySQL_Cluster.oo ClickHouse_Authentication.oo ClickHouse_Server.oo ProxySQL_Statistics.oo Chart_bundle_js.oo ProxySQL_HTTP_Server.oo ProxySQL_RESTAPI_Server.oo font-awesome.min.css.oo main-bundle.min.css.oo set_parser.oo +_OBJ_CXX = ProxySQL_GloVars.oo network.oo debug.oo configfile.oo Query_Cache.oo SpookyV2.oo MySQL_Authentication.oo gen_utils.oo sqlite3db.oo mysql_connection.oo MySQL_HostGroups_Manager.oo mysql_data_stream.oo MySQL_Thread.oo MySQL_Session.oo MySQL_Protocol.oo mysql_backend.oo Query_Processor.oo ProxySQL_Admin.oo ProxySQL_Config.oo ProxySQL_Restapi.oo MySQL_Monitor.oo MySQL_Logger.oo thread.oo MySQL_PreparedStatement.oo ProxySQL_Cluster.oo ClickHouse_Authentication.oo ClickHouse_Server.oo ProxySQL_Statistics.oo Chart_bundle_js.oo ProxySQL_HTTP_Server.oo ProxySQL_RESTAPI_Server.oo font-awesome.min.css.oo main-bundle.min.css.oo set_parser.oo OBJ_CXX = $(patsubst %,$(ODIR)/%,$(_OBJ_CXX)) HEADERS = ../include/*.h ../include/*.hpp diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index 303ce52e6..f76ec125c 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -6,6 +6,7 @@ #include "re2/regexp.h" #include "proxysql.h" #include "proxysql_config.h" +#include "proxysql_restapi.h" #include "cpp.h" #include "MySQL_Data_Stream.h" @@ -362,6 +363,10 @@ static int http_handler(void *cls, struct MHD_Connection *connection, const char #define ADMIN_SQLITE_TABLE_MYSQL_COLLATIONS "CREATE TABLE mysql_collations (Id INTEGER NOT NULL PRIMARY KEY , Collation VARCHAR NOT NULL , Charset VARCHAR NOT NULL , `Default` VARCHAR NOT NULL)" +#define ADMIN_SQLITE_TABLE_RESTAPI_ROUTES "CREATE TABLE restapi_routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , method VARCHAR NOT NULL CHECK (UPPER(method) IN ('GET','POST')) , uri VARCHAR NOT NULL , script VARCHAR NOT NULL , comment VARCHAR NOT NULL DEFAULT '')" + +#define ADMIN_SQLITE_TABLE_RUNTIME_RESTAPI_ROUTES "CREATE TABLE runtime_restapi_routes (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , method VARCHAR NOT NULL CHECK (UPPER(method) IN ('GET','POST')) , uri VARCHAR NOT NULL , script VARCHAR NOT NULL , comment VARCHAR NOT NULL DEFAULT '')" + #define ADMIN_SQLITE_TABLE_SCHEDULER "CREATE TABLE scheduler (id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL , active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1 , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , comment VARCHAR NOT NULL DEFAULT '')" #define ADMIN_SQLITE_TABLE_SCHEDULER_V1_2_0 "CREATE TABLE scheduler (id INTEGER NOT NULL , interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL , filename VARCHAR NOT NULL , arg1 VARCHAR , arg2 VARCHAR , arg3 VARCHAR , arg4 VARCHAR , arg5 VARCHAR , PRIMARY KEY(id))" @@ -564,6 +569,11 @@ ProxySQL_Config& ProxySQL_Admin::proxysql_config() { return instance; } +ProxySQL_Restapi& ProxySQL_Admin::proxysql_restapi() { + static ProxySQL_Restapi instance = ProxySQL_Restapi(admindb); + return instance; +} + int ProxySQL_Admin::FlushDigestTableToDisk(SQLite3DB *_db) { int r = 0; if (!GloQPro) return 0; @@ -1296,6 +1306,101 @@ bool admin_handler_command_load_or_save(char *query_no_space, unsigned int query } #endif /* DEBUG */ + if ((query_no_space_length>13) && ( (!strncasecmp("SAVE RESTAPI ", query_no_space, 13)) || (!strncasecmp("LOAD RESTAPI ", query_no_space, 13))) ) { + + if ( + (query_no_space_length==strlen("LOAD RESTAPI TO MEMORY") && !strncasecmp("LOAD RESTAPI TO MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD RESTAPI TO MEM") && !strncasecmp("LOAD RESTAPI TO MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD RESTAPI FROM DISK") && !strncasecmp("LOAD RESTAPI FROM DISK",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->proxysql_restapi().flush_restapi__from_disk_to_memory(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading restapi to to MEMORY\n"); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); + return false; + } + + if ( + (query_no_space_length==strlen("SAVE RESTAPI FROM MEMORY") && !strncasecmp("SAVE RESTAPI FROM MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE RESTAPI FROM MEM") && !strncasecmp("SAVE RESTAPI FROM MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE RESTAPI TO DISK") && !strncasecmp("SAVE RESTAPI TO DISK",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->proxysql_restapi().flush_restapi__from_memory_to_disk(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saving restapi to DISK\n"); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); + return false; + } + + if ( + (query_no_space_length==strlen("LOAD RESTAPI FROM MEMORY") && !strncasecmp("LOAD RESTAPI FROM MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD RESTAPI FROM MEM") && !strncasecmp("LOAD RESTAPI FROM MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD RESTAPI TO RUNTIME") && !strncasecmp("LOAD RESTAPI TO RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("LOAD RESTAPI TO RUN") && !strncasecmp("LOAD RESTAPI TO RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->proxysql_restapi().load_restapi_to_runtime(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded restapito RUNTIME\n"); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); + return false; + } + + if ( + (query_no_space_length==strlen("LOAD RESTAPI FROM CONFIG") && !strncasecmp("LOAD RESTAPI FROM CONFIG",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + if (GloVars.configfile_open) { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loading from file %s\n", GloVars.config_file); + if (GloVars.confFile->OpenFile(NULL)==true) { + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + int rows=0; + rows=SPA->proxysql_config().Read_Restapi_from_configfile(); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Loaded restapi from CONFIG\n"); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL, rows); + GloVars.confFile->CloseFile(); + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unable to open or parse config file %s\n", GloVars.config_file); + char *s=(char *)"Unable to open or parse config file %s"; + char *m=(char *)malloc(strlen(s)+strlen(GloVars.config_file)+1); + sprintf(m,s,GloVars.config_file); + SPA->send_MySQL_ERR(&sess->client_myds->myprot, m); + free(m); + } + } else { + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Unknown config file\n"); + SPA->send_MySQL_ERR(&sess->client_myds->myprot, (char *)"Config file unknown"); + } + return false; + } + + if ( + (query_no_space_length==strlen("SAVE RESTAPI TO MEMORY") && !strncasecmp("SAVE RESTAPI TO MEMORY",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE RESTAPI TO MEM") && !strncasecmp("SAVE RESTAPI TO MEM",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE RESTAPI FROM RUNTIME") && !strncasecmp("SAVE RESTAPI FROM RUNTIME",query_no_space, query_no_space_length)) + || + (query_no_space_length==strlen("SAVE RESTAPI FROM RUN") && !strncasecmp("SAVE RESTAPI FROM RUN",query_no_space, query_no_space_length)) + ) { + proxy_info("Received %s command\n", query_no_space); + ProxySQL_Admin *SPA=(ProxySQL_Admin *)pa; + SPA->save_scheduler_runtime_to_database(false); + proxy_debug(PROXY_DEBUG_ADMIN, 4, "Saved scheduler from RUNTIME\n"); + SPA->send_MySQL_OK(&sess->client_myds->myprot, NULL); + return false; + } + } + if ((query_no_space_length>15) && ( (!strncasecmp("SAVE SCHEDULER ", query_no_space, 15)) || (!strncasecmp("LOAD SCHEDULER ", query_no_space, 15))) ) { if ( @@ -2516,6 +2621,7 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign bool dump_global_variables=false; bool runtime_scheduler=false; + bool runtime_restapi_routes=false; bool runtime_mysql_users=false; bool runtime_mysql_firewall=false; bool runtime_mysql_ldap_mapping=false; @@ -2653,6 +2759,9 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign if (strstr(query_no_space,"runtime_scheduler")) { runtime_scheduler=true; refresh=true; } + if (strstr(query_no_space,"runtime_restapi_routes")) { + runtime_restapi_routes=true; refresh=true; + } if (strstr(query_no_space,"runtime_proxysql_servers")) { runtime_proxysql_servers=true; refresh=true; } @@ -2775,6 +2884,9 @@ bool ProxySQL_Admin::GenericRefreshStatistics(const char *query_no_space, unsign if (runtime_scheduler) { save_scheduler_runtime_to_database(true); } + if (runtime_restapi_routes) { + proxysql_restapi().save_restapi_runtime_to_database(true); + } if (runtime_checksums_values) { dump_checksums_values_table(); } @@ -3808,6 +3920,7 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { rc = pa->proxysql_config().Write_MySQL_Query_Rules_to_configfile(data); rc = pa->proxysql_config().Write_MySQL_Servers_to_configfile(data); rc = pa->proxysql_config().Write_Scheduler_to_configfile(data); + rc = pa->proxysql_config().Write_Restapi_to_configfile(data); rc = pa->proxysql_config().Write_ProxySQL_Servers_to_configfile(data); if (rc) { std::stringstream ss; @@ -3847,6 +3960,7 @@ void admin_session_handler(MySQL_Session *sess, void *_pa, PtrSize_t *pkt) { rc = pa->proxysql_config().Write_MySQL_Query_Rules_to_configfile(data); rc = pa->proxysql_config().Write_MySQL_Servers_to_configfile(data); rc = pa->proxysql_config().Write_Scheduler_to_configfile(data); + rc = pa->proxysql_config().Write_Restapi_to_configfile(data); rc = pa->proxysql_config().Write_ProxySQL_Servers_to_configfile(data); if (rc) { std::stringstream ss; @@ -4782,6 +4896,8 @@ bool ProxySQL_Admin::init() { insert_into_tables_defs(tables_defs_admin,"runtime_mysql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_FIREWALL_WHITELIST_RULES); insert_into_tables_defs(tables_defs_admin,"mysql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS); insert_into_tables_defs(tables_defs_admin,"runtime_mysql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_RUNTIME_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS); + insert_into_tables_defs(tables_defs_admin, "restapi_routes", ADMIN_SQLITE_TABLE_RESTAPI_ROUTES); + insert_into_tables_defs(tables_defs_admin, "runtime_restapi_routes", ADMIN_SQLITE_TABLE_RUNTIME_RESTAPI_ROUTES); #ifdef DEBUG insert_into_tables_defs(tables_defs_admin,"debug_levels", ADMIN_SQLITE_TABLE_DEBUG_LEVELS); #endif /* DEBUG */ @@ -4808,6 +4924,7 @@ bool ProxySQL_Admin::init() { insert_into_tables_defs(tables_defs_config,"mysql_firewall_whitelist_users", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_USERS); insert_into_tables_defs(tables_defs_config,"mysql_firewall_whitelist_rules", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_RULES); insert_into_tables_defs(tables_defs_config,"mysql_firewall_whitelist_sqli_fingerprints", ADMIN_SQLITE_TABLE_MYSQL_FIREWALL_WHITELIST_SQLI_FINGERPRINTS); + insert_into_tables_defs(tables_defs_config, "restapi_routes", ADMIN_SQLITE_TABLE_RESTAPI_ROUTES); #ifdef DEBUG insert_into_tables_defs(tables_defs_config,"debug_levels", ADMIN_SQLITE_TABLE_DEBUG_LEVELS); #endif /* DEBUG */ @@ -4890,6 +5007,7 @@ bool ProxySQL_Admin::init() { // workaround for issue #708 statsdb->execute("INSERT OR IGNORE INTO global_variables VALUES('mysql-max_allowed_packet',4194304)"); + #ifdef DEBUG if (GloVars.global.gdbg==false && GloVars.__cmd_proxysql_gdbg) { proxy_debug(PROXY_DEBUG_ADMIN, 4, "Enabling GloVars.global.gdbg because GloVars.__cmd_proxysql_gdbg==%d\n", GloVars.__cmd_proxysql_gdbg); @@ -4906,6 +5024,7 @@ bool ProxySQL_Admin::init() { proxysql_config().Read_Global_Variables_from_configfile("admin"); proxysql_config().Read_Global_Variables_from_configfile("mysql"); proxysql_config().Read_Scheduler_from_configfile(); + proxysql_config().Read_Restapi_from_configfile(); proxysql_config().Read_ProxySQL_Servers_from_configfile(); __insert_or_replace_disktable_select_maintable(); } @@ -4942,7 +5061,7 @@ bool ProxySQL_Admin::init() { #ifdef DEBUG std::cerr << "Admin initialized in "; #endif - return true; +return true; }; @@ -8103,6 +8222,7 @@ void ProxySQL_Admin::__insert_or_replace_maintable_select_disktable() { admindb->execute("INSERT OR REPLACE INTO main.mysql_firewall_whitelist_sqli_fingerprints SELECT * FROM disk.mysql_firewall_whitelist_sqli_fingerprints"); admindb->execute("INSERT OR REPLACE INTO main.global_variables SELECT * FROM disk.global_variables"); admindb->execute("INSERT OR REPLACE INTO main.scheduler SELECT * FROM disk.scheduler"); + admindb->execute("INSERT OR REPLACE INTO main.restapi_routes SELECT * FROM disk.restapi_routes"); admindb->execute("INSERT OR REPLACE INTO main.proxysql_servers SELECT * FROM disk.proxysql_servers"); #ifdef DEBUG admindb->execute("INSERT OR REPLACE INTO main.debug_levels SELECT * FROM disk.debug_levels"); @@ -10398,7 +10518,6 @@ void ProxySQL_Admin::disk_upgrade_mysql_users() { configdb->execute("PRAGMA foreign_keys = ON"); } - Scheduler_Row::Scheduler_Row(unsigned int _id, bool _is_active, unsigned int _in, char *_f, char *a1, char *a2, char *a3, char *a4, char *a5, char *_comment) { int i; id=_id; diff --git a/lib/ProxySQL_Config.cpp b/lib/ProxySQL_Config.cpp index b7e40b4b5..aeca63411 100644 --- a/lib/ProxySQL_Config.cpp +++ b/lib/ProxySQL_Config.cpp @@ -338,6 +338,116 @@ int ProxySQL_Config::Read_Scheduler_from_configfile() { return rows; } +int ProxySQL_Config::Write_Restapi_to_configfile(std::string& data) { + char* error = NULL; + int cols = 0; + int affected_rows = 0; + SQLite3_result* sqlite_resultset = NULL; + + char *query=(char *)"SELECT * FROM restapi_routes"; + admindb->execute_statement(query, &error, &cols, &affected_rows, &sqlite_resultset); + if (error) { + proxy_error("Error on read from restapi_router: %s\n", error); + return -1; + } else { + if (sqlite_resultset) { + data += "restapi:\n(\n"; + bool isNext = false; + for (auto r : sqlite_resultset->rows) { + if (isNext) + data += ",\n"; + data += "\t{\n"; + addField(data, "id", r->fields[0], ""); + addField(data, "active", r->fields[1], ""); + addField(data, "interval_ms", r->fields[2], ""); + addField(data, "method", r->fields[3], ""); + addField(data, "uri", r->fields[4]); + addField(data, "script", r->fields[5]); + addField(data, "comment", r->fields[6]); + + data += "\t}"; + isNext = true; + } + data += "\n)\n"; + } + } + + if (sqlite_resultset) + delete sqlite_resultset; + + return 0; +} + +int ProxySQL_Config::Read_Restapi_from_configfile() { + const Setting& root = GloVars.confFile->cfg.getRoot(); + if (root.exists("restapi")==false) return 0; + const Setting &routes = root["restapi"]; + int count = routes.getLength(); + //fprintf(stderr, "Found %d users\n",count); + int i; + int rows=0; + admindb->execute("PRAGMA foreign_keys = OFF"); + char *q=(char *)"INSERT OR REPLACE INTO restapi VALUES (%d, %d, %d, '%s', '%s', '%s', '%s')"; + for (i=0; i< count; i++) { + const Setting &route = routes[i]; + int id; + int active=1; + // variable for parsing interval_ms + int interval_ms=0; + + std::string method; + std::string uri; + std::string script; + std::string comment=""; + + // validate arguments + if (route.lookupValue("id", id)==false) { + proxy_error("Admin: detected a restapi route in config file without a mandatory id\n"); + continue; + } + route.lookupValue("active", active); + route.lookupValue("interval_ms", interval_ms); + if (route.lookupValue("method", method)==false) { + proxy_error("Admin: detected a restapi route in config file without a mandatory method\n"); + continue; + } + if (route.lookupValue("uri", uri)==false) { + proxy_error("Admin: detected a restapi route in config file without a mandatory uri\n"); + continue; + } + if (route.lookupValue("script", script)==false) { + proxy_error("Admin: detected a restapi route in config file without a mandatory script\n"); + continue; + } + route.lookupValue("comment", comment); + + int query_len=0; + query_len+=strlen(q) + + strlen(std::to_string(id).c_str()) + + strlen(std::to_string(active).c_str()) + + strlen(std::to_string(interval_ms).c_str()) + + strlen(method.c_str()) + + strlen(uri.c_str()) + + strlen(script.c_str()) + + strlen(comment.c_str()) + + 40; + char *query=(char *)malloc(query_len); + sprintf(query, q, + id, active, + interval_ms, + method.c_str(), + uri.c_str(), + script.c_str(), + comment.c_str() + ); + admindb->execute(query); + free(query); + rows++; + } + admindb->execute("PRAGMA foreign_keys = ON"); + return rows; +} + int ProxySQL_Config::Write_MySQL_Query_Rules_to_configfile(std::string& data) { char* error = NULL; int cols = 0; diff --git a/lib/ProxySQL_RESTAPI_Server.cpp b/lib/ProxySQL_RESTAPI_Server.cpp index 30b278074..c61bf6bb1 100644 --- a/lib/ProxySQL_RESTAPI_Server.cpp +++ b/lib/ProxySQL_RESTAPI_Server.cpp @@ -2,6 +2,9 @@ #include "cpp.h" #include "httpserver.hpp" +#include + + #include "ProxySQL_RESTAPI_Server.hpp" using namespace httpserver; @@ -13,15 +16,219 @@ using namespace httpserver; #endif /* DEBUG */ #define PROXYSQL_RESTAPI_SERVER_VERSION "2.0.1121" DEB -class hello_world_resource : public http_resource { -public: - const std::shared_ptr render_GET(const http_request&) { - return std::shared_ptr(new string_response("GET: Hello, World!")); +extern ProxySQL_Admin *GloAdmin; + +class sync_resource : public http_resource { +private: + void add_headers(std::shared_ptr &response) { + response->with_header("Content-Type", "application/json"); + response->with_header("Access-Control-Allow-Origin", "*"); + } + + const std::shared_ptr find_script(const http_request& req, std::string& script, int &interval_ms) { + char *error=NULL; + std::stringstream ss; + ss << "SELECT * FROM runtime_restapi_routes WHERE uri='" << req.get_path_piece(1) << "' and method='" << req.get_method() << "' and active=1"; + std::unique_ptr resultset = std::unique_ptr(GloAdmin->admindb->execute_statement(ss.str().c_str(), &error)); + if (!resultset) { + proxy_error("Cannot query script for given method [%s] and uri [%s]\n", req.get_method().c_str(), req.get_path_piece(1).c_str()); + std::stringstream ss; + if (error) { + ss << "{\"error\":\"The script for method [" << req.get_method() << "] and route [" << req.get_path() << "] was not found. Error: " << error << "Error: \"}"; + proxy_error("Path %s, error %s\n", req.get_path().c_str(), error); + } + else { + ss << "{\"error\":\"The script for method [" << req.get_method() << "] and route [" << req.get_path() << "] was not found.\"}"; + proxy_error("Path %s\n", req.get_path().c_str()); + } + auto response = std::shared_ptr(new string_response(ss.str())); + add_headers(response); + return response; + } + if (resultset && resultset->rows_count != 1) { + std::stringstream ss; + ss << "{\"error\":\"The script for method [" << req.get_method() << "] and route [" << req.get_path() << "] was not found. Rows count returned [" << resultset->rows_count << "]\" }"; + proxy_error("Script for method [%s] and route [%s] was not found\n", req.get_method().c_str(), req.get_path().c_str()); + auto response = std::shared_ptr(new string_response(ss.str())); + add_headers(response); + return response; + } + script = resultset->rows[0]->fields[5]; + interval_ms = atoi(resultset->rows[0]->fields[2]); + return std::shared_ptr(nullptr); + } + + const std::shared_ptr process_request(const http_request& req, const std::string& _params) { + std::string params = req.get_content(); + if (params.empty()) + params = _params; + if (params.empty()) { + proxy_error("Empty parameters\n"); + + auto response = std::shared_ptr(new string_response("{\"error\":\"Empty parameters\"}")); + add_headers(response); + return response; + } + + try { + nlohmann::json valid=nlohmann::json::parse(params); + } + catch(nlohmann::json::exception& e) { + std::stringstream ss; + ss << "{\"type\":\"in\", \"error\":\"" << e.what() << "\"}"; + proxy_error("Error parsing input json parameters. %s\n", ss.str().c_str()); + + auto response = std::shared_ptr(new string_response(ss.str())); + add_headers(response); + return response; + } + + std::string script; + int interval_ms; + auto result=find_script(req, script, interval_ms); + + // result == nullpts means that script was found and we can execute it. continue. + if (nullptr!=result) + return result; + + int pipefd[2]; + if (pipe(pipefd) == -1) { + proxy_error("Cannot create pipe\n"); + + auto response = std::shared_ptr(new string_response("{\"error\":\"Cannot create pipe.\"}")); + add_headers(response); + return response; + } + + pid_t pid; + if ((pid=fork()) == -1) { + proxy_error("Cannot fork\n"); + + auto response = std::shared_ptr(new string_response("{\"error\":\"Cannot fork.\"}")); + add_headers(response); + return response; + } + + char buf[65536] = {0}; + if (pid == 0) { + dup2(pipefd[1], STDOUT_FILENO); + close(pipefd[0]); + close(pipefd[1]); + + char* const args[] = {const_cast(script.c_str()), const_cast(params.c_str()), NULL}; + if (execve(script.c_str(), args, NULL) == -1) { + char path_buffer[PATH_MAX]; + char* cwd = getcwd(path_buffer, sizeof(path_buffer)-1); + std::stringstream ss; + ss << "{\"error\":\"Error calling execve().\", \"cwd\":\"" << cwd << "\", \"file\":\"" << script << "\"}"; + proxy_error("%s\n", ss.str().c_str()); + + auto response = std::shared_ptr(new string_response(ss.str())); + add_headers(response); + return response; + } + exit(EXIT_SUCCESS); + } + else { + close(pipefd[1]); + + fd_set set; + + FD_ZERO(&set); + FD_SET(pipefd[0], &set); + + struct timeval timeout; + timeout.tv_sec=interval_ms/1000; + timeout.tv_usec=(interval_ms%1000)*1000; + + int rv = select(pipefd[0]+1,&set,NULL,NULL,&timeout); + if (rv == -1) { + proxy_error("Error calling select for path %s\n", req.get_path().c_str()); + std::stringstream ss; + ss << "{\"error\":\"Error calling select().\", \"path\":\"" << req.get_path() << "\"}"; + + auto response = std::shared_ptr(new string_response(ss.str())); + add_headers(response); + return response; + } + else if (rv == 0) { + proxy_error("Timeout reading script output %s\n", script.c_str()); + std::stringstream ss; + ss << "{\"error\":\"Timeout reading script output. Script file: " << script << "\"}"; + + auto response = std::shared_ptr(new string_response(ss.str())); + add_headers(response); + return response; + } + else { + int nbytes = read(pipefd[0], buf, sizeof(buf) - 1); + if (nbytes == -1) { + proxy_error("Error reading pipe\n"); + + auto response = std::shared_ptr(new string_response("{\"error\":\"Error reading pipe.\"}")); + add_headers(response); + return response; + } + + // validate json correctness + try { + nlohmann::json j=nlohmann::json::parse(buf); + } + catch(nlohmann::json::exception& e) { + std::stringstream ss; + ss << "{\"type\":\"out\", \"error\":\"" << e.what() << "\"}"; + proxy_error("Error parsing script output. %s\n", buf); + + auto response = std::shared_ptr(new string_response(ss.str())); + add_headers(response); + return response; + } + } + close(pipefd[0]); + + int status; + waitpid(pid, &status, 0); + } + auto response = std::shared_ptr(new string_response(buf)); + add_headers(response); + return response; } - const std::shared_ptr render(const http_request&) { - return std::shared_ptr(new string_response("OTHER: Hello, World!")); +public: + const std::shared_ptr render(const http_request& req) { + proxy_info("Render generic request with method %s for uri %s\n", req.get_method().c_str(), req.get_path().c_str()); + std::stringstream ss; + ss << "{\"error\":\"HTTP method " << req.get_method().c_str() << " is not implemented\"}"; + + auto response = std::shared_ptr(new string_response(ss.str().c_str())); + response->with_header("Content-Type", "application/json"); + response->with_header("Access-Control-Allow-Origin", "*"); + return response; } + + const std::shared_ptr render_GET(const http_request& req) { + auto args = req.get_args(); + + size_t last = 0; + std::stringstream params; + params << "{"; + for (auto arg : args) { + params << "\"" << arg.first << "\":\"" << arg.second << "\""; + if (last < args.size()-1) { + params << ","; + last++; + } + } + params << "}"; + + return process_request(req, params.str()); + } + + const std::shared_ptr render_POST(const http_request& req) { + std::string params=req.get_content(); + return process_request(req, params); + } + }; void * restapi_server_thread(void *arg) { @@ -31,24 +238,16 @@ void * restapi_server_thread(void *arg) { } ProxySQL_RESTAPI_Server::ProxySQL_RESTAPI_Server(int p) { + ws = std::unique_ptr(new webserver(create_webserver(p))); + auto sr = new sync_resource(); + + endpoint = std::unique_ptr(sr); - // for now, this is COMPLETELY DISABLED - // just adding a POC - return; - - //ws = NULL; - port = p; - ws = new webserver(create_webserver(p)); - //hello_world_resource hwr; - hr = new hello_world_resource(); - //ws->register_resource("/hello", &hwr); - ws->register_resource("/hello", hr); - if (pthread_create(&thread_id, NULL, restapi_server_thread, ws) !=0 ) { - perror("Thread creation"); - exit(EXIT_FAILURE); + ws->register_resource("/sync", endpoint.get(), true); + if (pthread_create(&thread_id, NULL, restapi_server_thread, ws.get()) !=0 ) { + perror("Thread creation"); + exit(EXIT_FAILURE); } - //webserver ws2 = create_webserver(8080); - //webserws = create_webserver(8080); } ProxySQL_RESTAPI_Server::~ProxySQL_RESTAPI_Server() { diff --git a/lib/ProxySQL_Restapi.cpp b/lib/ProxySQL_Restapi.cpp new file mode 100644 index 000000000..2c851c377 --- /dev/null +++ b/lib/ProxySQL_Restapi.cpp @@ -0,0 +1,106 @@ +#include "proxysql_restapi.h" +#include "proxysql.h" +#include "proxysql_atomic.h" +#include "cpp.h" + +#include + +Restapi_Row::Restapi_Row(unsigned int _id, bool _is_active, unsigned int _in, const std::string& _method, const std::string& _uri, const std::string& _script, const std::string& _comment) : + id(_id), is_active(_is_active), interval_ms(_in), method(_method), uri(_uri), script(_script), comment(_comment) {} + +ProxySQL_Restapi::ProxySQL_Restapi(SQLite3DB* db) { + assert(db); + + admindb = db; +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_init(&rwlock,NULL); +#else + spinlock_rwlock_init(&rwlock); +#endif + version=0; +} + + +ProxySQL_Restapi::~ProxySQL_Restapi() {} + +void ProxySQL_Restapi::update_restapi_table(SQLite3_result *resultset) { +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_wrlock(&rwlock); +#else + spin_wrlock(&rwlock); +#endif + // delete all current rows + Restapi_Rows.clear(); + for (auto r : resultset->rows) { + unsigned int id = strtoul(r->fields[0], NULL, 10); + bool is_active=false; + if (atoi(r->fields[1])) { + is_active=true; + } + unsigned int interval_ms=strtoul(r->fields[2], NULL, 10); + Restapi_Rows.push_back({id, is_active, interval_ms, r->fields[3], r->fields[4], r->fields[5], r->fields[6]}); + } + + // increase version + __sync_fetch_and_add(&version,1); + // unlock +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_unlock(&rwlock); +#else + spin_wrunlock(&rwlock); +#endif +} + +void ProxySQL_Restapi::flush_restapi__from_disk_to_memory() { + admindb->wrlock(); + admindb->execute("DELETE FROM main.restapi_routes"); + admindb->execute("INSERT INTO main.restapi_routes SELECT * FROM disk.restapi_routes"); + admindb->wrunlock(); +} + +void ProxySQL_Restapi::flush_restapi__from_memory_to_disk() { + admindb->wrlock(); + admindb->execute("DELETE FROM disk.restapi_routes"); + admindb->execute("INSERT INTO disk.restapi_routes SELECT * FROM main.restapi_routes"); + admindb->wrunlock(); +} + +void ProxySQL_Restapi::save_restapi_runtime_to_database(bool _runtime) { + const char *query = _runtime ? "DELETE FROM main.runtime_restapi_routes" : "DELETE FROM main.restapi_routes"; + proxy_debug(PROXY_DEBUG_ADMIN, 4, "%s\n", query); + admindb->execute(query); + // read lock the scheduler +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_rdlock(&rwlock); +#else + spin_rdlock(&rwlock); +#endif + const char* table = _runtime ? " runtime_restapi_routes " : " restapi_routes "; + for (auto r : Restapi_Rows) { + std::stringstream ss; + ss << "INSERT INTO " << table << " VALUES(" << r.id << "," << r.is_active << "," + << r.interval_ms << ",'" << r.method << "','" << r.uri << "','" << r.script << "','" << r.comment << "')"; + + admindb->execute(ss.str().c_str()); + } + // unlock the scheduler +#ifdef PA_PTHREAD_MUTEX + pthread_rwlock_unlock(&rwlock); +#else + spin_rdunlock(&rwlock); +#endif +} + +void ProxySQL_Restapi::load_restapi_to_runtime() { + char *error=NULL; + char *query=(char *)"SELECT * FROM restapi_routes"; + std::unique_ptr resultset = std::unique_ptr(admindb->execute_statement(query, &error)); + if (error) { + proxy_error("Error on %s : %s\n", query, error); + } else { + update_restapi_table(resultset.get()); + save_restapi_runtime_to_database(true); + } +} + + diff --git a/scripts/export_users.py b/scripts/export_users.py new file mode 100755 index 000000000..8d3e1e796 --- /dev/null +++ b/scripts/export_users.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# +#./scripts/export_users.py '{"db":{"user":"root", "password":"a", "port":"3306", "host":"127.0.0.1"},"admin":{"user":"admin","password":"admin","port":"6032","host":"127.0.0.1"}}' +# + +import sys +import subprocess +import json +from MySQLdb import _mysql + +if len(sys.argv) > 1: + params=json.loads(sys.argv[1]) + db_mysql=_mysql.connect(host=params['db']['host'],user=params['db']['user'],passwd=params['db']['password'],port=int(params['db']['port'])) + db_mysql.query('SELECT user, authentication_string from mysql.user') + records=db_mysql.store_result().fetch_row(maxrows=0) + db_proxy_admin=_mysql.connect(host=params['admin']['host'],user=params['admin']['user'],passwd=params['admin']['password'],port=int(params['admin']['port'])) + for row in records: + db_proxy_admin.query('INSERT OR REPLACE INTO mysql_users (username, password) values ("'+str(row[0])+'","'+str(row[1])+'")') + + result='{"num_records":"'+str(len(records))+'"}' + try: + subprocess.check_output(['mysql', '-u'+params['admin']['user'], '-p'+params['admin']['password'], '-h'+params['admin']['host'], '-P'+params['admin']['port'], '-e', 'load mysql users to runtime'],stderr= subprocess.STDOUT) + except subprocess.CalledProcessError as e: + result='"Error calling mysql: ' + e.output.replace("'", "") + '"' + + print('{"params":'+str(sys.argv[1])+', "result":'+result+'}') + diff --git a/scripts/metrics.py b/scripts/metrics.py new file mode 100755 index 000000000..49fc50ce2 --- /dev/null +++ b/scripts/metrics.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python + +import sys +import subprocess +import json + +if len(sys.argv) > 1: + params=json.loads(sys.argv[1]) + out='' + try: + out=subprocess.check_output(['mysql', '-u'+params['user'], '-p'+params['password'], '-h'+params['host'], '-P'+params['port'], '-e', 'select * from stats.stats_memory_metrics'],stderr= subprocess.STDOUT) + except subprocess.CalledProcessError as e: + out="Error calling mysql: " + e.output.replace("'", "") + print('{"params":'+sys.argv[1].encode('string-escape')+', "result":"'+out.encode('string-escape')+'"}') + diff --git a/src/main.cpp b/src/main.cpp index 3c047d9f5..6009f8894 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -17,6 +17,7 @@ #include "query_processor.h" #include "MySQL_Authentication.hpp" #include "MySQL_LDAP_Authentication.hpp" +#include "proxysql_restapi.h" #include @@ -1261,6 +1262,7 @@ void ProxySQL_Main_init_phase3___start_all() { GloAdmin->init_mysql_servers(); GloAdmin->init_proxysql_servers(); GloAdmin->load_scheduler_to_runtime(); + GloAdmin->proxysql_restapi().load_restapi_to_runtime(); #ifdef DEBUG std::cerr << "Main phase3 : GloAdmin initialized in "; #endif diff --git a/test/tap/tests/proxysql_reference_select_config_file.cnf b/test/tap/tests/proxysql_reference_select_config_file.cnf index cc2f68f79..71f3bc75b 100644 --- a/test/tap/tests/proxysql_reference_select_config_file.cnf +++ b/test/tap/tests/proxysql_reference_select_config_file.cnf @@ -358,6 +358,18 @@ scheduler: comment="comment" } ) +restapi: +( + { + id=1 + active=1 + interval_ms=1000 + method=GET + uri="test" + script="./scripts/script.py" + comment="comment" + } +) proxysql_servers: ( { diff --git a/test/tap/tests/select_config_file-t.cpp b/test/tap/tests/select_config_file-t.cpp index 64aa84aed..5c6459106 100644 --- a/test/tap/tests/select_config_file-t.cpp +++ b/test/tap/tests/select_config_file-t.cpp @@ -62,11 +62,14 @@ int main(int argc, char** argv) { MYSQL_QUERY(mysql, "delete from mysql_galera_hostgroups"); MYSQL_QUERY(mysql, "delete from mysql_aws_aurora_hostgroups"); MYSQL_QUERY(mysql, "delete from scheduler"); + MYSQL_QUERY(mysql, "delete from restapi_routes"); MYSQL_QUERY(mysql, "delete from proxysql_servers"); MYSQL_QUERY(mysql, "insert into proxysql_servers (hostname, port, weight, comment) values ('hostname', 3333, 12, 'comment');"); MYSQL_QUERY(mysql, "insert into scheduler (id, active, interval_ms, filename, arg1, arg2, arg3, arg4, arg5, comment) values " " (1,1,1000,'filename','a1','a2','a3','a4','a5','comment');"); + MYSQL_QUERY(mysql, "insert into restapi_routes values " + " (1,1,1000,'GET', 'test','./scripts/script.py','comment');"); MYSQL_QUERY(mysql, "insert into mysql_aws_aurora_hostgroups (writer_hostgroup, reader_hostgroup, active, aurora_port, " " domain_name, max_lag_ms, check_interval_ms, check_timeout_ms, writer_is_also_reader, new_reader_weight, " " add_lag_ms, min_lag_ms, lag_num_checks, comment) " @@ -107,6 +110,13 @@ int main(int argc, char** argv) { std::string str = strStream.str(); //str holds the content of the file ok(!str.compare(resultset), "Files are equal"); +#if 0 + std::ofstream f; + f.open("out.txt", std::ios::out); + f << resultset; + f.close(); +#endif + } MYSQL_QUERY(mysql, "load mysql variables from disk"); @@ -114,6 +124,7 @@ int main(int argc, char** argv) { MYSQL_QUERY(mysql, "load mysql users from disk"); MYSQL_QUERY(mysql, "load mysql servers from disk"); MYSQL_QUERY(mysql, "load scheduler from disk"); + MYSQL_QUERY(mysql, "load restapi from disk"); MYSQL_QUERY(mysql, "load proxysql servers from disk"); mysql_close(mysql);