From b1f8045ecb03a00c8da0fa0eb1e47ab41841e688 Mon Sep 17 00:00:00 2001 From: iprunache Date: Thu, 25 Feb 2016 17:07:13 +0200 Subject: [PATCH] T87 Added proxysql-consul script to make install step --- Makefile | 14 +- integrations/consul/consul.cfg | 2 +- integrations/consul/proxysql-consul | 182 ++++++++++++++++++++++++ integrations/consul/proxysql-consul.cfg | 5 - integrations/consul/proxysql-consul.cnf | 8 ++ integrations/consul/proxysql-consul.py | 4 +- lib/ProxySQL_Admin.cpp | 2 +- 7 files changed, 206 insertions(+), 11 deletions(-) create mode 100755 integrations/consul/proxysql-consul delete mode 100644 integrations/consul/proxysql-consul.cfg create mode 100644 integrations/consul/proxysql-consul.cnf mode change 100755 => 100644 integrations/consul/proxysql-consul.py diff --git a/Makefile b/Makefile index 9f3632308..a66fe8f6d 100644 --- a/Makefile +++ b/Makefile @@ -168,7 +168,7 @@ cleanall: rm binaries/*deb || true rm binaries/*rpm || true -install: src/proxysql +install: src/proxysql install-proxysql-consul install -m 0755 src/proxysql /usr/local/bin install -m 0600 etc/proxysql.cnf /etc install -m 0755 etc/init.d/proxysql /etc/init.d @@ -176,10 +176,20 @@ install: src/proxysql update-rc.d proxysql defaults .PHONY: install -uninstall: +install-proxysql-consul: + install -m 0755 integrations/consul/proxysql-consul /usr/local/bin + install -m 0600 integrations/consul/proxysql-consul.cnf /etc +.PHONY: install-proxysql-consul + +uninstall: uninstall-proxysql-consul rm /etc/init.d/proxysql rm /etc/proxysql.cnf rm /usr/local/bin/proxysql rmdir /var/lib/proxysql 2>/dev/null || true update-rc.d proxysql remove .PHONY: uninstall + +uninstall-proxysql-consul: + rm /usr/local/bin/proxysql-consul + rm /etc/proxysql-consul.cnf +.PHONY: uninstall-proxysql-consul \ No newline at end of file diff --git a/integrations/consul/consul.cfg b/integrations/consul/consul.cfg index 3cd5a1c86..2a579104c 100644 --- a/integrations/consul/consul.cfg +++ b/integrations/consul/consul.cfg @@ -3,7 +3,7 @@ { "type": "keyprefix", "prefix": "proxysql/", - "handler": "/home/ionut/workspace/proxysql/proxysql/integrations/consul/proxysql-consul.py update" + "handler": "/home/ionut/workspace/proxysql/proxysql/integrations/consul/proxysql-consul update" } ] } diff --git a/integrations/consul/proxysql-consul b/integrations/consul/proxysql-consul new file mode 100755 index 000000000..d4da2ae2d --- /dev/null +++ b/integrations/consul/proxysql-consul @@ -0,0 +1,182 @@ +#!/usr/bin/env python + +import base64 +import json +import os +import requests +import sys + +import MySQLdb + +USAGE = """ +proxysql-consul put tablename +proxysql update < config +""" + +CFG_PATH = '/etc/proxysql-consul.cnf' +CFG_CONSUL_IFACE = 'consul_iface' +CFG_CONSUL_PORT = 'consul_port' +CFG_PROXY_IFACE = 'proxysql_admin_iface' +CFG_PROXY_PORT = 'proxysql_admin_port' +CFG_PROXY_USERNAME = 'proxysql_admin_username' +CFG_PROXY_PASSWORD = 'proxysql_admin_password' + +# Proxysql config types to Consul key mapping +TABLE_TO_KEY = { + 'mysql_servers': 'proxysql/mysql_servers', + 'mysql_query_rules': 'proxysql/mysql_query_rules' + } + +SAVE_QUERY_BY_TABLE = { + 'mysql_servers': 'SAVE MYSQL SERVERS TO MEMORY', + 'mysql_query_rules': 'SAVE MYSQL QUERY RULES TO MEMORY' + } + +LOAD_QUERY_BY_TABLE = { + 'mysql_servers': 'LOAD MYSQL SERVERS FROM MEMORY', + 'mysql_query_rules': 'LOAD MYSQL QUERY RULES FROM MEMORY' + } + +config = {} + + +def read_config(): + global config + with open(CFG_PATH) as config_file: + config_data = config_file.read() + config = json.loads(config_data) + + # TODO validate config + + +def fetch_proxysql_config(table): + admin_connection = MySQLdb.connect(config[CFG_PROXY_IFACE], + config[CFG_PROXY_USERNAME], + config[CFG_PROXY_PASSWORD], + port=config[CFG_PROXY_PORT], + db='main') + + # save runtime config to memory so we can read them + cursor = admin_connection.cursor() + save_query = SAVE_QUERY_BY_TABLE[table] + cursor.execute(save_query) + cursor.close() + + # read runtime config from memory + cursor = admin_connection.cursor() + cursor.execute('SELECT * FROM %s' % table) + rows = cursor.fetchall() + cursor.close() + admin_connection.close() + return rows + + +def put_config(table): + if table not in TABLE_TO_KEY: + print 'Unknown config table. Exiting.' + exit(1) + + rows = fetch_proxysql_config(table) + consul_data = {} + consul_data['table'] = table + consul_data['rows'] = rows + consul_data_json = json.dumps(consul_data) + + key = TABLE_TO_KEY[table] + consul_iface = config[CFG_CONSUL_IFACE] + consul_port = config[CFG_CONSUL_PORT] + + url = 'http://%s:%s/v1/kv/%s' % (consul_iface, consul_port, key) + r = requests.put(url, data=consul_data_json) + + +def build_multivalue_insert(table, rows): + # add quotes arround all values to make them strings in the sql query + quoted_values = [['"' + x + '"' for x in row] for row in rows] + # join each row in a values() group + row_join = [','.join(x) for x in quoted_values] + query = 'INSERT INTO %s VALUES(%s)' % (table, '),('.join(row_join)) + return query + +def update_proxysql_config(table, rows): + admin_connection = MySQLdb.connect(config[CFG_PROXY_IFACE], + config[CFG_PROXY_USERNAME], + config[CFG_PROXY_PASSWORD], + port=config[CFG_PROXY_PORT], + db='main') + + # clear table + cursor = admin_connection.cursor() + cursor.execute('DELETE FROM %s' % table) + cursor.close() + + # insert values from Consul + insert_query = build_multivalue_insert(table, rows) + cursor = admin_connection.cursor() + cursor.execute(insert_query) + cursor.close() + + # commit changes to runtine + load_query = LOAD_QUERY_BY_TABLE[table] + cursor = admin_connection.cursor() + cursor.execute(load_query) + cursor.close() + + admin_connection.close() + + +def update_config(): + """ + Reads all input from stdin that is passed by Consul and extracts the config + that was modified. + + Consul is configured to watch all proxysql keys and returns all their values + when one key is modified. To determine the value that was actually modified + we use the CONSUL_INDEX env var written by Consul. This should match the + 'ModifyIndex' field of the value that triggered the watch. + + Values are returned as a JSON array. The actual value content is stored in + the 'Value' field, base64 encoded. + """ + if 'CONSUL_INDEX' not in os.environ: + print 'Missing consul index on update request' + exit(1) + consul_index = int(os.environ['CONSUL_INDEX']) + consul_data = sys.stdin.read() + + updated_value = None + values = json.loads(consul_data) + for value in values: + if 'ModifyIndex' in value and value['ModifyIndex'] == consul_index: + updated_value = value + break + if not updated_value: + print 'Failed to determine updated config from Consul data' + exit(1) + + proxysql_config_json = base64.b64decode(updated_value['Value']) + proxysql_config = json.loads(proxysql_config_json) + table = proxysql_config['table'] + rows = proxysql_config['rows'] + if not rows: + print 'Empty config set for table: %s.' % table + return + update_proxysql_config(table, rows) + +if __name__ == '__main__': + read_config() + + if len(sys.argv) > 1: + mode = sys.argv[1] + if mode == 'put' and len(sys.argv) > 2: + table = sys.argv[2] + put_config(table) + print 'Configs pushed successfully.' + exit(0) + elif mode == 'update': + update_config() + print 'Configs updated successfully.' + exit(0) + + print USAGE + exit(1) diff --git a/integrations/consul/proxysql-consul.cfg b/integrations/consul/proxysql-consul.cfg deleted file mode 100644 index f1616381a..000000000 --- a/integrations/consul/proxysql-consul.cfg +++ /dev/null @@ -1,5 +0,0 @@ -{ - "consul_iface": "127.0.0.1", - "consul_port": "8500" -} - diff --git a/integrations/consul/proxysql-consul.cnf b/integrations/consul/proxysql-consul.cnf new file mode 100644 index 000000000..3021a9506 --- /dev/null +++ b/integrations/consul/proxysql-consul.cnf @@ -0,0 +1,8 @@ +{ + "consul_iface": "127.0.0.1", + "consul_port": "8500", + "proxysql_admin_iface": "127.0.0.1", + "proxysql_admin_port": 6032, + "proxysql_admin_username": "admin", + "proxysql_admin_password": "admin" +} diff --git a/integrations/consul/proxysql-consul.py b/integrations/consul/proxysql-consul.py old mode 100755 new mode 100644 index 60b50f7c7..9d901ab0c --- a/integrations/consul/proxysql-consul.py +++ b/integrations/consul/proxysql-consul.py @@ -8,12 +8,12 @@ import sys import MySQLdb -USAGE = """ +USAGE = """Usage: proxysql-consul put tablename proxysql update < config """ -CFG_PATH = '/etc/proxysql-consul.cfg' +CFG_PATH = '/etc/proxysql-consul.cnf' CFG_CONSUL_IFACE = 'consul_iface' CFG_CONSUL_PORT = 'consul_port' CFG_PROXY_IFACE = 'proxysql_admin_iface' diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index f64a4005d..a223f080c 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -3313,7 +3313,7 @@ int ProxySQL_Admin::save_config_to_cluster(char *tablename) { if (pid == 0) { // child errno = 0; - int exec_status = execl("/usr/local/bin/proxysql-consul.py", "/usr/local/bin/proxysql-consul.py", "put", tablename, (char *) 0); + int exec_status = execlp("/usr/local/bin/proxysql-consul", "/usr/local/bin/proxysql-consul", "put", tablename, (char *) 0); if (exec_status == -1) { proxy_error("Exec failed for config save script with errno: %d.\n", errno); }