diff --git a/Makefile b/Makefile index fb0c3b2c1..40fb09db1 100644 --- a/Makefile +++ b/Makefile @@ -10,8 +10,13 @@ DEBUG=${ALL_DEBUG} #export DEBUG #export OPTZ #export EXTRALINK -CURVER=1.2.3 +CURVER=1.2.4 DISTRO := $(shell gawk -F= '/^NAME/{print $$2}' /etc/os-release) +ifeq ($(wildcard /usr/lib/systemd/systemd), /usr/lib/systemd/systemd) + SYSTEMD=1 +else + SYSTEMD=0 +endif .PHONY: default default: build_deps build_lib build_src @@ -354,8 +359,13 @@ cleanall: install: src/proxysql install -m 0755 src/proxysql /usr/local/bin install -m 0600 etc/proxysql.cnf /etc - install -m 0755 etc/init.d/proxysql /etc/init.d if [ ! -d /var/lib/proxysql ]; then mkdir /var/lib/proxysql ; fi +ifeq ($(SYSTEMD), 1) + install -m 0644 systemd/proxysql.service /usr/lib/systemd/system/ + systemctl daemon-reload + systemctl enable proxysql.service +else + install -m 0755 etc/init.d/proxysql /etc/init.d ifeq ($(DISTRO),"CentOS Linux") chkconfig --level 0123456 proxysql on else @@ -365,12 +375,17 @@ else update-rc.d proxysql defaults endif endif +endif .PHONY: install uninstall: rm /etc/proxysql.cnf rm /usr/local/bin/proxysql rmdir /var/lib/proxysql 2>/dev/null || true +ifeq ($(SYSTEMD), 1) + systemctl stop proxysql.service + rm /usr/lib/systemd/system/proxysql.service +else ifeq ($(DISTRO),"CentOS Linux") chkconfig --level 0123456 proxysql off rm /etc/init.d/proxysql @@ -383,4 +398,5 @@ else update-rc.d proxysql remove endif endif +endif .PHONY: uninstall diff --git a/deps/Makefile b/deps/Makefile index f7c6ff7b6..b1c671350 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -31,6 +31,7 @@ mariadb-client-library/mariadb_client/include/my_config.h: cd mariadb-client-library/mariadb_client && patch libmariadb/net.c < ../net.c.patch # cd mariadb-client-library/mariadb_client && patch libmariadb/mysql_async.c < ../mysql_async.c.patch cd mariadb-client-library/mariadb_client && patch libmariadb/password.c < ../password.c.patch + cd mariadb-client-library/mariadb_client && patch libmariadb/ma_secure.c < ../ma_secure.c.patch cd mariadb-client-library/mariadb_client && patch include/mysql.h < ../mysql.h.patch cd mariadb-client-library/mariadb_client && CC=${CC} CXX=${CXX} ${MAKE} # cd mariadb-client-library/mariadb_client/include && make my_config.h diff --git a/deps/mariadb-client-library/ma_secure.c.patch b/deps/mariadb-client-library/ma_secure.c.patch new file mode 100644 index 000000000..ded95e792 --- /dev/null +++ b/deps/mariadb-client-library/ma_secure.c.patch @@ -0,0 +1,47 @@ +301,338d300 +< static int my_verify_callback(int ok, X509_STORE_CTX *ctx) +< { +< X509 *check_cert; +< SSL *ssl; +< MYSQL *mysql; +< DBUG_ENTER("my_verify_callback"); +< +< ssl = X509_STORE_CTX_get_ex_data(ctx, SSL_get_ex_data_X509_STORE_CTX_idx()); +< DBUG_ASSERT(ssl != NULL); +< mysql= (MYSQL *)SSL_get_app_data(ssl); +< DBUG_ASSERT(mysql != NULL); +< +< /* skip verification if no ca_file/path was specified */ +< if (!mysql->options.ssl_ca && !mysql->options.ssl_capath) +< { +< ok= 1; +< DBUG_RETURN(1); +< } +< +< if (!ok) +< { +< uint depth; +< if (!(check_cert= X509_STORE_CTX_get_current_cert(ctx))) +< DBUG_RETURN(0); +< depth= X509_STORE_CTX_get_error_depth(ctx); +< if (depth == 0) +< ok= 1; +< } +< +< /* +< my_set_error(mysql, CR_SSL_CONNECTION_ERROR, SQLSTATE_UNKNOWN, +< ER(CR_SSL_CONNECTION_ERROR), +< X509_verify_cert_error_string(ctx->error)); +< */ +< DBUG_RETURN(ok); +< } +< +< +352d313 +< int verify; +372,376d332 +< verify= (!mysql->options.ssl_ca && !mysql->options.ssl_capath) ? +< SSL_VERIFY_NONE : SSL_VERIFY_PEER; +< +< SSL_CTX_set_verify(SSL_context, verify, my_verify_callback); +< SSL_CTX_set_verify_depth(SSL_context, 1); diff --git a/diamond/ProxySQLCollector.conf b/diamond/ProxySQLCollector.conf new file mode 100644 index 000000000..12ddccf00 --- /dev/null +++ b/diamond/ProxySQLCollector.conf @@ -0,0 +1,7 @@ +enabled=True +db = None +host = 127.0.0.1 +passwd = stats +port = 6032 +user = stats + diff --git a/diamond/proxysqlstat.py b/diamond/proxysqlstat.py new file mode 100644 index 000000000..e74413cd5 --- /dev/null +++ b/diamond/proxysqlstat.py @@ -0,0 +1,191 @@ +# coding=utf-8 + +import diamond.collector +from diamond.collector import str_to_bool +import re +import time + +try: + import MySQLdb + from MySQLdb import MySQLError +except ImportError: + MySQLdb = None + MySQLError = ValueError + + +class ProxySQLCollector(diamond.collector.Collector): + + _GAUGE_KEYS = [ + 'Active_Transactions', + 'Client_Connections_connected', + 'ConnPool_memory_bytes', + 'MySQL_Monitor_Workers', + 'MySQL_Thread_Workers', + 'Query_Cache_Entries', + 'Query_Cache_Memory_bytes', + 'SQLite3_memory_bytes', + 'Server_Connections_connected', + 'mysql_backend_buffers_bytes', + 'mysql_frontend_buffers_bytes', + 'mysql_session_internal_bytes', + ] + + def __init__(self, *args, **kwargs): + super(ProxySQLCollector, self).__init__(*args, **kwargs) + + def process_config(self): + super(ProxySQLCollector, self).process_config() + if self.config['hosts'].__class__.__name__ != 'list': + self.config['hosts'] = [self.config['hosts']] + + # Move legacy config format to new format + if 'host' in self.config: + hoststr = "%s:%s@%s:%s/%s" % ( + self.config['user'], + self.config['passwd'], + self.config['host'], + self.config['port'], + self.config['db'], + ) + self.config['hosts'].append(hoststr) + + self.db = None + + def get_default_config_help(self): + config_help = super(ProxySQLCollector, self).get_default_config_help() + config_help.update({ + 'publish': + "Which rows of 'SHOW MYSQL STATUS' you would " + + "like to publish. Leave unset to publish all", + 'hosts': 'List of hosts to collect from. Format is ' + + 'yourusername:yourpassword@host:port/db[/nickname]' + + 'use db "None" to avoid connecting to a particular db' + }) + return config_help + + def get_default_config(self): + """ + Returns the default collector settings + """ + config = super(ProxySQLCollector, self).get_default_config() + config.update({ + 'path': 'proxysql', + # Connection settings + 'hosts': [], + + }) + return config + + def get_db_stats(self, query): + cursor = self.db.cursor(cursorclass=MySQLdb.cursors.DictCursor) + + try: + cursor.execute(query) + return cursor.fetchall() + except MySQLError, e: + self.log.error('ProxySQLCollector could not get db stats', e) + return () + + def connect(self, params): + try: + self.db = MySQLdb.connect(**params) + self.log.debug('ProxySQLCollector: Connected to database.') + except MySQLError, e: + self.log.error('ProxySQLCollector couldnt connect to database %s', e) + return False + return True + + def disconnect(self): + self.db.close() + + def get_db_global_status(self): + return self.get_db_stats('SHOW MYSQL STATUS') + + def get_stats(self, params): + metrics = {'status': {}} + + if not self.connect(params): + return metrics + + rows = self.get_db_global_status() + for row in rows: + try: + metrics['status'][row['Variable_name']] = float(row['Value']) + except: + pass + + self.disconnect() + + return metrics + + def _publish_stats(self, nickname, metrics): + + for key in metrics: + for metric_name in metrics[key]: + metric_value = metrics[key][metric_name] + + if type(metric_value) is not float: + continue + + if metric_name not in self._GAUGE_KEYS: + metric_value = self.derivative(nickname + metric_name, + metric_value) + if key == 'status': + if (('publish' not in self.config or + metric_name in self.config['publish'])): + self.publish(nickname + metric_name, metric_value) + else: + self.publish(nickname + metric_name, metric_value) + + def collect(self): + + if MySQLdb is None: + self.log.error('Unable to import MySQLdb') + return False + + for host in self.config['hosts']: + matches = re.search( + '^([^:]*):([^@]*)@([^:]*):?([^/]*)/([^/]*)/?(.*)', host) + + if not matches: + self.log.error( + 'Connection string not in required format, skipping: %s', + host) + continue + + params = {} + + params['host'] = matches.group(3) + try: + params['port'] = int(matches.group(4)) + except ValueError: + params['port'] = 3306 + params['db'] = matches.group(5) + params['user'] = matches.group(1) + params['passwd'] = matches.group(2) + + nickname = matches.group(6) + if len(nickname): + nickname += '.' + + if params['db'] == 'None': + del params['db'] + + try: + metrics = self.get_stats(params=params) + except Exception, e: + try: + self.disconnect() + except MySQLdb.ProgrammingError: + pass + self.log.error('Collection failed for %s %s', nickname, e) + continue + + # Warn if publish contains an unknown variable + if 'publish' in self.config and metrics['status']: + for k in self.config['publish'].split(): + if k not in metrics['status']: + self.log.error("No such key '%s' available, issue " + + "'show global status' for a full " + + "list", k) + self._publish_stats(nickname, metrics) diff --git a/doc/admin_tables.md b/doc/admin_tables.md index 01bdbf507..65580d90e 100644 --- a/doc/admin_tables.md +++ b/doc/admin_tables.md @@ -155,13 +155,13 @@ CREATE TABLE mysql_replication_hostgroups ( ``` Each row in `mysql_replication_hostgroups` represent a pair of *writer_hostgroup* and *reader_hostgroup* . -ProxySQL will monitor the value of read_only for all the servers in specified hostgroups, and based on the value of read_only will assign the server to the writer or reader hostgroups. +ProxySQL will monitor the value of `read_only` for all the servers in specified hostgroups, and based on the value of `read_only` will assign the server to the writer or reader hostgroups. The field `comment` can be used to store any arbitrary data. ## mysql_query_rules -Here is the statement used to create the `mysql_users` table: +Here is the statement used to create the `mysql_query_rules` table: ```sql CREATE TABLE mysql_query_rules ( diff --git a/doc/global_variables.md b/doc/global_variables.md index 27195efa7..f55acddff 100644 --- a/doc/global_variables.md +++ b/doc/global_variables.md @@ -298,7 +298,7 @@ This variable controls whether ProxySQL should use a cached (and less accurate) Default value: `true` -### `mysql-ping_interval_server` +### `mysql-ping_interval_server_msec` The interval at which the proxy should ping backend connections in order to maintain them alive, even though there is no outgoing traffic. The purpose here is to keep some connections alive in order to reduce the latency of new queries towards a less frequently used destination backend server. diff --git a/doc/release_notes/ProxySQL_v1.2.3.md b/doc/release_notes/ProxySQL_v1.2.3.md new file mode 100644 index 000000000..cd1b9c213 --- /dev/null +++ b/doc/release_notes/ProxySQL_v1.2.3.md @@ -0,0 +1,37 @@ +# ProxySQL v1.2.3 + +Release date: 2016-09-20 + +## Performance improvement + +None + + +## Usability improvement + +* Admin: introduced new table `runtime_mysql_users` [#691](../../../../issues/691) +* Compile: new packages avaiable for Fedora24 +* Compile: new packages avaiable for Ubuntu16 +* Doc: updated documentation on passwords +* General: added suppot for systemd (yet not included in binaries) + + +## New features + +* Admin: introduced new variable `admin-hash_passwords` to automatically hash mysql passwords [#676](../../../../issues/676) +* Query Cache: aggressive memory purging when 90% of memory limit is reached [#690](../../../../issues/690) +* Query Processor: added parsing for several SQL commands + + +## Bug fixes + +* Mirroring: fixes several bugs related to errors handling +* Mirroring: fixes crashing bug +* Query Cache: memory used was computed incorrectly +* Connection Pool: a failed `CHANGE_USER` could cause a crash [#682](../../../../issues/682) + + +## Contributors + +Thanks to contributors, in alphabetical order: +* @dveeden diff --git a/doc/scheduler.md b/doc/scheduler.md index ab9937f31..be6e9141b 100644 --- a/doc/scheduler.md +++ b/doc/scheduler.md @@ -1,6 +1,6 @@ # Scheduler -Scheduler is a feature introduced in v1.2.0. +Scheduler is a feature introduced in v1.2.0 . Scheduler is a cron-like implementation integrated inside ProxySQL with millisecond granularity. It is possible to be configured only through the Admin interface: configuration from config file is not supported yet and not in the roadmap. @@ -28,44 +28,48 @@ To enter into details: Table `scheduler` has the following structure: -```sql +```mysql Admin> SHOW CREATE TABLE scheduler\G *************************** 1. row *************************** table: scheduler Create Table: CREATE TABLE scheduler ( -id INTEGER NOT NULL, -interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL, -filename VARCHAR NOT NULL, -arg1 VARCHAR, -arg2 VARCHAR, -arg3 VARCHAR, -arg4 VARCHAR, -arg5 VARCHAR, -PRIMARY KEY(id)) + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1, + interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL, + filename VARCHAR NOT NULL, + arg1 VARCHAR, + arg2 VARCHAR, + arg3 VARCHAR, + arg4 VARCHAR, + arg5 VARCHAR, + comment VARCHAR NOT NULL DEFAULT '') 1 row in set (0.00 sec) ``` In details: * `id` : unique identifier of the scheduler job +* `active` : if set to 1, the job is active. Otherwise is not * `interval_ms` : how often (in millisecond) the job will be started. Minimum interval_ms is 100 milliseconds * `filename` : full path of the executable to be executed * `arg1` to `arg5` : arguments (maximum 5) that can be passed to the job +* `comment` : an free form text field to annotate the purpose of the job -For reference only, table `runtime_scheduler` has the same identical tructure: +For reference only, table `runtime_scheduler` has the same identical structure: ```sql Admin> SHOW CREATE TABLE runtime_scheduler\G *************************** 1. row *************************** table: runtime_scheduler Create Table: CREATE TABLE runtime_scheduler ( -id INTEGER NOT NULL, -interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL, -filename VARCHAR NOT NULL, -arg1 VARCHAR, -arg2 VARCHAR, -arg3 VARCHAR, -arg4 VARCHAR, -arg5 VARCHAR, -PRIMARY KEY(id)) + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + active INT CHECK (active IN (0,1)) NOT NULL DEFAULT 1, + interval_ms INTEGER CHECK (interval_ms>=100 AND interval_ms<=100000000) NOT NULL, + filename VARCHAR NOT NULL, + arg1 VARCHAR, + arg2 VARCHAR, + arg3 VARCHAR, + arg4 VARCHAR, + arg5 VARCHAR, + comment VARCHAR NOT NULL DEFAULT '') 1 row in set (0.00 sec) ``` @@ -76,4 +80,4 @@ For this reason ProxySQL has new commands to support Scheduler: * `SAVE SCHEDULER FROM RUNTIME` and `SAVE SCHEDULER TO MEMORY` : save the configuration from runtime to `main`.`scheduler`; * `SAVE SCHEDULER FROM MEMORY` and `SAVE SCHEDULER TO DISK` : save the configuration from `main`.`scheduler` to `disk`.`scheduler`, and becomes persistent across restart. -The scheduler is implemented calling `fork()` and then `excve()`. If `execve()` fails the error is reported into error log. +The scheduler is implemented calling `fork()` and then `execve()`. If `execve()` fails the error is reported into error log. diff --git a/docker/images/proxysql/centos67-build/proxysql.spec b/docker/images/proxysql/centos67-build/proxysql.spec index 694210102..1c1e0c279 100644 --- a/docker/images/proxysql/centos67-build/proxysql.spec +++ b/docker/images/proxysql/centos67-build/proxysql.spec @@ -7,7 +7,7 @@ Summary: A high-performance MySQL proxy Name: proxysql -Version: 1.2.3 +Version: 1.2.4 Release: 1 License: GPL+ Group: Development/Tools diff --git a/docker/images/proxysql/centos7-build/proxysql.spec b/docker/images/proxysql/centos7-build/proxysql.spec index 694210102..1c1e0c279 100644 --- a/docker/images/proxysql/centos7-build/proxysql.spec +++ b/docker/images/proxysql/centos7-build/proxysql.spec @@ -7,7 +7,7 @@ Summary: A high-performance MySQL proxy Name: proxysql -Version: 1.2.3 +Version: 1.2.4 Release: 1 License: GPL+ Group: Development/Tools diff --git a/docker/images/proxysql/debian-7.8-build/proxysql.ctl b/docker/images/proxysql/debian-7.8-build/proxysql.ctl index 8475d93a5..2b2fd3357 100644 --- a/docker/images/proxysql/debian-7.8-build/proxysql.ctl +++ b/docker/images/proxysql/debian-7.8-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.2.3 +Version: 1.2.4 Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md diff --git a/docker/images/proxysql/debian-8.2-build/proxysql.ctl b/docker/images/proxysql/debian-8.2-build/proxysql.ctl index 8475d93a5..2b2fd3357 100644 --- a/docker/images/proxysql/debian-8.2-build/proxysql.ctl +++ b/docker/images/proxysql/debian-8.2-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.2.3 +Version: 1.2.4 Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md diff --git a/docker/images/proxysql/fedora24-build/proxysql.spec b/docker/images/proxysql/fedora24-build/proxysql.spec index 694210102..1c1e0c279 100644 --- a/docker/images/proxysql/fedora24-build/proxysql.spec +++ b/docker/images/proxysql/fedora24-build/proxysql.spec @@ -7,7 +7,7 @@ Summary: A high-performance MySQL proxy Name: proxysql -Version: 1.2.3 +Version: 1.2.4 Release: 1 License: GPL+ Group: Development/Tools diff --git a/docker/images/proxysql/ubuntu-12.04-build/proxysql.ctl b/docker/images/proxysql/ubuntu-12.04-build/proxysql.ctl index 8475d93a5..2b2fd3357 100644 --- a/docker/images/proxysql/ubuntu-12.04-build/proxysql.ctl +++ b/docker/images/proxysql/ubuntu-12.04-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.2.3 +Version: 1.2.4 Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md diff --git a/docker/images/proxysql/ubuntu-14.04-build/proxysql.ctl b/docker/images/proxysql/ubuntu-14.04-build/proxysql.ctl index 8475d93a5..2b2fd3357 100644 --- a/docker/images/proxysql/ubuntu-14.04-build/proxysql.ctl +++ b/docker/images/proxysql/ubuntu-14.04-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.2.3 +Version: 1.2.4 Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md diff --git a/docker/images/proxysql/ubuntu-15.10-build/proxysql.ctl b/docker/images/proxysql/ubuntu-15.10-build/proxysql.ctl index 8475d93a5..2b2fd3357 100644 --- a/docker/images/proxysql/ubuntu-15.10-build/proxysql.ctl +++ b/docker/images/proxysql/ubuntu-15.10-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.2.3 +Version: 1.2.4 Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md diff --git a/docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl b/docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl index 8475d93a5..2b2fd3357 100644 --- a/docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl +++ b/docker/images/proxysql/ubuntu-16.04-build/proxysql.ctl @@ -4,7 +4,7 @@ Homepage: http://www.proxysql.com Standards-Version: 3.9.2 Package: proxysql -Version: 1.2.3 +Version: 1.2.4 Maintainer: Rene Cannao Architecture: amd64 # Changelog: CHANGELOG.md diff --git a/include/MySQL_Thread.h b/include/MySQL_Thread.h index e739bb227..459c6d9fd 100644 --- a/include/MySQL_Thread.h +++ b/include/MySQL_Thread.h @@ -174,6 +174,9 @@ class MySQL_Thread unsigned long long mysql_backend_buffers_bytes; unsigned long long mysql_frontend_buffers_bytes; unsigned long long mysql_session_internal_bytes; + unsigned long long ConnPool_get_conn_immediate; + unsigned long long ConnPool_get_conn_success; + unsigned long long ConnPool_get_conn_failure; unsigned int active_transactions; } status_variables; @@ -377,6 +380,9 @@ class MySQL_Threads_Handler unsigned long long get_mysql_backend_buffers_bytes(); unsigned long long get_mysql_frontend_buffers_bytes(); unsigned long long get_mysql_session_internal_bytes(); + unsigned long long get_ConnPool_get_conn_immediate(); + unsigned long long get_ConnPool_get_conn_success(); + unsigned long long get_ConnPool_get_conn_failure(); iface_info *MLM_find_iface_from_fd(int fd) { return MLM->find_iface_from_fd(fd); } diff --git a/lib/MySQL_Monitor.cpp b/lib/MySQL_Monitor.cpp index 91d9cd008..f57b771e0 100644 --- a/lib/MySQL_Monitor.cpp +++ b/lib/MySQL_Monitor.cpp @@ -942,7 +942,8 @@ __end_monitor_connect_loop: if (mysql_thread___monitor_ping_interval < 3600000) mysql_thread___monitor_history = mysql_thread___monitor_ping_interval * (mysql_thread___monitor_ping_max_failures + 1 ); } - rc=sqlite3_bind_int64(statement, 1, start_time-mysql_thread___monitor_history*1000); assert(rc==SQLITE_OK); + unsigned long long time_now=realtime_time(); + rc=sqlite3_bind_int64(statement, 1, time_now-mysql_thread___monitor_history*1000); assert(rc==SQLITE_OK); SAFE_SQLITE3_STEP(statement); rc=sqlite3_clear_bindings(statement); assert(rc==SQLITE_OK); rc=sqlite3_reset(statement); assert(rc==SQLITE_OK); @@ -1052,7 +1053,8 @@ __end_monitor_ping_loop: if (mysql_thread___monitor_ping_interval < 3600000) mysql_thread___monitor_history = mysql_thread___monitor_ping_interval * (mysql_thread___monitor_ping_max_failures + 1 ); } - rc=sqlite3_bind_int64(statement, 1, start_time-mysql_thread___monitor_history*1000); assert(rc==SQLITE_OK); + unsigned long long time_now=realtime_time(); + rc=sqlite3_bind_int64(statement, 1, time_now-mysql_thread___monitor_history*1000); assert(rc==SQLITE_OK); SAFE_SQLITE3_STEP(statement); rc=sqlite3_clear_bindings(statement); assert(rc==SQLITE_OK); rc=sqlite3_reset(statement); assert(rc==SQLITE_OK); @@ -1271,7 +1273,8 @@ __end_monitor_read_only_loop: if (mysql_thread___monitor_ping_interval < 3600000) mysql_thread___monitor_history = mysql_thread___monitor_ping_interval * (mysql_thread___monitor_ping_max_failures + 1 ); } - rc=sqlite3_bind_int64(statement, 1, start_time-mysql_thread___monitor_history*1000); assert(rc==SQLITE_OK); + unsigned long long time_now=realtime_time(); + rc=sqlite3_bind_int64(statement, 1, time_now-mysql_thread___monitor_history*1000); assert(rc==SQLITE_OK); SAFE_SQLITE3_STEP(statement); rc=sqlite3_clear_bindings(statement); assert(rc==SQLITE_OK); rc=sqlite3_reset(statement); assert(rc==SQLITE_OK); @@ -1386,7 +1389,8 @@ __end_monitor_replication_lag_loop: if (mysql_thread___monitor_ping_interval < 3600000) mysql_thread___monitor_history = mysql_thread___monitor_ping_interval * (mysql_thread___monitor_ping_max_failures + 1 ); } - rc=sqlite3_bind_int64(statement, 1, start_time-mysql_thread___monitor_history*1000); assert(rc==SQLITE_OK); + unsigned long long time_now=realtime_time(); + rc=sqlite3_bind_int64(statement, 1, time_now-mysql_thread___monitor_history*1000); assert(rc==SQLITE_OK); SAFE_SQLITE3_STEP(statement); rc=sqlite3_clear_bindings(statement); assert(rc==SQLITE_OK); rc=sqlite3_reset(statement); assert(rc==SQLITE_OK); diff --git a/lib/MySQL_Protocol.cpp b/lib/MySQL_Protocol.cpp index 90bc4c603..1694fed8a 100644 --- a/lib/MySQL_Protocol.cpp +++ b/lib/MySQL_Protocol.cpp @@ -1908,6 +1908,8 @@ unsigned long long MySQL_ResultSet::current_size() { unsigned long long intsize=0; intsize+=sizeof(MySQL_ResultSet); intsize+=RESULTSET_BUFLEN; // size of buffer + if (PSarrayOUT==NULL) // see bug #699 + return intsize; intsize+=sizeof(PtrSizeArray); intsize+=(PSarrayOUT->size*sizeof(PtrSize_t *)); unsigned int i; diff --git a/lib/MySQL_Session.cpp b/lib/MySQL_Session.cpp index 91efe2f0d..bb72f9ae9 100644 --- a/lib/MySQL_Session.cpp +++ b/lib/MySQL_Session.cpp @@ -999,8 +999,16 @@ bool MySQL_Session::handler_again___status_CHANGING_SCHEMA(int *_rc) { bool MySQL_Session::handler_again___status_CONNECTING_SERVER(int *_rc) { //fprintf(stderr,"CONNECTING_SERVER\n"); + if (mirror) { + mybe->server_myds->connect_retries_on_failure=0; // no try for mirror + mybe->server_myds->wait_until=thread->curtime+mysql_thread___connect_timeout_server*1000; + pause_until=0; + } if (mybe->server_myds->max_connect_time) { if (thread->curtime >= mybe->server_myds->max_connect_time) { + if (mirror) { + PROXY_TRACE(); + } char buf[256]; sprintf(buf,"Max connect timeout reached while reaching hostgroup %d after %llums", current_hostgroup, (thread->curtime - CurrentQuery.start_time)/1000 ); client_myds->myprot.generate_pkt_ERR(true,NULL,NULL,1,1045,(char *)"#28000",buf); @@ -1016,6 +1024,10 @@ bool MySQL_Session::handler_again___status_CONNECTING_SERVER(int *_rc) { if (mybe->server_myds->myconn) { //mybe->server_myds->destroy_MySQL_Connection(); mybe->server_myds->destroy_MySQL_Connection_From_Pool(false); + if (mirror) { + PROXY_TRACE(); + NEXT_IMMEDIATE_NEW(WAITING_CLIENT_DATA); + } } mybe->server_myds->max_connect_time=0; NEXT_IMMEDIATE_NEW(WAITING_CLIENT_DATA); @@ -1024,6 +1036,12 @@ bool MySQL_Session::handler_again___status_CONNECTING_SERVER(int *_rc) { if (mybe->server_myds->myconn==NULL) { handler___client_DSS_QUERY_SENT___server_DSS_NOT_INITIALIZED__get_connection(); } + if (mybe->server_myds->myconn==NULL) { + if (mirror) { + PROXY_TRACE(); + NEXT_IMMEDIATE_NEW(WAITING_CLIENT_DATA); + } + } if (mybe->server_myds->myconn==NULL) { pause_until=thread->curtime+mysql_thread___connect_retries_delay*1000; //goto __exit_DSS__STATE_NOT_INITIALIZED; @@ -1050,11 +1068,17 @@ bool MySQL_Session::handler_again___status_CONNECTING_SERVER(int *_rc) { //mybe->server_myds->myprot.init(&mybe->server_myds, mybe->server_myds->myconn->userinfo, this); /* */ assert(myconn->async_state_machine!=ASYNC_IDLE); + if (mirror) { + PROXY_TRACE(); + } rc=myconn->async_connect(myds->revents); if (myds->mypolls==NULL) { // connection yet not in mypolls myds->assign_fd_from_mysql_conn(); thread->mypolls.add(POLLIN|POLLOUT, mybe->server_myds->fd, mybe->server_myds, curtime); + if (mirror) { + PROXY_TRACE(); + } } switch (rc) { case 0: @@ -1078,6 +1102,9 @@ bool MySQL_Session::handler_again___status_CONNECTING_SERVER(int *_rc) { if (myds->connect_retries_on_failure >0 ) { myds->connect_retries_on_failure--; //myds->destroy_MySQL_Connection(); + if (mirror) { + PROXY_TRACE(); + } myds->destroy_MySQL_Connection_From_Pool(false); NEXT_IMMEDIATE_NEW(CONNECTING_SERVER); } else { @@ -1100,6 +1127,9 @@ bool MySQL_Session::handler_again___status_CONNECTING_SERVER(int *_rc) { previous_status.pop(); } //myds->destroy_MySQL_Connection(); + if (mirror) { + PROXY_TRACE(); + } myds->destroy_MySQL_Connection_From_Pool( myerr ? true : false ); myds->max_connect_time=0; NEXT_IMMEDIATE_NEW(WAITING_CLIENT_DATA); @@ -2769,9 +2799,14 @@ void MySQL_Session::handler___client_DSS_QUERY_SENT___server_DSS_NOT_INITIALIZED mc=thread->get_MyConn_local(mybe->hostgroup_id); // experimental , #644 if (mc==NULL) { mc=MyHGM->get_MyConn_from_pool(mybe->hostgroup_id); + } else { + thread->status_variables.ConnPool_get_conn_immediate++; } if (mc) { mybe->server_myds->attach_connection(mc); + thread->status_variables.ConnPool_get_conn_success++; + } else { + thread->status_variables.ConnPool_get_conn_failure++; } #endif // } diff --git a/lib/MySQL_Thread.cpp b/lib/MySQL_Thread.cpp index 372dea21b..44038be3e 100644 --- a/lib/MySQL_Thread.cpp +++ b/lib/MySQL_Thread.cpp @@ -4,6 +4,10 @@ #include "cpp.h" #include "MySQL_Thread.h" +#ifdef DEBUG +MySQL_Session *sess_stopat; +#endif + #define PROXYSQL_LISTEN_LEN 1024 extern Query_Processor *GloQPro; @@ -2038,6 +2042,11 @@ void MySQL_Thread::process_all_sessions() { } for (n=0; nlen; n++) { MySQL_Session *sess=(MySQL_Session *)mysql_sessions->index(n); +#ifdef DEBUG + if(sess==sess_stopat) { + sess_stopat=sess; + } +#endif if (sess->mirror==true) { // this is a mirror session if (sess->status==WAITING_CLIENT_DATA) { // the mirror session has completed unregister_session(n); @@ -2229,6 +2238,9 @@ MySQL_Thread::MySQL_Thread() { status_variables.mysql_backend_buffers_bytes=0; status_variables.mysql_frontend_buffers_bytes=0; status_variables.mysql_session_internal_bytes=0; + status_variables.ConnPool_get_conn_immediate=0; + status_variables.ConnPool_get_conn_success=0; + status_variables.ConnPool_get_conn_failure=0; status_variables.active_transactions=0; } @@ -2474,6 +2486,24 @@ SQLite3_result * MySQL_Threads_Handler::SQL3_GlobalStatus() { result->add_row(pta); } } + { // ConnPool_get_conn_immediate + pta[0]=(char *)"ConnPool_get_conn_immediate"; + sprintf(buf,"%llu",get_ConnPool_get_conn_immediate()); + pta[1]=buf; + result->add_row(pta); + } + { // ConnPool_get_conn_success + pta[0]=(char *)"ConnPool_get_conn_success"; + sprintf(buf,"%llu",get_ConnPool_get_conn_success()); + pta[1]=buf; + result->add_row(pta); + } + { // ConnPool_get_conn_failure + pta[0]=(char *)"ConnPool_get_conn_failure"; + sprintf(buf,"%llu",get_ConnPool_get_conn_failure()); + pta[1]=buf; + result->add_row(pta); + } free(pta); return result; } @@ -2934,3 +2964,42 @@ void MySQL_Thread::return_local_connections() { MyHGM->push_MyConn_to_pool_array(ca); free(ca); } + +unsigned long long MySQL_Threads_Handler::get_ConnPool_get_conn_immediate() { + unsigned long long q=0; + unsigned int i; + for (i=0;istatus_variables.ConnPool_get_conn_immediate,0); + } + } + return q; +} + +unsigned long long MySQL_Threads_Handler::get_ConnPool_get_conn_success() { + unsigned long long q=0; + unsigned int i; + for (i=0;istatus_variables.ConnPool_get_conn_success,0); + } + } + return q; +} + +unsigned long long MySQL_Threads_Handler::get_ConnPool_get_conn_failure() { + unsigned long long q=0; + unsigned int i; + for (i=0;istatus_variables.ConnPool_get_conn_failure,0); + } + } + return q; +} diff --git a/lib/ProxySQL_Admin.cpp b/lib/ProxySQL_Admin.cpp index f8f69718d..fadbec0ec 100644 --- a/lib/ProxySQL_Admin.cpp +++ b/lib/ProxySQL_Admin.cpp @@ -535,7 +535,9 @@ bool is_valid_global_variable(const char *var_name) { bool admin_handler_command_set(char *query_no_space, unsigned int query_no_space_length, MySQL_Session *sess, ProxySQL_Admin *pa, char **q, unsigned int *ql) { if (!strstr(query_no_space,(char *)"password")) { // issue #599 proxy_debug(PROXY_DEBUG_ADMIN, 4, "Received command %s\n", query_no_space); - proxy_info("Received command %s\n", query_no_space); + if (strcmp(query_no_space,(char *)"set autocommit=0")) { + proxy_info("Received command %s\n", query_no_space); + } } // Get a pointer to the beginnig of var=value entry and split to get var name and value char *set_entry = query_no_space + strlen("SET "); diff --git a/lib/mysql_data_stream.cpp b/lib/mysql_data_stream.cpp index 3879ba9fb..9d17c6e33 100644 --- a/lib/mysql_data_stream.cpp +++ b/lib/mysql_data_stream.cpp @@ -598,13 +598,16 @@ void MySQL_Data_Stream::generate_compressed_packet() { while (ilen && total_sizeindex(i); total_size+=p->size; - if (i==0) { - mysql_hdr hdr; - memcpy(&hdr,p->ptr,sizeof(mysql_hdr)); - if (hdr.pkt_id==0) { - myconn->compression_pkt_id=-1; - } - } +// The following 7 lines of code are commented as they seem responsible for bug reported in +// https://github.com/sysown/proxysql/issues/297#issuecomment-247152748 +// They seems logically completely incorrect! +// if (i==0) { +// mysql_hdr hdr; +// memcpy(&hdr,p->ptr,sizeof(mysql_hdr)); +// if (hdr.pkt_id==0) { +// myconn->compression_pkt_id=-1; +// } +// } i++; } if (i>=2) { diff --git a/systemd/proxysql.service b/systemd/proxysql.service new file mode 100644 index 000000000..7ea7df229 --- /dev/null +++ b/systemd/proxysql.service @@ -0,0 +1,12 @@ +[Unit] +Description=High Performance Advanced Proxy for MySQL +After=network.target + +[Service] +LimitNOFILE=102400 +LimitCORE=1073741824 +ExecStart=/usr/local/bin/proxysql -f +Restart=always + +[Install] +WantedBy=default.target