From 8d805c387793ec067ad11a5e2e954cecbc9e2aaf Mon Sep 17 00:00:00 2001 From: Andrei Ismail Date: Fri, 22 May 2015 09:19:56 +0300 Subject: [PATCH 01/19] #285 Add docker compose YML for setting up 1 proxy and 1 MySQL backend --- docker/1backend/docker-compose.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 docker/1backend/docker-compose.yml diff --git a/docker/1backend/docker-compose.yml b/docker/1backend/docker-compose.yml new file mode 100644 index 000000000..0f67246c6 --- /dev/null +++ b/docker/1backend/docker-compose.yml @@ -0,0 +1,14 @@ +proxysql: + build: ./proxysql + links: + - backend1hostgroup0 + ports: + - "127.0.0.1:6032:6032" + - "6033:6033" + +backend1hostgroup0: + build: ./mysql + environment: + MYSQL_ROOT_PASSWORD: root + expose: + - "3306" From e45ad15bf32ac4f24b9ad1b494d00fb0753644de Mon Sep 17 00:00:00 2001 From: Andrei Ismail Date: Fri, 22 May 2015 09:21:01 +0300 Subject: [PATCH 02/19] #285 Delete Dockerfile -- we will have one per test setup --- docker/Dockerfile | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 docker/Dockerfile diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 0d2935ebb..000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -# We're using Ubuntu 14:04 because ProxySQL compilation needs one of the latest -# g++ compilers. Also, it's a long term release. -FROM ubuntu:14.04 -MAINTAINER Andrei Ismail -RUN apt-get update && apt-get install -y\ - cmake\ - make\ - g++\ - gcc\ - git\ - libssl-dev\ - libmysqlclient-dev - -RUN cd /opt; git clone https://github.com/sysown/proxysql-0.2.git -RUN cd /opt/proxysql-0.2; make clean && make -RUN mkdir -p /var/run/proxysql -RUN cp /opt/proxysql-0.2/proxysql.cfg /etc \ No newline at end of file From 13511276c77f1883cecfdc96a0b7adb92868b0fc Mon Sep 17 00:00:00 2001 From: Andrei Ismail Date: Fri, 22 May 2015 09:21:34 +0300 Subject: [PATCH 03/19] #285 Add Dockerfile for MySQL instance We are extending the base Dockerfile with a schema dump and a script that is able to install that. --- docker/1backend/mysql/Dockerfile | 7 +++++++ docker/1backend/mysql/import_schema.sh | 1 + docker/1backend/mysql/schema.sql | 5 +++++ 3 files changed, 13 insertions(+) create mode 100644 docker/1backend/mysql/Dockerfile create mode 100644 docker/1backend/mysql/import_schema.sh create mode 100644 docker/1backend/mysql/schema.sql diff --git a/docker/1backend/mysql/Dockerfile b/docker/1backend/mysql/Dockerfile new file mode 100644 index 000000000..7600b1b01 --- /dev/null +++ b/docker/1backend/mysql/Dockerfile @@ -0,0 +1,7 @@ +# We are creating a custom Dockerfile for MySQL as there is no easy way to +# move a file from host into the container. In our case, it's schema.sql +# There is a proposed improvement to "docker cp" but it's still being +# discussed (https://github.com/docker/docker/issues/5846). +FROM mysql:latest +ADD ./schema.sql /tmp/ +ADD ./import_schema.sh /tmp/ \ No newline at end of file diff --git a/docker/1backend/mysql/import_schema.sh b/docker/1backend/mysql/import_schema.sh new file mode 100644 index 000000000..fe993ee7e --- /dev/null +++ b/docker/1backend/mysql/import_schema.sh @@ -0,0 +1 @@ +cat /tmp/schema.sql | mysql -h 127.0.0.1 -u root -proot \ No newline at end of file diff --git a/docker/1backend/mysql/schema.sql b/docker/1backend/mysql/schema.sql new file mode 100644 index 000000000..ff75fee5e --- /dev/null +++ b/docker/1backend/mysql/schema.sql @@ -0,0 +1,5 @@ +CREATE DATABASE test; + +USE test; + +CREATE TABLE strings(value LONGTEXT); \ No newline at end of file From 71c0fb2722224de4870d0c81d1f146df4ea53ab2 Mon Sep 17 00:00:00 2001 From: Andrei Ismail Date: Fri, 22 May 2015 09:22:17 +0300 Subject: [PATCH 04/19] #285 Add base class for ProxySQL tests Proxy config population is still not working because I'm not able to access the admin port on MacOS (via boot2docker). Moving development into the cloud .. :) --- test/__init__.py | 0 test/proxysql_test.py | 82 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 test/__init__.py create mode 100644 test/proxysql_test.py diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/proxysql_test.py b/test/proxysql_test.py new file mode 100644 index 000000000..ea070d048 --- /dev/null +++ b/test/proxysql_test.py @@ -0,0 +1,82 @@ +import subprocess +import time +from unittest import TestCase + +from docker import Client +from docker.utils import kwargs_from_env +import MySQLdb + +class ProxySQLTest(TestCase): + + DOCKER_COMPOSE_FILE = "./docker/1backend" + + @classmethod + def _startup_docker_services(cls): + # We have to perform docker-compose build + docker-compose up, + # instead of just doing the latter because of a bug which will give a + # 500 internal error for the Docker bug. When this is fixed, we should + # remove this first extra step. + subprocess.call(["docker-compose", "build"], cwd=cls.DOCKER_COMPOSE_FILE) + subprocess.call(["docker-compose", "up", "-d"], cwd=cls.DOCKER_COMPOSE_FILE) + + @classmethod + def _shutdown_docker_services(cls): + subprocess.call(["docker-compose", "stop"], cwd=cls.DOCKER_COMPOSE_FILE) + subprocess.call(["docker-compose", "rm", "--force"], cwd=cls.DOCKER_COMPOSE_FILE) + + @classmethod + def _get_proxysql_container(cls): + containers = Client(**kwargs_from_env()).containers() + for container in containers: + if 'proxysql' in container['image']: + return container + + @classmethod + def _get_mysql_containers(cls): + result = [] + containers = Client(**kwargs_from_env()).containers() + for container in containers: + if 'proxysql' not in container['Image']: + result.append(container) + return result + + @classmethod + def _populate_mysql_containers_with_dump(cls): + mysql_containers = cls._get_mysql_containers() + # We have already added the SQL dump to the container by using + # the ADD mysql command in the Dockerfile for mysql -- check it + # out. The standard agreed location is at /tmp/schema.sql. + # + # Unfortunately we can't do this step at runtime due to limitations + # on how transfer between host and container is supposed to work by + # design. See the Dockerfile for MySQL for more details. + for mysql_container in mysql_containers: + container_id = mysql_container['Names'][0][1:] + subprocess.call(["docker", "exec", container_id, "bash", "/tmp/import_schema.sh"]) + + @classmethod + def _populate_proxy_configuration_with_backends(cls): + pass + + @classmethod + def setUpClass(cls): + cls._shutdown_docker_services() + cls._startup_docker_services() + + # TODO(andrei): figure out a more reliable method to wait for + # MySQL to start up within the container. Otherwise, there will be + # an error when we try to initialize the MySQL instance with the dump. + time.sleep(30) + cls._populate_mysql_containers_with_dump() + + cls._populate_proxy_configuration_with_backends() + + @classmethod + def tearDownClass(cls): + pass + + def test_asd(self): + self.assertEqual(1, 1) + + def test_asdf(self): + self.assertEqual(1, 1) \ No newline at end of file From f9c70f11f8974c5e3da674a5dd645dd0b00130ae Mon Sep 17 00:00:00 2001 From: Andrei Ismail Date: Fri, 22 May 2015 09:22:50 +0300 Subject: [PATCH 05/19] #285 Add requirements.txt for running the Python test suite Docker compose and requests version are particularly important. --- requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..592c34ed3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +nose==1.3.6 +docker-compose==1.1.0 +requests==2.4.3 +MySQL-python==1.2.5 \ No newline at end of file From 5db205365d81c3cee09b003babc44ba090099837 Mon Sep 17 00:00:00 2001 From: Andrei Ismail Date: Fri, 22 May 2015 09:32:19 +0300 Subject: [PATCH 06/19] #285 Add Dockerfile for ProxySQL instance w/ one backend. --- docker/1backend/proxysql/Dockerfile | 20 ++++++++++++++++++++ docker/1backend/proxysql/proxysql.cfg | 18 ++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 docker/1backend/proxysql/Dockerfile create mode 100644 docker/1backend/proxysql/proxysql.cfg diff --git a/docker/1backend/proxysql/Dockerfile b/docker/1backend/proxysql/Dockerfile new file mode 100644 index 000000000..4405f4f78 --- /dev/null +++ b/docker/1backend/proxysql/Dockerfile @@ -0,0 +1,20 @@ +# We're using Ubuntu 14:04 because ProxySQL compilation needs one of the latest +# g++ compilers. Also, it's a long term release. +FROM ubuntu:14.04 +MAINTAINER Andrei Ismail +RUN apt-get update && apt-get install -y\ + cmake\ + make\ + g++\ + gcc\ + git\ + libssl-dev\ + libmysqlclient-dev + +RUN cd /opt; git clone https://github.com/sysown/proxysql-0.2.git proxysql +RUN cd /opt/proxysql; make clean && make +RUN mkdir -p /var/run/proxysql +ADD ./proxysql.cfg /etc/ + +WORKDIR /opt/proxysql/src +CMD ["/opt/proxysql/src/proxysql", "--initial"] \ No newline at end of file diff --git a/docker/1backend/proxysql/proxysql.cfg b/docker/1backend/proxysql/proxysql.cfg new file mode 100644 index 000000000..85b385f0a --- /dev/null +++ b/docker/1backend/proxysql/proxysql.cfg @@ -0,0 +1,18 @@ +datadir="/tmp" + +admin_variables = +{ + admin_credentials="admin:admin" + mysql_ifaces="127.0.0.1:6032" + refresh_interval=2000 + debug=true +} + +mysql_users = +( + { + username = "root" + password = "root" + default_hostgroup = 0 + } +) \ No newline at end of file From 5f4ae7e00990211d21138b5efc7f413c305f041a Mon Sep 17 00:00:00 2001 From: Andrei Ismail Date: Fri, 22 May 2015 17:11:18 +0300 Subject: [PATCH 07/19] #285 Stop mapping ProxySQL admin port 127.0.0.1 interface The reason for which I was trying to do this was that it wasn't being correctly forwarded by Docker. It works correctly if I bind the forwarded port to 0.0.0.0 instead of 127.0.0.1 on the original container. --- docker/1backend/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/1backend/docker-compose.yml b/docker/1backend/docker-compose.yml index 0f67246c6..1d5c2cc55 100644 --- a/docker/1backend/docker-compose.yml +++ b/docker/1backend/docker-compose.yml @@ -3,7 +3,7 @@ proxysql: links: - backend1hostgroup0 ports: - - "127.0.0.1:6032:6032" + - "6032:6032" - "6033:6033" backend1hostgroup0: From d78acb1cbfd8c98873d476cc92ac9c9e139892a7 Mon Sep 17 00:00:00 2001 From: Andrei Ismail Date: Fri, 22 May 2015 17:11:40 +0300 Subject: [PATCH 08/19] #285 Populate db with 4 strings for playing around with them --- docker/1backend/mysql/schema.sql | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/1backend/mysql/schema.sql b/docker/1backend/mysql/schema.sql index ff75fee5e..cc6ea878d 100644 --- a/docker/1backend/mysql/schema.sql +++ b/docker/1backend/mysql/schema.sql @@ -2,4 +2,9 @@ CREATE DATABASE test; USE test; -CREATE TABLE strings(value LONGTEXT); \ No newline at end of file +CREATE TABLE strings(value LONGTEXT); + +INSERT INTO strings(value) VALUES('a'); +INSERT INTO strings(value) VALUES('ab'); +INSERT INTO strings(value) VALUES('abc'); +INSERT INTO strings(value) VALUES('abcd'); \ No newline at end of file From c7d199bd0663c4d02860b9948b71ee37e8dd07e0 Mon Sep 17 00:00:00 2001 From: Andrei Ismail Date: Fri, 22 May 2015 17:12:49 +0300 Subject: [PATCH 09/19] #285 Fix config file name .cfg wasn't being picked up by ProxySQL. What led me into believing it was .cfg was that the config file in the root of the repo is named like that. --- docker/1backend/proxysql/Dockerfile | 2 +- docker/1backend/proxysql/{proxysql.cfg => proxysql.cnf} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename docker/1backend/proxysql/{proxysql.cfg => proxysql.cnf} (86%) diff --git a/docker/1backend/proxysql/Dockerfile b/docker/1backend/proxysql/Dockerfile index 4405f4f78..c4514f6ee 100644 --- a/docker/1backend/proxysql/Dockerfile +++ b/docker/1backend/proxysql/Dockerfile @@ -14,7 +14,7 @@ RUN apt-get update && apt-get install -y\ RUN cd /opt; git clone https://github.com/sysown/proxysql-0.2.git proxysql RUN cd /opt/proxysql; make clean && make RUN mkdir -p /var/run/proxysql -ADD ./proxysql.cfg /etc/ +ADD ./proxysql.cnf /etc/ WORKDIR /opt/proxysql/src CMD ["/opt/proxysql/src/proxysql", "--initial"] \ No newline at end of file diff --git a/docker/1backend/proxysql/proxysql.cfg b/docker/1backend/proxysql/proxysql.cnf similarity index 86% rename from docker/1backend/proxysql/proxysql.cfg rename to docker/1backend/proxysql/proxysql.cnf index 85b385f0a..b50345d01 100644 --- a/docker/1backend/proxysql/proxysql.cfg +++ b/docker/1backend/proxysql/proxysql.cnf @@ -3,7 +3,7 @@ datadir="/tmp" admin_variables = { admin_credentials="admin:admin" - mysql_ifaces="127.0.0.1:6032" + mysql_ifaces="0.0.0.0:6032" refresh_interval=2000 debug=true } From f4f191c963f37fc0d5c76d8671500b1d2179d0f3 Mon Sep 17 00:00:00 2001 From: Andrei Ismail Date: Fri, 22 May 2015 17:13:15 +0300 Subject: [PATCH 10/19] #285 Add base class for ProxySQL tests based on a fleet of containers w/ Docker --- ...proxysql_test.py => proxysql_base_test.py} | 68 ++++++++++++++++--- 1 file changed, 58 insertions(+), 10 deletions(-) rename test/{proxysql_test.py => proxysql_base_test.py} (54%) diff --git a/test/proxysql_test.py b/test/proxysql_base_test.py similarity index 54% rename from test/proxysql_test.py rename to test/proxysql_base_test.py index ea070d048..6a5590ad4 100644 --- a/test/proxysql_test.py +++ b/test/proxysql_base_test.py @@ -1,3 +1,4 @@ +import re import subprocess import time from unittest import TestCase @@ -6,9 +7,15 @@ from docker import Client from docker.utils import kwargs_from_env import MySQLdb -class ProxySQLTest(TestCase): +class ProxySQLBaseTest(TestCase): - DOCKER_COMPOSE_FILE = "./docker/1backend" + DOCKER_COMPOSE_FILE = None + PROXYSQL_ADMIN_PORT = 6032 + PROXYSQL_ADMIN_USERNAME = "admin" + PROXYSQL_ADMIN_PASSWORD = "admin" + PROXYSQL_RW_PORT = 6033 + PROXYSQL_RW_USERNAME = "root" + PROXYSQL_RW_PASSWORD = "root" @classmethod def _startup_docker_services(cls): @@ -28,7 +35,7 @@ class ProxySQLTest(TestCase): def _get_proxysql_container(cls): containers = Client(**kwargs_from_env()).containers() for container in containers: - if 'proxysql' in container['image']: + if 'proxysql' in container['Image']: return container @classmethod @@ -54,9 +61,54 @@ class ProxySQLTest(TestCase): container_id = mysql_container['Names'][0][1:] subprocess.call(["docker", "exec", container_id, "bash", "/tmp/import_schema.sh"]) + @classmethod + def _extract_hostgroup_from_container_name(cls, container_name): + service_name = container_name.split('_')[1] + return int(re.search(r'BACKEND(\d+)HOSTGROUP(\d+)', service_name).group(2)) + + @classmethod + def _extract_port_number_from_uri(cls, uri): + return int(uri.split(':')[2]) + + @classmethod + def _get_environment_variables_from_container(cls, container_name): + output = Client(**kwargs_from_env()).execute(container_name, 'env') + result = {} + lines = output.split('\n') + for line in lines: + line = line.strip() + if len(line) == 0: + continue + (k, v) = line.split('=') + result[k] = v + return result + @classmethod def _populate_proxy_configuration_with_backends(cls): - pass + proxysql_container = cls._get_proxysql_container() + mysql_containers = cls._get_mysql_containers() + environment_variables = cls._get_environment_variables_from_container( + proxysql_container['Names'][0][1:]) + + proxy_admin_connection = MySQLdb.connect("127.0.0.1", + cls.PROXYSQL_ADMIN_USERNAME, + cls.PROXYSQL_ADMIN_PASSWORD, + port=cls.PROXYSQL_ADMIN_PORT) + cursor = proxy_admin_connection.cursor() + + for mysql_container in mysql_containers: + container_name = mysql_container['Names'][0][1:].upper() + port_uri = environment_variables['%s_PORT' % container_name] + port_no = cls._extract_port_number_from_uri(port_uri) + ip = environment_variables['%s_PORT_%d_TCP_ADDR' % (container_name, port_no)] + hostgroup = cls._extract_hostgroup_from_container_name(container_name) + cursor.execute("INSERT INTO mysql_servers(hostgroup_id, hostname, port, status) " + "VALUES(%d, '%s', %d, 'ONLINE')" % + (hostgroup, ip, port_no)) + + cursor.execute("LOAD MYSQL SERVERS TO RUNTIME") + cursor.close() + proxy_admin_connection.close() @classmethod def setUpClass(cls): @@ -73,10 +125,6 @@ class ProxySQLTest(TestCase): @classmethod def tearDownClass(cls): - pass - - def test_asd(self): - self.assertEqual(1, 1) + cls._shutdown_docker_services() - def test_asdf(self): - self.assertEqual(1, 1) \ No newline at end of file + \ No newline at end of file From 04fe577032d1bee913c169920065189f33e51cbe Mon Sep 17 00:00:00 2001 From: Andrei Ismail Date: Fri, 22 May 2015 17:13:40 +0300 Subject: [PATCH 11/19] #285 Implement one simple test that selects strings and checks for all of them being present --- test/one_backend_test.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 test/one_backend_test.py diff --git a/test/one_backend_test.py b/test/one_backend_test.py new file mode 100644 index 000000000..887aaf22f --- /dev/null +++ b/test/one_backend_test.py @@ -0,0 +1,21 @@ +import MySQLdb + +from proxysql_base_test import ProxySQLBaseTest + +class OneBackendTest(ProxySQLBaseTest): + + DOCKER_COMPOSE_FILE = "./docker/1backend" + + def test_select_strings_returns_correct_result(self): + proxy_connection = MySQLdb.connect("127.0.0.1", + ProxySQLBaseTest.PROXYSQL_RW_USERNAME, + ProxySQLBaseTest.PROXYSQL_RW_PASSWORD, + port=ProxySQLBaseTest.PROXYSQL_RW_PORT) + cursor = proxy_connection.cursor() + cursor.execute("USE test") + cursor.execute("SELECT * FROM strings") + rows = cursor.fetchall() + self.assertEqual(set([row[0] for row in rows]), + set(['a', 'ab', 'abc', 'abcd'])) + cursor.close() + proxy_connection.close() \ No newline at end of file From 3edc0ab63f6dc442d2a554e9eb74738b16600aba Mon Sep 17 00:00:00 2001 From: Andrei Ismail Date: Tue, 2 Jun 2015 14:20:53 +0300 Subject: [PATCH 12/19] #285 Expose the MySQL backend publicly so that the tests too can connect to it. --- docker/1backend/docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/1backend/docker-compose.yml b/docker/1backend/docker-compose.yml index 1d5c2cc55..98a8735ac 100644 --- a/docker/1backend/docker-compose.yml +++ b/docker/1backend/docker-compose.yml @@ -12,3 +12,5 @@ backend1hostgroup0: MYSQL_ROOT_PASSWORD: root expose: - "3306" + ports: + - "13306:3306" From e3b03dbaba3d643c5c8e08e639a54c8ff9d208bd Mon Sep 17 00:00:00 2001 From: Andrei Ismail Date: Tue, 2 Jun 2015 14:21:35 +0300 Subject: [PATCH 13/19] #285 Make sure to always drop the database before populating the schema The reason is that the previous tests might have left the database in an inconsistent state. --- docker/1backend/mysql/schema.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/1backend/mysql/schema.sql b/docker/1backend/mysql/schema.sql index cc6ea878d..19785d9af 100644 --- a/docker/1backend/mysql/schema.sql +++ b/docker/1backend/mysql/schema.sql @@ -1,3 +1,5 @@ +DROP DATABASE IF EXISTS test; + CREATE DATABASE test; USE test; From 0d2667d78bc5b18f6cdab32bdf69e147e95eb43b Mon Sep 17 00:00:00 2001 From: Andrei Ismail Date: Tue, 2 Jun 2015 14:22:17 +0300 Subject: [PATCH 14/19] #285 Added utility methods for running SQL queries against the proxy or against a MySQL backend --- test/proxysql_base_test.py | 78 +++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 2 deletions(-) diff --git a/test/proxysql_base_test.py b/test/proxysql_base_test.py index 6a5590ad4..6d2768c59 100644 --- a/test/proxysql_base_test.py +++ b/test/proxysql_base_test.py @@ -1,3 +1,4 @@ +import random import re import subprocess import time @@ -126,5 +127,78 @@ class ProxySQLBaseTest(TestCase): @classmethod def tearDownClass(cls): cls._shutdown_docker_services() - - \ No newline at end of file + + def run_query_proxysql(self, query, db, return_result=True): + """Run a query against the ProxySQL proxy and optionally return its + results as a set of rows.""" + proxy_connection = MySQLdb.connect("127.0.0.1", + ProxySQLBaseTest.PROXYSQL_RW_USERNAME, + ProxySQLBaseTest.PROXYSQL_RW_PASSWORD, + port=ProxySQLBaseTest.PROXYSQL_RW_PORT, + db=db) + cursor = proxy_connection.cursor() + cursor.execute(query) + if return_result: + rows = cursor.fetchall() + cursor.close() + proxy_connection.close() + if return_result: + return rows + + def run_query_mysql(self, query, db, return_result=True, hostgroup=0): + """Run a query against the MySQL backend and optionally return its + results as a set of rows. + + IMPORTANT: since the queries are actually ran against the MySQL backend, + that backend needs to expose its MySQL port to the outside through + docker compose's port mapping mechanism. + + This will actually parse the docker-compose configuration file to + retrieve the available backends and hostgroups and will pick a backend + from the specified hostgroup.""" + + # Figure out which are the containers for the specified hostgroup + mysql_backends = ProxySQLBaseTest._get_mysql_containers() + mysql_backends_in_hostgroup = [] + for backend in mysql_backends: + container_name = backend['Names'][0][1:].upper() + backend_hostgroup = ProxySQLBaseTest._extract_hostgroup_from_container_name(container_name) + + mysql_port_exposed=False + if not backend.get('Ports'): + continue + for exposed_port in backend.get('Ports', []): + if exposed_port['PrivatePort'] == 3306: + mysql_port_exposed = True + + if backend_hostgroup == hostgroup and mysql_port_exposed: + mysql_backends_in_hostgroup.append(backend) + + if len(mysql_backends_in_hostgroup) == 0: + raise Exception('No backends with a publicly exposed port were ' + 'found in hostgroup %d' % hostgroup) + + # Pick a random container, extract its connection details + container = random.choice(mysql_backends_in_hostgroup) + for exposed_port in container.get('Ports', []): + if exposed_port['PrivatePort'] == 3306: + mysql_port = exposed_port['PublicPort'] + + mysql_connection = MySQLdb.connect("127.0.0.1", + # Warning: this assumes that ProxySQL + # and all the backends have the same + # credentials. + # TODO(andrei): revisit this assumption + # in authentication tests. + ProxySQLBaseTest.PROXYSQL_RW_USERNAME, + ProxySQLBaseTest.PROXYSQL_RW_PASSWORD, + port=mysql_port, + db=db) + cursor = mysql_connection.cursor() + cursor.execute(query) + if return_result: + rows = cursor.fetchall() + cursor.close() + mysql_connection.close() + if return_result: + return rows \ No newline at end of file From e0388cd932803700077cfbfad83e7130d7c8a349 Mon Sep 17 00:00:00 2001 From: Andrei Ismail Date: Tue, 2 Jun 2015 14:22:50 +0300 Subject: [PATCH 15/19] #285 Document the methods of the ProxySQL base test class --- test/proxysql_base_test.py | 72 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/test/proxysql_base_test.py b/test/proxysql_base_test.py index 6d2768c59..4332d3a72 100644 --- a/test/proxysql_base_test.py +++ b/test/proxysql_base_test.py @@ -20,6 +20,12 @@ class ProxySQLBaseTest(TestCase): @classmethod def _startup_docker_services(cls): + """Start up all the docker services necessary to start this test. + + They are specified in the docker compose file specified in the variable + cls.DOCKER_COMPOSE_FILE. + """ + # We have to perform docker-compose build + docker-compose up, # instead of just doing the latter because of a bug which will give a # 500 internal error for the Docker bug. When this is fixed, we should @@ -29,11 +35,24 @@ class ProxySQLBaseTest(TestCase): @classmethod def _shutdown_docker_services(cls): + """Shut down all the docker services necessary to start this test. + + They are specified in the docker compose file specified in the variable + cls.DOCKER_COMPOSE_FILE. + """ + subprocess.call(["docker-compose", "stop"], cwd=cls.DOCKER_COMPOSE_FILE) subprocess.call(["docker-compose", "rm", "--force"], cwd=cls.DOCKER_COMPOSE_FILE) @classmethod def _get_proxysql_container(cls): + """Out of all the started docker containers, select the one which + represents the proxy instance. + + Note that this only supports one proxy instance for now. This method + relies on interogating the Docker daemon via its REST API. + """ + containers = Client(**kwargs_from_env()).containers() for container in containers: if 'proxysql' in container['Image']: @@ -41,6 +60,12 @@ class ProxySQLBaseTest(TestCase): @classmethod def _get_mysql_containers(cls): + """Out of all the started docker containers, select the ones which + represent the MySQL backend instances. + + This method relies on interogating the Docker daemon via its REST API. + """ + result = [] containers = Client(**kwargs_from_env()).containers() for container in containers: @@ -50,6 +75,13 @@ class ProxySQLBaseTest(TestCase): @classmethod def _populate_mysql_containers_with_dump(cls): + """Populates the started MySQL backend containers with the specified + SQL dump file. + + The reason for doing this __after__ the containers are started is + because we want to keep them as generic as possible. + """ + mysql_containers = cls._get_mysql_containers() # We have already added the SQL dump to the container by using # the ADD mysql command in the Dockerfile for mysql -- check it @@ -64,15 +96,33 @@ class ProxySQLBaseTest(TestCase): @classmethod def _extract_hostgroup_from_container_name(cls, container_name): + """MySQL backend containers are named using a naming convention: + backendXhostgroupY, where X and Y can be multi-digit numbers. + This extracts the value of the hostgroup from the container name. + + I made this choice because I wasn't able to find another easy way to + associate arbitrary metadata with a Docker container through the + docker compose file. + """ + service_name = container_name.split('_')[1] return int(re.search(r'BACKEND(\d+)HOSTGROUP(\d+)', service_name).group(2)) @classmethod def _extract_port_number_from_uri(cls, uri): + """Given a Docker container URI (exposed as an environment variable by + the host linking mechanism), extract the TCP port number from it.""" return int(uri.split(':')[2]) @classmethod def _get_environment_variables_from_container(cls, container_name): + """Retrieve the environment variables from the given container. + + This is useful because the host linking mechanism will expose + connectivity information to the linked hosts by the use of environment + variables. + """ + output = Client(**kwargs_from_env()).execute(container_name, 'env') result = {} lines = output.split('\n') @@ -86,6 +136,18 @@ class ProxySQLBaseTest(TestCase): @classmethod def _populate_proxy_configuration_with_backends(cls): + """Populate ProxySQL's admin information with the MySQL backends + and their associated hostgroups. + + This is needed because I do not want to hardcode this into the ProxySQL + config file of the test scenario, as it leaves more room for quick + iteration. + + In order to configure ProxySQL with the correct backends, we are using + the MySQL admin interface of ProxySQL, and inserting rows into the + `mysql_servers` table, which contains a list of which servers go into + which hostgroup. + """ proxysql_container = cls._get_proxysql_container() mysql_containers = cls._get_mysql_containers() environment_variables = cls._get_environment_variables_from_container( @@ -113,12 +175,16 @@ class ProxySQLBaseTest(TestCase): @classmethod def setUpClass(cls): + # Always shutdown docker services because the previous test might have + # left them in limbo. cls._shutdown_docker_services() + cls._startup_docker_services() - # TODO(andrei): figure out a more reliable method to wait for - # MySQL to start up within the container. Otherwise, there will be - # an error when we try to initialize the MySQL instance with the dump. + # Sleep for 30 seconds because we want to populate the MySQL containers + # with SQL dumps, but there is a race condition because we do not know + # when the MySQL daemons inside them have actually started or not. + # TODO(andrei): find a better solution time.sleep(30) cls._populate_mysql_containers_with_dump() From 8121f19dafcde86393700e83bc1680a7f37837b7 Mon Sep 17 00:00:00 2001 From: Andrei Ismail Date: Tue, 2 Jun 2015 14:23:15 +0300 Subject: [PATCH 16/19] #285 Simplify the example test to use the utility method that runs queries against the proxy --- test/one_backend_test.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/test/one_backend_test.py b/test/one_backend_test.py index 887aaf22f..3bd6379cd 100644 --- a/test/one_backend_test.py +++ b/test/one_backend_test.py @@ -7,15 +7,7 @@ class OneBackendTest(ProxySQLBaseTest): DOCKER_COMPOSE_FILE = "./docker/1backend" def test_select_strings_returns_correct_result(self): - proxy_connection = MySQLdb.connect("127.0.0.1", - ProxySQLBaseTest.PROXYSQL_RW_USERNAME, - ProxySQLBaseTest.PROXYSQL_RW_PASSWORD, - port=ProxySQLBaseTest.PROXYSQL_RW_PORT) - cursor = proxy_connection.cursor() - cursor.execute("USE test") - cursor.execute("SELECT * FROM strings") - rows = cursor.fetchall() + + rows = self.run_query_proxysql("SELECT * FROM strings", "test") self.assertEqual(set([row[0] for row in rows]), - set(['a', 'ab', 'abc', 'abcd'])) - cursor.close() - proxy_connection.close() \ No newline at end of file + set(['a', 'ab', 'abc', 'abcd'])) \ No newline at end of file From 020dd5b044f14762af9363ba1c593ce2a856603f Mon Sep 17 00:00:00 2001 From: Andrei Ismail Date: Wed, 3 Jun 2015 13:15:42 +0300 Subject: [PATCH 17/19] #285 Move ProxySQL docker container to a "base" folder, from where it can be used "as is" --- docker/1backend/docker-compose.yml | 2 +- docker/{1backend => base}/proxysql/Dockerfile | 0 docker/{1backend => base}/proxysql/proxysql.cnf | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename docker/{1backend => base}/proxysql/Dockerfile (100%) rename docker/{1backend => base}/proxysql/proxysql.cnf (100%) diff --git a/docker/1backend/docker-compose.yml b/docker/1backend/docker-compose.yml index 98a8735ac..7a3bb3607 100644 --- a/docker/1backend/docker-compose.yml +++ b/docker/1backend/docker-compose.yml @@ -1,5 +1,5 @@ proxysql: - build: ./proxysql + build: ../base/proxysql links: - backend1hostgroup0 ports: diff --git a/docker/1backend/proxysql/Dockerfile b/docker/base/proxysql/Dockerfile similarity index 100% rename from docker/1backend/proxysql/Dockerfile rename to docker/base/proxysql/Dockerfile diff --git a/docker/1backend/proxysql/proxysql.cnf b/docker/base/proxysql/proxysql.cnf similarity index 100% rename from docker/1backend/proxysql/proxysql.cnf rename to docker/base/proxysql/proxysql.cnf From dc19cb9ff4b8bfe5353e34324507944554fdf735 Mon Sep 17 00:00:00 2001 From: Andrei Ismail Date: Wed, 3 Jun 2015 13:29:59 +0300 Subject: [PATCH 18/19] #285 Rename "docker" top-level folder to scenarios --- {docker => scenarios}/1backend/docker-compose.yml | 0 {docker => scenarios}/1backend/mysql/Dockerfile | 0 {docker => scenarios}/1backend/mysql/import_schema.sh | 0 {docker => scenarios}/1backend/mysql/schema.sql | 0 {docker => scenarios}/base/proxysql/Dockerfile | 0 {docker => scenarios}/base/proxysql/proxysql.cnf | 0 test/one_backend_test.py | 2 +- 7 files changed, 1 insertion(+), 1 deletion(-) rename {docker => scenarios}/1backend/docker-compose.yml (100%) rename {docker => scenarios}/1backend/mysql/Dockerfile (100%) rename {docker => scenarios}/1backend/mysql/import_schema.sh (100%) rename {docker => scenarios}/1backend/mysql/schema.sql (100%) rename {docker => scenarios}/base/proxysql/Dockerfile (100%) rename {docker => scenarios}/base/proxysql/proxysql.cnf (100%) diff --git a/docker/1backend/docker-compose.yml b/scenarios/1backend/docker-compose.yml similarity index 100% rename from docker/1backend/docker-compose.yml rename to scenarios/1backend/docker-compose.yml diff --git a/docker/1backend/mysql/Dockerfile b/scenarios/1backend/mysql/Dockerfile similarity index 100% rename from docker/1backend/mysql/Dockerfile rename to scenarios/1backend/mysql/Dockerfile diff --git a/docker/1backend/mysql/import_schema.sh b/scenarios/1backend/mysql/import_schema.sh similarity index 100% rename from docker/1backend/mysql/import_schema.sh rename to scenarios/1backend/mysql/import_schema.sh diff --git a/docker/1backend/mysql/schema.sql b/scenarios/1backend/mysql/schema.sql similarity index 100% rename from docker/1backend/mysql/schema.sql rename to scenarios/1backend/mysql/schema.sql diff --git a/docker/base/proxysql/Dockerfile b/scenarios/base/proxysql/Dockerfile similarity index 100% rename from docker/base/proxysql/Dockerfile rename to scenarios/base/proxysql/Dockerfile diff --git a/docker/base/proxysql/proxysql.cnf b/scenarios/base/proxysql/proxysql.cnf similarity index 100% rename from docker/base/proxysql/proxysql.cnf rename to scenarios/base/proxysql/proxysql.cnf diff --git a/test/one_backend_test.py b/test/one_backend_test.py index 3bd6379cd..1a3cae11e 100644 --- a/test/one_backend_test.py +++ b/test/one_backend_test.py @@ -4,7 +4,7 @@ from proxysql_base_test import ProxySQLBaseTest class OneBackendTest(ProxySQLBaseTest): - DOCKER_COMPOSE_FILE = "./docker/1backend" + DOCKER_COMPOSE_FILE = "./scenarios/1backend" def test_select_strings_returns_correct_result(self): From bd17e0a2fdb016051691bc27850fd126f5fd39c7 Mon Sep 17 00:00:00 2001 From: Andrei Ismail Date: Wed, 3 Jun 2015 13:30:10 +0300 Subject: [PATCH 19/19] #285 Add document that describes how to write a test --- test/how_to.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 test/how_to.md diff --git a/test/how_to.md b/test/how_to.md new file mode 100644 index 000000000..1af9899f8 --- /dev/null +++ b/test/how_to.md @@ -0,0 +1,46 @@ +# How are the tests built? + +First off, a few words about how the infrastructure of the tests looks like. + +Tests are written in Python, and the services needed for running a test +(a ProxySQL instance and one or more MySQL instances) are specified in a +docker-compose.yml file and are started by using Docker's docker-compose. + +Tests are ran using nosetests (https://nose.readthedocs.org/en/latest/), +Python's de facto leader in terms of how tests are written and ran. The command +to run the tests is, from the root of the repository: + +```python +nosetests --nocapture +``` + +The "--nocapture" flag is present in order to have detailed output on what is +going on. Otherwise, the output will be suppressed by nosetests to give you only +a high-level report of how many tests passed and failed. + +# Where can I find the tests? + +Tests are grouped in scenarios. A __scenario__ specifies a configuration of +ProxySQL and MySQL backends, together with initial data to populate the MySQL +instances (a text file containing SQL queries). + +The folder "scenarios" found in the root folder of the repository contains +these scenarios. There is a "base" folder with common utilities, and then there +is one folder for each scenario. For example, "1backend" is the name for the +scenario of 1 ProxySQL proxy, and 1 MySQL backend. + +To create such a scenario, the simplest way to go about, is to copy-paste the +"1backend" scenario and modify it. Some of the important things to modify: +- docker-compose.yml. This is where the list of services is described, and + where you actuall specify how many MySQL backends there are, and which ports + they expose and how. Be careful, there is a naming convention +- mysql/schema.sql. This is where the MySQL backends get populated + +# How do I write a test? + +It's pretty simple. Once you have a working scenario, you write a class in +the top-level "test" folder, which inherits from ProxySQLBaseTest. One such +example is one_backend_test.py. The only thing which you should specify is +the docker-compose filename, and then start querying both the proxy and the +MySQL backends and testing assertions by using the `run_query_proxysql` and +`run_query_mysql' class methods. \ No newline at end of file