From edb1754b5e61df55ee4529be63428636a1bab425 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Canna=C3=B2?= Date: Tue, 3 Nov 2015 01:51:17 +0000 Subject: [PATCH] Drafing a new testing environment --- .../5backends-replication/docker-compose.yml | 4 +- docker/scenarios/repl1/app.sh | 51 +++++++ docker/scenarios/repl1/configure.sh | 134 ++++++++++++++++++ .../repl1/configure_repl_hostgroup.sh | 87 ++++++++++++ docker/scenarios/repl1/mysql.sh | 59 ++++++++ docker/scenarios/repl1/mysql_add.cnf_ | 19 +++ docker/scenarios/repl1/proxy.sh | 52 +++++++ docker/scenarios/repl1/test1.py | 98 +++++++++++++ docker/scenarios/repl1/test1_.py | 89 ++++++++++++ docker/scenarios/repl1/vars | 7 + docker/scenarios/repl1/vars2 | 23 +++ 11 files changed, 621 insertions(+), 2 deletions(-) create mode 100755 docker/scenarios/repl1/app.sh create mode 100755 docker/scenarios/repl1/configure.sh create mode 100755 docker/scenarios/repl1/configure_repl_hostgroup.sh create mode 100755 docker/scenarios/repl1/mysql.sh create mode 100644 docker/scenarios/repl1/mysql_add.cnf_ create mode 100755 docker/scenarios/repl1/proxy.sh create mode 100644 docker/scenarios/repl1/test1.py create mode 100644 docker/scenarios/repl1/test1_.py create mode 100644 docker/scenarios/repl1/vars create mode 100755 docker/scenarios/repl1/vars2 diff --git a/docker/scenarios/5backends-replication/docker-compose.yml b/docker/scenarios/5backends-replication/docker-compose.yml index d9cac0903..7bae1730d 100644 --- a/docker/scenarios/5backends-replication/docker-compose.yml +++ b/docker/scenarios/5backends-replication/docker-compose.yml @@ -1,5 +1,5 @@ proxysql: - image: proxysql:{{proxysql_image}} + image: proxysql:proxysql links: - backend1hostgroup0 - backend1hostgroup1 @@ -120,4 +120,4 @@ backend4hostgroup1: "vendor": "proxysql" "com.proxysql.hostgroup": "1" volumes: - - ./slave4-conf.d:/etc/mysql/conf.d \ No newline at end of file + - ./slave4-conf.d:/etc/mysql/conf.d diff --git a/docker/scenarios/repl1/app.sh b/docker/scenarios/repl1/app.sh new file mode 100755 index 000000000..d3d873258 --- /dev/null +++ b/docker/scenarios/repl1/app.sh @@ -0,0 +1,51 @@ +set -e +. ./vars + +destroy() { + for i in `seq 1 ${NUMAPPS}` ; do + echo -n "Destroying container app$i ... " + ID=`$USESUDO docker ps -a -f name=app$i -q` + if [ -z "$ID" ]; then + echo "not found" + else + $USESUDO docker rm app$i + echo "done" + fi + done +} + +prepare() { + for i in `seq 1 ${NUMAPPS}` ; do + echo Creating and running container app$i + $USESUDO docker create --hostname=app$i --name=app$i renecannao/proxysql:dev /bin/sh -c "while true; do sleep 3600; done" + $USESUDO docker start app$i + done +} + +shutdown() { + for i in `seq 1 ${NUMAPPS}` ; do + echo -n "Stopping container app$i ... " + ID=`$USESUDO docker ps -f name=app$i -q` + if [ -z "$ID" ]; then + echo "not found" + else + $USESUDO docker stop app$i + echo "done" + fi + done +} + +case $1 in + destroy) + destroy + ;; + prepare) + prepare + ;; + shutdown) + shutdown + ;; + *) + echo "Invalid argument" + ;; +esac diff --git a/docker/scenarios/repl1/configure.sh b/docker/scenarios/repl1/configure.sh new file mode 100755 index 000000000..f1be7b01c --- /dev/null +++ b/docker/scenarios/repl1/configure.sh @@ -0,0 +1,134 @@ +set -e +. ./vars + +declare -a AIPS +declare -a DIPS +declare -a PIPS + + +echo Retrieving IPs +for i in `seq 1 ${NUMSERVERS}` ; do + DIPS[$i]=`$USESUDO docker inspect -f '{{.NetworkSettings.IPAddress}}' db$i` +done + + +configure_mysql() { + echo Setting my.cnf + for i in `seq 1 ${NUMSERVERS}` ; do + $USESUDO docker exec db$i bash -c "echo -e \"[client]\npassword=$ROOTPASS\" > /root/.my.cnf" + done + + + echo "retrieving GTID executed" + GES=`$USESUDO docker exec -it db1 mysql -e "SHOW GLOBAL VARIABLES LIKE 'gtid_executed'" -NB | awk '{print $2}'` + + echo Setting grants on db1 + $USESUDO docker exec db1 mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO root@$DOCNET IDENTIFIED BY \"$ROOTPASS\" WITH GRANT OPTION" + for h in ${DIPS[*]} ; do + $USESUDO docker exec db1 mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO root@$h IDENTIFIED BY \"$ROOTPASS\" WITH GRANT OPTION" + $USESUDO docker exec db1 mysql -u root -e "GRANT REPLICATION SLAVE ON *.* TO replication@$h IDENTIFIED BY \"$REPLPASS\"" + done + + echo Setting replication + for i in `seq 2 ${NUMSERVERS}` ; do + $USESUDO docker exec db$i mysql -u root -e "STOP SLAVE" + $USESUDO docker exec db$i mysql -u root -e "RESET SLAVE" + $USESUDO docker exec db$i mysql -u root -e "RESET MASTER" + $USESUDO docker exec db$i mysql -u root -e "SET GLOBAL gtid_purged=\"$GES\"" + $USESUDO docker exec db$i mysql -u root -e "CHANGE MASTER TO MASTER_HOST=\"${DIPS[1]}\" , MASTER_USER='replication', MASTER_PASSWORD=\"$REPLPASS\" , MASTER_AUTO_POSITION=1" + $USESUDO docker exec db$i mysql -u root -e "START SLAVE" + done +} + +install_proxy_app() { + echo "Installing standard ProxySQL on apps" + for i in `seq 1 ${NUMAPPS}` ; do + $USESUDO docker cp proxysql*deb app$i:/tmp + $USESUDO docker exec app$i sh -c "dpkg -i /tmp/proxysql*deb" + done +} + +install_proxy_cluster() { + echo "Installing standard ProxySQL on cluster layer" + for i in `seq 1 ${NUMPROXIES}` ; do + $USESUDO docker cp proxysql*deb proxy$i:/tmp + $USESUDO docker exec proxy$i sh -c "dpkg -i /tmp/proxysql*deb" + done +## This is optional + echo "Replacing binaries in cluster layer" + for i in `seq 1 ${NUMPROXIES}` ; do + $USESUDO docker cp proxysql proxy$i:/usr/bin/proxysql + done +} + +start_proxy_cluster() { + echo "Starting ProxySQL in cluster layer" + for i in `seq 1 ${NUMPROXIES}` ; do + $USESUDO docker exec proxy$i service proxysql start + sleep 3 # wait some time for proxysql to start + ## this is already an important testing: proxysql allows to change listening port at runtime + $USESUDO docker exec proxy$i mysql -u admin -padmin -h 127.0.0.1 -P6032 -e "UPDATE global_variables SET variable_value='0.0.0.0:6032' WHERE variable_name='admin-mysql_ifaces'; SAVE ADMIN VARIABLES TO DISK; LOAD ADMIN VARIABLES TO RUNTIME;" + done + sleep 1 # wait some time for proxysql to bind on new port +} + +start_proxy_app() { + echo "Starting ProxySQL on Apps" + for i in `seq 1 ${NUMAPPS}` ; do + $USESUDO docker exec app$i service proxysql start + sleep 3 # wait some time for proxysql to start + ## this is already an important testing: proxysql allows to change listening port at runtime + $USESUDO docker exec app$i mysql -u admin -padmin -h 127.0.0.1 -P6032 -e "UPDATE global_variables SET variable_value='0.0.0.0:6032' WHERE variable_name='admin-mysql_ifaces'; SAVE ADMIN VARIABLES TO DISK; LOAD ADMIN VARIABLES TO RUNTIME;" + done + sleep 1 # wait some time for proxysql to bind on new port +} + + +configure_proxy_app() { + echo Configure ProxySQL on Apps + for i in ${AIPS[*]} ; do + echo "DELETE FROM mysql_servers;" | mysql -u admin -h $i -P6032 + for h in ${PIPS[*]} ; do + echo "INSERT INTO mysql_servers (hostgroup_id, hostname, port) VALUES (0,\"$h\",6033);" | mysql -u admin -h $i -P6032 + done + echo "LOAD MYSQL SERVERS TO RUNTIME; SAVE MYSQL SERVERS TO DISK;" | mysql -u admin -h $i -P6032 + echo "DELETE FROM mysql_users; INSERT INTO mysql_users (username, password) VALUES ('root',\"$ROOTPASS\"); LOAD MYSQL USERS TO RUNTIME; SAVE MYSQL USERS TO DISK;" | mysql -u admin -h $i -P6032 + done +} + +configure_proxy_cluster() { + echo Configure ProxySQL on Cluster Layer + for i in ${PIPS[*]} ; do + echo "DELETE FROM mysql_servers;" | mysql -u admin -h $i -P6032 + for h in ${DIPS[*]} ; do + echo "INSERT INTO mysql_servers (hostgroup_id, hostname, port) VALUES (0,\"$h\",3306);" | mysql -u admin -h $i -P6032 + done + echo "LOAD MYSQL SERVERS TO RUNTIME; SAVE MYSQL SERVERS TO DISK;" | mysql -u admin -h $i -P6032 + echo "DELETE FROM mysql_users; INSERT INTO mysql_users (username, password) VALUES ('root',\"$ROOTPASS\"); LOAD MYSQL USERS TO RUNTIME; SAVE MYSQL USERS TO DISK;" | mysql -u admin -h $i -P6032 + done +} + +echo Retrieving Application IPs +for i in `seq 1 ${NUMAPPS}` ; do + AIPS[$i]=`$USESUDO docker inspect -f '{{.NetworkSettings.IPAddress}}' app$i` +done + +echo Retrieving ProxySQL IPs +for i in `seq 1 ${NUMPROXIES}` ; do + PIPS[$i]=`$USESUDO docker inspect -f '{{.NetworkSettings.IPAddress}}' proxy$i` +done + + +#configure_mysql +#install_proxy_app +#install_proxy_cluster +#start_proxy_cluster +#start_proxy_app + +export MYSQL_PWD="admin" + +echo APP IPs: ${AIPS[*]} +echo ProxySQL IPs: ${PIPS[*]} + +configure_proxy_app +configure_proxy_cluster diff --git a/docker/scenarios/repl1/configure_repl_hostgroup.sh b/docker/scenarios/repl1/configure_repl_hostgroup.sh new file mode 100755 index 000000000..cffd050c1 --- /dev/null +++ b/docker/scenarios/repl1/configure_repl_hostgroup.sh @@ -0,0 +1,87 @@ +set -e +. ./vars + +declare -a AIPS +declare -a DIPS +declare -a PIPS + + +echo Retrieving IPs +for i in `seq 1 ${NUMSERVERS}` ; do + DIPS[$i]=`$USESUDO docker inspect -f '{{.NetworkSettings.IPAddress}}' db$i` +done + +echo Retrieving Application IPs +for i in `seq 1 ${NUMAPPS}` ; do + AIPS[$i]=`$USESUDO docker inspect -f '{{.NetworkSettings.IPAddress}}' app$i` +done + +echo Retrieving ProxySQL IPs +for i in `seq 1 ${NUMPROXIES}` ; do + PIPS[$i]=`$USESUDO docker inspect -f '{{.NetworkSettings.IPAddress}}' proxy$i` +done + + +configure_ro_dbs() { + export MYSQL_PWD=$ROOTPASS + for i in ${DIPS[*]} ; do + echo "Configuring read_only=$1 on $i" + mysql -u root -h $i -P3306 -e "SET GLOBAL read_only=$1" + done +} + + +dump_ro_proxies() { + echo "Dumping mysql_servers table" + for i in ${PIPS[*]} ; do + mysql -u admin -h $i -P6032 -e "SELECT * FROM mysql_servers" + done +} + +configure_repl_hostgroup() { + echo Configure Replication topology on Cluster Layer + for i in ${PIPS[*]} ; do + echo "Processing ProxySQL on $i" + echo "Configuring monitoring users" + echo "UPDATE global_variables SET variable_value='root' WHERE variable_name='mysql-monitor_username';" | mysql -u admin -h $i -P6032 + echo "UPDATE global_variables SET variable_value=\"$ROOTPASS\" WHERE variable_name='mysql-monitor_password';" | mysql -u admin -h $i -P6032 + echo "LOAD MYSQL VARIABLES TO RUNTIME" | mysql -u admin -h $i -P6032 + echo "SAVE MYSQL VARIABLES TO DISK" | mysql -u admin -h $i -P6032 + echo "Setting hostgroups" + echo "DELETE FROM mysql_replication_hostgroups;" | mysql -u admin -h $i -P6032 + echo "INSERT INTO mysql_replication_hostgroups VALUES(0,1);" | mysql -u admin -h $i -P6032 + echo "LOAD MYSQL SERVERS TO RUNTIME" | mysql -u admin -h $i -P6032 + echo "SAVE MYSQL SERVERS TO DISK" | mysql -u admin -h $i -P6032 + done +} + + +export MYSQL_PWD="admin" +configure_repl_hostgroup + +configure_ro_dbs 0 +export MYSQL_PWD="admin" +echo "ProxySQL should reconfigure the servers" +echo "All servers must be at least in hostgroup 0 , and optionally in hostgroup 1" +echo "We sleep few seconds..." +sleep 5 +dump_ro_proxies + +configure_ro_dbs 1 +export MYSQL_PWD="admin" +echo "ProxySQL should reconfigure the servers" +echo "All servers must be in hostgroup 1 , **NONE** in hostgroup 0" +echo "We sleep few seconds..." +sleep 5 +dump_ro_proxies + + + + + +#echo APP IPs: ${AIPS[*]} +#echo ProxySQL IPs: ${PIPS[*]} + + +configure_repl_hostgroup + diff --git a/docker/scenarios/repl1/mysql.sh b/docker/scenarios/repl1/mysql.sh new file mode 100755 index 000000000..6f4d5daf7 --- /dev/null +++ b/docker/scenarios/repl1/mysql.sh @@ -0,0 +1,59 @@ +set -e +. ./vars + +destroy() { + for i in `seq 1 ${NUMSERVERS}` ; do + echo -n "Destroying container db$i ... " + ID=`$USESUDO docker ps -a -f name=db$i -q` + if [ -z "$ID" ]; then + echo "not found" + else + $USESUDO docker rm db$i + echo "done" + fi + done +} + +prepare() { + for i in `seq 1 ${NUMSERVERS}` ; do + echo Creating container db$i + $USESUDO docker create --hostname=db$i --name=db$i -e MYSQL_ROOT_PASSWORD=${ROOTPASS} mysql:latest + RANID=$(($RANDOM*32768+$RANDOM)) + echo "Using random server_id $RANID" + sed -e "s/XXXX/$RANID/" mysql_add.cnf_ > mysql_add.cnf + $USESUDO docker cp mysql_add.cnf db$i:/etc/mysql/conf.d/ + done + rm mysql_add.cnf + for i in `seq 1 ${NUMSERVERS}` ; do + echo Starting container db$i + $USESUDO docker start db$i + done +} + +shutdown() { + for i in `seq 1 ${NUMSERVERS}` ; do + echo -n "Stopping container db$i ... " + ID=`$USESUDO docker ps -f name=db$i -q` + if [ -z "$ID" ]; then + echo "not found" + else + $USESUDO docker stop db$i + echo "done" + fi + done +} + +case $1 in + destroy) + destroy + ;; + prepare) + prepare + ;; + shutdown) + shutdown + ;; + *) + echo "Invalid argument" + ;; +esac diff --git a/docker/scenarios/repl1/mysql_add.cnf_ b/docker/scenarios/repl1/mysql_add.cnf_ new file mode 100644 index 000000000..236afdc89 --- /dev/null +++ b/docker/scenarios/repl1/mysql_add.cnf_ @@ -0,0 +1,19 @@ +[mysqld] +server_id = XXXX +innodb_file_per_table +innodb_flush_log_at_trx_commit=2 +query_cache_size = 0 +sync_binlog = 0 +log_bin +binlog_format = MIXED +gtid_mode = ON +log-slave-updates +enforce-gtid-consistency +expire_logs_days = 3 +max_binlog_size = 100M +bind_address = 0.0.0.0 +max_connections = 5000 +skip_name_resolve + +[mysqld_safe] +open_files_limit = 102400 diff --git a/docker/scenarios/repl1/proxy.sh b/docker/scenarios/repl1/proxy.sh new file mode 100755 index 000000000..f609570a0 --- /dev/null +++ b/docker/scenarios/repl1/proxy.sh @@ -0,0 +1,52 @@ +set -e +. ./vars + + +destroy() { + for i in `seq 1 ${NUMPROXIES}` ; do + echo -n "Destroying container proxy$i ... " + ID=`$USESUDO docker ps -a -f name=proxy$i -q` + if [ -z "$ID" ]; then + echo "not found" + else + $USESUDO docker rm proxy$i + echo "done" + fi + done +} + +prepare() { + for i in `seq 1 ${NUMPROXIES}` ; do + echo Creating and running container proxy$i + $USESUDO docker create --hostname=proxy$i --name=proxy$i renecannao/proxysql:dev /bin/sh -c "while true; do sleep 3600; done" + $USESUDO docker start proxy$i + done +} + +shutdown() { + for i in `seq 1 ${NUMPROXIES}` ; do + echo -n "Stopping container proxy$i ... " + ID=`$USESUDO docker ps -f name=proxy$i -q` + if [ -z "$ID" ]; then + echo "not found" + else + $USESUDO docker stop proxy$i + echo "done" + fi + done +} + +case $1 in + destroy) + destroy + ;; + prepare) + prepare + ;; + shutdown) + shutdown + ;; + *) + echo "Invalid argument" + ;; +esac diff --git a/docker/scenarios/repl1/test1.py b/docker/scenarios/repl1/test1.py new file mode 100644 index 000000000..82e2a0e3f --- /dev/null +++ b/docker/scenarios/repl1/test1.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +import MySQLdb +import os +import sys +import thread +import time +import signal +import threading + +qcounter = 0 +qcounter_lock = threading.Lock() +qOK = 0 +qOK_lock = threading.Lock() +qERR = 0 +qERR_lock = threading.Lock() + + + +# config +mysqluser = os.getenv('MYSQL_USER') +mysqlpass = os.getenv('MYSQL_PASS') +app_ips = os.getenv('APP_IPS') +database_ips = os.getenv('DATABASE_IPS') +proxy_ips = os.getenv('PROXY_IPS') + +app_list = [] +database_list = [] +proxy_list = [] + +def sigterm_handler(_signo, _stack_frame): + # Raises SystemExit(0): + sys.exit(0) + + + +def print_exception(e): + line_number = sys.exc_info()[2].tb_lineno + print "Line: " + str(line_number) + print e + +def thread_client_conn(app_ip): + conn = None + try: + conn = MySQLdb.connect(host=app_ip, port=6033, user=mysqluser, passwd=mysqlpass) + #conn = MySQLdb.connect(host="172.17.0.139", port=6033, user=mysqluser, passwd=mysqlpass) + + cur = conn.cursor() + while True: +# for x in range(0, 100): + global qcounter + global qOK + global qERR + qcounter_lock.acquire() + qcounter += 1 + x = qcounter + qcounter_lock.release() + time.sleep(1) + query = "SELECT " + str(x) + try: + cur.execute(query) + res = cur.fetchone() +# print res[0] + qOK_lock.acquire() + qOK += 1 + qOK_lock.release() + except Exception, e: + qERR_lock.acquire() + qERR += 1 + qERR_lock.release() + #print "Query failed" + except Exception, e: + print "Failed to connect" + print_exception(e) + finally: + if conn: + conn.close() + +def main(): +# signal.signal(signal.SIGTERM, sigterm_handler) + if not mysqluser or not mysqlpass or not database_ips or not app_ips or not proxy_ips: + sys.exit("environment incorrectly configured; aborting!") + app_list = app_ips.split() + database_list = database_ips.split() + proxy_list = proxy_ips.split() + try: + threads = [threading.Thread(target=thread_client_conn) for t in range(10)] + for t in threads: + t.setDaemon(True) + t.start() +# for t in threads: +# t.join() + while True: + time.sleep(1) + finally: + print "Queries result. OK: " , qOK , " ERR: " , qERR + +if __name__ == '__main__': + main() diff --git a/docker/scenarios/repl1/test1_.py b/docker/scenarios/repl1/test1_.py new file mode 100644 index 000000000..6bb65f0ad --- /dev/null +++ b/docker/scenarios/repl1/test1_.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python +import MySQLdb +import os +import sys +import thread +import time +import signal +import threading + +qcounter = 0 +qcounter_lock = threading.Lock() +qOK = 0 +qOK_lock = threading.Lock() +qERR = 0 +qERR_lock = threading.Lock() + + + +# config +mysqluser = os.getenv('MYSQL_USER') +mysqlpass = os.getenv('MYSQL_PASS') + + +def sigterm_handler(_signo, _stack_frame): + # Raises SystemExit(0): + sys.exit(0) + + + +def print_exception(e): + line_number = sys.exc_info()[2].tb_lineno + print "Line: " + str(line_number) + print e + +def thread_client_conn(): + conn = None + try: + conn = MySQLdb.connect(host="127.0.0.1", port=6033, user=mysqluser, passwd=mysqlpass) + #conn = MySQLdb.connect(host="172.17.0.139", port=6033, user=mysqluser, passwd=mysqlpass) + + cur = conn.cursor() + while True: +# for x in range(0, 100): + global qcounter + global qOK + global qERR + qcounter_lock.acquire() + qcounter += 1 + x = qcounter + qcounter_lock.release() + time.sleep(1) + query = "SELECT " + str(x) + try: + cur.execute(query) + res = cur.fetchone() +# print res[0] + qOK_lock.acquire() + qOK += 1 + qOK_lock.release() + except Exception, e: + qERR_lock.acquire() + qERR += 1 + qERR_lock.release() + #print "Query failed" + except Exception, e: + print "Failed to connect" + print_exception(e) + finally: + if conn: + conn.close() + +def main(): + signal.signal(signal.SIGTERM, sigterm_handler) + if not mysqluser or not mysqlpass: + sys.exit("environment incorrectly configured; aborting!") + try: + threads = [threading.Thread(target=thread_client_conn) for t in range(10)] + for t in threads: + t.setDaemon(True) + t.start() +# for t in threads: +# t.join() + while True: + time.sleep(1) + finally: + print "Queries result. OK: " , qOK , " ERR: " , qERR + +if __name__ == '__main__': + main() diff --git a/docker/scenarios/repl1/vars b/docker/scenarios/repl1/vars new file mode 100644 index 000000000..b6a0200c3 --- /dev/null +++ b/docker/scenarios/repl1/vars @@ -0,0 +1,7 @@ +export ROOTPASS=rootpass +export REPLPASS=replpass +export NUMAPPS=2 +export NUMSERVERS=5 +export NUMPROXIES=2 +export USESUDO=sudo +export DOCNET=`ifconfig docker0| grep "inet addr" | awk -F ':' '{print $2}' | awk '{print $1}'` diff --git a/docker/scenarios/repl1/vars2 b/docker/scenarios/repl1/vars2 new file mode 100755 index 000000000..7f6156598 --- /dev/null +++ b/docker/scenarios/repl1/vars2 @@ -0,0 +1,23 @@ +. ./vars + +declare -a AIPS +declare -a DIPS +declare -a PIPS + + +for i in `seq 1 ${NUMSERVERS}` ; do + DIPS[$i]=`$USESUDO docker inspect -f '{{.NetworkSettings.IPAddress}}' db$i` +done + + +for i in `seq 1 ${NUMAPPS}` ; do + AIPS[$i]=`$USESUDO docker inspect -f '{{.NetworkSettings.IPAddress}}' app$i` +done + +for i in `seq 1 ${NUMPROXIES}` ; do + PIPS[$i]=`$USESUDO docker inspect -f '{{.NetworkSettings.IPAddress}}' proxy$i` +done + +export APP_IPS="${AIPS[*]}" +export DATABASE_IPS="${DIPS[*]}" +export PROXY_IPS="${PIPS[*]}"