From ecdcf72e0e61496beb68dfa8d428ea3de3df1ba5 Mon Sep 17 00:00:00 2001 From: Timothy Messier Date: Tue, 24 Aug 2021 14:48:11 -0400 Subject: [PATCH] ci: Update Makefile with new targets for testing This updates the test-ci target to startup the test database container first, wait for it to be ready, and then start running the tests. It also creates some new targets for local devs. Prior to running any tests either via `make test` or `go test`, the database should be started with: make test-database-up It can take a few seconds for the database to fully initialize, the status can be checked with: docker logs -f boundary-sql-tests This database can be stopped and destroyed with: make test-database-down In general it should only need to be restarted when new migrations are added. But it might also be necessary to restart if some tests fail in an unexpected way and fail to do cleanup steps. --- .circleci/config.yml | 22 +++++-- .circleci/config/jobs/build.yml | 14 ++++- .circleci/config/jobs/test-sql.yml | 2 +- CONTRIBUTING.md | 97 +++++++++++++++++++++++++++++- Makefile | 12 +++- 5 files changed, 136 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 781135b83b..049f97ccb7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ version: 2 jobs: test-sql-13-alpine: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-2004:202107-02 steps: - checkout - run: @@ -249,7 +249,7 @@ jobs: - .buildcache/packages/store test-sql-11-alpine: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-2004:202107-02 steps: - checkout - run: @@ -887,7 +887,7 @@ jobs: - .buildcache/packages/store test-sql-12-alpine: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-2004:202107-02 steps: - checkout - run: @@ -969,14 +969,26 @@ jobs: - .buildcache/packages/store build: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-2004:202107-02 + resource_class: large steps: - checkout + - run: + command: | + which pg_isready || sudo apt-get update && sudo apt-get install -y postgresql-client + make DOCKER_ARGS='-d' -C testing/dbtest/docker database-up + until pg_isready -h 127.0.0.1; do docker container inspect boundary-sql-tests &> /dev/null || exit -1; sleep 1; done + name: Initialize Test Database - run: command: | make test-ci name: Run Acceptance Tests no_output_timeout: 15m + - run: + command: | + make -C testing/dbtest/docker clean + name: Cleanup + when: always - run: command: | echo 'export SLACK_BUILD_STATUS="fail"' >> $BASH_ENV @@ -1467,7 +1479,7 @@ jobs: - .buildcache/packages/store test-sql-latest: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-2004:202107-02 steps: - checkout - run: diff --git a/.circleci/config/jobs/build.yml b/.circleci/config/jobs/build.yml index 2289ec8c14..87a76947a6 100644 --- a/.circleci/config/jobs/build.yml +++ b/.circleci/config/jobs/build.yml @@ -1,13 +1,25 @@ +resource_class: large machine: - image: 'ubuntu-1604:201903-01' + image: 'ubuntu-2004:202107-02' working_directory: ~/boundary steps: - checkout +- run: + name: "Initialize Test Database" + command: | + which pg_isready || sudo apt-get update && sudo apt-get install -y postgresql-client + make DOCKER_ARGS='-d' -C testing/dbtest/docker database-up + until pg_isready -h 127.0.0.1; do docker container inspect boundary-sql-tests &> /dev/null || exit -1; sleep 1; done - run: name: "Run Acceptance Tests" no_output_timeout: 15m command: | make test-ci +- run: + name: "Cleanup" + when: always + command: | + make -C testing/dbtest/docker clean - slack/status: fail_only: true only_for_branches: master diff --git a/.circleci/config/jobs/test-sql.yml b/.circleci/config/jobs/test-sql.yml index c651d2d5e0..f82933aa11 100644 --- a/.circleci/config/jobs/test-sql.yml +++ b/.circleci/config/jobs/test-sql.yml @@ -1,5 +1,5 @@ machine: - image: 'ubuntu-1604:201903-01' + image: 'ubuntu-2004:202107-02' working_directory: ~/boundary parameters: postgres-version: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b06d3bfb8d..6cd3dfcd65 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -70,7 +70,40 @@ Any bug fixes fall into this section. ## Testing -To run the entire test suite run this command in the root of the project: +Most tests require a postgres database instance to successfully run. +This is provided via docker +by running a customized postgres image that is optimized for boundary tests. + +Before running the test suite, this docker container must be started: + +``` +$ make test-database-up +``` + +This can take a few seconds to initialize as it will +create a template database with the boundary migrations. +The progress can be checked b running: + +``` +$ docker logs -f boundary-sql-tests +``` + +Once a log line like the following is seen, the container is ready for running +tests: + +``` +PostgreSQL init process complete; ready for start up. +``` + +Alternatively if the `pg_isready` command is installed, it can be used to +determine if the container is ready, i.e.: + +``` +$ until pg_isready -h 127.0.0.1; do sleep 1; done +``` + +To run the entire test suite run this command in the root of the project +once the test database is ready: ``` $ make test @@ -88,3 +121,65 @@ run: ``` $ go test -run TestAuthTokenAuthenticator -v ./internal/auth ``` + +### Stopping the test database container + +The test database container can be shutdown using: + +``` +$ make test-database-down +``` + +Note that the container does *not* need to be shutdown between each run of +`make test` or `go test`. + +### Test database container options + +By default the container uses the host port of 5432. +This can changed using an environment variable: + +``` +$ export TEST_DB_PORT=5433 +$ make test-database-up +$ make test +``` + +The default docker image is built using the `postgres:11` base image. +The image can be changed using a make option to test against other versions: + +``` +$ make IMAGE_TAG=docker.io/hashicorpboundary/postgres:12-alpine test-database-up +$ make IMAGE_TAG=docker.io/hashicorpboundary/postgres:13-alpine test-database-up +$ make IMAGE_TAG=docker.io/hashicorpboundary/postgres:alpine test-database-up +``` + +Additional options can be passed to postgres to customize and override the +configuration in the config file of the docker image. +See the troubleshooting section below for more details. + +### Troubleshooting test database container + +The postgres configuration file included in the image +is optimized to support running the full test suite in parallel in CI. +As such, there may be issues starting the container locally, +especially in cases where the container has less then 4GB of memory. + +This is likely the case if the output of `docker logs boundary-sql-tests` shows: + +``` +pg_ctl: could not start server +``` + +In this case adjust the +[max_connections](https://www.postgresql.org/docs/11/runtime-config-connection.html#GUC-MAX-CONNECTIONS) +and/or +[shared_buffers](https://www.postgresql.org/docs/11/runtime-config-resource.html#GUC-SHARED-BUFFERS): + +``` +make PG_OPTS="-c max_connections=1000" test-database-up +``` + +Note that if `max_connections` is set too low, it may result in sporadic test +failures if a connection cannot be established. In this case, reduce the number +of concurrent tests via `GOMAXPROCS` or selectively run tests. + diff --git a/Makefile b/Makefile index b0bc24aef0..9a5e4c2e4b 100644 --- a/Makefile +++ b/Makefile @@ -157,14 +157,20 @@ website-install: website-start: @npm start --prefix website/ +test-database-up: + make -C testing/dbtest/docker database-up + +test-database-down: + make -C testing/dbtest/docker clean + test-ci: install-go ~/.go/bin/go test ./... -v $(TESTARGS) -timeout 120m test-sql: $(MAKE) -C internal/db/sqltest/ test -test: - ~/.go/bin/go test ./... -timeout 30m +test: + go test ./... -timeout 30m install-go: ./ci/goinstall.sh @@ -207,7 +213,7 @@ docker-publish: docker push $(IMAGE_TAG) docker push hashicorp/boundary:latest -.PHONY: api cli tools gen migrations proto website ci-config ci-verify set-ui-version docker docker-build docker-build-dev docker-publish +.PHONY: api cli tools gen migrations proto website ci-config ci-verify set-ui-version docker docker-build docker-build-dev docker-publish test-database-up test-database-down .NOTPARALLEL: