--- name: enos on: # Only trigger this working using workflow_call. It assumes that secrets are # being inherited from the caller. workflow_call: inputs: artifact-name: required: true type: string edition: required: true type: string go-version: required: true type: string docker-image-file: required: false type: string jobs: setup: outputs: cache-go-mod: ${{ steps.go-cache-paths.outputs.go-mod }} go-mod-cache-key: ${{ steps.go-mod-cache-key.outputs.key }} runs-on: ${{ fromJSON(vars.RUNNER) }} steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: fetch-depth: '0' - name: Set up Go uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 with: go-version: ${{ inputs.go-version }} cache: false - name: Determine Go cache paths id: go-cache-paths run: | echo "go-mod=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" - name: Determine Go modules cache key id: go-mod-cache-key run: | echo "key=go-mod-${{ hashFiles('**/go.sum') }}" >> "$GITHUB_OUTPUT" - name: Set up Go modules cache uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | ${{ steps.go-cache-paths.outputs.go-mod }} key: ${{ steps.go-mod-cache-key.outputs.key }} restore-keys: | go-mod - name: Download Go modules run: | go mod download enos: name: test needs: - setup # Enos jobs are still a bit flaky, ensure they don't fail the workflow. continue-on-error: true strategy: fail-fast: false # don't fail as that can skip required cleanup steps for jobs matrix: include: - filter: 'e2e_aws builder:crt ip_version:4' - filter: 'e2e_database' - filter: 'e2e_docker_base builder:crt' - filter: 'e2e_docker_base_connect builder:crt' - filter: 'e2e_docker_base_plus builder:crt' - filter: 'e2e_docker_base_with_gcp builder:crt' - filter: 'e2e_docker_base_with_vault builder:crt' - filter: 'e2e_docker_base_with_worker builder:crt' - filter: 'e2e_docker_base_with_worker_version builder:crt' - filter: 'e2e_docker_worker_registration_controller_led builder:crt' - filter: 'e2e_docker_worker_registration_worker_led builder:crt' runs-on: ${{ fromJSON(vars.RUNNER_LARGE) }} env: GITHUB_TOKEN: ${{ secrets.SERVICE_USER_GITHUB_TOKEN }} # Scenario variables ENOS_DEBUG_DATA_ROOT_DIR: ./enos/support/debug-data ENOS_VAR_aws_region: us-east-1 ENOS_VAR_aws_ssh_keypair_name: ${{ github.event.repository.name }}-ci-ssh-key ENOS_VAR_aws_ssh_private_key_path: ./support/private_key.pem ENOS_VAR_local_boundary_dir: ./support/boundary ENOS_VAR_local_boundary_src_dir: ${{ github.workspace }} ENOS_VAR_local_boundary_ui_src_dir: ./support/src/boundary-ui ENOS_VAR_crt_bundle_path: ./support/boundary.zip ENOS_VAR_test_email: ${{ secrets.SERVICE_USER_EMAIL }} ENOS_VAR_boundary_edition: ${{ inputs.edition }} ENOS_VAR_boundary_docker_image_file: ./support/boundary_docker_image.tar ENOS_VAR_go_version: ${{ inputs.go-version }} ENOS_VAR_gcp_project_id: ${{ secrets.GCP_PROJECT_ID_CI }} ENOS_VAR_gcp_client_email: ${{ secrets.GCP_CLIENT_EMAIL_CI }} ENOS_VAR_gcp_private_key_id: ${{ secrets.GCP_PRIVATE_KEY_ID_CI }} ENOS_VAR_gcp_private_key: ${{ secrets.GCP_PRIVATE_KEY_CI }} steps: - name: Checkout uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Set up Go uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0 with: go-version: ${{ inputs.go-version }} cache: false - name: Set up Go modules cache id: go-mod-cache uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: | ${{ needs.setup.outputs.cache-go-mod }} key: ${{ needs.setup.outputs.go-mod-cache-key }} restore-keys: | go-mod - name: Downloads Go modules if cache miss if: steps.go-mod-cache.outputs.cache-hit != 'true' run: | go mod download - name: Set up Terraform uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # TSCCR: loading action configs: failed to query HEAD reference: failed to get advertised references: authorization failed with: # the terraform wrapper will break Terraform execution in enos because # it changes the output to text when we expect it to be JSON. terraform_wrapper: false - name: Import GPG key for Boundary pass keystore id: import_gpg uses: crazy-max/ghaction-import-gpg@e89d40939c28e39f97cf32126055eeae86ba74ec # v6.3.0 with: gpg_private_key: ${{ secrets.ENOS_GPG_PRIVATE_KEY }} passphrase: ${{ secrets.ENOS_GPG_PASSPHRASE }} - name: Trust the pass keystore GPG key id: trust_gpg run: | gpg -a --encrypt -r ${{ secrets.ENOS_GPG_UID }} --trust-model always echo "trusted-key ${{ secrets.ENOS_GPG_UID }}" >> ~/.gnupg/gpg.conf cat ~/.gnupg/gpg.conf - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@00943011d9042930efac3dcd3a170e4273319bc8 # v5.1.0 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_CI }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_CI }} aws-region: us-east-1 role-to-assume: ${{ secrets.AWS_ROLE_ARN_CI }} role-skip-session-tagging: true role-duration-seconds: 3600 - name: Configure GCP credentials if: contains(matrix.filter, 'gcp') id: gcp_auth uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 with: credentials_json: ${{ secrets.GCP_CREDENTIALS }} access_token_lifetime: '3600s' project_id: ${{ secrets.GCP_PROJECT_ID_CI }} - name: 'Set up GCP Cloud SDK' if: contains(matrix.filter, 'gcp') uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db # v3.0.1 - name: Set up Enos uses: hashicorp/action-setup-enos@v1 # TSCCR: loading action configs: failed to query HEAD reference: failed to get advertised references: authorization failed with: github-token: ${{ secrets.SERVICE_USER_GITHUB_TOKEN }} - name: Prepare scenario dependencies id: prepare_scenario run: | mkdir -p ./enos/support echo "${{ secrets.SSH_KEY_PRIVATE_CI }}" > ./enos/support/private_key.pem chmod 600 ./enos/support/private_key.pem echo "debug_data_artifact_name=enos-debug-data_$(echo ${{ matrix.filter }} | sed -e 's/ /_/g' | sed -e 's/:/=/g')" >> "$GITHUB_OUTPUT" - name: Set up dependency cache id: dep-cache uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: /tmp/test-deps key: enos-bats-cli-ui-deps-jq-1.6-password-store-1.7.4-vault-1.12.2 - name: Debug dep-cache run: | mkdir -p /tmp/test-deps ls -la /tmp/test-deps - name: Download jq if: steps.dep-cache.outputs.cache-hit != 'true' # NOTE: if you update the jq version make sure to update the dep cache key run: | mkdir -p /tmp/test-deps wget https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -O /tmp/test-deps/jq-bin - name: Install jq run: | chmod +x /tmp/test-deps/jq-bin sudo cp /tmp/test-deps/jq-bin /usr/local/bin/jq - name: Download and unzip pass for Boundary keyring if: steps.dep-cache.outputs.cache-hit != 'true' # NOTE: if you update the password store version make sure to update the dep cache key run: | mkdir -p /tmp/test-deps/pass wget https://github.com/zx2c4/password-store/archive/refs/tags/1.7.4.tar.gz -O /tmp/test-deps/pass/pass.tar.gz cd /tmp/test-deps/pass tar -xvf pass.tar.gz - name: Install pass for Boundary keyring run: | cd /tmp/test-deps/pass/password-store-1.7.4 sudo make install pass init ${{ secrets.ENOS_GPG_UID }} - name: Download Vault AMD64 binary if: steps.dep-cache.outputs.cache-hit != 'true' run: | wget https://releases.hashicorp.com/vault/1.12.2/vault_1.12.2_linux_amd64.zip -O /tmp/test-deps/vault.zip - name: Install Vault CLI if: contains(matrix.filter, 'vault') || contains(matrix.filter, 'e2e_docker') || matrix.filter == 'e2e_database' run: | unzip /tmp/test-deps/vault.zip -d /usr/local/bin - name: GH fix for localhost resolution if: github.repository == 'hashicorp/boundary' && contains(matrix.filter, 'e2e_docker') run: | cat /etc/hosts && echo "-----------" sudo sed -i 's/::1 *localhost ip6-localhost ip6-loopback/::1 ip6 -localhost ip6-loopback/g' /etc/hosts cat /etc/hosts ssh -V - name: Download Boundary Linux AMD64 bundle id: download uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: ${{ inputs.artifact-name }} path: ./enos/support/downloads - name: Unzip and rename Boundary bundle run: | unzip ${{steps.download.outputs.download-path}}/*.zip -d enos/support/boundary mv ${{steps.download.outputs.download-path}}/*.zip enos/support/boundary.zip - name: Download Boundary Linux AMD64 docker image if: contains(matrix.filter, 'e2e_docker') uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 id: download-docker with: name: ${{ inputs.docker-image-file }} path: ./enos/support/downloads - name: Rename docker image file if: contains(matrix.filter, 'e2e_docker') run: | mv ${{ steps.download-docker.outputs.download-path }}/*.tar enos/support/boundary_docker_image.tar - name: Install other test dependencies if: contains(matrix.filter, 'e2e_docker_base_plus') run: | # mongosh wget -q https://github.com/mongodb-js/mongosh/releases/download/v2.7.0/mongosh-2.7.0-linux-x64.tgz -O /tmp/mongosh.tgz tar -xzf /tmp/mongosh.tgz -C "$HOME" echo "$HOME/mongosh-2.7.0-linux-x64/bin" >> "$GITHUB_PATH" # cqlsh (cassandra CLI) wget -q https://archive.apache.org/dist/cassandra/4.1.10/apache-cassandra-4.1.10-bin.tar.gz -O /tmp/cassandra.tar.gz tar -xzf /tmp/cassandra.tar.gz -C "$HOME" echo "$HOME/apache-cassandra-4.1.10/bin" >> "$GITHUB_PATH" # redis sudo apt update && sudo apt install -y redis-tools mysql-client - name: Output Terraform version info run: | mkdir -p ./enos/terraform-plugin-cache export ENOS_VAR_enos_user=$GITHUB_ACTOR && \ enos scenario check --chdir ./enos ${{ matrix.filter }} && \ enos scenario exec --chdir ./enos ${{ matrix.filter }} --cmd "version" - name: Determine boundary version to test against previous worker version # Get the Boundary version number and decrement the minor version by 1 if: contains(matrix.filter, 'e2e_docker_base_with_worker_version') run: | version_num=$(./enos/support/boundary/boundary version | awk -F'[. ]' '/Version Number/ {print $(NF-1)}') export ENOS_VAR_worker_version="$((version_num - 1))" echo "ENOS_VAR_worker_version=0.$ENOS_VAR_worker_version" >> "$GITHUB_ENV" - name: Run Enos scenario id: run # Continue once and retry continue-on-error: true run: | mkdir -p ./enos/terraform-plugin-cache export ENOS_VAR_enos_user=$GITHUB_ACTOR && \ enos scenario launch --timeout 60m0s --chdir ./enos ${{ matrix.filter }} - name: Rename e2e tests output continue-on-error: true run: | pushd enos scenario="${{ matrix.filter }}" count=$(find ./*.log 2>/dev/null | wc -l | xargs) if [ "$count" != 0 ] then for f in *.log; do mv -- "$f" "${f%.log}_${scenario%% *}.log"; done fi popd - name: Split matrix filter name id: split run: | SCENARIO=$(echo "${{ matrix.filter }}" | cut -d' ' -f1,3 | sed 's/:/_/g') echo fragment="${SCENARIO}" >> "$GITHUB_OUTPUT" - name: Upload e2e tests output uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: test-${{ steps.split.outputs.fragment }} path: enos/*.log retention-days: 5 - name: Get logs from controller container # Retrieve logs from the worker container on a failed # run to help diagnose a deadlock issue if: contains(matrix.filter, 'e2e_docker') && steps.run.outcome == 'failure' run: | docker logs boundary - name: Get logs from worker container # Retrieve logs from the worker container on a failed # run to help diagnose a deadlock issue if: contains(matrix.filter, 'e2e_docker') && contains(matrix.filter, 'worker') && steps.run.outcome == 'failure' run: | docker logs worker - name: Get logs from postgres container # Retrieve logs from the postgres container on a failed # run to help diagnose a deadlock issue if: contains(matrix.filter, 'e2e_docker') && steps.run.outcome == 'failure' run: | docker logs database - name: Retry Enos scenario id: run_retry if: steps.run.outcome == 'failure' run: | export ENOS_VAR_enos_user=$GITHUB_ACTOR && \ enos scenario launch --timeout 60m0s --chdir ./enos ${{ matrix.filter }} - name: Upload Debug Data if: ${{ always() && steps.run_retry.outcome == 'failure' }} uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: # The name of the artifact is the same as the matrix scenario name with the spaces replaced with underscores and colons replaced by equals. name: ${{ steps.prepare_scenario.outputs.debug_data_artifact_name }} path: ${{ env.ENOS_DEBUG_DATA_ROOT_DIR }} retention-days: 30 - name: Destroy Enos scenario id: destroy continue-on-error: true if: ${{ always() }} run: | export ENOS_VAR_enos_user=$GITHUB_ACTOR && \ enos scenario destroy --timeout 60m0s --chdir ./enos ${{ matrix.filter }} - name: Get logs for aws dependencies error # Retrieve logs from the terraform to help diagnose some aws cleanup issues if: ${{ always() && steps.destroy.outcome == 'failure' }} continue-on-error: true run: | enos scenario exec --cmd graph --chdir ./enos ${{ matrix.filter }} TF_DIR=$(find ./enos/.enos/ -type d -mindepth 1 -maxdepth 1 | tail -1) pushd "${TF_DIR}" terraform state list terraform state show module.create_base_infra.aws_route.igw popd - name: Destroy Enos scenario (Retry) if: ${{ always() && steps.destroy.outcome == 'failure' }} run: | export ENOS_VAR_enos_user=$GITHUB_ACTOR && \ enos scenario destroy --timeout 60m0s --chdir ./enos ${{ matrix.filter }} - name: Output Enos debug information on failure if: ${{ failure() }} run: | env find ./enos -name "scenario.tf" -exec cat {} \; - name: Send Slack message if Run and Retry fails (or if something else went wrong) uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 # steps.run.outcome reports as failure when there is an error in `Run Enos scenario` # failure() captures errors before `Run Enos scenario` # failure() does not capture errors in `Run Enos scenario` due to continue-on-error if: ${{ failure() || (steps.run.outcome == 'failure' && steps.run_retry.outcome == 'failure') }} with: method: chat.postMessage token: ${{ secrets.SLACK_BOUNDARY_TEST_BOT_TOKEN }} payload: | channel: ${{ secrets.SLACK_BOUNDARY_TEST_BOT_CHANNEL_ID }} text: ":x: e2e tests failed (${{ matrix.filter }}): ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\n*Branch:* ${{ github.repository }}:${{ github.head_ref || github.ref_name }}" - name: Send Slack message if Run but Retry passes uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1 if: ${{ steps.run.outcome == 'failure' && steps.run_retry.outcome != 'failure' }} with: method: chat.postMessage token: ${{ secrets.SLACK_BOUNDARY_TEST_BOT_TOKEN }} payload: | channel: ${{ secrets.SLACK_BOUNDARY_TEST_BOT_CHANNEL_ID }} text: ":warning: e2e tests passed, but needed retry (${{ matrix.filter }}): ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\n*Branch:* ${{ github.repository }}:${{ github.head_ref || github.ref_name }}"