mirror of https://github.com/hashicorp/terraform
Execute equivalence tests on every pull request (#35976)
Liam Cervante
1 year ago
committed by
GitHub
parent
c816275625
commit
449aa908b9
@ -1,45 +0,0 @@
|
||||
name: crt-hook-equivalence-tests
|
||||
|
||||
on:
|
||||
repository_dispatch:
|
||||
types:
|
||||
- crt-hook-equivalence-tests::terraform::*
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
parse-metadata:
|
||||
name: "Parse metadata.json"
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.parse.outputs.version }}
|
||||
target-branch: ${{ steps.parse.outputs.target-branch }}
|
||||
steps:
|
||||
- name: parse
|
||||
id: parse
|
||||
env:
|
||||
METADATA_PAYLOAD: ${{ toJSON(github.event.client_payload.payload) }}
|
||||
run: |
|
||||
VERSION=$(echo ${METADATA_PAYLOAD} | jq -r '.version')
|
||||
TARGET_BRANCH=$(./.github/scripts/equivalence-test.sh get-target-branch "$VERSION")
|
||||
|
||||
echo "target-branch=$TARGET_BRANCH" >> "GITHUB_OUTPUT"
|
||||
echo "version=$VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
run-equivalence-tests:
|
||||
runs-on: ubuntu-latest
|
||||
name: "Run equivalence tests"
|
||||
needs:
|
||||
- parse-metadata
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
ref: ${{ needs.parse-metadata.outputs.target-branch }}
|
||||
- uses: ./.github/actions/equivalence-test
|
||||
with:
|
||||
target-terraform-version: ${{ needs.parse-metadata.outputs.version }}
|
||||
target-terraform-branch: ${{ needs.parse-metadata.outputs.target-branch }}
|
||||
target-equivalence-test-version: 0.3.0
|
||||
target-os: linux
|
||||
target-arch: amd64
|
||||
@ -0,0 +1,70 @@
|
||||
name: equivalence-test-diff
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types:
|
||||
- synchronize
|
||||
- ready_for_review
|
||||
- reopened
|
||||
- synchronize
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
equivalence-test-diff:
|
||||
name: "Equivalence Test Diff"
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Fetch source code
|
||||
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
|
||||
- name: Determine Go version
|
||||
id: go
|
||||
uses: ./.github/actions/go-version
|
||||
|
||||
- name: Install Go toolchain
|
||||
uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
|
||||
with:
|
||||
go-version: ${{ steps.go.outputs.version }}
|
||||
cache-dependency-path: go.sum
|
||||
|
||||
- name: Download testing framework
|
||||
shell: bash
|
||||
run: |
|
||||
./.github/scripts/equivalence-test.sh download_equivalence_test_binary \
|
||||
0.3.0 \
|
||||
./bin/equivalence-tests \
|
||||
linux \
|
||||
amd64
|
||||
|
||||
- name: Build terraform
|
||||
shell: bash
|
||||
run: ./.github/scripts/equivalence-test.sh build_terraform_binary ./bin/terraform
|
||||
|
||||
- name: Run equivalence tests
|
||||
id: equivalence-tests
|
||||
shell: bash {0} # we want to capture the exit code
|
||||
run: |
|
||||
./bin/equivalence-tests diff \
|
||||
--tests=testing/equivalence-tests/tests \
|
||||
--goldens=testing/equivalence-tests/outputs \
|
||||
--binary=$(pwd)/bin/terraform
|
||||
echo "exit-code=$?" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Equivalence tests failed
|
||||
if: steps.equivalence-tests.outputs.exit-code == 1 # 1 is the exit code for failure
|
||||
shell: bash
|
||||
run: |
|
||||
gh pr comment ${{ github.event.pull_request.number }} \
|
||||
--body "The equivalence tests failed. Please investigate [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${{ github.job }})."
|
||||
exit 1 # fail the job
|
||||
|
||||
- name: Equivalence tests changed
|
||||
if: steps.equivalence-tests.outputs.exit-code == 2 # 2 is the exit code for changed
|
||||
shell: bash
|
||||
run: |
|
||||
gh pr comment ${{ github.event.pull_request.number }} \
|
||||
--body "The equivalence tests will be updated. Please verify the changes [here](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/job/${{ github.job }})."
|
||||
@ -0,0 +1,66 @@
|
||||
name: equivalence-test-update
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
types: [ closed ]
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: "Should run equivalence tests?"
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
should_run: ${{ steps.target_branch.outputs.should_run }}
|
||||
steps:
|
||||
- name: target_branch
|
||||
id: target_branch
|
||||
run: |
|
||||
merged = ${{ github.event.pull_request.merged }}
|
||||
target_branch = ${{ github.event.pull_request.base.ref }}
|
||||
|
||||
targets_release_branch = false
|
||||
if [ "$target_branch" == "main" ]; then
|
||||
targets_release_branch = true
|
||||
elif [ "$target_branch" =~ ^v[0-9]+\.[0-9]+$ ]; then
|
||||
targets_release_branch = true
|
||||
fi
|
||||
|
||||
should_run=false
|
||||
if [ "$merged" == "true" ] && [ "$targets_release_branch" == "true" ]; then
|
||||
should_run=true
|
||||
fi
|
||||
|
||||
echo "should_run=$should_run" >> ${GITHUB_OUTPUT}
|
||||
run-equivalence-tests:
|
||||
name: "Run equivalence tests"
|
||||
needs:
|
||||
- check
|
||||
if: needs.check.outputs.should_run == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||
with:
|
||||
ref: ${{ inputs.target-branch }}
|
||||
|
||||
- name: Determine Go version
|
||||
id: go
|
||||
uses: ./.github/actions/go-version
|
||||
|
||||
- name: Install Go toolchain
|
||||
uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
|
||||
with:
|
||||
go-version: ${{ steps.go.outputs.version }}
|
||||
cache-dependency-path: go.sum
|
||||
|
||||
- uses: ./.github/actions/equivalence-test
|
||||
with:
|
||||
target-equivalence-test-version: ${{ inputs.equivalence-test-version }}
|
||||
target-os: linux
|
||||
target-arch: amd64
|
||||
current-branch: ${{ github.event.pull_request.base.ref }}
|
||||
new-branch: equivalence-testing/${{ github.event.pull_request.head.ref }}
|
||||
reviewers: ${{ github.event.pull_request.merged_by.login }}
|
||||
message: "Update equivalence test golden files after ${{ github.event.pull_request.url }}."
|
||||
@ -0,0 +1,46 @@
|
||||
# Equivalence testing
|
||||
|
||||
This directory contains the test cases for the equivalence testing. The
|
||||
Terraform equivalence tests are E2E tests that are used to verify that the
|
||||
output of Terraform commands doesn't change in unexpected ways. The tests are
|
||||
run by comparing the output of the Terraform commands before and after a change
|
||||
to the codebase.
|
||||
|
||||
## Running the tests
|
||||
|
||||
The equivalence tests are executed by the Terraform equivalence testing
|
||||
framework. This is built in [github.com/hashicorp/terraform-equivalence-testing](https://github.com/hashicorp/terraform-equivalence-testing).
|
||||
|
||||
To execute the tests you must download the `terraform-equivalence-testing`
|
||||
binary and execute either the `diff` or `update` command. The `diff` command
|
||||
will run the tests and output the differences between the current and previous
|
||||
run. The `update` command will run the tests and update the reference output
|
||||
files.
|
||||
|
||||
You can also execute the tests directly using the `equivalence-tests-manual`
|
||||
GitHub action. This action will run the tests against a given branch and
|
||||
open a PR with the results.
|
||||
|
||||
## Automated testing
|
||||
|
||||
The equivalence tests are run automatically by the Terraform CI system. The
|
||||
tests are run when every pull request is opened and when every pull request
|
||||
is closed.
|
||||
|
||||
When pull requests are opened, the tests run the diff command and will comment
|
||||
on the PR with the results. PR authors should validate any changes to the output
|
||||
files and make sure that the changes are expected.
|
||||
|
||||
When pull requests are closed, the tests run the update command and open a new
|
||||
PR with the updated reference output files. PR authors should review the changes
|
||||
and make sure that the changes are expected before merging the automated PR.
|
||||
|
||||
If the framework detects no changes, the process should be invisible to the PR
|
||||
author. No comments will be made on the PR and no new PRs will be opened.
|
||||
|
||||
## Writing new tests
|
||||
|
||||
New tests should be written into the `tests` directory. Each test should be
|
||||
written in a separate directory and should follow the guidelines in the
|
||||
equivalence testing framework documentation. Any tests added to this directory
|
||||
will be picked up the CI system and run automatically.
|
||||
Loading…
Reference in new issue