diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 08f396a114..913653edca 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -359,3 +359,90 @@ jobs:
       artifact-name: "boundary_${{ needs.product-metadata.outputs.product-version }}_linux_amd64.zip"
       go-version: ${{ needs.product-metadata.outputs.go-version }}
     secrets: inherit
+
+  bats:
+    runs-on: ubuntu-latest
+    name: CLI tests
+    if: "! github.event.pull_request.head.repo.fork"
+    needs:
+      - product-metadata
+      - build-linux
+    steps:
+      - uses: actions/checkout@v3
+      - name: Import GPG key for Boundary pass keystore
+        id: import_gpg
+        uses: crazy-max/ghaction-import-gpg@v5
+        with:
+          gpg_private_key: ${{ secrets.ENOS_GPG_PRIVATE_KEY }}
+          passphrase: ${{ secrets.ENOS_GPG_PASSPHRASE }}
+      - name: Trust the pass keystore GPG key
+        id: trust_gpg
+        run: |
+          gpg -a --encrypt -r ${{ secrets.ENOS_GPG_UID }} --trust-model always
+          echo "trusted-key ${{ secrets.ENOS_GPG_UID }}" >> ~/.gnupg/gpg.conf
+          cat ~/.gnupg/gpg.conf
+      - name: Set up Bats CLI UI tests dependency cache
+        id: dep-cache
+        uses: actions/cache@v3
+        with:
+          path: /tmp/bats-cli-ui-deps
+          key: enos-bats-cli-ui-deps-jq-1.6-password-store-1.7.4
+      - name: Set up Node for Bats install
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+      - name: Install Bats via NPM
+        # Use npm so this workflow is portable on multiple runner distros
+        run: npm install --location=global bats
+      - name: Download jq for Bats CLI UI tests
+        if: steps.dep-cache.outputs.cache-hit != 'true'
+        # NOTE: if you update the jq version make sure to update the dep cache key
+        run: |
+          mkdir -p /tmp/bats-cli-ui-deps
+          wget https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -O /tmp/bats-cli-ui-deps/jq-bin
+      - name: Install jq for Bats CLI UI tests
+        run: |
+          chmod +x /tmp/bats-cli-ui-deps/jq-bin
+          sudo cp /tmp/bats-cli-ui-deps/jq-bin /usr/local/bin/jq
+      - name: Download and unzip pass for Boundary keyring
+        if: steps.dep-cache.outputs.cache-hit != 'true'
+        # NOTE: if you update the password store version make sure to update the dep cache key
+        run: |
+          mkdir -p /tmp/bats-cli-ui-deps/pass
+          wget https://git.zx2c4.com/password-store/snapshot/password-store-1.7.4.tar.xz -O /tmp/bats-cli-ui-deps/pass/pass.tar.xz
+          cd /tmp/bats-cli-ui-deps/pass
+          tar -xvf pass.tar.xz
+      - name: Install pass for Boundary keyring
+        run: |
+          cd /tmp/bats-cli-ui-deps/pass/password-store-1.7.4
+          sudo make install
+          pass init ${{ secrets.ENOS_GPG_UID }}
+      - name: Download Linux AMD64 Boundary bundle
+        id: download
+        uses: actions/download-artifact@v3
+        with:
+          name: boundary_${{ needs.product-metadata.outputs.product-version }}_linux_amd64.zip
+          path: /tmp
+      - name: Unpack boundary bundle
+        run: |
+          unzip ${{steps.download.outputs.download-path}}/boundary_${{ needs.product-metadata.outputs.product-version }}_linux_amd64.zip -d /usr/local/bin
+          rm ${{steps.download.outputs.download-path}}/boundary_${{ needs.product-metadata.outputs.product-version }}_linux_amd64.zip
+      - name: Versions
+        run: |
+          echo "go version:"
+          go version
+          echo "bats version:"
+          bats --version
+          echo "jq version:"
+          jq --version
+          echo "gpg version:"
+          gpg --version
+          echo "pass version:"
+          pass --version
+          echo "bash version:"
+          bash --version
+          echo "boundary version:"
+          boundary version
+      - name: Run cli bats tests
+        run: |
+          make test-cli
diff --git a/Makefile b/Makefile
index 7cc4cbcbe3..c62ee20d05 100644
--- a/Makefile
+++ b/Makefile
@@ -266,6 +266,10 @@ test-sdk:
 test-api:
 	$(MAKE) -C api/ test
 
+.PHONY: test-cli
+test-cli:
+	$(MAKE) -C internal/tests/cli test
+
 .PHONY: test-all
 test-all: test-sdk test-api test
 
diff --git a/internal/tests/cli/Makefile b/internal/tests/cli/Makefile
new file mode 100644
index 0000000000..f153e4e374
--- /dev/null
+++ b/internal/tests/cli/Makefile
@@ -0,0 +1,5 @@
+all: test
+
+.PHONY: test
+test:
+	./test.sh
diff --git a/internal/tests/cli/README.md b/internal/tests/cli/README.md
index 0b290c2306..36c4704594 100644
--- a/internal/tests/cli/README.md
+++ b/internal/tests/cli/README.md
@@ -1,9 +1,9 @@
 # Boundary CLI Tests
 
-This directory contains [bats tests](https://github.com/bats-core/bats-core) for testing the Boundary CLI against arbitrary Boundary deployments. 
+This directory contains [bats tests](https://github.com/bats-core/bats-core) for testing the Boundary CLI against arbitrary Boundary deployments.
 The tests are meant to mimic common workflows such as creating resources, and connecting to targets. Currently, the tests rely heavily on
 generated resources when running Boundary in `dev` mode. In the future, we hope to remove this dependency and generate all resources through
-the Boundary CLI from the outset. 
+the Boundary CLI from the outset.
 
 The tests are designed to be idempotent.
 
@@ -13,6 +13,7 @@ The tests are designed to be idempotent.
 
 - [jq](https://stedolan.github.io/jq/)
 - [bats](https://github.com/bats-core/bats-core)
+- [netcat](https://nc110.sourceforge.io/)
 - [boundary](https://github.com/hashicorp/boundary)
 
 #### Running Tests
@@ -44,7 +45,7 @@ IS_VERSION=true bats -p boundary/
 You can run this suite against an arbitrary cluster by overriding the following env vars:
 
 ```bash
-DEFAULT_LOGIN=<my_login> \ 
+DEFAULT_LOGIN=<my_login> \
 DEFAULT_PASSWORD=<my_password> \
 BOUNDARY_ADDR=http://boundary-test-controller-salmon-b70f2710539143b4.elb.us-east-1.amazonaws.com:9200 \
 bats boundary/
@@ -52,4 +53,4 @@ bats boundary/
 
 ~> Note that these tests currently expect generated resources to exist. Failure to run against a Boundary deployment that does
 not have generated resources will result in a lot of test failures. Future work includes breaking thses tests out in a way
-that makes them not dependant on generated resources.
+that makes them not dependent on generated resources.
diff --git a/internal/tests/cli/test.sh b/internal/tests/cli/test.sh
new file mode 100755
index 0000000000..7789fc5b51
--- /dev/null
+++ b/internal/tests/cli/test.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# TERM isn't set automatically in CI so we need to make sure it's always there.
+export TERM=${TERM:=dumb}
+
+function die {
+    echo $*
+    exit -1
+}
+
+# error out early if missing a command
+which boundary || die "missing boundary"
+which jq       || die "missing jq"
+which bats     || die "missing bats"
+which nc       || die "missing nc"
+
+echo "starting boundary dev in background"
+boundary dev &>/dev/null &
+boundary_pid=$!
+
+function cleanup {
+    rv=$?
+    echo "stopping boundary dev"
+    if [[ -n ${boundary_pid} ]]; then
+        kill ${boundary_pid}
+    fi
+    exit $rv
+}
+
+trap cleanup EXIT
+
+until boundary scopes list; do
+    echo 'waiting for boundary to be up'
+    sleep 1
+done
+
+echo "running bats tests"
+bats -p ./boundary