chore(vitest): run tests from the repo root with balanced shards

.github/workflows/develop-tests.yml

@@ -0,0 +1,158 @@
+name: Develop tests
+
+# Runs the full unit, integration, and e2e suites from the repository root on
+# every develop push and keeps two by-products that PR runs cannot produce
+# themselves (ADR 0024):
+#
+# - one merged coverage report, since coverage left the PR jobs;
+# - per-file test durations, merged across shards into
+#   `.vitest/shard-weights.json` and saved to the Actions cache. PR runs of
+#   test.yml restore that file and hand it to every shard as a run artifact,
+#   so the root sequencer balances shards on develop's latest numbers. Caches
+#   saved from PR or merge-queue runs are invisible to other branches, which
+#   is why this has to run on develop itself.
+on:
+  push:
+    branches:
+      - develop
+    paths-ignore:
+      - "apps/docs/**"
+      - "release-notes/**"
+      - "**/*.md"
+
+permissions:
+  contents: read
+  actions: read
+
+concurrency:
+  group: develop-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  unit-integration:
+    name: Unit and integration with coverage
+    runs-on: blacksmith-8vcpu-ubuntu-2404
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Setup
+        uses: ./.github/actions/setup
+        with:
+          dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }}
+
+      - name: Run unit and integration tests with coverage
+        run: pnpm run test --coverage.enabled --reporter=default --reporter=github-actions --reporter=json --outputFile=.vitest/report-unit-integration.json
+
+      - name: Upload coverage report
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: coverage-${{ github.sha }}
+          path: coverage/
+          retention-days: 30
+
+      - name: Upload test report
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: test-report-unit-integration
+          path: .vitest/report-unit-integration.json
+          retention-days: 7
+
+  e2e:
+    name: End-to-end (shard ${{ matrix.shard }}/3)
+    runs-on: blacksmith-8vcpu-ubuntu-2404
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3]
+    steps:
+      - name: Checkout
+        uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1.0.0-beta
+        with:
+          fetch-depth: 0
+
+      - name: Setup
+        uses: ./.github/actions/setup
+        with:
+          dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }}
+
+      - name: Cache Go CLI binary
+        id: cache-go-binary
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: apps/cli-go/supabase-go
+          key: go-cli-${{ runner.os }}-${{ hashFiles('apps/cli-go/**/*.go',
+            'apps/cli-go/go.mod', 'apps/cli-go/go.sum') }}
+
+      - name: Build Go CLI
+        if: steps.cache-go-binary.outputs.cache-hit != 'true'
+        run: go build -o supabase-go .
+        working-directory: apps/cli-go
+
+      - name: Run end-to-end tests
+        run: pnpm run test:e2e --shard=${{ matrix.shard }}/3 --reporter=default --reporter=github-actions --reporter=json --outputFile=.vitest/report-e2e-${{ matrix.shard }}.json
+        env:
+          CLI_HARNESS_TARGET: ts-legacy
+          SUPABASE_GO_BINARY: ${{ github.workspace }}/apps/cli-go/supabase-go
+
+      - name: Upload test report
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: test-report-e2e-${{ matrix.shard }}
+          path: .vitest/report-e2e-${{ matrix.shard }}.json
+          retention-days: 7
+
+  shard-weights:
+    name: Merge shard weights
+    # Merge whatever reports exist even when a shard failed: a failed shard
+    # still reports durations for the files it ran, and files without a fresh
+    # duration keep their previous weight.
+    if: always()
+    needs: [unit-integration, e2e]
+    runs-on: blacksmith-4vcpu-ubuntu-2404
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Setup
+        uses: ./.github/actions/setup
+        with:
+          dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }}
+
+      - name: Download test reports
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: test-report-*
+          path: .vitest/reports
+          merge-multiple: true
+
+      - name: Restore previous shard weights
+        id: previous
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: .vitest/shard-weights.json
+          key: vitest-shard-weights-${{ github.sha }}
+          restore-keys: |
+            vitest-shard-weights-
+
+      - name: Merge durations into shard weights
+        run: |
+          previous=""
+          if [ -f .vitest/shard-weights.json ]; then
+            mv .vitest/shard-weights.json .vitest/previous-shard-weights.json
+            previous="--previous .vitest/previous-shard-weights.json"
+          fi
+          # shellcheck disable=SC2086
+          bun tools/test-shard-weights.ts merge --out .vitest/shard-weights.json $previous .vitest/reports/*.json
+
+      - name: Save shard weights
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: .vitest/shard-weights.json
+          key: vitest-shard-weights-${{ github.sha }}

.github/workflows/test.yml

@@ -99,6 +99,7 @@ jobs:
       ts: ${{ steps.filter.outputs.ts }}
       go: ${{ steps.filter.outputs.go }}
       ci: ${{ steps.filter.outputs.ci }}
+      shard-weights: ${{ steps.shard-weights.outputs.cache-matched-key != '' }}
     steps:
       - name: Checkout
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -130,40 +131,45 @@ jobs:
             ci:
               - ".github/**"
 
-  test-unit:
-    needs: changes
-    if: |
-      !startsWith(github.head_ref, 'release-notes/') &&
-      (github.event_name == 'merge_group' ||
-        inputs.force ||
-        github.event.pull_request.draft == false) &&
-      (needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true')
-    name: Run unit tests
-    runs-on: blacksmith-4vcpu-ubuntu-2404
-    steps:
-      - name: Checkout
-        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      # Shard weights are develop's last per-file durations (develop-tests.yml).
+      # Restore them once here and hand them to every shard as a run artifact:
+      # shards must partition from identical input, and a prefix restore in
+      # each shard could pick up different cache generations. No cache means
+      # the sequencer falls back to dealing files by count.
+      - name: Restore shard weights
+        id: shard-weights
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
-          persist-credentials: false
+          path: .vitest/shard-weights.json
+          key: vitest-shard-weights-${{ github.sha }}
+          restore-keys: |
+            vitest-shard-weights-
 
-      - name: Setup
-        uses: ./.github/actions/setup
+      - name: Publish shard weights for this run
+        if: steps.shard-weights.outputs.cache-matched-key != ''
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }}
+          name: shard-weights
+          path: .vitest/shard-weights.json
+          retention-days: 1
 
-      - name: Run unit tests
-        run: pnpm run test:unit --coverage.enabled
-
-  test-integration:
+  # One root Vitest run over the unit and integration kinds of every package,
+  # split across two runners by the balanced sequencer in vitest.config.mts.
+  # The summary job below keeps the required check name stable (ADR 0024).
+  test:
     needs: changes
     if: |
       !startsWith(github.head_ref, 'release-notes/') &&
       (github.event_name == 'merge_group' ||
         inputs.force ||
         github.event.pull_request.draft == false) &&
       (needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true')
-    name: Run integration tests
+    name: Run unit and integration tests (shard ${{ matrix.shard }}/2)
     runs-on: blacksmith-8vcpu-ubuntu-2404
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2]
     steps:
       - name: Checkout
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -175,8 +181,23 @@ jobs:
         with:
           dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }}
 
-      - name: Run integration tests
-        run: pnpm run test:integration --coverage.enabled
+      - name: Restore Vitest module cache
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: node_modules/.vitest-cache
+          key: vitest-cache-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}-${{ github.sha }}
+          restore-keys: |
+            vitest-cache-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}-
+
+      - name: Use shard weights from develop
+        if: needs.changes.outputs.shard-weights == 'true'
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: shard-weights
+          path: .vitest
+
+      - name: Run unit and integration tests
+        run: pnpm run test --shard=${{ matrix.shard }}/2
 
   test-summary:
     if: |
@@ -186,7 +207,7 @@ jobs:
         inputs.force ||
         github.event.pull_request.draft == false)
     name: Run unit and integration tests
-    needs: [changes, test-unit, test-integration]
+    needs: [changes, test]
     runs-on: ubuntu-latest
     steps:
       - name: Verify unit and integration tests succeeded
@@ -195,12 +216,11 @@ jobs:
             echo "::error ::Path gate did not succeed: changes=${{ needs.changes.result }}"
             exit 1
           fi
-          if [ "${{ needs.test-unit.result }}" = "failure" ] || [ "${{ needs.test-unit.result }}" = "cancelled" ] || \
-            [ "${{ needs.test-integration.result }}" = "failure" ] || [ "${{ needs.test-integration.result }}" = "cancelled" ]; then
-            echo "::error ::Unit or integration tests failed: unit=${{ needs.test-unit.result }}, integration=${{ needs.test-integration.result }}"
+          if [ "${{ needs.test.result }}" = "failure" ] || [ "${{ needs.test.result }}" = "cancelled" ]; then
+            echo "::error ::One or more unit/integration shards failed: ${{ needs.test.result }}"
             exit 1
           fi
-          echo "Unit and integration tests reported: unit=${{ needs.test-unit.result }}, integration=${{ needs.test-integration.result }}"
+          echo "All unit/integration shards reported: ${{ needs.test.result }}"
 
   test-e2e:
     needs: changes
@@ -240,15 +260,28 @@ jobs:
         run: go build -o supabase-go .
         working-directory: apps/cli-go
 
+      - name: Restore Vitest module cache
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: node_modules/.vitest-cache
+          key: vitest-cache-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}-${{ github.sha }}
+          restore-keys: |
+            vitest-cache-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}-
+
       # The ts-legacy harness invokes `node apps/cli/dist/supabase.js` with
       # `SUPABASE_CLI_BINARY_OVERRIDE` pointing at the compiled legacy binary
-      # in `apps/cli/dist/`. Build the CLI explicitly before invoking every
-      # package-local e2e suite.
-      - name: Build CLI
-        run: pnpm exec turbo run supabase#build
+      # in `apps/cli/dist/`. The root e2e task depends on `supabase#build`, so
+      # Turbo builds the CLI first; one root Vitest run then covers every
+      # package's e2e projects and the balanced sequencer decides the shard.
+      - name: Use shard weights from develop
+        if: needs.changes.outputs.shard-weights == 'true'
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: shard-weights
+          path: .vitest
 
       - name: Run end-to-end tests
-        run: pnpm exec turbo run test:e2e:run --only --concurrency=1 -- --shard=${{ matrix.shard }}/3
+        run: pnpm run test:e2e --shard=${{ matrix.shard }}/3
         env:
           CLI_HARNESS_TARGET: ts-legacy
           SUPABASE_GO_BINARY: ${{ github.workspace }}/apps/cli-go/supabase-go

AGENTS.md

@@ -49,14 +49,17 @@ Expected exceptions:
 
 **vitest.config.ts:**
 
-Package configs build on the repo-root `vitest.shared.ts` preset. `definePackageConfig` merges the
+Package configs build on the repo-root `vitest.shared.mts` preset. `definePackageConfig` merges the
 shared defaults (bun export-condition resolution for workspace packages, v8 coverage,
-`passWithNoTests`, console output only from failing tests), and `testProject("unit" | "integration" |
-"e2e" | "live", overrides)` declares one inline project per test kind with the repo's file-suffix
-convention baked in. Package-specific settings such as timeouts, setup files, serial execution, or
-Vite plugins go in the overrides. The root `vitest.config.mts` loads every package config as a nested
-project group, so `bun --bun vitest run --project '*(unit)'` from the repo root runs one kind across
-all workspaces while `pnpm test:unit` inside a package still works standalone.
+`passWithNoTests`, console output only from failing tests, the file-system module cache), and
+`testProject("unit" | "integration" | "e2e" | "e2e-stack" | "live", overrides)` declares one inline
+project per test kind with the repo's file-suffix convention baked in; `e2e-stack` runs serially by
+default. Package-specific settings such as timeouts, setup files, or Vite plugins go in the
+overrides. Package scripts call Vitest directly (`test`, `test:unit`, `test:integration`, `test:e2e`
+where applicable). The root `vitest.config.mts` loads every package config as a nested project
+group, so the root `pnpm test` runs unit and integration across all workspaces in one process and
+`--project '*(unit)'` or `--project 'supabase (integration)'` selects a slice; the root `test:e2e`
+goes through the Turbo task `//#test:e2e:run` because e2e needs the built CLI (ADR 0024).
 
 ## Config Naming Vocabulary