diff --git a/.github/workflows/develop-tests.yml b/.github/workflows/develop-tests.yml new file mode 100644 index 0000000000..b3986decb4 --- /dev/null +++ b/.github/workflows/develop-tests.yml @@ -0,0 +1,158 @@ +name: Develop tests + +# Runs the full unit, integration, and e2e suites from the repository root on +# every develop push and keeps two by-products that PR runs cannot produce +# themselves (ADR 0024): +# +# - one merged coverage report, since coverage left the PR jobs; +# - per-file test durations, merged across shards into +# `.vitest/shard-weights.json` and saved to the Actions cache. PR runs of +# test.yml restore that file and hand it to every shard as a run artifact, +# so the root sequencer balances shards on develop's latest numbers. Caches +# saved from PR or merge-queue runs are invisible to other branches, which +# is why this has to run on develop itself. +on: + push: + branches: + - develop + paths-ignore: + - "apps/docs/**" + - "release-notes/**" + - "**/*.md" + +permissions: + contents: read + actions: read + +concurrency: + group: develop-tests-${{ github.ref }} + cancel-in-progress: true + +jobs: + unit-integration: + name: Unit and integration with coverage + runs-on: blacksmith-8vcpu-ubuntu-2404 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + + - name: Run unit and integration tests with coverage + run: pnpm run test --coverage.enabled --reporter=default --reporter=github-actions --reporter=json --outputFile=.vitest/report-unit-integration.json + + - name: Upload coverage report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: coverage-${{ github.sha }} + path: coverage/ + retention-days: 30 + + - name: Upload test report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-report-unit-integration + path: .vitest/report-unit-integration.json + retention-days: 7 + + e2e: + name: End-to-end (shard ${{ matrix.shard }}/3) + runs-on: blacksmith-8vcpu-ubuntu-2404 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + steps: + - name: Checkout + uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1.0.0-beta + with: + fetch-depth: 0 + + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + + - name: Cache Go CLI binary + id: cache-go-binary + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: apps/cli-go/supabase-go + key: go-cli-${{ runner.os }}-${{ hashFiles('apps/cli-go/**/*.go', + 'apps/cli-go/go.mod', 'apps/cli-go/go.sum') }} + + - name: Build Go CLI + if: steps.cache-go-binary.outputs.cache-hit != 'true' + run: go build -o supabase-go . + working-directory: apps/cli-go + + - name: Run end-to-end tests + run: pnpm run test:e2e --shard=${{ matrix.shard }}/3 --reporter=default --reporter=github-actions --reporter=json --outputFile=.vitest/report-e2e-${{ matrix.shard }}.json + env: + CLI_HARNESS_TARGET: ts-legacy + SUPABASE_GO_BINARY: ${{ github.workspace }}/apps/cli-go/supabase-go + + - name: Upload test report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-report-e2e-${{ matrix.shard }} + path: .vitest/report-e2e-${{ matrix.shard }}.json + retention-days: 7 + + shard-weights: + name: Merge shard weights + # Merge whatever reports exist even when a shard failed: a failed shard + # still reports durations for the files it ran, and files without a fresh + # duration keep their previous weight. + if: always() + needs: [unit-integration, e2e] + runs-on: blacksmith-4vcpu-ubuntu-2404 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup + uses: ./.github/actions/setup + with: + dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + + - name: Download test reports + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: test-report-* + path: .vitest/reports + merge-multiple: true + + - name: Restore previous shard weights + id: previous + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .vitest/shard-weights.json + key: vitest-shard-weights-${{ github.sha }} + restore-keys: | + vitest-shard-weights- + + - name: Merge durations into shard weights + run: | + previous="" + if [ -f .vitest/shard-weights.json ]; then + mv .vitest/shard-weights.json .vitest/previous-shard-weights.json + previous="--previous .vitest/previous-shard-weights.json" + fi + # shellcheck disable=SC2086 + bun tools/test-shard-weights.ts merge --out .vitest/shard-weights.json $previous .vitest/reports/*.json + + - name: Save shard weights + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .vitest/shard-weights.json + key: vitest-shard-weights-${{ github.sha }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3037e901e4..c4e8738139 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -99,6 +99,7 @@ jobs: ts: ${{ steps.filter.outputs.ts }} go: ${{ steps.filter.outputs.go }} ci: ${{ steps.filter.outputs.ci }} + shard-weights: ${{ steps.shard-weights.outputs.cache-matched-key != '' }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -130,31 +131,32 @@ jobs: ci: - ".github/**" - test-unit: - needs: changes - if: | - !startsWith(github.head_ref, 'release-notes/') && - (github.event_name == 'merge_group' || - inputs.force || - github.event.pull_request.draft == false) && - (needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true') - name: Run unit tests - runs-on: blacksmith-4vcpu-ubuntu-2404 - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Shard weights are develop's last per-file durations (develop-tests.yml). + # Restore them once here and hand them to every shard as a run artifact: + # shards must partition from identical input, and a prefix restore in + # each shard could pick up different cache generations. No cache means + # the sequencer falls back to dealing files by count. + - name: Restore shard weights + id: shard-weights + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: - persist-credentials: false + path: .vitest/shard-weights.json + key: vitest-shard-weights-${{ github.sha }} + restore-keys: | + vitest-shard-weights- - - name: Setup - uses: ./.github/actions/setup + - name: Publish shard weights for this run + if: steps.shard-weights.outputs.cache-matched-key != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} + name: shard-weights + path: .vitest/shard-weights.json + retention-days: 1 - - name: Run unit tests - run: pnpm run test:unit --coverage.enabled - - test-integration: + # One root Vitest run over the unit and integration kinds of every package, + # split across two runners by the balanced sequencer in vitest.config.mts. + # The summary job below keeps the required check name stable (ADR 0024). + test: needs: changes if: | !startsWith(github.head_ref, 'release-notes/') && @@ -162,8 +164,12 @@ jobs: inputs.force || github.event.pull_request.draft == false) && (needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true') - name: Run integration tests + name: Run unit and integration tests (shard ${{ matrix.shard }}/2) runs-on: blacksmith-8vcpu-ubuntu-2404 + strategy: + fail-fast: false + matrix: + shard: [1, 2] steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -175,8 +181,23 @@ jobs: with: dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} - - name: Run integration tests - run: pnpm run test:integration --coverage.enabled + - name: Restore Vitest module cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: node_modules/.vitest-cache + key: vitest-cache-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}-${{ github.sha }} + restore-keys: | + vitest-cache-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}- + + - name: Use shard weights from develop + if: needs.changes.outputs.shard-weights == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: shard-weights + path: .vitest + + - name: Run unit and integration tests + run: pnpm run test --shard=${{ matrix.shard }}/2 test-summary: if: | @@ -186,7 +207,7 @@ jobs: inputs.force || github.event.pull_request.draft == false) name: Run unit and integration tests - needs: [changes, test-unit, test-integration] + needs: [changes, test] runs-on: ubuntu-latest steps: - name: Verify unit and integration tests succeeded @@ -195,12 +216,11 @@ jobs: echo "::error ::Path gate did not succeed: changes=${{ needs.changes.result }}" exit 1 fi - if [ "${{ needs.test-unit.result }}" = "failure" ] || [ "${{ needs.test-unit.result }}" = "cancelled" ] || \ - [ "${{ needs.test-integration.result }}" = "failure" ] || [ "${{ needs.test-integration.result }}" = "cancelled" ]; then - echo "::error ::Unit or integration tests failed: unit=${{ needs.test-unit.result }}, integration=${{ needs.test-integration.result }}" + if [ "${{ needs.test.result }}" = "failure" ] || [ "${{ needs.test.result }}" = "cancelled" ]; then + echo "::error ::One or more unit/integration shards failed: ${{ needs.test.result }}" exit 1 fi - echo "Unit and integration tests reported: unit=${{ needs.test-unit.result }}, integration=${{ needs.test-integration.result }}" + echo "All unit/integration shards reported: ${{ needs.test.result }}" test-e2e: needs: changes @@ -240,15 +260,28 @@ jobs: run: go build -o supabase-go . working-directory: apps/cli-go + - name: Restore Vitest module cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: node_modules/.vitest-cache + key: vitest-cache-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}-${{ github.sha }} + restore-keys: | + vitest-cache-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}- + # The ts-legacy harness invokes `node apps/cli/dist/supabase.js` with # `SUPABASE_CLI_BINARY_OVERRIDE` pointing at the compiled legacy binary - # in `apps/cli/dist/`. Build the CLI explicitly before invoking every - # package-local e2e suite. - - name: Build CLI - run: pnpm exec turbo run supabase#build + # in `apps/cli/dist/`. The root e2e task depends on `supabase#build`, so + # Turbo builds the CLI first; one root Vitest run then covers every + # package's e2e projects and the balanced sequencer decides the shard. + - name: Use shard weights from develop + if: needs.changes.outputs.shard-weights == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: shard-weights + path: .vitest - name: Run end-to-end tests - run: pnpm exec turbo run test:e2e:run --only --concurrency=1 -- --shard=${{ matrix.shard }}/3 + run: pnpm run test:e2e --shard=${{ matrix.shard }}/3 env: CLI_HARNESS_TARGET: ts-legacy SUPABASE_GO_BINARY: ${{ github.workspace }}/apps/cli-go/supabase-go diff --git a/AGENTS.md b/AGENTS.md index 03348ca725..af1ce8070b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,14 +49,17 @@ Expected exceptions: **vitest.config.ts:** -Package configs build on the repo-root `vitest.shared.ts` preset. `definePackageConfig` merges the +Package configs build on the repo-root `vitest.shared.mts` preset. `definePackageConfig` merges the shared defaults (bun export-condition resolution for workspace packages, v8 coverage, -`passWithNoTests`, console output only from failing tests), and `testProject("unit" | "integration" | -"e2e" | "live", overrides)` declares one inline project per test kind with the repo's file-suffix -convention baked in. Package-specific settings such as timeouts, setup files, serial execution, or -Vite plugins go in the overrides. The root `vitest.config.mts` loads every package config as a nested -project group, so `bun --bun vitest run --project '*(unit)'` from the repo root runs one kind across -all workspaces while `pnpm test:unit` inside a package still works standalone. +`passWithNoTests`, console output only from failing tests, the file-system module cache), and +`testProject("unit" | "integration" | "e2e" | "e2e-stack" | "live", overrides)` declares one inline +project per test kind with the repo's file-suffix convention baked in; `e2e-stack` runs serially by +default. Package-specific settings such as timeouts, setup files, or Vite plugins go in the +overrides. Package scripts call Vitest directly (`test`, `test:unit`, `test:integration`, `test:e2e` +where applicable). The root `vitest.config.mts` loads every package config as a nested project +group, so the root `pnpm test` runs unit and integration across all workspaces in one process and +`--project '*(unit)'` or `--project 'supabase (integration)'` selects a slice; the root `test:e2e` +goes through the Turbo task `//#test:e2e:run` because e2e needs the built CLI (ADR 0024). ## Config Naming Vocabulary diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 56fb1098a4..7da7af88fd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -150,7 +150,7 @@ Standard TypeScript workspaces (`apps/cli-e2e`, `apps/cli`, `packages/api`, `pac | `test:e2e` | Run end-to-end tests where applicable | | `types:check` | Type-check with `tsc --noEmit` | -The test and type-check scripts are declared in each package's `package.json`, so package-local commands are directly discoverable and can be sharded independently. +The test and type-check scripts are declared in each package's `package.json` and call Vitest directly, so package-local commands are discoverable with no orchestration layer in between. Linting, formatting, and unused-code analysis are repo-wide rather than per-package: `oxlint`, `oxfmt`, and `knip` read `.oxlintrc.json`, `.oxfmtrc.json`, and `knip.json` at the repo root (knip's config maps each workspace under its `workspaces` key). The root `check:all`/`fix:all` scripts are the sole repo-wide quality entrypoints and use Turbo to run the package type checks and root-owned quality scripts. Package-local work can run `pnpm types:check` and the package's test scripts. Running the tools directly from the repository root also just works: @@ -169,22 +169,34 @@ pnpm run test:unit # If this package declares an integration suite: pnpm run test:integration -# From the workspace root — repo-wide quality and all-project test fan-out: +# From the workspace root — repo-wide quality and one Vitest run across every package: pnpm run check:all pnpm run fix:all -pnpm run test:unit && pnpm run test:integration +pnpm run test # unit + integration, all packages +pnpm run test:unit # one kind across all packages +pnpm run test:e2e # builds the CLI via Turbo, then every e2e project ``` -The root unit and integration scripts use Turbo to fan out the package-local -`test:*:run` tasks across the standard TypeScript/Vitest workspaces. The Go -workspace remains package-local because its tests run directly through Go: -`pnpm --dir apps/cli-go run test:unit`. Go tests are covered by the dedicated -Go CI workflow. Unit and integration tasks are uncached for now; e2e tasks are -also uncached and run one package at a time. Within a package, plain -`*.e2e.test.ts` files run in parallel while `*.stack.e2e.test.ts` files (those -that start a local stack or run Docker containers) run serially in the -`e2e-stack` project. Forward a Vitest shard to every e2e package with -`pnpm run test:e2e --shard=1/3`. +The root test scripts run a single Vitest process from the repository root: +`vitest.config.mts` loads every package config as a nested project group, so +`--project 'supabase (integration)'` or `--project '*(unit)'` selects any slice +of the repo and one report covers the run. `test:e2e` is the exception that +still goes through Turbo (`//#test:e2e:run`), because e2e depends on the +built CLI and Turbo owns that build graph. Within a run, plain `*.e2e.test.ts` +files run in parallel while `*.stack.e2e.test.ts` files (those that start a +local stack or run Docker containers) run one at a time in the `e2e-stack` +projects. CI shards the root runs with `--shard=N/M`. The root config's sequencer balances +shards from per-file durations recorded by the develop run +(`.github/workflows/develop-tests.yml` writes `.vitest/shard-weights.json`; PR +runs receive it as an artifact) and falls back to dealing files by count when +that file is absent, which is the case locally. To reproduce CI's partition, +download the `shard-weights` artifact into `.vitest/` or point +`VITEST_SHARD_WEIGHTS` at it. Pass +Vitest flags straight after the script name: pnpm forwards a literal `--` to the +script, and Vitest treats everything after `--` as file filters. +The Go workspace remains package-local because its tests run directly through +Go: `pnpm --dir apps/cli-go run test:unit`. Go tests are covered by the +dedicated Go CI workflow. See ADR 0024 for the rationale. ## E2E Compatibility Test Suite @@ -228,7 +240,7 @@ Live CI is manual or daily scheduled and is not PR-blocking; run it manually on ```sh # Replay mode — fast, no credentials needed -pnpm exec turbo run @supabase/cli-e2e#test:e2e:run # ts-legacy target +pnpm --filter @supabase/cli-e2e run test:e2e # ts-legacy target ``` ### Recording fixtures @@ -246,7 +258,7 @@ Review the generated files in `apps/cli-e2e/fixtures/recorded/` before committin After recording, replay must pass with no changes against the freshly committed fixtures: ```sh -pnpm exec turbo run @supabase/cli-e2e#test:e2e:run +pnpm --filter @supabase/cli-e2e run test:e2e ``` A test failing only after a recording session usually means an assertion needs updating to match the CLI's current real-world output, not the fixture. diff --git a/apps/cli-e2e/AGENTS.md b/apps/cli-e2e/AGENTS.md index 0066ef9e38..0dd0d399bd 100644 --- a/apps/cli-e2e/AGENTS.md +++ b/apps/cli-e2e/AGENTS.md @@ -147,7 +147,7 @@ Run the following orchestration commands from the repository root. ```sh # Replay (no credentials needed) -pnpm exec turbo run @supabase/cli-e2e#test:e2e:run # ts-legacy target +pnpm --filter @supabase/cli-e2e run test:e2e # ts-legacy target # Record (requires staging access) SUPABASE_ACCESS_TOKEN=sbp_... SUPABASE_STAGING_URL=https://api.supabase.green \ @@ -162,19 +162,24 @@ After recording, replay must pass with no changes between the two commands. ### Sharding (replay only) -CI splits the replay suite across 3 parallel shards via vitest's `--shard` -flag (https://vitest.dev/guide/improving-performance.html#sharding). -Locally, invoke vitest directly so the flag isn't eaten by a `--` -passthrough quirk in package-script argument forwarding: +CI runs every package's e2e projects from the repository root in one Vitest +process and splits that run across 3 shards with vitest's `--shard` flag +(https://vitest.dev/guide/improving-performance.html#sharding): + +```sh +pnpm run test:e2e --shard=1/3 # from the repo root; builds the CLI first +``` + +To shard only this package, invoke vitest directly: ```sh pnpm --filter @supabase/cli-e2e exec bun --bun vitest run --shard=1/3 -pnpm --filter @supabase/cli-e2e exec bun --bun vitest run --shard=2/3 -pnpm --filter @supabase/cli-e2e exec bun --bun vitest run --shard=3/3 ``` -The custom file sequencer in `vitest.config.ts` (lexicographic) runs -per-process, so each shard still has deterministic intra-shard ordering. +File order is lexicographic either way: the repo-root `vitest.config.mts` +sequencer sorts files by path within each project for root runs, and this +package's `vitest.config.ts` sequencer does the same for standalone runs, so +every shard has deterministic intra-shard ordering. **Sharding is replay-only — never shard a recording run.** The recorder is a single-job operation; parallel shards would race on the shared @@ -192,7 +197,7 @@ Build the Go CLI from source and point `SUPABASE_GO_BINARY` at it: # Replay SUPABASE_GO_BINARY=/tmp/supabase-test-binary \ - pnpm exec turbo run @supabase/cli-e2e#test:e2e:run + pnpm --filter @supabase/cli-e2e run test:e2e # Record SUPABASE_GO_BINARY=/tmp/supabase-test-binary \ diff --git a/apps/cli-e2e/package.json b/apps/cli-e2e/package.json index 131a48256e..d49c065822 100644 --- a/apps/cli-e2e/package.json +++ b/apps/cli-e2e/package.json @@ -4,13 +4,12 @@ "private": true, "type": "module", "scripts": { + "record": "RECORD=true CLI_HARNESS_TARGET=ts-legacy bun --bun vitest run", + "types:check": "tsc --noEmit", "test": "pnpm run test:e2e", - "test:e2e": "pnpm exec turbo run supabase#build && pnpm exec turbo run @supabase/cli-e2e#test:e2e:run --only --", - "test:e2e:run": "bun --bun vitest run", + "test:e2e": "pnpm exec turbo run supabase#build && bun --bun vitest run", "test:legacy": "CLI_HARNESS_TARGET=ts-legacy pnpm run test:e2e", - "test:next": "CLI_HARNESS_TARGET=ts-next pnpm run test:e2e", - "record": "RECORD=true CLI_HARNESS_TARGET=ts-legacy pnpm run test:e2e:run", - "types:check": "tsc --noEmit" + "test:next": "CLI_HARNESS_TARGET=ts-next pnpm run test:e2e" }, "dependencies": { "@supabase/cli-test-helpers": "workspace:*" diff --git a/apps/cli-e2e/vitest.config.ts b/apps/cli-e2e/vitest.config.ts index 337992723e..7157327740 100644 --- a/apps/cli-e2e/vitest.config.ts +++ b/apps/cli-e2e/vitest.config.ts @@ -1,5 +1,5 @@ import { BaseSequencer, type TestSpecification } from "vitest/node"; -import { definePackageConfig, testProject } from "../../vitest.shared.ts"; +import { definePackageConfig, testProject } from "../../vitest.shared.mts"; export default definePackageConfig({ test: { diff --git a/apps/cli-go/package.json b/apps/cli-go/package.json index 197015c913..48d634cbe8 100644 --- a/apps/cli-go/package.json +++ b/apps/cli-go/package.json @@ -5,7 +5,6 @@ "build": "go build -o supabase-go .", "lint:check": "golangci-lint run --timeout 5m", "lint:fix": "golangci-lint run --fix", - "test:unit": "pnpm exec turbo run @supabase/cli-go#test:unit:run --", - "test:unit:run": "go test ./..." + "test:unit": "go test ./..." } } diff --git a/apps/cli/AGENTS.md b/apps/cli/AGENTS.md index 7e2695c68d..7ce0f4be08 100644 --- a/apps/cli/AGENTS.md +++ b/apps/cli/AGENTS.md @@ -459,9 +459,9 @@ Rules: ## Testing -Use `pnpm run test` to run tests. The `package.json` `test` script runs the unit, integration, and e2e Vitest projects, with coverage enabled for unit and integration. +Use `pnpm run test` to run tests. The `package.json` `test` script runs the unit and integration Vitest projects; add `--coverage.enabled` for a coverage report. `pnpm run test:e2e` builds the CLI through Turbo and then runs the `e2e` and `e2e-stack` projects: stackless subprocess tests in parallel, then the stack-backed files one at a time. -Use `pnpm run test:unit && pnpm run test:integration` for the main in-process suite, and `pnpm run test:e2e` for the sequential subprocess suite. +Use `pnpm run test:unit` or `pnpm run test:integration` for one kind. From the repository root, `bun --bun vitest run --project 'supabase (integration)'` runs the same project inside the repo-wide Vitest process. Always run the relevant unit and integration tests automatically for the command or workspace you changed. Do not run the full e2e suite automatically. Only run e2e when the user asks, or when you need extra confidence for the command you touched. diff --git a/apps/cli/package.json b/apps/cli/package.json index abae7fec98..2dda3412f8 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -30,16 +30,13 @@ "build:shim": "bun build src/shared/cli/bin.ts --outfile dist/supabase.js --target node", "docs:spec": "bun scripts/generate-docs-spec.ts", "dev:legacy": "pnpm exec bun src/legacy/main.ts", - "test": "pnpm run test:unit --coverage.enabled && pnpm run test:integration --coverage.enabled && pnpm run test:e2e", - "test:unit": "pnpm exec turbo run supabase#test:unit:run --", - "test:unit:run": "bun --bun vitest run --project unit --coverage.reportsDirectory=coverage/unit", - "test:integration": "pnpm exec turbo run supabase#test:integration:run --", - "test:integration:run": "bun --bun vitest run --project integration --coverage.reportsDirectory=coverage/integration", - "test:e2e": "pnpm exec turbo run supabase#build && pnpm exec turbo run supabase#test:e2e:run --only --", - "test:e2e:run": "bun --bun vitest run --project e2e --project e2e-stack", + "types:check": "tsc --noEmit", + "test": "bun --bun vitest run --project unit --project integration", + "test:unit": "bun --bun vitest run --project unit", + "test:integration": "bun --bun vitest run --project integration", + "test:e2e": "pnpm exec turbo run supabase#build && bun --bun vitest run --project e2e --project e2e-stack", "test:live": "bun --bun vitest run --project live", - "test:smoke": "bun run tests/smoke-test.ts", - "types:check": "tsc --noEmit" + "test:smoke": "bun run tests/smoke-test.ts" }, "dependencies": { "eciesjs": "^0.5.0", diff --git a/apps/cli/vitest.config.ts b/apps/cli/vitest.config.ts index ed76c3a096..d32224963c 100644 --- a/apps/cli/vitest.config.ts +++ b/apps/cli/vitest.config.ts @@ -1,5 +1,5 @@ import { readFileSync } from "node:fs"; -import { definePackageConfig, testProject } from "../../vitest.shared.ts"; +import { definePackageConfig, testProject } from "../../vitest.shared.mts"; // `src/shared/services/dockerfile-images.ts` imports the Go CLI's Dockerfile // with Bun's `{ type: "text" }` import attribute; Vite needs a loader for it. diff --git a/docs/adr/0024-test-execution-topology.md b/docs/adr/0024-test-execution-topology.md index 16df8fd33f..bf56640d9d 100644 --- a/docs/adr/0024-test-execution-topology.md +++ b/docs/adr/0024-test-execution-topology.md @@ -15,9 +15,9 @@ Vitest 5 changed what is possible: a root config can reference package configs t ## Decision -1. **One root Vitest run is the unit of execution for unit, integration, and e2e tests.** The repo-root `vitest.config.mts` loads every package config as a nested project group named ` ()`. CI and the root scripts run that process with `--project` and `--shard` filters. Package configs remain runnable standalone for local work and share one preset, `vitest.shared.ts`. +1. **One root Vitest run is the unit of execution for unit, integration, and e2e tests.** The repo-root `vitest.config.mts` loads every package config as a nested project group named ` ()`. CI and the root scripts run that process with `--project` and `--shard` filters. Package configs remain runnable standalone for local work and share one preset, `vitest.shared.mts`. 2. **E2e tests that start a local stack are a distinct sub-kind, marked by the `*.stack.e2e.test.ts` suffix**, and run in their own serial `e2e-stack` test project. All other e2e files run with normal file parallelism. A file claims the stack-backed suffix when it starts a Supabase stack, Docker or native; a file that only spawns the CLI against an isolated temporary home does not. -3. **Shards are balanced by weight class, not duration data.** The root config's sequencer deals stack-backed files round-robin across shards by sorted path, then stackless files the same way. Every shard computes the same partition independently. +3. **Shards are balanced from develop's own durations, with a count-based fallback.** A develop-push workflow runs the full suite with Vitest's `json` reporter, merges per-file durations across shards into `.vitest/shard-weights.json`, and saves it to the Actions cache. A PR run restores that file once, in its gate job, and hands it to every shard as a run artifact; the root sequencer then assigns files largest-first to the least-loaded shard within each class, stack-backed first. Without the file, files are dealt round-robin within each test project. Every shard computes the same partition from identical input. 4. **Turbo keeps the dependency graph and nothing else in test execution.** E2e is a Turbo root task, `//#test:e2e`, that depends on `supabase#build`; live and smoke stay Turbo tasks for the same reason. Unit and integration no longer pass through Turbo. Job-level skipping is done with path rules in the workflow, not `turbo --affected`. 5. **Coverage leaves PR runs.** A develop-push workflow produces one merged root report; PR jobs run uninstrumented. @@ -39,7 +39,7 @@ These are the canonical names for the concepts above; use them in configs, scrip - **Wall-clock is the objective**, agreed ahead of compute and configuration simplicity. Only a process that sees all e2e files can balance them, and only a project split can let stackless files run in parallel while stack-backed files stay serial. Vitest schedules a serial project after the parallel groups within one run, so the split needs no custom scheduling. Simulated on the measured durations, the worst shard's serial work drops from about 7.9 to about 4.5 minutes with three shards. - **Turbo's test-specific strengths were not in use and could not be made sound cheaply.** Every test task was `cache: false`, CI had no remote cache, and `.turbo` was not persisted, so no test caching existed to lose. Making per-package test caching honest would require declaring sibling packages' sources, the built binary, Docker images, and environment as inputs. Turbo's package graph also does not know the CLI shells out to the Go sidecar (that link exists only in the task graph via `supabase#build`), so `--affected` would wrongly skip e2e on Go-only changes. Path rules cost seconds and encode the three real cases: docs-only, Go-only, everything else. - **A filename suffix is the existing vocabulary.** Test kinds are already chosen by suffix and colocated; extending that to a sub-kind keeps the resource claim visible in the file name, greppable, and reviewable, without adopting tags or breaking colocation. -- **Weight classes over a duration manifest.** Durations range from 0.4s to 150s within the stack-backed class, so a manifest would balance better, by roughly one more minute, but it is data that drifts and needs a job to refresh. The class-based deal needs no data and can be upgraded later if imbalance persists. +- **Durations, sourced automatically.** Durations range from 0.4s to 150s within the stack-backed class, and neither file count nor file size predicts them: measured on a develop run, count dealing left the worst shard at 305s of serial work and size weighting made it worse, while true durations give 196 / 195 / 195s. A committed manifest would need a human to refresh it; taking the numbers from the develop run that already exists makes them self-maintaining. The one-restore-then-artifact hop exists because shards must partition from byte-identical input, and a prefix cache restore in each shard can pick up different generations. Caches saved from PR or merge-queue runs are invisible to other branches, which is why the writer must run on develop. ## Consequences @@ -56,13 +56,15 @@ These are the canonical names for the concepts above; use them in configs, scrip - The cli-e2e package's lexicographic sequencer no longer applies when its files run from the root; the root sequencer provides the equivalent tiebreak. - Renaming stack-backed files churns blame once and adds a rule authors must know: starting a stack means taking the suffix. - Per-package test caching is foreclosed for as long as tests run from the root. Revisit only if a remote cache arrives and test inputs can be declared honestly. +- Every develop push runs the e2e suite once more, about fifteen runner-minutes, to keep durations current. +- The unit/integration class is bounded by its single slowest file (a 228s legacy `start` integration test at the time of writing); no sequencer can shorten that, only the test can. ## Alternatives Considered 1. **Keep Turbo fan-out and only split e2e projects**: smallest diff, keeps package-level `--affected` for e2e, but cannot balance shards across packages and rarely skips anything on this graph because the CLI depends on every other package. 2. **Separate serial stack job and parallel stackless job**: the stack job alone is about 10.6 minutes serial, worse than today, unless it is itself sharded, which collapses into the chosen design. 3. **Vitest tags or a directory for stack-backed files**: tags hide the claim inside the file and introduce a mechanism the repo does not use; a directory breaks colocation. -4. **Committed duration manifest**: better balance for more maintenance; deferred, see above. +4. **Committed duration manifest**: the same balance as the automated durations, but regenerated by hand; rejected for the maintenance. **File-size weighting** was also measured and rejected: size does not track duration here (a 1.2 KB start test takes 42s, a 20 KB native createStack test 9s). **Vitest's own results cache** as a direct sharding input was rejected because shards could restore different generations and silently drop files. 5. **`isolate: false` for unit tests**: about 40 test files mutate env, cwd, or globals or use `vi.mock`; a semantic change that needs an audit before any measurement exists. 6. **`turbo --affected` as the CI gate**: unsound without adding the Go sidecar to the CLI's package.json dependencies, and a gate job pays an install before any test job starts. diff --git a/docs/superpowers/plans/2026-09-04-test-execution-topology.md b/docs/superpowers/plans/2026-09-04-test-execution-topology.md index 741329a4ca..b40b730e45 100644 --- a/docs/superpowers/plans/2026-09-04-test-execution-topology.md +++ b/docs/superpowers/plans/2026-09-04-test-execution-topology.md @@ -1,6 +1,6 @@ # Test execution topology: Turbo + Vitest 5 -Status: agreed 2026-09-04, implementation in three PRs. Vocabulary and the decisions worth keeping after this plan is done live in +Status: agreed 2026-09-04, implementation in three PRs (A: #6472, B: #6473, C: stacked on B). Vocabulary and the decisions worth keeping after this plan is done live in ADR 0024. ## Goals and how ties break @@ -26,21 +26,21 @@ skipping exists today. ## Decisions -| # | Decision | Chosen | -| --- | ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | Tie-breaker between goals | PR CI wall-clock | -| 2 | Compute budget | Soft; wall-clock wins within ~1.5x minutes | -| 3 | Unit/integration runner | One root Vitest process; Turbo drops out of unit/integration | -| 4 | E2e parallelism | Stackless e2e files run in parallel; stack-backed files stay serial | -| 5 | Stack-backed marker | File suffix `*.stack.e2e.test.ts` | -| 6 | E2e CI topology | One root Vitest run per shard, as Turbo root task `//#test:e2e` depending on `supabase#build` | -| 7 | Shard balance | Custom sequencer deals stack-backed files round-robin by sorted path, then stackless; no duration data | -| 8 | Unit/integration CI shape | One root run over both kinds, `--shard=N/2`, matrix of two jobs | -| 9 | Coverage | Removed from PR runs; develop-push workflow produces one merged root report | -| 10 | Job gating | Path rules: docs/release-notes only skips all tests; Go-only skips unit/integration, keeps e2e; `.github` or any TS workspace runs everything | -| 11 | Scripts | Packages: direct Vitest, one hop, no `:run` layer. Root: `test`, `test:unit`, `test:integration` are root Vitest; `test:e2e`, `test:live` stay Turbo tasks | -| 12 | Runtime tuning | `fsModuleCache` on, `node_modules/.vite` restored in CI, one `vitest doctor` run recorded; no isolation or pool changes | -| 13 | Delivery | Three PRs; Vitest-4-safe work first because Vitest 5.0.0 is firewall-quarantined | +| # | Decision | Chosen | +| --- | ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | Tie-breaker between goals | PR CI wall-clock | +| 2 | Compute budget | Soft; wall-clock wins within ~1.5x minutes | +| 3 | Unit/integration runner | One root Vitest process; Turbo drops out of unit/integration | +| 4 | E2e parallelism | Stackless e2e files run in parallel; stack-backed files stay serial | +| 5 | Stack-backed marker | File suffix `*.stack.e2e.test.ts` | +| 6 | E2e CI topology | One root Vitest run per shard, as Turbo root task `//#test:e2e` depending on `supabase#build` | +| 7 | Shard balance | Durations from the develop run (`develop-tests.yml` merges shard JSON reports into `.vitest/shard-weights.json`, cached); the PR gate job publishes them as a run artifact; the sequencer assigns largest-first per class and deals by count when the file is absent | +| 8 | Unit/integration CI shape | One root run over both kinds, `--shard=N/2`, matrix of two jobs | +| 9 | Coverage | Removed from PR runs; develop-push workflow produces one merged root report | +| 10 | Job gating | Path rules: docs/release-notes only skips all tests; Go-only skips unit/integration, keeps e2e; `.github` or any TS workspace runs everything | +| 11 | Scripts | Packages: direct Vitest, one hop, no `:run` layer. Root: `test`, `test:unit`, `test:integration` are root Vitest; `test:e2e`, `test:live` stay Turbo tasks | +| 12 | Runtime tuning | `fsModuleCache` on, `node_modules/.vitest-cache` restored in CI, one `vitest doctor` run recorded; no isolation or pool changes | +| 13 | Delivery | Three PRs; Vitest-4-safe work first because Vitest 5.0.0 is firewall-quarantined | Considered and rejected: per-package Turbo test caching (test inputs span sibling packages, the built binary, Docker, and env; CI cache is cold anyway); @@ -67,7 +67,7 @@ declarative`, `db schema declarative sync`, `shadow-cache`, the stack suffix and keeps them. `packages/stack` becomes `e2e-stack` only. `apps/cli-e2e` stays `e2e`. Both keep the existing global setup and timeouts. -3. Extend `vitest.shared.ts` (or, on Vitest 4, the inline configs) so the kind +3. Extend `vitest.shared.mts` (or, on Vitest 4, the inline configs) so the kind list knows `e2e-stack`; update the kind table in `AGENTS.md` and the e2e section of `CONTRIBUTING.md`. 4. Add the path-rule gate to `.github/workflows/test.yml`: a `changes` job using @@ -81,7 +81,7 @@ declarative`, `db schema declarative sync`, `shadow-cache`, This is the content of #6457 rebased onto develop after PR A: Vitest 5, the coverage-provider bump, the `@effect/vitest` peer rule, the knip root plugin -change, `.gitignore`, and `vitest.shared.ts` with `definePackageConfig` and +change, `.gitignore`, and `vitest.shared.mts` with `definePackageConfig` and `testProject`. Re-run `pnpm install --frozen-lockfile` in CI as the readiness check; the blocker is `firewall.depthfirst.com` returning 451 for the 5.0.0 tarballs. @@ -109,7 +109,7 @@ tarballs. 5. **CI**: unit and integration become one matrix job, `shard: [1, 2]`, running the root `test` script with `--shard`; the existing summary job keeps the required check name. The e2e step becomes `pnpm exec turbo run //#test:e2e --- --shard=N/3`. Add `actions/cache` for `node_modules/.vite` keyed on the +--shard=N/3`. Add `actions/cache` for `node_modules/.vitest-cache` keyed on the lockfile plus a version salt. Remove `--coverage.enabled` from PR jobs. 6. **Coverage on develop**: root `coverage` config (istanbul, include `{apps,packages}/*/src/**/*.ts`, the CLI's exclude list prefixed with diff --git a/package.json b/package.json index 88822d379d..ef552ea275 100644 --- a/package.json +++ b/package.json @@ -9,10 +9,6 @@ "record": "pnpm exec turbo run @supabase/cli-e2e#record --", "test:smoke": "pnpm exec turbo run supabase#test:smoke --", "dev:docs": "pnpm exec turbo run @supabase/docs#dev", - "test:unit": "pnpm exec turbo run test:unit:run --filter=!@supabase/cli-go --", - "test:integration": "pnpm exec turbo run test:integration:run --", - "test:e2e": "pnpm exec turbo run supabase#build && pnpm exec turbo run test:e2e:run --only --concurrency=1 --", - "test:vitest": "bun --bun vitest run --project '!supabase (live)'", "check:all": "pnpm exec turbo run types:check lint:check fmt:check knip:check lint:effect:check", "fix:all": "pnpm exec turbo run lint:fix fmt:fix knip:fix && pnpm run lint:effect:fix", "lint:check": "oxlint --config .oxlintrc.json", @@ -27,7 +23,12 @@ "repos:install": "git submodule update --init --recursive", "repos:pull": "git submodule update --remote", "local-registry": "bun tools/release/local-registry.ts", - "cli-release": "bun tools/release/local-release.ts" + "cli-release": "bun tools/release/local-release.ts", + "test": "bun --bun vitest run --project \"*(unit)\" --project \"*(integration)\"", + "test:unit": "bun --bun vitest run --project \"*(unit)\"", + "test:integration": "bun --bun vitest run --project \"*(integration)\"", + "test:e2e": "pnpm exec turbo run //#test:e2e:run --", + "test:e2e:run": "bun --bun vitest run --project \"*(e2e)\" --project \"*(e2e-stack)\"" }, "devDependencies": { "@effect/tsgo": "catalog:", diff --git a/packages/api/package.json b/packages/api/package.json index 13bdbaa8cf..af0f7f8bf9 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -15,10 +15,9 @@ "generate:spec": "bun run scripts/download-openapi.ts", "generate": "bun run generate:spec && bun run scripts/generate.ts", "generate:check": "pnpm generate && pnpm --workspace-root run fmt:fix && git diff --exit-code -- src/generated scripts/openapi-source.json", - "test": "pnpm run test:unit", - "test:unit": "pnpm exec turbo run @supabase/api#test:unit:run --", - "test:unit:run": "bun --bun vitest run --project unit --coverage.reportsDirectory=coverage/unit", - "types:check": "tsc --noEmit" + "types:check": "tsc --noEmit", + "test": "bun --bun vitest run --project unit", + "test:unit": "bun --bun vitest run --project unit" }, "dependencies": { "@effect/platform-bun": "catalog:", diff --git a/packages/api/vitest.config.ts b/packages/api/vitest.config.ts index 62fffebcfa..7001a2bd13 100644 --- a/packages/api/vitest.config.ts +++ b/packages/api/vitest.config.ts @@ -1,4 +1,4 @@ -import { definePackageConfig, testProject } from "../../vitest.shared.ts"; +import { definePackageConfig, testProject } from "../../vitest.shared.mts"; export default definePackageConfig({ test: { projects: [testProject("unit")] }, diff --git a/packages/cli-test-helpers/package.json b/packages/cli-test-helpers/package.json index c080b70f69..70a385333d 100644 --- a/packages/cli-test-helpers/package.json +++ b/packages/cli-test-helpers/package.json @@ -7,10 +7,9 @@ ".": "./src/index.ts" }, "scripts": { - "test": "pnpm run test:unit", - "test:unit": "pnpm exec turbo run @supabase/cli-test-helpers#test:unit:run --", - "test:unit:run": "bun --bun vitest run --project unit --coverage.reportsDirectory=coverage/unit", - "types:check": "tsc --noEmit" + "types:check": "tsc --noEmit", + "test": "bun --bun vitest run --project unit", + "test:unit": "bun --bun vitest run --project unit" }, "devDependencies": { "@tsconfig/bun": "catalog:", diff --git a/packages/cli-test-helpers/vitest.config.ts b/packages/cli-test-helpers/vitest.config.ts index 62fffebcfa..7001a2bd13 100644 --- a/packages/cli-test-helpers/vitest.config.ts +++ b/packages/cli-test-helpers/vitest.config.ts @@ -1,4 +1,4 @@ -import { definePackageConfig, testProject } from "../../vitest.shared.ts"; +import { definePackageConfig, testProject } from "../../vitest.shared.mts"; export default definePackageConfig({ test: { projects: [testProject("unit")] }, diff --git a/packages/config/package.json b/packages/config/package.json index 8302128046..976c87d07d 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -67,9 +67,8 @@ "scripts": { "build": "bun run ./scripts/build.ts", "types:check": "tsc --noEmit", - "test": "pnpm run test:unit", - "test:unit": "pnpm exec turbo run @supabase/config#test:unit:run --", - "test:unit:run": "bun --bun vitest run --project unit --coverage.reportsDirectory=coverage/unit" + "test": "bun --bun vitest run --project unit", + "test:unit": "bun --bun vitest run --project unit" }, "dependencies": { "@standard-schema/spec": "^1.1.0", diff --git a/packages/config/vitest.config.ts b/packages/config/vitest.config.ts index 62fffebcfa..7001a2bd13 100644 --- a/packages/config/vitest.config.ts +++ b/packages/config/vitest.config.ts @@ -1,4 +1,4 @@ -import { definePackageConfig, testProject } from "../../vitest.shared.ts"; +import { definePackageConfig, testProject } from "../../vitest.shared.mts"; export default definePackageConfig({ test: { projects: [testProject("unit")] }, diff --git a/packages/process-compose/package.json b/packages/process-compose/package.json index 2fdc3c72b4..564d073da6 100644 --- a/packages/process-compose/package.json +++ b/packages/process-compose/package.json @@ -7,12 +7,10 @@ ".": "./src/index.ts" }, "scripts": { - "test": "pnpm run test:unit && pnpm run test:integration", - "test:unit": "pnpm exec turbo run @supabase/process-compose#test:unit:run --", - "test:unit:run": "bun --bun vitest run --project unit --coverage.reportsDirectory=coverage/unit", - "test:integration": "pnpm exec turbo run @supabase/process-compose#test:integration:run --", - "test:integration:run": "bun --bun vitest run --project integration --coverage.reportsDirectory=coverage/integration", - "types:check": "tsc --noEmit" + "types:check": "tsc --noEmit", + "test": "bun --bun vitest run --project unit --project integration", + "test:unit": "bun --bun vitest run --project unit", + "test:integration": "bun --bun vitest run --project integration" }, "dependencies": { "effect": "catalog:" diff --git a/packages/process-compose/vitest.config.ts b/packages/process-compose/vitest.config.ts index aefedecbd3..c83e489386 100644 --- a/packages/process-compose/vitest.config.ts +++ b/packages/process-compose/vitest.config.ts @@ -1,4 +1,4 @@ -import { definePackageConfig, testProject } from "../../vitest.shared.ts"; +import { definePackageConfig, testProject } from "../../vitest.shared.mts"; export default definePackageConfig({ test: { projects: [testProject("unit"), testProject("integration")] }, diff --git a/packages/stack/package.json b/packages/stack/package.json index 3ec22b4796..7c666617cf 100644 --- a/packages/stack/package.json +++ b/packages/stack/package.json @@ -22,16 +22,13 @@ "./daemon-bun": "./src/daemon-bun.ts" }, "scripts": { - "test": "pnpm run test:unit && pnpm run test:integration && pnpm run test:e2e", - "test:unit": "pnpm exec turbo run @supabase/stack#test:unit:run --", - "test:unit:run": "bun --bun vitest run --project unit --coverage.reportsDirectory=coverage/unit", - "test:integration": "pnpm exec turbo run @supabase/stack#test:integration:run --", - "test:integration:run": "bun --bun vitest run --project integration --coverage.reportsDirectory=coverage/integration", - "test:e2e": "pnpm exec turbo run @supabase/stack#test:e2e:run --", - "test:e2e:run": "bun --bun vitest run --project e2e-stack", - "test:e2e:warmup": "bun run tests/warmup-e2e.ts", "sync:versions": "bun run scripts/sync-versions-from-dockerfile.ts", - "types:check": "tsc --noEmit" + "types:check": "tsc --noEmit", + "test": "bun --bun vitest run --project unit --project integration", + "test:unit": "bun --bun vitest run --project unit", + "test:integration": "bun --bun vitest run --project integration", + "test:e2e": "bun --bun vitest run --project e2e-stack", + "test:e2e:warmup": "bun run tests/warmup-e2e.ts" }, "dependencies": { "@effect/platform-bun": "catalog:", diff --git a/packages/stack/vitest.config.ts b/packages/stack/vitest.config.ts index 3079bd4956..0306a45d16 100644 --- a/packages/stack/vitest.config.ts +++ b/packages/stack/vitest.config.ts @@ -1,4 +1,4 @@ -import { definePackageConfig, testProject } from "../../vitest.shared.ts"; +import { definePackageConfig, testProject } from "../../vitest.shared.mts"; export default definePackageConfig({ test: { diff --git a/tools/test-shard-weights.ts b/tools/test-shard-weights.ts new file mode 100644 index 0000000000..13ef0f6f1f --- /dev/null +++ b/tools/test-shard-weights.ts @@ -0,0 +1,87 @@ +/** + * Builds `.vitest/shard-weights.json`, the per-file test durations that the + * root Vitest sequencer uses to balance CI shards (ADR 0024). + * + * Usage: + * bun tools/test-shard-weights.ts merge --out [--previous ] ... + * + * Inputs are Vitest `json` reporter files (`--reporter=json --outputFile=...`), + * one per shard of the develop run. Each file's duration is its `endTime - + * startTime` in seconds, keyed by repo-relative path. Reports are merged with + * the previous weights so files that did not run this time (skipped, or on a + * shard that failed) keep their last known duration; a report always wins over + * the previous file for the files it contains. + * + * Run by `.github/workflows/develop-tests.yml` on every develop push; the + * result is saved to the Actions cache and handed to PR shards by `test.yml`. + */ +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, relative, resolve } from "node:path"; + +interface JsonReport { + readonly testResults: ReadonlyArray<{ + readonly name: string; + readonly startTime: number; + readonly endTime: number; + }>; +} + +interface ShardWeights { + readonly version: 1; + readonly generatedAt: string; + readonly weights: Record; +} + +function parseArgs(argv: ReadonlyArray) { + const [command, ...rest] = argv; + if (command !== "merge") { + throw new Error(`Unknown command ${JSON.stringify(command)}; expected "merge"`); + } + let out: string | undefined; + let previous: string | undefined; + const reports: string[] = []; + for (let i = 0; i < rest.length; i++) { + const arg = rest[i]; + if (arg === "--out") out = rest[++i]; + else if (arg === "--previous") previous = rest[++i]; + else if (arg !== undefined) reports.push(arg); + } + if (out === undefined) throw new Error("--out is required"); + return { out, previous, reports }; +} + +function readJson(path: string): T { + return JSON.parse(readFileSync(path, "utf8")) as T; +} + +function main(argv: ReadonlyArray): number { + const { out, previous, reports } = parseArgs(argv); + const root = process.cwd(); + const weights: Record = {}; + if (previous !== undefined) { + Object.assign(weights, readJson(previous).weights); + } + let recorded = 0; + for (const report of reports) { + for (const result of readJson(report).testResults) { + const seconds = (result.endTime - result.startTime) / 1000; + if (!Number.isFinite(seconds) || seconds < 0) continue; + weights[relative(root, resolve(result.name))] = Math.round(seconds * 100) / 100; + recorded++; + } + } + const sorted = Object.fromEntries(Object.entries(weights).sort(([a], [b]) => a.localeCompare(b))); + const output: ShardWeights = { + version: 1, + generatedAt: new Date().toISOString(), + weights: sorted, + }; + mkdirSync(dirname(out), { recursive: true }); + writeFileSync(out, `${JSON.stringify(output, null, 2)}\n`); + console.log( + `wrote ${out}: ${Object.keys(sorted).length} files (${recorded} from ${reports.length} report${reports.length === 1 ? "" : "s"})`, + ); + return 0; +} + +process.exitCode = main(process.argv.slice(2)); diff --git a/turbo.json b/turbo.json index 9574c0d82d..a968a722dd 100644 --- a/turbo.json +++ b/turbo.json @@ -35,18 +35,6 @@ "cache": true, "outputs": [".source/**"] }, - "test:unit:run": { - "cache": false, - "passThroughEnv": ["*"] - }, - "test:integration:run": { - "cache": false, - "passThroughEnv": ["*"] - }, - "test:e2e:run": { - "cache": false, - "passThroughEnv": ["*"] - }, "@supabase/cli-go#build": { "cache": true, "inputs": ["$TURBO_DEFAULT$", "$TURBO_ROOT$/mise.toml", "$TURBO_ROOT$/mise.lock"], @@ -125,12 +113,7 @@ "dependsOn": ["@supabase/docs#generate"], "persistent": true }, - "supabase#test:e2e:run": { - "cache": false, - "passThroughEnv": ["*"], - "dependsOn": ["supabase#build"] - }, - "@supabase/cli-e2e#test:e2e:run": { + "//#test:e2e:run": { "cache": false, "passThroughEnv": ["*"], "dependsOn": ["supabase#build"] diff --git a/vitest.config.mts b/vitest.config.mts index 5e36ef95a6..35bfdcb0b6 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -1,15 +1,153 @@ +import { existsSync, readFileSync } from "node:fs"; +import { relative } from "node:path"; import { defineConfig } from "vitest/config"; -import { runDefaults } from "./vitest.shared.ts"; +import { BaseSequencer, type TestSpecification } from "vitest/node"; +import { runDefaults } from "./vitest.shared.mts"; // Every package config becomes a nested project group named after the package, // so `vitest --project 'supabase (unit)'` or `--project '*(integration)'` selects -// slices of the whole repo from one process. Root-only options such as `silent` +// slices of the whole repo from one process. Run-level options such as `silent` // are not inherited by these file-referenced projects; they come from -// `vitest.shared.ts` through each package config for standalone runs and are +// `vitest.shared.mts` through each package config for standalone runs and are // repeated here for root runs. + +const byPath = (a: TestSpecification, b: TestSpecification) => a.moduleId.localeCompare(b.moduleId); +const isStackBacked = (spec: TestSpecification) => spec.project.name.endsWith("(e2e-stack)"); + +/** + * Per-file durations from the last develop run, produced by + * `tools/test-shard-weights.ts` and delivered to CI shards as a run artifact + * (see `.github/workflows/develop-tests.yml` and `test.yml`). Keyed by + * repo-relative path in seconds. Absent locally and on a cold cache. + */ +const SHARD_WEIGHTS_FILE = process.env["VITEST_SHARD_WEIGHTS"] ?? ".vitest/shard-weights.json"; + +function loadShardWeights(): ReadonlyMap | undefined { + if (!existsSync(SHARD_WEIGHTS_FILE)) return undefined; + const parsed: unknown = JSON.parse(readFileSync(SHARD_WEIGHTS_FILE, "utf8")); + if (typeof parsed !== "object" || parsed === null || !("weights" in parsed)) return undefined; + const weights = (parsed as { weights: Record }).weights; + return new Map( + Object.entries(weights).filter( + (entry): entry is [string, number] => typeof entry[1] === "number" && entry[1] >= 0, + ), + ); +} + +/** + * Deterministic shard balancing (ADR 0024). + * + * Vitest's default `shard()` sorts files by a path hash and cuts contiguous + * slices, so the serial stack-backed e2e files land on shards by luck. Files + * are balanced here in two classes, stack-backed first, then everything else, + * because a serial file and a parallel file do not cost the same. + * + * With develop's per-file durations available, each class is assigned + * largest-first to the least-loaded shard; files without a recorded duration + * get the class median so a new file degrades balance slightly rather than + * breaking it. Without durations, files are dealt round-robin within each + * test project with a running offset, which still halves every project across + * shards (sorting every file by path would pair each integration file with + * its unit sibling and hand one shard all of one kind). + * + * Every shard computes the same partition independently, so both paths depend + * only on the file list and the weights file, and every ordering is total. + * + * `sort()` keeps Vitest's project grouping but orders files lexicographically + * within a project, which is what the compatibility e2e suite in apps/cli-e2e + * relies on for deterministic replay ordering. + */ +class BalancedSequencer extends BaseSequencer { + override async shard(files: TestSpecification[]): Promise { + const { index, count } = this.ctx.config.shard ?? { index: 1, count: 1 }; + const weights = loadShardWeights(); + const classes = [files.filter(isStackBacked), files.filter((spec) => !isStackBacked(spec))]; + const selected: TestSpecification[] = []; + for (const specs of classes) { + selected.push( + ...(weights === undefined + ? dealByProject(specs, index, count) + : assignByWeight(specs, index, count, weights)), + ); + } + return selected; + } + + override async sort(files: TestSpecification[]): Promise { + return [...files].sort((a, b) => a.project.name.localeCompare(b.project.name) || byPath(a, b)); + } +} + +function dealByProject(specs: TestSpecification[], index: number, count: number) { + const byProject = new Map(); + for (const spec of specs) { + byProject.set(spec.project.name, [...(byProject.get(spec.project.name) ?? []), spec]); + } + const selected: TestSpecification[] = []; + let offset = 0; + for (const name of [...byProject.keys()].sort()) { + const projectSpecs = [...(byProject.get(name) ?? [])].sort(byPath); + selected.push( + ...projectSpecs.filter((_, position) => (position + offset) % count === index - 1), + ); + offset += projectSpecs.length; + } + return selected; +} + +function assignByWeight( + specs: TestSpecification[], + index: number, + count: number, + weights: ReadonlyMap, +) { + const root = process.cwd(); + const known = specs + .map((spec) => weights.get(relative(root, spec.moduleId))) + .filter((weight): weight is number => weight !== undefined) + .sort((a, b) => a - b); + const fallback = known.length === 0 ? 1 : (known[Math.floor(known.length / 2)] ?? 1); + const weightOf = (spec: TestSpecification) => + weights.get(relative(root, spec.moduleId)) ?? fallback; + const loads = Array.from({ length: count }, () => 0); + const selected: TestSpecification[] = []; + for (const spec of [...specs].sort((a, b) => weightOf(b) - weightOf(a) || byPath(a, b))) { + let target = 0; + for (let shard = 1; shard < count; shard++) { + if ((loads[shard] ?? 0) < (loads[target] ?? 0)) target = shard; + } + loads[target] = (loads[target] ?? 0) + weightOf(spec); + if (target === index - 1) selected.push(spec); + } + return selected; +} + export default defineConfig({ test: { ...runDefaults, projects: ["apps/*/vitest.config.ts", "packages/*/vitest.config.ts"], + sequence: { sequencer: BalancedSequencer }, + // Root runs produce one merged report across packages. Disabled by default; + // the develop-push coverage workflow enables it. Include patterns are + // repo-relative because Vitest 5 matches coverage globs without a + // "contains" fallback. + coverage: { + enabled: false, + provider: "v8", + include: ["apps/*/src/**/*.ts", "packages/*/src/**/*.ts"], + exclude: [ + "**/*.test.ts", + "**/tests/**", + "**/scripts/**", + "apps/cli-e2e/**", + "apps/cli/src/**/*.command.ts", + "apps/cli/src/app.ts", + "apps/cli/src/bin.ts", + "apps/cli/src/index.ts", + "apps/cli/src/supabase.ts", + ], + reporter: ["text-summary", "lcov"], + reportsDirectory: "coverage", + }, }, }); diff --git a/vitest.shared.ts b/vitest.shared.mts similarity index 95% rename from vitest.shared.ts rename to vitest.shared.mts index ba6b4a20c7..23b1b12e9b 100644 --- a/vitest.shared.ts +++ b/vitest.shared.mts @@ -73,6 +73,9 @@ export const runDefaults = { passWithNoTests: true, // Console output from passing tests is noise; failing tests still print theirs. silent: "passed-only", + // Persist Vite's transform output under node_modules/.vitest-cache so reruns + // and separate processes reuse it. CI restores that directory between runs. + fsModuleCache: true, } as const satisfies TestUserConfig; const packageDefaults: ViteUserConfig = defineConfig({