diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index de3dec7ff4..3037e901e4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -78,12 +78,66 @@ jobs: continue-on-error: true run: pnpm run check:config-api - test-unit: + # Path gate for the test jobs. Docs-only and release-notes-only changes skip + # every test job; Go-only changes skip unit and integration but keep e2e, + # which shells out to the Go sidecar binary; anything under .github or a + # TypeScript workspace runs everything. `check` is never gated. The summary + # jobs fail if this job does not succeed, so a broken gate cannot turn into + # a false green (ADR 0024). + changes: if: | !startsWith(github.head_ref, 'release-notes/') && (github.event_name == 'merge_group' || inputs.force || github.event.pull_request.draft == false) + name: Detect changed areas + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + ts: ${{ steps.filter.outputs.ts }} + go: ${{ steps.filter.outputs.go }} + ci: ${{ steps.filter.outputs.ci }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Classify changed paths + id: filter + uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 + with: + filters: | + ts: + - "apps/cli/**" + - "apps/cli-e2e/**" + - "packages/**" + - "tools/**" + - "patches/**" + - "package.json" + - "pnpm-lock.yaml" + - "pnpm-workspace.yaml" + - "turbo.json" + - "tsconfig.json" + - "mise.toml" + - "mise.lock" + - ".bun-version" + - "vitest.*" + go: + - "apps/cli-go/**" + ci: + - ".github/**" + + test-unit: + needs: changes + if: | + !startsWith(github.head_ref, 'release-notes/') && + (github.event_name == 'merge_group' || + inputs.force || + github.event.pull_request.draft == false) && + (needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true') name: Run unit tests runs-on: blacksmith-4vcpu-ubuntu-2404 steps: @@ -101,11 +155,13 @@ jobs: run: pnpm run test:unit --coverage.enabled test-integration: + needs: changes if: | !startsWith(github.head_ref, 'release-notes/') && (github.event_name == 'merge_group' || inputs.force || - github.event.pull_request.draft == false) + github.event.pull_request.draft == false) && + (needs.changes.outputs.ts == 'true' || needs.changes.outputs.ci == 'true') name: Run integration tests runs-on: blacksmith-8vcpu-ubuntu-2404 steps: @@ -130,11 +186,15 @@ jobs: inputs.force || github.event.pull_request.draft == false) name: Run unit and integration tests - needs: [test-unit, test-integration] + needs: [changes, test-unit, test-integration] runs-on: ubuntu-latest steps: - name: Verify unit and integration tests succeeded run: | + if [ "${{ needs.changes.result }}" != "success" ]; then + echo "::error ::Path gate did not succeed: changes=${{ needs.changes.result }}" + exit 1 + fi if [ "${{ needs.test-unit.result }}" = "failure" ] || [ "${{ needs.test-unit.result }}" = "cancelled" ] || \ [ "${{ needs.test-integration.result }}" = "failure" ] || [ "${{ needs.test-integration.result }}" = "cancelled" ]; then echo "::error ::Unit or integration tests failed: unit=${{ needs.test-unit.result }}, integration=${{ needs.test-integration.result }}" @@ -143,11 +203,13 @@ jobs: echo "Unit and integration tests reported: unit=${{ needs.test-unit.result }}, integration=${{ needs.test-integration.result }}" test-e2e: + needs: changes if: | !startsWith(github.head_ref, 'release-notes/') && (github.event_name == 'merge_group' || inputs.force || - github.event.pull_request.draft == false) + github.event.pull_request.draft == false) && + (needs.changes.outputs.ts == 'true' || needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true') name: Run end-to-end tests (shard ${{ matrix.shard }}/3) runs-on: blacksmith-8vcpu-ubuntu-2404 strategy: @@ -204,11 +266,15 @@ jobs: inputs.force || github.event.pull_request.draft == false) name: Run end-to-end tests - needs: test-e2e + needs: [changes, test-e2e] runs-on: ubuntu-latest steps: - name: Verify all shards succeeded run: | + if [ "${{ needs.changes.result }}" != "success" ]; then + echo "::error ::Path gate did not succeed: changes=${{ needs.changes.result }}" + exit 1 + fi if [ "${{ needs.test-e2e.result }}" = "failure" ] || [ "${{ needs.test-e2e.result }}" = "cancelled" ]; then echo "::error ::One or more e2e shards failed: ${{ needs.test-e2e.result }}" exit 1 diff --git a/AGENTS.md b/AGENTS.md index 9e313dc366..fa33c3d460 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -294,6 +294,7 @@ See `apps/cli/src/commands/login/` as the canonical example. - `*.unit.test.ts` — unit tests, colocated next to source - `*.integration.test.ts` — integration tests, colocated next to source - `*.e2e.test.ts` — end-to-end tests, colocated next to source +- `*.stack.e2e.test.ts` — stack-backed end-to-end tests: e2e files that start a local Supabase stack or run Docker containers. Still e2e, but they run one file at a time while plain e2e files run in parallel (ADR 0024) - `tests/` — shared test helpers (for example `tests/helpers/cli.ts`) ### Testing pyramid for CLI commands @@ -309,6 +310,7 @@ See `apps/cli/src/commands/login/` as the canonical example. - Do not use e2e tests for help text, argument normalization, dry-run payloads, schema rendering, projection formatting, or similar detail coverage unless the real subprocess boundary itself is the thing being validated. - If an assertion can be expressed faithfully in an integration test, it should generally live there instead of in e2e. - When in doubt, move coverage down the pyramid: e2e -> integration -> unit. +- An e2e test that starts a local stack (`supabase start`, `db start`, `createStack`) or runs a Docker container must use the `*.stack.e2e.test.ts` suffix so it joins the serial `e2e-stack` project. A test that only spawns the CLI against an isolated temporary home keeps the plain suffix and runs in parallel. ### Test execution policy diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c37e341d0d..56fb1098a4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -180,8 +180,11 @@ The root unit and integration scripts use Turbo to fan out the package-local workspace remains package-local because its tests run directly through Go: `pnpm --dir apps/cli-go run test:unit`. Go tests are covered by the dedicated Go CI workflow. Unit and integration tasks are uncached for now; e2e tasks are -also uncached and run one package at a time. Forward a Vitest shard to every -e2e package with `pnpm run test:e2e --shard=1/3`. +also uncached and run one package at a time. Within a package, plain +`*.e2e.test.ts` files run in parallel while `*.stack.e2e.test.ts` files (those +that start a local stack or run Docker containers) run serially in the +`e2e-stack` project. Forward a Vitest shard to every e2e package with +`pnpm run test:e2e --shard=1/3`. ## E2E Compatibility Test Suite diff --git a/apps/cli/AGENTS.md b/apps/cli/AGENTS.md index 60c28994cb..7e2695c68d 100644 --- a/apps/cli/AGENTS.md +++ b/apps/cli/AGENTS.md @@ -478,6 +478,7 @@ Read https://www.effect.solutions/testing for Effect testing patterns. Note that - `*.unit.test.ts` belongs to the `unit` Vitest project and is the default for unit-style and other fast in-process tests. - `*.integration.test.ts` belongs to the `integration` project and is for in-process integration tests that exercise real handler or service behavior with layered dependency replacement. - `*.e2e.test.ts` belongs to the `e2e` Vitest project and is for black-box CLI subprocess tests. +- `*.stack.e2e.test.ts` belongs to the `e2e-stack` Vitest project: black-box CLI subprocess tests that start a local stack or run Docker containers. They run one file at a time; plain `e2e` files run in parallel. Use this suffix whenever the test invokes `start`, `db start`, `stop`, `status`, or otherwise needs a running stack or a container (ADR 0024). - `*.live.test.ts` belongs to the `live` Vitest project and is for black-box CLI subprocess tests whose asserted command reaches a real Supabase platform or project data plane — see "Live tests" below. ### Testing policy diff --git a/apps/cli/package.json b/apps/cli/package.json index 3e493c5179..abae7fec98 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -36,7 +36,7 @@ "test:integration": "pnpm exec turbo run supabase#test:integration:run --", "test:integration:run": "bun --bun vitest run --project integration --coverage.reportsDirectory=coverage/integration", "test:e2e": "pnpm exec turbo run supabase#build && pnpm exec turbo run supabase#test:e2e:run --only --", - "test:e2e:run": "bun --bun vitest run --project e2e", + "test:e2e:run": "bun --bun vitest run --project e2e --project e2e-stack", "test:live": "bun --bun vitest run --project live", "test:smoke": "bun run tests/smoke-test.ts", "types:check": "tsc --noEmit" diff --git a/apps/cli/src/legacy/commands/db/diff/diff.declarative.e2e.test.ts b/apps/cli/src/legacy/commands/db/diff/diff.declarative.stack.e2e.test.ts similarity index 100% rename from apps/cli/src/legacy/commands/db/diff/diff.declarative.e2e.test.ts rename to apps/cli/src/legacy/commands/db/diff/diff.declarative.stack.e2e.test.ts diff --git a/apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.e2e.test.ts b/apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.stack.e2e.test.ts similarity index 100% rename from apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.e2e.test.ts rename to apps/cli/src/legacy/commands/db/schema/declarative/sync/sync.stack.e2e.test.ts diff --git a/apps/cli/src/legacy/commands/db/start/start.e2e.test.ts b/apps/cli/src/legacy/commands/db/start/start.stack.e2e.test.ts similarity index 100% rename from apps/cli/src/legacy/commands/db/start/start.e2e.test.ts rename to apps/cli/src/legacy/commands/db/start/start.stack.e2e.test.ts diff --git a/apps/cli/src/legacy/commands/start/start.lifecycle.e2e.test.ts b/apps/cli/src/legacy/commands/start/start.lifecycle.stack.e2e.test.ts similarity index 100% rename from apps/cli/src/legacy/commands/start/start.lifecycle.e2e.test.ts rename to apps/cli/src/legacy/commands/start/start.lifecycle.stack.e2e.test.ts diff --git a/apps/cli/src/legacy/commands/start/start.slim-images.e2e.test.ts b/apps/cli/src/legacy/commands/start/start.slim-images.stack.e2e.test.ts similarity index 100% rename from apps/cli/src/legacy/commands/start/start.slim-images.e2e.test.ts rename to apps/cli/src/legacy/commands/start/start.slim-images.stack.e2e.test.ts diff --git a/apps/cli/src/legacy/commands/start/start.e2e.test.ts b/apps/cli/src/legacy/commands/start/start.stack.e2e.test.ts similarity index 100% rename from apps/cli/src/legacy/commands/start/start.e2e.test.ts rename to apps/cli/src/legacy/commands/start/start.stack.e2e.test.ts diff --git a/apps/cli/src/legacy/commands/status/status.e2e.test.ts b/apps/cli/src/legacy/commands/status/status.stack.e2e.test.ts similarity index 100% rename from apps/cli/src/legacy/commands/status/status.e2e.test.ts rename to apps/cli/src/legacy/commands/status/status.stack.e2e.test.ts diff --git a/apps/cli/src/legacy/commands/stop/stop.e2e.test.ts b/apps/cli/src/legacy/commands/stop/stop.stack.e2e.test.ts similarity index 100% rename from apps/cli/src/legacy/commands/stop/stop.e2e.test.ts rename to apps/cli/src/legacy/commands/stop/stop.stack.e2e.test.ts diff --git a/apps/cli/src/legacy/shared/db-bootstrap/shadow-cache.e2e.test.ts b/apps/cli/src/legacy/shared/db-bootstrap/shadow-cache.stack.e2e.test.ts similarity index 100% rename from apps/cli/src/legacy/shared/db-bootstrap/shadow-cache.e2e.test.ts rename to apps/cli/src/legacy/shared/db-bootstrap/shadow-cache.stack.e2e.test.ts diff --git a/apps/cli/src/shared/functions/serve-main-offline.e2e.test.ts b/apps/cli/src/shared/functions/serve-main-offline.stack.e2e.test.ts similarity index 100% rename from apps/cli/src/shared/functions/serve-main-offline.e2e.test.ts rename to apps/cli/src/shared/functions/serve-main-offline.stack.e2e.test.ts diff --git a/apps/cli/vitest.config.ts b/apps/cli/vitest.config.ts index 8718c893fc..13c4fb42bb 100644 --- a/apps/cli/vitest.config.ts +++ b/apps/cli/vitest.config.ts @@ -80,8 +80,30 @@ export default defineConfig({ ssr: { resolve: workspacePackageSsrResolve }, plugins: [dockerfileTextPlugin()], test: { + // Stackless e2e: black-box CLI subprocesses against an isolated + // temporary home. Safe under file-level parallelism (see the flake + // policy in AGENTS.md), so Vitest's default parallelism applies. name: "e2e", include: ["**/*.e2e.test.ts"], + exclude: ["**/*.stack.e2e.test.ts", "**/node_modules/**"], + globalSetup: ["tests/e2e-global-setup.ts"], + setupFiles: ["tests/e2e-setup.ts"], + testTimeout: 120_000, + hookTimeout: 120_000, + }, + }, + { + resolve: workspacePackageResolve, + ssr: { resolve: workspacePackageSsrResolve }, + plugins: [dockerfileTextPlugin()], + test: { + // Stack-backed e2e: files that start a local Supabase stack or run + // Docker containers claim machine-level resources, so they run one + // file at a time. Vitest schedules this serial group after the + // parallel projects finish. Opt in with the `*.stack.e2e.test.ts` + // suffix (ADR 0024). + name: "e2e-stack", + include: ["**/*.stack.e2e.test.ts"], fileParallelism: false, maxWorkers: 1, globalSetup: ["tests/e2e-global-setup.ts"], diff --git a/docs/adr/0024-test-execution-topology.md b/docs/adr/0024-test-execution-topology.md new file mode 100644 index 0000000000..16df8fd33f --- /dev/null +++ b/docs/adr/0024-test-execution-topology.md @@ -0,0 +1,77 @@ +# 0024. Test Execution Topology: Root Vitest Runs, Stack-Backed E2e, and Turbo's Role + +**Status**: proposed +**Date**: 2026-09-04 + +## Problem Statement + +Tests in this monorepo were run by Turbo fanning out to one Vitest process per package for each test kind, with every test task uncached. That shape had three costs that grew with the suite: + +1. **CI wall-clock was decided by luck.** The e2e suite was sharded per package by Vitest's path-hash slicing, so the 13 files that start a local Supabase stack or run a Docker container (about 72% of all e2e time) landed on shards at random. Recent runs spent 7.9, 5.6, and 3.3 minutes in the three shards' test steps. +2. **Every e2e project ran files strictly serially**, a setting from the first architecture commit that no comment defended and that contradicts the repository's own flake policy, which requires tests to be correct under file-level parallelism. Serial execution meant 58 stackless CLI e2e files queued behind the stack-backed ones instead of fanning out. +3. **Configuration and scripts repeated themselves.** Seven package configs restated coverage, include globs, and export-condition resolution; each test kind reached Vitest through three layers of scripts (`test:unit` to Turbo to `test:unit:run` to Vitest), and six processes produced six interleaved reports for one repository-wide run. + +Vitest 5 changed what is possible: a root config can reference package configs that declare their own `projects`, inline projects inherit their declaring config, and one process can filter, shard, and sequence across every package. + +## Decision + +1. **One root Vitest run is the unit of execution for unit, integration, and e2e tests.** The repo-root `vitest.config.mts` loads every package config as a nested project group named ` ()`. CI and the root scripts run that process with `--project` and `--shard` filters. Package configs remain runnable standalone for local work and share one preset, `vitest.shared.ts`. +2. **E2e tests that start a local stack are a distinct sub-kind, marked by the `*.stack.e2e.test.ts` suffix**, and run in their own serial `e2e-stack` test project. All other e2e files run with normal file parallelism. A file claims the stack-backed suffix when it starts a Supabase stack, Docker or native; a file that only spawns the CLI against an isolated temporary home does not. +3. **Shards are balanced by weight class, not duration data.** The root config's sequencer deals stack-backed files round-robin across shards by sorted path, then stackless files the same way. Every shard computes the same partition independently. +4. **Turbo keeps the dependency graph and nothing else in test execution.** E2e is a Turbo root task, `//#test:e2e`, that depends on `supabase#build`; live and smoke stay Turbo tasks for the same reason. Unit and integration no longer pass through Turbo. Job-level skipping is done with path rules in the workflow, not `turbo --affected`. +5. **Coverage leaves PR runs.** A develop-push workflow produces one merged root report; PR jobs run uninstrumented. + +## Vocabulary + +These are the canonical names for the concepts above; use them in configs, scripts, docs, and PR descriptions. + +- **Test kind**: one of unit, integration, e2e, or live. Every test file belongs to exactly one kind, chosen by its file suffix and colocated with the code it covers. A kind is a promise about what the test touches: unit is pure in-process logic, integration runs handlers against mocked services, e2e drives a real CLI subprocess, live drives a real CLI subprocess against a real Supabase platform and is never part of the default loop. +- **Stack-backed e2e**: an e2e test whose file starts a local Supabase stack, Docker or native, or runs a Docker container, and so claims machine-level resources. Marked by the `*.stack.e2e.test.ts` suffix and run one file at a time. An e2e file that only spawns the CLI against an isolated temporary home is **stackless** and keeps the plain `*.e2e.test.ts` suffix. +- **Test project**: a Vitest project, a named slice of the suite with its own include pattern and runtime settings. Each package declares one per test kind it has. Always say "test project"; in this repo an unqualified "project" is a hosted Supabase project (see `ProjectConfig`, ADR 0020). From the repository root a test project is addressed as ` ()`, for example `supabase (unit)` or `@supabase/stack (e2e-stack)`; inside a package, by kind alone. +- **Package config**: a package's `vitest.config.ts`. It plays two roles: the run root for a standalone run, and one member of the root run's project list. +- **Standalone run**: Vitest started inside one package. Only that package's test projects are visible; run-level options come from the package config. +- **Root run**: Vitest started from the repository root. Every package config is loaded as a group of test projects, so one process can select any set of kinds or packages. Run-level options come from the root config; the package configs' copies are ignored. +- **Run-level option**: a Vitest option read only from the run's root config: coverage, reporters, `silent`, `passWithNoTests`, sequencing, and sharding among them. Package configs still set them for standalone runs. +- **Compatibility e2e suite**: the record-and-replay suite in `apps/cli-e2e`. An e2e test project like any other, but it exercises the CLI against recorded Management API traffic rather than a local stack. + +## Rationale + +- **Wall-clock is the objective**, agreed ahead of compute and configuration simplicity. Only a process that sees all e2e files can balance them, and only a project split can let stackless files run in parallel while stack-backed files stay serial. Vitest schedules a serial project after the parallel groups within one run, so the split needs no custom scheduling. Simulated on the measured durations, the worst shard's serial work drops from about 7.9 to about 4.5 minutes with three shards. +- **Turbo's test-specific strengths were not in use and could not be made sound cheaply.** Every test task was `cache: false`, CI had no remote cache, and `.turbo` was not persisted, so no test caching existed to lose. Making per-package test caching honest would require declaring sibling packages' sources, the built binary, Docker images, and environment as inputs. Turbo's package graph also does not know the CLI shells out to the Go sidecar (that link exists only in the task graph via `supabase#build`), so `--affected` would wrongly skip e2e on Go-only changes. Path rules cost seconds and encode the three real cases: docs-only, Go-only, everything else. +- **A filename suffix is the existing vocabulary.** Test kinds are already chosen by suffix and colocated; extending that to a sub-kind keeps the resource claim visible in the file name, greppable, and reviewable, without adopting tags or breaking colocation. +- **Weight classes over a duration manifest.** Durations range from 0.4s to 150s within the stack-backed class, so a manifest would balance better, by roughly one more minute, but it is data that drifts and needs a job to refresh. The class-based deal needs no data and can be upgraded later if imbalance persists. + +## Consequences + +### Positive + +- E2e wall-clock is bounded by the balanced stack-backed work per shard, not by hash luck. +- One report per CI job and per local root run; `--project '*(unit)'` addresses a kind across the repo, `--project 'supabase (integration)'` a kind within a package. +- Package configs shrink to a list of projects plus genuinely local settings; scripts are one hop from any entry point to Vitest. +- The flake policy and the e2e configuration agree. + +### Negative + +- Run-level options (`silent`, `passWithNoTests`, coverage, sequencing) must be declared in the root config and repeated through the preset for standalone runs; Vitest never inherits them into file-referenced projects. +- The cli-e2e package's lexicographic sequencer no longer applies when its files run from the root; the root sequencer provides the equivalent tiebreak. +- Renaming stack-backed files churns blame once and adds a rule authors must know: starting a stack means taking the suffix. +- Per-package test caching is foreclosed for as long as tests run from the root. Revisit only if a remote cache arrives and test inputs can be declared honestly. + +## Alternatives Considered + +1. **Keep Turbo fan-out and only split e2e projects**: smallest diff, keeps package-level `--affected` for e2e, but cannot balance shards across packages and rarely skips anything on this graph because the CLI depends on every other package. +2. **Separate serial stack job and parallel stackless job**: the stack job alone is about 10.6 minutes serial, worse than today, unless it is itself sharded, which collapses into the chosen design. +3. **Vitest tags or a directory for stack-backed files**: tags hide the claim inside the file and introduce a mechanism the repo does not use; a directory breaks colocation. +4. **Committed duration manifest**: better balance for more maintenance; deferred, see above. +5. **`isolate: false` for unit tests**: about 40 test files mutate env, cwd, or globals or use `vi.mock`; a semantic change that needs an audit before any measurement exists. +6. **`turbo --affected` as the CI gate**: unsound without adding the Go sidecar to the CLI's package.json dependencies, and a gate job pays an install before any test job starts. + +## Related Decisions + +- ADR 0013: Live E2E Tests Bypass the Replay Server (live remains a separate, never-default kind) +- ADR 0017: Simplified Managed Stack Architecture (what a stack-backed e2e test starts) + +## See Also + +- `docs/superpowers/plans/2026-09-04-test-execution-topology.md` for the implementation plan and measured baseline +- Vitest 5 projects guide: https://vitest.dev/guide/projects diff --git a/docs/adr/README.md b/docs/adr/README.md index bd5817e110..ab8f2d698a 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -64,6 +64,7 @@ When an ADR becomes outdated, mark it as `deprecated` or reference the supersedi | 0021 | [ProjectConfig Convergence Semantics](0021-projectconfig-convergence-semantics.md) | accepted | | 0022 | [Config Diff Classification and Managed Surface](0022-config-diff-classification-and-managed-surface.md) | accepted | | 0023 | [Config Pull Write Strategy and Scope Resolution](0023-config-pull-write-strategy-and-scope-resolution.md) | accepted | +| 0024 | [Test Execution Topology](0024-test-execution-topology.md) | proposed | ## Template diff --git a/docs/superpowers/plans/2026-09-04-test-execution-topology.md b/docs/superpowers/plans/2026-09-04-test-execution-topology.md new file mode 100644 index 0000000000..741329a4ca --- /dev/null +++ b/docs/superpowers/plans/2026-09-04-test-execution-topology.md @@ -0,0 +1,136 @@ +# Test execution topology: Turbo + Vitest 5 + +Status: agreed 2026-09-04, implementation in three PRs. Vocabulary and the decisions worth keeping after this plan is done live in +ADR 0024. + +## Goals and how ties break + +1. **PR CI wall-clock** is the objective. Compute is a soft budget (roughly + 1.5x today's runner minutes is acceptable when it shortens the longest job). +2. **Local dev loop** second: one hop from any script to Vitest, filter by kind + or package from anywhere, quiet output from passing tests. +3. **Configuration simplicity** is a constraint, not a goal in itself. + +## Measured baseline (develop, 2026-09-03) + +| Job | Setup | Work | Notes | +| ------------------ | ----- | ------------------ | --------------------------------------------------------------------------------------------- | +| Unit | 1.5m | 1.8m | six Turbo-launched Vitest processes; CLI package is 73s of the 74s | +| Integration | 1.3m | 4.0m | | +| E2e shard 1/2/3 | ~1.5m | 7.9m / 5.6m / 3.3m | 889s total; 13 stack-backed files (develop) are about 633s of it; every e2e project is serial | +| Check code quality | 1.9m | 0.4m | | + +Turbo's cache is cold on every CI run (no remote cache, `.turbo` not persisted) +and every test task is `cache: false`, so no test caching or `--affected` +skipping exists today. + +## Decisions + +| # | Decision | Chosen | +| --- | ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | Tie-breaker between goals | PR CI wall-clock | +| 2 | Compute budget | Soft; wall-clock wins within ~1.5x minutes | +| 3 | Unit/integration runner | One root Vitest process; Turbo drops out of unit/integration | +| 4 | E2e parallelism | Stackless e2e files run in parallel; stack-backed files stay serial | +| 5 | Stack-backed marker | File suffix `*.stack.e2e.test.ts` | +| 6 | E2e CI topology | One root Vitest run per shard, as Turbo root task `//#test:e2e` depending on `supabase#build` | +| 7 | Shard balance | Custom sequencer deals stack-backed files round-robin by sorted path, then stackless; no duration data | +| 8 | Unit/integration CI shape | One root run over both kinds, `--shard=N/2`, matrix of two jobs | +| 9 | Coverage | Removed from PR runs; develop-push workflow produces one merged root report | +| 10 | Job gating | Path rules: docs/release-notes only skips all tests; Go-only skips unit/integration, keeps e2e; `.github` or any TS workspace runs everything | +| 11 | Scripts | Packages: direct Vitest, one hop, no `:run` layer. Root: `test`, `test:unit`, `test:integration` are root Vitest; `test:e2e`, `test:live` stay Turbo tasks | +| 12 | Runtime tuning | `fsModuleCache` on, `node_modules/.vite` restored in CI, one `vitest doctor` run recorded; no isolation or pool changes | +| 13 | Delivery | Three PRs; Vitest-4-safe work first because Vitest 5.0.0 is firewall-quarantined | + +Considered and rejected: per-package Turbo test caching (test inputs span +sibling packages, the built binary, Docker, and env; CI cache is cold anyway); +`turbo --affected` as the job gate (Turbo's package graph does not know the CLI +depends on the Go sidecar, and a gate job would add ~1.3m install to every PR); +a committed duration manifest for sharding (data that drifts, for about one +more minute); `isolate: false` (about 40 test files depend on per-file +isolation). + +## PR A: e2e split and job gating (Vitest 4, land now) + +Independent of the Vitest 5 upgrade. Expected effect: worst e2e shard test step +from ~7.9m toward ~5m even with today's per-package hash sharding, because the +58 stackless CLI files stop queueing behind stack-backed ones. + +1. Rename the 10 CLI files and 3 stack-package files that start a local stack to + `*.stack.e2e.test.ts` (`git mv`, no content changes). The list is in + `apps/cli/src/**` (`start`, `stop`, `status`, `db start`, `db diff +declarative`, `db schema declarative sync`, `shadow-cache`, + `serve-main-offline`) and `packages/stack/tests/createStack*`. +2. In `apps/cli/vitest.config.ts`, split the `e2e` project: `e2e` includes + `**/*.e2e.test.ts`, excludes `**/*.stack.e2e.test.ts`, and drops + `fileParallelism: false` and `maxWorkers: 1`; new `e2e-stack` includes only + the stack suffix and keeps them. `packages/stack` becomes `e2e-stack` only. + `apps/cli-e2e` stays `e2e`. Both keep the existing global setup and + timeouts. +3. Extend `vitest.shared.ts` (or, on Vitest 4, the inline configs) so the kind + list knows `e2e-stack`; update the kind table in `AGENTS.md` and the e2e + section of `CONTRIBUTING.md`. +4. Add the path-rule gate to `.github/workflows/test.yml`: a `changes` job using + a paths filter with three outputs (`ts`, `go`, `ci`); `test-unit` and + `test-integration` need `ts || ci`; `test-e2e` needs `ts || go || ci`; the + summary jobs treat skipped as success (they already do). Keep `check` always + on. +5. Measure: per-shard test-step time on the next three develop merges. + +## PR B: Vitest 5 and the shared preset (when the firewall clears) + +This is the content of #6457 rebased onto develop after PR A: Vitest 5, the +coverage-provider bump, the `@effect/vitest` peer rule, the knip root plugin +change, `.gitignore`, and `vitest.shared.ts` with `definePackageConfig` and +`testProject`. Re-run `pnpm install --frozen-lockfile` in CI as the readiness +check; the blocker is `firewall.depthfirst.com` returning 451 for the 5.0.0 +tarballs. + +## PR C: root runs, balanced shards, one-hop scripts (after PR B) + +1. **Balanced sequencer** in `vitest.config.mts`: subclass `BaseSequencer`; + `shard()` partitions specs by `project.name.endsWith("(e2e-stack)")`, sorts + each partition by `moduleId`, and takes every `count`-th file starting at + `index - 1`; `sort()` keeps Vitest's default and breaks ties + lexicographically, replacing the cli-e2e package's own sequencer. +2. **Turbo root task** `//#test:e2e` in `turbo.json`: `dependsOn: +["supabase#build"]`, `cache: false`, `passThroughEnv: ["*"]`. Root script + `test:e2e:run`: `bun --bun vitest run --project '*(e2e*)'`. Delete the three + per-package `test:e2e:run` Turbo entries and their `supabase#build` + dependencies. `test:live` and `test:smoke` are unchanged. +3. **Root scripts**: `test` = `vitest run --project '*(unit)' --project +'*(integration)'`; `test:unit`, `test:integration` filter by kind; `test:e2e` + = `turbo run //#test:e2e --`. Delete `test:vitest`. Remove the + `test:unit:run` and `test:integration:run` Turbo tasks. +4. **Package scripts**: `test` runs unit and integration standalone; + `test:unit`, `test:integration`, `test:e2e` are single Vitest calls with + `--project`; delete every `:run` script and the Turbo wrappers. Go stays + package-local as today. +5. **CI**: unit and integration become one matrix job, `shard: [1, 2]`, running + the root `test` script with `--shard`; the existing summary job keeps the + required check name. The e2e step becomes `pnpm exec turbo run //#test:e2e +-- --shard=N/3`. Add `actions/cache` for `node_modules/.vite` keyed on the + lockfile plus a version salt. Remove `--coverage.enabled` from PR jobs. +6. **Coverage on develop**: root `coverage` config (istanbul, include + `{apps,packages}/*/src/**/*.ts`, the CLI's exclude list prefixed with + `apps/cli/`), enabled only by a develop-push workflow that uploads the merged + lcov as an artifact. +7. **fsModuleCache**: `fsModuleCache: true` in the preset and root config. +8. **Doctor**: run `vitest doctor` against `supabase (unit)` once and paste the + output into the PR description; act on nothing that changes isolation or + pool. +9. **Docs**: `AGENTS.md` Package Structure and Testing sections, + `CONTRIBUTING.md` test commands, `apps/cli-e2e/AGENTS.md` sequencer note, + `packages/config/AGENTS.md` run instructions. Update the ADR 0024 vocabulary if any + term shifts. +10. Measure the same table as the baseline on three develop merges. Expected: + unit/integration jobs ~4.4m each, e2e shards ~4.5m test step each, plus + whatever `fsModuleCache` returns. + +## Follow-ups explicitly out of scope + +- More e2e shards (cheap to add if the balanced result still shows skew). +- Duration-aware sharding from real data. +- `isolate: false` for unit projects after auditing env/cwd/global/`vi.mock` + users. +- Turbo remote cache; revisit per-package test caching only if it arrives. diff --git a/packages/stack/package.json b/packages/stack/package.json index 37162aeb5d..3ec22b4796 100644 --- a/packages/stack/package.json +++ b/packages/stack/package.json @@ -28,7 +28,7 @@ "test:integration": "pnpm exec turbo run @supabase/stack#test:integration:run --", "test:integration:run": "bun --bun vitest run --project integration --coverage.reportsDirectory=coverage/integration", "test:e2e": "pnpm exec turbo run @supabase/stack#test:e2e:run --", - "test:e2e:run": "bun --bun vitest run --project e2e", + "test:e2e:run": "bun --bun vitest run --project e2e-stack", "test:e2e:warmup": "bun run tests/warmup-e2e.ts", "sync:versions": "bun run scripts/sync-versions-from-dockerfile.ts", "types:check": "tsc --noEmit" diff --git a/packages/stack/tests/createStack-docker.e2e.test.ts b/packages/stack/tests/createStack-docker.stack.e2e.test.ts similarity index 100% rename from packages/stack/tests/createStack-docker.e2e.test.ts rename to packages/stack/tests/createStack-docker.stack.e2e.test.ts diff --git a/packages/stack/tests/createStack-native.e2e.test.ts b/packages/stack/tests/createStack-native.stack.e2e.test.ts similarity index 100% rename from packages/stack/tests/createStack-native.e2e.test.ts rename to packages/stack/tests/createStack-native.stack.e2e.test.ts diff --git a/packages/stack/tests/createStack.e2e.test.ts b/packages/stack/tests/createStack.stack.e2e.test.ts similarity index 100% rename from packages/stack/tests/createStack.e2e.test.ts rename to packages/stack/tests/createStack.stack.e2e.test.ts diff --git a/packages/stack/vitest.config.ts b/packages/stack/vitest.config.ts index ffffff51a2..d251303904 100644 --- a/packages/stack/vitest.config.ts +++ b/packages/stack/vitest.config.ts @@ -27,8 +27,11 @@ export default defineConfig({ }, { test: { - name: "e2e", - include: ["**/*.e2e.test.ts"], + // Every e2e file here starts a local stack, so the whole kind is + // stack-backed and serial. See ADR 0024 for the `*.stack.e2e.test.ts` + // convention. + name: "e2e-stack", + include: ["**/*.stack.e2e.test.ts"], fileParallelism: false, globalSetup: ["./tests/global-setup.ts"], },