feat(playwright-1): add reusable Playwright E2E foundation

.github/workflows/test-build.yml

@@ -245,4 +245,97 @@ jobs:
           AWS_REGION: 'us-west-2'
           ENCRYPTION_KEY: '7cf672e460e430c1fba707575c2b0e2ad5a99dddf9b7b7e3b5646e630861db1c' # dummy key for CI only
           TURBO_CACHE_DIR: .turbo
-        run: bunx turbo run build --filter=sim
+        run: bunx turbo run build --filter=sim
+
+  settings-e2e:
+    name: Settings E2E (informational)
+    runs-on: blacksmith-8vcpu-ubuntu-2404
+    timeout-minutes: 45
+    continue-on-error: true
+
+    services:
+      postgres:
+        image: pgvector/pgvector:pg17
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2
+        with:
+          bun-version: 1.3.13
+
+      - name: Setup Node
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
+        with:
+          node-version: 22
+
+      - name: Mount Bun cache (Sticky Disk)
+        uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
+        with:
+          key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
+          path: ~/.bun/install/cache
+
+      - name: Mount node_modules (Sticky Disk)
+        uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
+        with:
+          key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
+          path: ./node_modules
+
+      - name: Mount Playwright browsers (Sticky Disk)
+        uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
+        with:
+          key: ${{ github.repository }}-playwright-browsers-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
+          path: ~/.cache/ms-playwright
+
+      - name: Restore E2E Next.js build cache
+        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
+        with:
+          path: ./apps/sim/.next/cache
+          key: ${{ runner.os }}-nextjs-e2e-hosted-billing-${{ hashFiles('bun.lock') }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-nextjs-e2e-hosted-billing-${{ hashFiles('bun.lock') }}-
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Configure E2E hostname
+        run: echo "127.0.0.1 e2e.sim.ai" | sudo tee -a /etc/hosts
+
+      - name: Install Chromium
+        working-directory: apps/sim
+        run: bun run test:e2e:install-browsers -- --with-deps
+
+      - name: Run settings E2E foundation
+        timeout-minutes: 40
+        working-directory: apps/sim
+        env:
+          CI: 'true'
+          E2E_PG_ADMIN_URL: 'postgresql://postgres:postgres@127.0.0.1:5432/postgres'
+        run: bun run test:e2e
+
+      - name: Upload E2E diagnostics
+        if: failure() || cancelled()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: settings-e2e-${{ github.run_id }}
+          path: |
+            apps/sim/playwright-report/
+            apps/sim/test-results/
+            apps/sim/e2e/.runs/
+            !apps/sim/e2e/.runs/**/auth/**
+          if-no-files-found: ignore
+          include-hidden-files: true
+          retention-days: 7

.gitignore

@@ -18,6 +18,10 @@ package-lock.json
 # testing
 /coverage
 /apps/**/coverage
+/apps/**/playwright-report/
+/apps/**/test-results/
+/apps/**/e2e/.runs/
+/apps/**/e2e/.auth/
 
 # next.js
 /.next/
@@ -31,6 +35,7 @@ package-lock.json
 **/dist/
 **/standalone/
 sim-standalone.tar.gz
+/.artifacts/
 
 # redis
 dump.rdb

apps/realtime/src/env.ts

@@ -14,6 +14,7 @@ const EnvSchema = z.object({
   INTERNAL_API_SECRET: z.string().min(32),
   NEXT_PUBLIC_APP_URL: z.string().url(),
   ALLOWED_ORIGINS: z.string().optional(),
+  REALTIME_HOST: z.string().min(1).default('0.0.0.0'),
   PORT: z.coerce.number().int().positive().default(3002),
   SIM_DB_ROLE: z.enum(['web', 'trigger', 'realtime']).optional(),
   DISABLE_AUTH: z

apps/realtime/src/index.ts

@@ -31,6 +31,7 @@ async function createRoomManager(io: SocketIOServer): Promise<IRoomManager> {
 async function main() {
   const httpServer = createServer()
   const PORT = env.PORT
+  const HOST = env.REALTIME_HOST
 
   logger.info('Starting Socket.IO server...', {
     port: PORT,
@@ -96,9 +97,9 @@ async function main() {
 
   await assertSchemaCompatibility()
 
-  httpServer.listen(PORT, '0.0.0.0', () => {
-    logger.info(`Socket.IO server running on port ${PORT}`)
-    logger.info(`Health check available at: http://localhost:${PORT}/health`)
+  httpServer.listen(PORT, HOST, () => {
+    logger.info(`Socket.IO server running on ${HOST}:${PORT}`)
+    logger.info(`Health check available at: http://${HOST}:${PORT}/health`)
   })
 
   const shutdown = async () => {

apps/realtime/src/routes/http.ts

@@ -69,6 +69,7 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) {
             status: 'ok',
             timestamp: new Date().toISOString(),
             connections,
+            ...(process.env.E2E_RUN_ID ? { runId: process.env.E2E_RUN_ID } : {}),
           })
         )
       } catch (error) {

apps/sim/app/api/health/route.test.ts

@@ -1,7 +1,7 @@
 /**
  * @vitest-environment node
  */
-import { describe, expect, it } from 'vitest'
+import { describe, expect, it, vi } from 'vitest'
 import { GET } from '@/app/api/health/route'
 
 describe('GET /api/health', () => {
@@ -14,4 +14,17 @@ describe('GET /api/health', () => {
       timestamp: expect.any(String),
     })
   })
+
+  it('returns the E2E run identity when configured', async () => {
+    vi.stubEnv('E2E_RUN_ID', 'run-health-check')
+    try {
+      const response = await GET()
+      await expect(response.json()).resolves.toMatchObject({
+        status: 'ok',
+        runId: 'run-health-check',
+      })
+    } finally {
+      vi.unstubAllEnvs()
+    }
+  })
 })

apps/sim/app/api/health/route.ts

@@ -1,11 +1,14 @@
 /**
  * Health check endpoint for deployment platforms and container probes.
  */
+export const dynamic = 'force-dynamic'
+
 export async function GET(): Promise<Response> {
   return Response.json(
     {
       status: 'ok',
       timestamp: new Date().toISOString(),
+      ...(process.env.E2E_RUN_ID ? { runId: process.env.E2E_RUN_ID } : {}),
     },
     { status: 200 }
   )

apps/sim/e2e/README.md

@@ -0,0 +1,100 @@
+# Sim browser E2E
+
+This directory contains the reusable full-stack Playwright harness. It runs the
+production Next.js app, realtime, deterministic external fakes, and a migrated
+per-run pgvector database.
+
+## One-time setup
+
+0. Install Node 22 and Bun. Playwright workers require Node 22; set
+   `E2E_NODE_BINARY` to an alternate Node 22 executable when `node` on `PATH`
+   points elsewhere.
+
+1. Map the hosted E2E origin to loopback:
+
+   ```bash
+   echo "127.0.0.1 e2e.sim.ai" | sudo tee -a /etc/hosts
+   ```
+
+   The runner refuses to start unless every resolved address is loopback and an
+   IPv4 `127.0.0.1` result is present. Chromium and Node are configured to prefer
+   that IPv4 mapping, so CI environments that also synthesize `::1` remain safe.
+
+2. Start a local pgvector/Postgres admin instance:
+
+   ```bash
+   docker run --rm --name sim-e2e-postgres \
+     -e POSTGRES_USER=postgres \
+     -e POSTGRES_PASSWORD=postgres \
+     -e POSTGRES_DB=postgres \
+     -p 5432:5432 \
+     pgvector/pgvector:pg17
+   ```
+
+   Cleanup uses `DROP DATABASE ... WITH (FORCE)`, which requires PostgreSQL 13
+   or newer and is supported by the pinned pgvector/PostgreSQL 17 image.
+
+3. Install Chromium from `apps/sim`:
+
+   ```bash
+   bun run test:e2e:install-browsers
+   ```
+
+## Run the foundation
+
+From `apps/sim`:
+
+```bash
+E2E_PG_ADMIN_URL=postgresql://postgres:postgres@127.0.0.1:5432/postgres \
+  bun run test:e2e
+```
+
+The runner creates a unique `sim_e2e_<runId>` database, migrates it, starts the
+Stripe fake and realtime, builds and starts Next.js, runs Playwright on Node 22,
+then stops services and drops only that guarded database.
+
+On interruption, the runner launches a detached cleanup supervisor before
+exiting. It terminates managed process groups, force-drops the guarded database,
+and removes temporary auth/cloud-config directories even if another Ctrl-C
+terminates the foreground package runner.
+
+Pass Playwright arguments after `--`:
+
+```bash
+bun run test:e2e -- --project=hosted-billing-chromium-navigation
+bun run test:e2e -- --grep "unauthenticated"
+```
+
+Do not invoke `playwright test` directly. Raw Playwright bypasses environment,
+database, process, sharding, and teardown guards; the config rejects runs that
+were not launched by the orchestrator. Report and trace viewer commands remain
+safe because they do not execute tests.
+
+Sharding is supported only for the navigation project. The runner rejects
+`--shard` for `hosted-billing-chromium-workflows`.
+
+## Diagnostics
+
+- HTML report: `playwright-report/`
+- Traces and screenshots: `test-results/`
+- App, realtime, migration, and fake logs: `e2e/.runs/<runId>/logs/`
+
+Open the report:
+
+```bash
+node ../../node_modules/@playwright/test/cli.js show-report playwright-report
+```
+
+Open a trace:
+
+```bash
+node ../../node_modules/@playwright/test/cli.js show-trace test-results/<test>/trace.zip
+```
+
+The runner starts every child process from a fresh environment. It allowlists
+only deterministic E2E values and shadows keys found in local `.env*` files, so
+developer credentials are not used as test state or written to reports.
+
+Provider log scans are diagnostic tripwires, not proof of zero egress. The
+primary boundaries are the default-deny child environment, provider disabling,
+loopback-only service bindings, and guarded Stripe transport.