From dd124d321a08ff4a8f3cd55ad36abedc264c68e1 Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Mon, 20 Jul 2026 15:44:23 -0700 Subject: [PATCH 01/10] test(e2e): add reusable Playwright foundation Establish a hermetic full-stack browser harness so settings and future surfaces can exercise real authentication, APIs, and isolated database state safely. --- .github/workflows/test-build.yml | 99 ++- .gitignore | 4 + apps/sim/e2e/README.md | 80 +++ apps/sim/e2e/fakes/stripe/server.ts | 582 ++++++++++++++++++ apps/sim/e2e/foundation/admin-api.spec.ts | 17 + apps/sim/e2e/foundation/runtime.spec.ts | 7 + apps/sim/e2e/foundation/safety.spec.ts | 83 +++ apps/sim/e2e/foundation/stripe-fake.spec.ts | 38 ++ apps/sim/e2e/scripts/options.ts | 46 ++ apps/sim/e2e/scripts/run.ts | 181 ++++++ .../e2e/settings/smoke/authenticated.spec.ts | 107 ++++ .../settings/smoke/unauthenticated.spec.ts | 8 + apps/sim/e2e/support/database.ts | 74 +++ apps/sim/e2e/support/deployment-profile.ts | 123 ++++ apps/sim/e2e/support/env.ts | 104 ++++ apps/sim/e2e/support/hosts.ts | 35 ++ apps/sim/e2e/support/paths.ts | 11 + apps/sim/e2e/support/process.ts | 71 +++ apps/sim/e2e/support/readiness.ts | 31 + apps/sim/e2e/support/stack.ts | 119 ++++ apps/sim/lib/auth/auth.ts | 5 +- .../lib/billing/stripe-client-config.test.ts | 123 ++++ apps/sim/lib/billing/stripe-client-config.ts | 114 ++++ apps/sim/lib/billing/stripe-client.ts | 5 +- apps/sim/lib/core/config/env.ts | 2 + apps/sim/package.json | 4 + apps/sim/playwright.config.ts | 48 ++ apps/sim/vitest.config.ts | 2 +- biome.json | 3 + bun.lock | 9 + 30 files changed, 2127 insertions(+), 8 deletions(-) create mode 100644 apps/sim/e2e/README.md create mode 100644 apps/sim/e2e/fakes/stripe/server.ts create mode 100644 apps/sim/e2e/foundation/admin-api.spec.ts create mode 100644 apps/sim/e2e/foundation/runtime.spec.ts create mode 100644 apps/sim/e2e/foundation/safety.spec.ts create mode 100644 apps/sim/e2e/foundation/stripe-fake.spec.ts create mode 100644 apps/sim/e2e/scripts/options.ts create mode 100644 apps/sim/e2e/scripts/run.ts create mode 100644 apps/sim/e2e/settings/smoke/authenticated.spec.ts create mode 100644 apps/sim/e2e/settings/smoke/unauthenticated.spec.ts create mode 100644 apps/sim/e2e/support/database.ts create mode 100644 apps/sim/e2e/support/deployment-profile.ts create mode 100644 apps/sim/e2e/support/env.ts create mode 100644 apps/sim/e2e/support/hosts.ts create mode 100644 apps/sim/e2e/support/paths.ts create mode 100644 apps/sim/e2e/support/process.ts create mode 100644 apps/sim/e2e/support/readiness.ts create mode 100644 apps/sim/e2e/support/stack.ts create mode 100644 apps/sim/lib/billing/stripe-client-config.test.ts create mode 100644 apps/sim/lib/billing/stripe-client-config.ts create mode 100644 apps/sim/playwright.config.ts diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index f20cae5f7d6..0e25b9056c5 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -245,4 +245,101 @@ jobs: AWS_REGION: 'us-west-2' ENCRYPTION_KEY: '7cf672e460e430c1fba707575c2b0e2ad5a99dddf9b7b7e3b5646e630861db1c' # dummy key for CI only TURBO_CACHE_DIR: .turbo - run: bunx turbo run build --filter=sim \ No newline at end of file + run: bunx turbo run build --filter=sim + + settings-e2e: + name: Settings E2E (informational) + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 45 + continue-on-error: true + + services: + postgres: + image: pgvector/pgvector:pg17 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.13 + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: 22 + + - name: Mount Bun cache (Sticky Disk) + uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + with: + key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} + path: ~/.bun/install/cache + + - name: Mount node_modules (Sticky Disk) + uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + with: + key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} + path: ./node_modules + + - name: Mount E2E Turbo cache (Sticky Disk) + uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + with: + key: ${{ github.repository }}-turbo-cache-e2e-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} + path: ./.turbo + + - name: Mount Playwright browsers (Sticky Disk) + uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 + with: + key: ${{ github.repository }}-playwright-browsers-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} + path: ~/.cache/ms-playwright + + - name: Restore E2E Next.js build cache + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: ./apps/sim/.next/cache + key: ${{ runner.os }}-nextjs-e2e-hosted-billing-${{ hashFiles('bun.lock') }} + restore-keys: | + ${{ runner.os }}-nextjs-e2e-hosted-billing- + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Configure E2E hostname + run: echo "127.0.0.1 e2e.sim.ai" | sudo tee -a /etc/hosts + + - name: Install Chromium + working-directory: apps/sim + run: bun run test:e2e:install-browsers -- --with-deps + + - name: Run settings E2E foundation + working-directory: apps/sim + env: + CI: 'true' + E2E_PG_ADMIN_URL: 'postgresql://postgres:postgres@127.0.0.1:5432/postgres' + TURBO_CACHE_DIR: .turbo + run: bun run test:e2e + + - name: Upload E2E diagnostics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: settings-e2e-${{ github.run_id }} + path: | + apps/sim/playwright-report/ + apps/sim/test-results/ + apps/sim/e2e/.runs/ + if-no-files-found: ignore + retention-days: 7 \ No newline at end of file diff --git a/.gitignore b/.gitignore index a8a0d8e2fde..52fd13db03f 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,10 @@ package-lock.json # testing /coverage /apps/**/coverage +/apps/**/playwright-report/ +/apps/**/test-results/ +/apps/**/e2e/.runs/ +/apps/**/e2e/.auth/ # next.js /.next/ diff --git a/apps/sim/e2e/README.md b/apps/sim/e2e/README.md new file mode 100644 index 00000000000..c026698f89a --- /dev/null +++ b/apps/sim/e2e/README.md @@ -0,0 +1,80 @@ +# Sim browser E2E + +This directory contains the reusable full-stack Playwright harness. It runs the +production Next.js app, realtime, deterministic external fakes, and a migrated +per-run pgvector database. + +## One-time setup + +1. Map the hosted E2E origin to loopback: + + ```bash + echo "127.0.0.1 e2e.sim.ai" | sudo tee -a /etc/hosts + ``` + + The runner refuses to start unless every resolved address is loopback. + +2. Start a local pgvector/Postgres admin instance: + + ```bash + docker run --rm --name sim-e2e-postgres \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=postgres \ + -p 5432:5432 \ + pgvector/pgvector:pg17 + ``` + +3. Install Chromium from `apps/sim`: + + ```bash + bun run test:e2e:install-browsers + ``` + +## Run the foundation + +From `apps/sim`: + +```bash +E2E_PG_ADMIN_URL=postgresql://postgres:postgres@127.0.0.1:5432/postgres \ + bun run test:e2e +``` + +The runner creates a unique `sim_e2e_` database, migrates it, starts the +Stripe fake and realtime, builds and starts Next.js, runs Playwright on Node 22, +then stops services and drops only that guarded database. + +Pass Playwright arguments after `--`: + +```bash +bun run test:e2e -- --project=hosted-billing-chromium-navigation +bun run test:e2e -- --grep "unauthenticated" +``` + +For a local rerun after a successful build with the same hosted/public profile, +use `--skip-build`. CI and clean validation runs must always build. + +Sharding is supported only for the navigation project. The runner rejects +`--shard` for `hosted-billing-chromium-workflows`. + +## Diagnostics + +- HTML report: `playwright-report/` +- Traces and screenshots: `test-results/` +- App, realtime, migration, and fake logs: `e2e/.runs//logs/` + +Open the report: + +```bash +node ../../node_modules/@playwright/test/cli.js show-report playwright-report +``` + +Open a trace: + +```bash +node ../../node_modules/@playwright/test/cli.js show-trace test-results//trace.zip +``` + +The runner starts every child process from a fresh environment. It allowlists +only deterministic E2E values and shadows keys found in local `.env*` files, so +developer credentials are not used as test state or written to reports. diff --git a/apps/sim/e2e/fakes/stripe/server.ts b/apps/sim/e2e/fakes/stripe/server.ts new file mode 100644 index 00000000000..8d9bcd2bba4 --- /dev/null +++ b/apps/sim/e2e/fakes/stripe/server.ts @@ -0,0 +1,582 @@ +import { createHash, timingSafeEqual } from 'node:crypto' +import { + createServer, + type IncomingHttpHeaders, + type IncomingMessage, + type Server, + type ServerResponse, +} from 'node:http' +import type { AddressInfo } from 'node:net' + +export const STRIPE_FAKE_ENDPOINTS = { + health: '/health', + requestLog: '/__control/requests', + reset: '/__control/reset', +} as const + +const REDACTED = '[REDACTED]' +const DEFAULT_MAX_BODY_BYTES = 64 * 1024 +const FORM_CONTENT_TYPE = 'application/x-www-form-urlencoded' +const SENSITIVE_FIELD_PATTERN = /authorization|api.?key|secret|token|password|card|source/i + +type RecordedParameters = Record + +export interface StripeFakeRequestRecord { + sequence: number + method: string + path: string + query: RecordedParameters + headers: RecordedParameters + body: RecordedParameters | null + unexpected: boolean +} + +export interface StripeFakeServerOptions { + apiKey: string + hostname?: '127.0.0.1' | '::1' + maxBodyBytes?: number + port?: number +} + +export interface StripeFakeServer { + readonly baseUrl: string | null + readonly requestLog: readonly StripeFakeRequestRecord[] + start(): Promise + stop(): Promise + reset(): void +} + +interface FakeCustomer { + id: string + object: 'customer' + created: number + email: string | null + livemode: false + metadata: Record + name: string | null +} + +interface StripeError { + type: 'api_error' | 'invalid_request_error' + code: string + message: string +} + +class RequestBodyError extends Error { + constructor( + message: string, + readonly status: number + ) { + super(message) + } +} + +function isExpectedStripeRequest(method: string, path: string): boolean { + return ( + ((method === 'GET' || method === 'POST') && path === '/v1/customers/search') || + (method === 'GET' && path === '/v1/customers') || + (method === 'POST' && path === '/v1/customers') || + (method === 'GET' && /^\/v1\/customers\/cus_e2e_[a-f0-9]+$/.test(path)) + ) +} + +function secureEqual(actual: string | undefined, expected: string): boolean { + if (actual === undefined) return false + + const actualBuffer = Buffer.from(actual) + const expectedBuffer = Buffer.from(expected) + return ( + actualBuffer.length === expectedBuffer.length && timingSafeEqual(actualBuffer, expectedBuffer) + ) +} + +function appendParameter( + target: RecordedParameters, + key: string, + value: string +): RecordedParameters { + const recordedValue = SENSITIVE_FIELD_PATTERN.test(key) ? REDACTED : value + const existing = target[key] + if (existing === undefined) { + target[key] = recordedValue + } else if (Array.isArray(existing)) { + existing.push(recordedValue) + } else { + target[key] = [existing, recordedValue] + } + return target +} + +function redactParameters(parameters: URLSearchParams): RecordedParameters { + const redacted: RecordedParameters = {} + for (const [key, value] of parameters) { + appendParameter(redacted, key, value) + } + return redacted +} + +function redactHeaders(headers: IncomingHttpHeaders): RecordedParameters { + const redacted: RecordedParameters = {} + for (const key of ['authorization', 'content-type', 'idempotency-key', 'stripe-version']) { + const value = headers[key] + if (value === undefined) continue + redacted[key] = key === 'authorization' ? REDACTED : value + } + return redacted +} + +function cloneRequestLog(records: StripeFakeRequestRecord[]): StripeFakeRequestRecord[] { + return structuredClone(records) +} + +function sendJson( + response: ServerResponse, + status: number, + body: unknown, + requestId?: string +): void { + const serialized = JSON.stringify(body) + response.writeHead(status, { + 'content-length': Buffer.byteLength(serialized), + 'content-type': 'application/json; charset=utf-8', + ...(requestId ? { 'request-id': requestId } : {}), + }) + response.end(serialized) +} + +function sendStripeError( + response: ServerResponse, + status: number, + error: StripeError, + requestId: string +): void { + sendJson(response, status, { error }, requestId) +} + +async function readRequestBody(request: IncomingMessage, maxBodyBytes: number): Promise { + const chunks: Buffer[] = [] + let totalBytes = 0 + + for await (const chunk of request) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk) + totalBytes += buffer.length + if (totalBytes > maxBodyBytes) { + throw new RequestBodyError('Request body exceeds the Stripe fake limit', 413) + } + chunks.push(buffer) + } + + return Buffer.concat(chunks).toString('utf8') +} + +function parseFormBody(request: IncomingMessage, rawBody: string): URLSearchParams { + const contentType = request.headers['content-type']?.split(';', 1)[0]?.trim().toLowerCase() + if (contentType !== FORM_CONTENT_TYPE) { + throw new RequestBodyError(`Stripe fake requires ${FORM_CONTENT_TYPE}`, 400) + } + return new URLSearchParams(rawBody) +} + +function metadataFrom(parameters: URLSearchParams): Record { + const metadata: Record = {} + for (const [key, value] of parameters) { + const match = /^metadata\[([^\]]+)\]$/.exec(key) + if (match) metadata[match[1]] = value + } + return metadata +} + +function stableCustomerIdentity(parameters: URLSearchParams): string { + const metadata = metadataFrom(parameters) + const preferredIdentity = + metadata.userId || metadata.organizationId || parameters.get('email') || undefined + if (preferredIdentity) return preferredIdentity + + return [...parameters.entries()] + .sort(([leftKey, leftValue], [rightKey, rightValue]) => + `${leftKey}\u0000${leftValue}`.localeCompare(`${rightKey}\u0000${rightValue}`) + ) + .map(([key, value]) => `${key}=${value}`) + .join('&') +} + +function createDeterministicCustomer(parameters: URLSearchParams): FakeCustomer { + const digest = createHash('sha256').update(stableCustomerIdentity(parameters)).digest('hex') + return { + id: `cus_e2e_${digest.slice(0, 24)}`, + object: 'customer', + created: 1_700_000_000 + (Number.parseInt(digest.slice(0, 6), 16) % 1_000_000), + email: parameters.get('email'), + livemode: false, + metadata: metadataFrom(parameters), + name: parameters.get('name'), + } +} + +function unescapeSearchValue(value: string): string { + return value.replace(/\\(["\\])/g, '$1') +} + +function customerMatchesSearch(customer: FakeCustomer, query: string): boolean { + const emailMatch = /(?:^|\s)email:"((?:\\.|[^"])*)"/.exec(query) + if (emailMatch && customer.email !== unescapeSearchValue(emailMatch[1])) return false + + const metadataPattern = /(-?)metadata\["([^"]+)"\]:"((?:\\.|[^"])*)"/g + for (const match of query.matchAll(metadataPattern)) { + const isNegative = match[1] === '-' + const actual = customer.metadata[match[2]] + const expected = unescapeSearchValue(match[3]) + if (isNegative ? actual === expected : actual !== expected) return false + } + + return true +} + +function parseLimit(parameters: URLSearchParams): number { + const rawLimit = parameters.get('limit') + if (rawLimit === null) return 10 + + const limit = Number(rawLimit) + if (!Number.isInteger(limit) || limit < 1 || limit > 100) { + throw new RequestBodyError('Stripe fake requires limit to be an integer from 1 to 100', 400) + } + return limit +} + +function validateTestApiKey(apiKey: string): void { + if (!apiKey.startsWith('sk_test_') || apiKey.length === 'sk_test_'.length) { + throw new Error('Stripe fake apiKey must be a non-empty sk_test_ secret key') + } +} + +function validateServerOptions(options: StripeFakeServerOptions): void { + validateTestApiKey(options.apiKey) + if ( + options.port !== undefined && + (!Number.isInteger(options.port) || options.port < 0 || options.port > 65535) + ) { + throw new Error('Stripe fake port must be an integer from 0 to 65535') + } + if ( + options.maxBodyBytes !== undefined && + (!Number.isInteger(options.maxBodyBytes) || options.maxBodyBytes < 1) + ) { + throw new Error('Stripe fake maxBodyBytes must be a positive integer') + } +} + +/** + * Creates a loopback-only Stripe fake. The returned server is inert until start() + * is called, allowing an orchestrator to own lifecycle and failure handling. + */ +export function createStripeFakeServer(options: StripeFakeServerOptions): StripeFakeServer { + validateServerOptions(options) + + const hostname = options.hostname ?? '127.0.0.1' + const port = options.port ?? 0 + const maxBodyBytes = options.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES + const expectedAuthorization = `Bearer ${options.apiKey}` + const records: StripeFakeRequestRecord[] = [] + const customers = new Map() + let sequence = 0 + let baseUrl: string | null = null + + const reset = (): void => { + records.length = 0 + customers.clear() + sequence = 0 + } + + const handleControlRequest = ( + request: IncomingMessage, + response: ServerResponse, + method: string, + path: string + ): boolean => { + if (method === 'GET' && path === STRIPE_FAKE_ENDPOINTS.health) { + sendJson(response, 200, { status: 'ok', service: 'stripe-fake' }) + return true + } + + const isRequestLog = method === 'GET' && path === STRIPE_FAKE_ENDPOINTS.requestLog + const isReset = method === 'POST' && path === STRIPE_FAKE_ENDPOINTS.reset + if (!isRequestLog && !isReset) return false + + if (!secureEqual(request.headers.authorization, expectedAuthorization)) { + sendStripeError( + response, + 401, + { + type: 'invalid_request_error', + code: 'api_key_invalid', + message: 'Invalid test Bearer authorization for Stripe fake control endpoint.', + }, + 'req_e2e_control' + ) + return true + } + + if (isRequestLog) { + sendJson(response, 200, { requests: cloneRequestLog(records) }) + } else { + reset() + sendJson(response, 200, { reset: true }) + } + return true + } + + const handleStripeRequest = async ( + request: IncomingMessage, + response: ServerResponse, + method: string, + url: URL + ): Promise => { + sequence += 1 + const requestId = `req_e2e_${String(sequence).padStart(6, '0')}` + const expected = isExpectedStripeRequest(method, url.pathname) + let formBody: URLSearchParams | null = null + let recordedBody: RecordedParameters | null = null + + try { + if (method !== 'GET' && method !== 'HEAD') { + const rawBody = await readRequestBody(request, maxBodyBytes) + if (rawBody) { + if (request.headers['content-type']?.startsWith(FORM_CONTENT_TYPE)) { + formBody = new URLSearchParams(rawBody) + recordedBody = redactParameters(formBody) + } else { + recordedBody = { content: REDACTED } + } + } + } + } catch (error) { + records.push({ + sequence, + method, + path: url.pathname, + query: redactParameters(url.searchParams), + headers: redactHeaders(request.headers), + body: recordedBody, + unexpected: !expected, + }) + const bodyError = + error instanceof RequestBodyError + ? error + : new RequestBodyError('Unable to read Stripe fake request body', 400) + sendStripeError( + response, + bodyError.status, + { + type: 'invalid_request_error', + code: 'invalid_request_body', + message: bodyError.message, + }, + requestId + ) + return + } + + records.push({ + sequence, + method, + path: url.pathname, + query: redactParameters(url.searchParams), + headers: redactHeaders(request.headers), + body: recordedBody, + unexpected: !expected, + }) + + if (!secureEqual(request.headers.authorization, expectedAuthorization)) { + sendStripeError( + response, + 401, + { + type: 'invalid_request_error', + code: 'api_key_invalid', + message: 'Invalid test Bearer authorization for Stripe fake.', + }, + requestId + ) + return + } + + if (!expected) { + sendStripeError( + response, + 501, + { + type: 'api_error', + code: 'stripe_fake_unimplemented', + message: `Stripe fake does not implement ${method} ${url.pathname}.`, + }, + requestId + ) + return + } + + try { + if ((method === 'GET' || method === 'POST') && url.pathname === '/v1/customers/search') { + const parameters = + method === 'GET' ? url.searchParams : (formBody ?? parseFormBody(request, '')) + const query = parameters.get('query') + if (!query) throw new RequestBodyError('Stripe fake customer search requires query', 400) + + const data = [...customers.values()] + .filter((customer) => customerMatchesSearch(customer, query)) + .slice(0, parseLimit(parameters)) + sendJson( + response, + 200, + { + object: 'search_result', + data, + has_more: false, + next_page: null, + url: '/v1/customers/search', + }, + requestId + ) + return + } + + if (method === 'GET' && url.pathname === '/v1/customers') { + const email = url.searchParams.get('email') + const data = [...customers.values()] + .filter((customer) => email === null || customer.email === email) + .slice(0, parseLimit(url.searchParams)) + sendJson( + response, + 200, + { + object: 'list', + data, + has_more: false, + url: '/v1/customers', + }, + requestId + ) + return + } + + if (method === 'GET' && url.pathname.startsWith('/v1/customers/')) { + const customerId = decodeURIComponent(url.pathname.slice('/v1/customers/'.length)) + const customer = customers.get(customerId) + if (!customer) { + sendStripeError( + response, + 404, + { + type: 'invalid_request_error', + code: 'resource_missing', + message: `No such customer: ${customerId}`, + }, + requestId + ) + return + } + sendJson(response, 200, customer, requestId) + return + } + + const parameters = formBody ?? parseFormBody(request, '') + const candidate = createDeterministicCustomer(parameters) + const customer = customers.get(candidate.id) ?? candidate + customers.set(customer.id, customer) + sendJson(response, 200, customer, requestId) + } catch (error) { + const bodyError = + error instanceof RequestBodyError + ? error + : new RequestBodyError('Stripe fake rejected malformed request parameters', 400) + sendStripeError( + response, + bodyError.status, + { + type: 'invalid_request_error', + code: 'invalid_request', + message: bodyError.message, + }, + requestId + ) + } + } + + const server: Server = createServer((request, response) => { + const method = request.method?.toUpperCase() ?? 'UNKNOWN' + const url = new URL(request.url ?? '/', 'http://stripe-fake.invalid') + if (handleControlRequest(request, response, method, url.pathname)) return + + void handleStripeRequest(request, response, method, url).catch(() => { + if (!response.headersSent) { + sendStripeError( + response, + 500, + { + type: 'api_error', + code: 'stripe_fake_internal_error', + message: 'Stripe fake encountered an internal error.', + }, + 'req_e2e_internal' + ) + } else { + response.destroy() + } + }) + }) + + return { + get baseUrl() { + return baseUrl + }, + get requestLog() { + return cloneRequestLog(records) + }, + async start() { + if (baseUrl) return baseUrl + + await new Promise((resolve, reject) => { + const handleError = (error: Error) => { + server.off('listening', handleListening) + reject(error) + } + const handleListening = () => { + server.off('error', handleError) + resolve() + } + server.once('error', handleError) + server.once('listening', handleListening) + server.listen(port, hostname) + }) + + const address = server.address() as AddressInfo + const urlHostname = address.family === 'IPv6' ? `[${address.address}]` : address.address + baseUrl = `http://${urlHostname}:${address.port}` + return baseUrl + }, + async stop() { + if (!server.listening) { + baseUrl = null + return + } + + await new Promise((resolve, reject) => { + server.close((error) => { + if (error) reject(error) + else resolve() + }) + server.closeIdleConnections() + }) + baseUrl = null + }, + reset, + } +} + +/** Starts a Stripe fake in one call for orchestration code. */ +export async function startStripeFakeServer( + options: StripeFakeServerOptions +): Promise { + const server = createStripeFakeServer(options) + await server.start() + return server +} diff --git a/apps/sim/e2e/foundation/admin-api.spec.ts b/apps/sim/e2e/foundation/admin-api.spec.ts new file mode 100644 index 00000000000..44c9911c23f --- /dev/null +++ b/apps/sim/e2e/foundation/admin-api.spec.ts @@ -0,0 +1,17 @@ +import { expect, test } from '@playwright/test' + +test('Admin API is configured and rejects missing credentials', async ({ request }) => { + const adminKey = process.env.E2E_ADMIN_API_KEY + expect(adminKey, 'E2E_ADMIN_API_KEY must be provided to the Playwright worker').toBeTruthy() + + const unauthorized = await request.get('/api/v1/admin/users?limit=1&offset=0') + expect(unauthorized.status()).toBe(401) + + const authorized = await request.get('/api/v1/admin/users?limit=1&offset=0', { + headers: { 'x-admin-key': adminKey! }, + }) + const responseText = await authorized.text() + expect(authorized.status(), responseText).toBe(200) + const body = JSON.parse(responseText) as { data?: unknown[] } + expect(Array.isArray(body.data)).toBe(true) +}) diff --git a/apps/sim/e2e/foundation/runtime.spec.ts b/apps/sim/e2e/foundation/runtime.spec.ts new file mode 100644 index 00000000000..a3a394d107e --- /dev/null +++ b/apps/sim/e2e/foundation/runtime.spec.ts @@ -0,0 +1,7 @@ +import { expect, test } from '@playwright/test' + +test('Playwright workers run on Node 22', () => { + expect(process.release.name).toBe('node') + expect(Number(process.versions.node.split('.')[0])).toBe(22) + expect(process.execPath.toLowerCase()).not.toContain('bun') +}) diff --git a/apps/sim/e2e/foundation/safety.spec.ts b/apps/sim/e2e/foundation/safety.spec.ts new file mode 100644 index 00000000000..92f2447545d --- /dev/null +++ b/apps/sim/e2e/foundation/safety.spec.ts @@ -0,0 +1,83 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { loadEnvConfig } from '@next/env' +import { expect, test } from '@playwright/test' +import { parseRunOptions } from '../scripts/options' +import { + assertLoopbackPostgresUrl, + assertSafeDatabaseName, + buildRunDatabaseUrl, +} from '../support/database' +import { buildChildEnvironment, discoverEnvFileKeys } from '../support/env' +import { isLoopbackAddress } from '../support/hosts' + +test.describe('foundation safety guards', () => { + test('discovers env keys without leaking values and shadows unknown keys', () => { + const directory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-env-')) + try { + writeFileSync( + path.join(directory, '.env'), + 'OPENAI_API_KEY=do-not-leak-this\nSAFE_VALUE=from-file\n' + ) + expect(discoverEnvFileKeys(directory)).toEqual(['OPENAI_API_KEY', 'SAFE_VALUE']) + + const result = buildChildEnvironment({ + values: { REQUIRED_VALUE: 'configured' }, + required: ['REQUIRED_VALUE'], + allowedSensitiveKeys: new Set(), + envDirectory: directory, + }) + expect(result.env.OPENAI_API_KEY).toBe('') + expect(result.env.SAFE_VALUE).toBe('') + expect(JSON.stringify(result)).not.toContain('do-not-leak-this') + expect(JSON.stringify(result)).not.toContain('from-file') + } finally { + rmSync(directory, { recursive: true, force: true }) + } + }) + + test('@next/env preserves a pre-set process value over an env file', () => { + const directory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-next-env-')) + const key = `SIM_E2E_CANARY_${Date.now()}` + try { + writeFileSync(path.join(directory, '.env'), `${key}=file-value\n`) + process.env[key] = 'shadowed-value' + loadEnvConfig(directory, false, console, true) + expect(process.env[key]).toBe('shadowed-value') + } finally { + delete process.env[key] + rmSync(directory, { recursive: true, force: true }) + } + }) + + test('database guards reject shared or remote targets', () => { + expect(() => assertSafeDatabaseName('simstudio')).toThrow() + expect(() => assertSafeDatabaseName('sim_e2e_valid_run')).not.toThrow() + expect(() => + assertLoopbackPostgresUrl('postgresql://postgres:postgres@example.com/postgres') + ).toThrow() + expect( + buildRunDatabaseUrl( + 'postgresql://postgres:postgres@127.0.0.1:5432/postgres', + 'sim_e2e_valid_run' + ) + ).toContain('/sim_e2e_valid_run') + }) + + test('loopback detection rejects public addresses', () => { + expect(isLoopbackAddress('127.0.0.1')).toBe(true) + expect(isLoopbackAddress('127.10.20.30')).toBe(true) + expect(isLoopbackAddress('::1')).toBe(true) + expect(isLoopbackAddress('8.8.8.8')).toBe(false) + }) + + test('sharding is limited to the navigation project', () => { + expect(() => + parseRunOptions(['--project=hosted-billing-chromium-navigation', '--shard=1/2']) + ).not.toThrow() + expect(() => + parseRunOptions(['--project=hosted-billing-chromium-workflows', '--shard=1/2']) + ).toThrow(/coupled workflows must remain unsharded/) + }) +}) diff --git a/apps/sim/e2e/foundation/stripe-fake.spec.ts b/apps/sim/e2e/foundation/stripe-fake.spec.ts new file mode 100644 index 00000000000..12187b57535 --- /dev/null +++ b/apps/sim/e2e/foundation/stripe-fake.spec.ts @@ -0,0 +1,38 @@ +import { expect, test } from '@playwright/test' +import { startStripeFakeServer } from '../fakes/stripe/server' + +test('Stripe fake records allowlisted calls and rejects unknown routes', async () => { + const apiKey = 'sk_test_foundation_fake_spec' + const fake = await startStripeFakeServer({ apiKey }) + try { + const baseUrl = fake.baseUrl + expect(baseUrl).toBeTruthy() + + const health = await fetch(`${baseUrl}/health`) + expect(health.status).toBe(200) + + const customer = await fetch(`${baseUrl}/v1/customers`, { + method: 'POST', + headers: { + authorization: `Bearer ${apiKey}`, + 'content-type': 'application/x-www-form-urlencoded', + }, + body: new URLSearchParams({ + email: 'foundation@example.com', + 'metadata[userId]': 'user-foundation', + }), + }) + expect(customer.status).toBe(200) + expect((await customer.json()) as { id: string }).toMatchObject({ + id: expect.stringMatching(/^cus_e2e_/), + }) + + const unknown = await fetch(`${baseUrl}/v1/invoices`, { + headers: { authorization: `Bearer ${apiKey}` }, + }) + expect(unknown.status).toBe(501) + expect(fake.requestLog.some(({ unexpected }) => unexpected)).toBe(true) + } finally { + await fake.stop() + } +}) diff --git a/apps/sim/e2e/scripts/options.ts b/apps/sim/e2e/scripts/options.ts new file mode 100644 index 00000000000..71a895fb1fa --- /dev/null +++ b/apps/sim/e2e/scripts/options.ts @@ -0,0 +1,46 @@ +const NAVIGATION_PROJECT = 'hosted-billing-chromium-navigation' +const WORKFLOWS_PROJECT = 'hosted-billing-chromium-workflows' + +export interface E2eRunOptions { + playwrightArgs: string[] + skipBuild: boolean +} + +export function parseRunOptions(argv: string[]): E2eRunOptions { + const normalizedArgs = argv[0] === '--' ? argv.slice(1) : [...argv] + const skipBuild = normalizedArgs.includes('--skip-build') + const playwrightArgs = normalizedArgs.filter((arg) => arg !== '--skip-build') + const projects = getOptionValues(playwrightArgs, '--project') + const hasShard = hasOption(playwrightArgs, '--shard') + + if ( + hasShard && + (projects.length === 0 || + projects.includes(WORKFLOWS_PROJECT) || + projects.some((project) => project !== NAVIGATION_PROJECT)) + ) { + throw new Error( + `--shard is supported only with --project=${NAVIGATION_PROJECT}; coupled workflows must remain unsharded` + ) + } + + return { playwrightArgs, skipBuild } +} + +function hasOption(args: string[], name: string): boolean { + return args.some((arg) => arg === name || arg.startsWith(`${name}=`)) +} + +function getOptionValues(args: string[], name: string): string[] { + const values: string[] = [] + for (let index = 0; index < args.length; index += 1) { + const arg = args[index] + if (arg.startsWith(`${name}=`)) { + values.push(arg.slice(name.length + 1)) + } else if (arg === name && args[index + 1]) { + values.push(args[index + 1]) + index += 1 + } + } + return values +} diff --git a/apps/sim/e2e/scripts/run.ts b/apps/sim/e2e/scripts/run.ts new file mode 100644 index 00000000000..daade957034 --- /dev/null +++ b/apps/sim/e2e/scripts/run.ts @@ -0,0 +1,181 @@ +import { spawnSync } from 'node:child_process' +import { randomUUID } from 'node:crypto' +import { mkdirSync, writeFileSync } from 'node:fs' +import path from 'node:path' +import { type StripeFakeServer, startStripeFakeServer } from '../fakes/stripe/server' +import { + createRunDatabase, + createRunDatabaseName, + dropRunDatabase, + type RunDatabase, +} from '../support/database' +import { createHostedBillingProfile, E2E_ORIGIN, E2E_PROFILE } from '../support/deployment-profile' +import { formatRedactedEnvironmentSummary } from '../support/env' +import { assertE2eHostResolvesToLoopback } from '../support/hosts' +import { getRunDirectory } from '../support/paths' +import { type ManagedProcess, stopProcesses } from '../support/process' +import { buildApp, runMigrations, runPlaywright, startApp, startRealtime } from '../support/stack' +import { parseRunOptions } from './options' + +const DEFAULT_ADMIN_DATABASE_URL = 'postgresql://postgres:postgres@127.0.0.1:5432/postgres' +const STRIPE_TEST_KEY = 'sk_test_sim_e2e_foundation' + +async function main(): Promise { + const options = parseRunOptions(process.argv.slice(2)) + const runId = createRunId() + const runDirectory = getRunDirectory(runId) + const logsDirectory = path.join(runDirectory, 'logs') + const storageStatePath = path.join(runDirectory, 'auth', 'foundation.json') + mkdirSync(logsDirectory, { recursive: true }) + mkdirSync(path.dirname(storageStatePath), { recursive: true }) + + const nodeExecutable = resolveNode22() + const bunExecutable = resolveBunExecutable() + const adminDatabaseUrl = process.env.E2E_PG_ADMIN_URL ?? DEFAULT_ADMIN_DATABASE_URL + + let runDatabase: RunDatabase | null = null + let stripeFake: StripeFakeServer | null = null + const processes: ManagedProcess[] = [] + let failed = false + + try { + const hostAddresses = await assertE2eHostResolvesToLoopback() + console.info(`E2E host resolved to loopback: ${hostAddresses.join(', ')}`) + + runDatabase = await createRunDatabase(adminDatabaseUrl, createRunDatabaseName(runId)) + stripeFake = await startStripeFakeServer({ + apiKey: STRIPE_TEST_KEY, + hostname: '127.0.0.1', + port: 0, + }) + if (!stripeFake.baseUrl) throw new Error('Stripe fake did not expose a base URL') + + const profile = createHostedBillingProfile({ + runId, + databaseUrl: runDatabase.url, + stripeApiBaseUrl: stripeFake.baseUrl, + ci: process.env.CI === 'true', + }) + console.info(formatRedactedEnvironmentSummary(profile.id, profile.childEnvironment)) + + const stackOptions = { + bunExecutable, + nodeExecutable, + env: profile.childEnvironment.env, + logsDirectory, + } + + await runMigrations(stackOptions) + if (!options.skipBuild) await buildApp(stackOptions) + processes.push(await startRealtime(stackOptions)) + processes.push(await startApp(stackOptions)) + + const playwrightEnvironment = createPlaywrightEnvironment( + profile.childEnvironment.env, + runId, + stripeFake.baseUrl, + storageStatePath + ) + await runPlaywright( + { + nodeExecutable, + env: playwrightEnvironment, + logsDirectory, + }, + options.playwrightArgs + ) + } catch (error) { + failed = true + console.error(error) + process.exitCode = 1 + } finally { + await stopProcesses(processes) + + if (stripeFake) { + writeFileSync( + path.join(logsDirectory, 'stripe-requests.json'), + JSON.stringify(stripeFake.requestLog, null, 2) + ) + await stripeFake.stop() + } + + if (runDatabase) { + try { + await dropRunDatabase(adminDatabaseUrl, runDatabase.name) + } catch (error) { + failed = true + process.exitCode = 1 + console.error(`Failed to drop ${runDatabase.name}:`, error) + } + } + + console.info(`E2E ${failed ? 'failed' : 'completed'}; diagnostics: ${runDirectory}`) + } +} + +function createRunId(): string { + const timestamp = new Date().toISOString().replace(/\D/g, '').slice(0, 14) + const random = randomUUID().replace(/-/g, '').slice(0, 10) + return `${timestamp}_${random}` +} + +function resolveBunExecutable(): string { + if (!process.versions.bun) { + throw new Error('The E2E orchestrator must be started with Bun') + } + return process.execPath +} + +function resolveNode22(): string { + const executable = process.env.E2E_NODE_BINARY ?? 'node' + const result = spawnSync(executable, ['--version'], { + encoding: 'utf8', + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + }) + if (result.status !== 0) { + throw new Error(`Unable to execute Node for Playwright: ${result.stderr}`) + } + const version = result.stdout.trim() + if (!/^v22\./.test(version)) { + throw new Error(`Playwright E2E requires Node 22, received ${version}`) + } + return executable +} + +function createPlaywrightEnvironment( + stackEnvironment: Record, + runId: string, + stripeFakeUrl: string, + storageStatePath: string +): Record { + const keys = [ + 'PATH', + 'HOME', + 'USER', + 'SHELL', + 'TMPDIR', + 'TMP', + 'TEMP', + 'SYSTEMROOT', + 'CI', + 'GITHUB_ACTIONS', + 'PLAYWRIGHT_BROWSERS_PATH', + ] + const env: Record = {} + for (const key of keys) { + const value = stackEnvironment[key] + if (value !== undefined) env[key] = value + } + return { + ...env, + E2E_PROFILE, + E2E_RUN_ID: runId, + E2E_BASE_URL: E2E_ORIGIN, + E2E_ADMIN_API_KEY: stackEnvironment.ADMIN_API_KEY, + E2E_STRIPE_FAKE_URL: stripeFakeUrl, + E2E_STRIPE_FAKE_KEY: stackEnvironment.STRIPE_SECRET_KEY, + E2E_STORAGE_STATE_PATH: storageStatePath, + } +} + +await main() diff --git a/apps/sim/e2e/settings/smoke/authenticated.spec.ts b/apps/sim/e2e/settings/smoke/authenticated.spec.ts new file mode 100644 index 00000000000..a4b3f5fd8ed --- /dev/null +++ b/apps/sim/e2e/settings/smoke/authenticated.spec.ts @@ -0,0 +1,107 @@ +import { expect, test } from '@playwright/test' + +interface StripeRequestLog { + requests: Array<{ + method: string + path: string + unexpected: boolean + }> +} + +test('billing-enabled signup, login, and settings use real Sim boundaries', async ({ + browser, + page, + request, +}, testInfo) => { + test.slow() + + const runId = process.env.E2E_RUN_ID ?? `${Date.now()}` + const email = `e2e-foundation-${runId}@example.com` + const password = 'E2eFoundation1!' + const fakeUrl = requiredEnv('E2E_STRIPE_FAKE_URL') + const fakeKey = requiredEnv('E2E_STRIPE_FAKE_KEY') + const adminKey = requiredEnv('E2E_ADMIN_API_KEY') + const storageStatePath = + process.env.E2E_STORAGE_STATE_PATH ?? testInfo.outputPath('foundation-auth.json') + + await page.goto('/signup') + await expect(page.getByRole('heading', { name: 'Create an account' })).toBeVisible() + await page.getByLabel('Full name').fill('Playwright Foundation') + await page.getByLabel('Email').fill(email) + await page.getByRole('textbox', { name: 'Password' }).fill(password) + await page.getByRole('button', { name: 'Create account' }).click() + + await expect(page).toHaveURL(/\/workspace\/[^/]+\/(?:home|w(?:\/|$))/, { + timeout: 60_000, + }) + const workspaceId = new URL(page.url()).pathname.split('/')[2] + expect(workspaceId).toBeTruthy() + + await expect + .poll(async () => { + const response = await request.get(`${fakeUrl}/__control/requests`, { + headers: { authorization: `Bearer ${fakeKey}` }, + }) + expect(response.ok()).toBe(true) + const log = (await response.json()) as StripeRequestLog + return { + created: log.requests.some( + ({ method, path }) => method === 'POST' && path === '/v1/customers' + ), + unexpected: log.requests.filter(({ unexpected }) => unexpected).length, + } + }) + .toEqual({ created: true, unexpected: 0 }) + + const sessionResponse = await page.request.get('/api/auth/get-session') + expect(sessionResponse.ok()).toBe(true) + const session = (await sessionResponse.json()) as { user?: { id?: string } } + const userId = session.user?.id + expect(userId).toBeTruthy() + + const billingResponse = await request.get(`/api/v1/admin/users/${userId}/billing`, { + headers: { 'x-admin-key': adminKey }, + }) + expect(billingResponse.ok()).toBe(true) + const billing = (await billingResponse.json()) as { + data?: { stripeCustomerId?: string | null } + } + expect(billing.data?.stripeCustomerId).toMatch(/^cus_e2e_/) + + const settingsPath = `/workspace/${workspaceId}/settings/general` + await page.goto(settingsPath) + await assertGeneralSettings(page) + + await page.getByRole('button', { name: 'Sign out' }).click() + await expect(page).toHaveURL(/\/login\?fromLogout=true$/) + + await page.getByLabel('Email').fill(email) + await page.getByRole('textbox', { name: 'Password' }).fill(password) + await page.getByRole('button', { name: 'Sign in' }).click() + await expect(page).toHaveURL(/\/workspace(?:\/|$)/) + + await page.context().storageState({ path: storageStatePath }) + const restoredContext = await browser.newContext({ storageState: storageStatePath }) + try { + const restoredPage = await restoredContext.newPage() + await restoredPage.goto(`${requiredEnv('E2E_BASE_URL')}${settingsPath}`) + await assertGeneralSettings(restoredPage) + } finally { + await restoredContext.close() + } +}) + +async function assertGeneralSettings(page: import('@playwright/test').Page): Promise { + await expect(page).toHaveURL(/\/settings\/general$/) + await expect(page.getByRole('heading', { name: 'General', level: 1 })).toBeVisible() + await expect( + page.getByText('Manage your profile, appearance, and preferences.', { exact: true }) + ).toBeVisible() + await expect(page.getByText('Profile', { exact: true })).toBeVisible() +} + +function requiredEnv(key: string): string { + const value = process.env[key] + if (!value) throw new Error(`Missing required Playwright environment value: ${key}`) + return value +} diff --git a/apps/sim/e2e/settings/smoke/unauthenticated.spec.ts b/apps/sim/e2e/settings/smoke/unauthenticated.spec.ts new file mode 100644 index 00000000000..ab3aaca910f --- /dev/null +++ b/apps/sim/e2e/settings/smoke/unauthenticated.spec.ts @@ -0,0 +1,8 @@ +import { expect, test } from '@playwright/test' + +test('protected settings redirect unauthenticated users to login', async ({ page }) => { + await page.goto('/account/settings/general') + + await expect(page).toHaveURL(/\/login(?:\?|$)/) + await expect(page.getByRole('heading', { name: 'Sign in' })).toBeVisible() +}) diff --git a/apps/sim/e2e/support/database.ts b/apps/sim/e2e/support/database.ts new file mode 100644 index 00000000000..03e471b3ad0 --- /dev/null +++ b/apps/sim/e2e/support/database.ts @@ -0,0 +1,74 @@ +import postgres from 'postgres' +import { isLoopbackAddress } from './hosts' + +const DATABASE_NAME_PATTERN = /^sim_e2e_[a-z0-9_]+$/ +const PROTECTED_DATABASES = new Set(['postgres', 'simstudio', 'template0', 'template1']) + +export interface RunDatabase { + name: string + url: string +} + +export function createRunDatabaseName(runId: string): string { + const suffix = runId + .toLowerCase() + .replace(/[^a-z0-9]+/g, '_') + .replace(/^_+|_+$/g, '') + const name = `sim_e2e_${suffix}` + assertSafeDatabaseName(name) + return name +} + +export function assertSafeDatabaseName(name: string): void { + if (!DATABASE_NAME_PATTERN.test(name) || PROTECTED_DATABASES.has(name)) { + throw new Error(`Refusing unsafe E2E database name: ${name}`) + } +} + +export function assertLoopbackPostgresUrl(rawUrl: string): URL { + const url = new URL(rawUrl) + const hostname = url.hostname.replace(/^\[|\]$/g, '') + if (!(hostname === 'localhost' || hostname === '::1' || isLoopbackAddress(hostname))) { + throw new Error(`E2E PostgreSQL admin URL must be loopback, received ${url.hostname}`) + } + return url +} + +export function buildRunDatabaseUrl(adminUrl: string, databaseName: string): string { + assertSafeDatabaseName(databaseName) + const url = assertLoopbackPostgresUrl(adminUrl) + url.pathname = `/${databaseName}` + return url.toString() +} + +export async function createRunDatabase( + adminUrl: string, + databaseName: string +): Promise { + assertSafeDatabaseName(databaseName) + assertLoopbackPostgresUrl(adminUrl) + const sql = postgres(adminUrl, { max: 1, connect_timeout: 10 }) + try { + await sql.unsafe(`CREATE DATABASE "${databaseName}"`) + } finally { + await sql.end() + } + return { name: databaseName, url: buildRunDatabaseUrl(adminUrl, databaseName) } +} + +export async function dropRunDatabase(adminUrl: string, databaseName: string): Promise { + assertSafeDatabaseName(databaseName) + assertLoopbackPostgresUrl(adminUrl) + const sql = postgres(adminUrl, { max: 1, connect_timeout: 10 }) + try { + await sql` + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = ${databaseName} + AND pid <> pg_backend_pid() + ` + await sql.unsafe(`DROP DATABASE IF EXISTS "${databaseName}"`) + } finally { + await sql.end() + } +} diff --git a/apps/sim/e2e/support/deployment-profile.ts b/apps/sim/e2e/support/deployment-profile.ts new file mode 100644 index 00000000000..0419c801eab --- /dev/null +++ b/apps/sim/e2e/support/deployment-profile.ts @@ -0,0 +1,123 @@ +import { buildChildEnvironment, type ChildEnvironment } from './env' + +export const E2E_PROFILE = 'hosted-billing-chromium' +export const E2E_HOST = 'e2e.sim.ai' +export const E2E_ORIGIN = `http://${E2E_HOST}:3000` +export const E2E_SOCKET_ORIGIN = `http://${E2E_HOST}:3002` + +const REQUIRED_KEYS = [ + 'NODE_ENV', + 'NEXT_PUBLIC_APP_URL', + 'BETTER_AUTH_URL', + 'BETTER_AUTH_SECRET', + 'DATABASE_URL', + 'MIGRATION_DATABASE_URL', + 'ENCRYPTION_KEY', + 'API_ENCRYPTION_KEY', + 'INTERNAL_API_SECRET', + 'ADMIN_API_KEY', + 'BILLING_ENABLED', + 'NEXT_PUBLIC_BILLING_ENABLED', + 'STRIPE_SECRET_KEY', + 'STRIPE_API_BASE_URL', + 'E2E_PROFILE', + 'E2E_RUN_ID', +] as const + +const ALLOWED_SENSITIVE_KEYS = new Set([ + 'BETTER_AUTH_SECRET', + 'ENCRYPTION_KEY', + 'API_ENCRYPTION_KEY', + 'INTERNAL_API_SECRET', + 'ADMIN_API_KEY', + 'E2E_ADMIN_API_KEY', + 'EMAIL_PASSWORD_SIGNUP_ENABLED', + 'NEXT_PUBLIC_EMAIL_PASSWORD_SIGNUP_ENABLED', + 'STRIPE_SECRET_KEY', + 'STRIPE_WEBHOOK_SECRET', +]) + +export interface HostedBillingProfileOptions { + runId: string + databaseUrl: string + stripeApiBaseUrl: string + ci: boolean +} + +export interface HostedBillingProfile { + id: typeof E2E_PROFILE + origin: typeof E2E_ORIGIN + childEnvironment: ChildEnvironment +} + +export function createHostedBillingProfile({ + runId, + databaseUrl, + stripeApiBaseUrl, + ci, +}: HostedBillingProfileOptions): HostedBillingProfile { + const values: Record = { + NODE_ENV: 'production', + NODE_OPTIONS: '--no-warnings --max-old-space-size=8192', + NEXT_TELEMETRY_DISABLED: '1', + E2E_PROFILE, + E2E_RUN_ID: runId, + E2E_BASE_URL: E2E_ORIGIN, + NEXT_PUBLIC_APP_URL: E2E_ORIGIN, + BETTER_AUTH_URL: E2E_ORIGIN, + BETTER_AUTH_SECRET: 'e2e-better-auth-secret-at-least-32-characters-long', + DATABASE_URL: databaseUrl, + MIGRATION_DATABASE_URL: databaseUrl, + ENCRYPTION_KEY: '11'.repeat(32), + API_ENCRYPTION_KEY: '22'.repeat(32), + INTERNAL_API_SECRET: 'e2e-internal-api-secret-at-least-32-characters', + ADMIN_API_KEY: 'e2e-admin-api-key-at-least-32-characters-long', + E2E_ADMIN_API_KEY: 'e2e-admin-api-key-at-least-32-characters-long', + BILLING_ENABLED: 'true', + NEXT_PUBLIC_BILLING_ENABLED: 'true', + EMAIL_VERIFICATION_ENABLED: 'false', + EMAIL_PASSWORD_SIGNUP_ENABLED: 'true', + NEXT_PUBLIC_EMAIL_PASSWORD_SIGNUP_ENABLED: 'true', + DISABLE_REGISTRATION: 'false', + DISABLE_EMAIL_SIGNUP: 'false', + SIGNUP_MX_VALIDATION_ENABLED: 'false', + NEXT_PUBLIC_POSTHOG_ENABLED: 'false', + BLACKLISTED_PROVIDERS: 'ollama,ollama-cloud,vllm,litellm,openrouter,together,fireworks,baseten', + STRIPE_SECRET_KEY: 'sk_test_sim_e2e_foundation', + STRIPE_WEBHOOK_SECRET: 'whsec_sim_e2e_foundation', + STRIPE_FREE_PRICE_ID: 'price_e2e_free', + STRIPE_API_BASE_URL: stripeApiBaseUrl, + E2E_STRIPE_FAKE_URL: stripeApiBaseUrl, + SOCKET_SERVER_URL: 'http://127.0.0.1:3002', + NEXT_PUBLIC_SOCKET_URL: E2E_SOCKET_ORIGIN, + CI: ci ? 'true' : 'false', + } + + validateProfileValues(values) + + return { + id: E2E_PROFILE, + origin: E2E_ORIGIN, + childEnvironment: buildChildEnvironment({ + values, + required: REQUIRED_KEYS, + allowedSensitiveKeys: ALLOWED_SENSITIVE_KEYS, + }), + } +} + +function validateProfileValues(values: Record): void { + if (values.NEXT_PUBLIC_APP_URL !== E2E_ORIGIN || values.BETTER_AUTH_URL !== E2E_ORIGIN) { + throw new Error('E2E app and Better Auth origins must exactly match') + } + if (values.BILLING_ENABLED !== values.NEXT_PUBLIC_BILLING_ENABLED) { + throw new Error('BILLING_ENABLED and NEXT_PUBLIC_BILLING_ENABLED must match') + } + if (!values.DATABASE_URL.includes('/sim_e2e_')) { + throw new Error('E2E profile requires a sim_e2e_ database') + } + const stripeUrl = new URL(values.STRIPE_API_BASE_URL) + if (!['127.0.0.1', 'localhost', '::1', '[::1]'].includes(stripeUrl.hostname)) { + throw new Error('E2E Stripe API must use a loopback hostname') + } +} diff --git a/apps/sim/e2e/support/env.ts b/apps/sim/e2e/support/env.ts new file mode 100644 index 00000000000..66e09b6ec99 --- /dev/null +++ b/apps/sim/e2e/support/env.ts @@ -0,0 +1,104 @@ +import { existsSync, readdirSync, readFileSync } from 'node:fs' +import path from 'node:path' +import { SIM_APP_DIR } from './paths' + +const ENV_KEY_PATTERN = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=/ +const SENSITIVE_KEY_PATTERN = + /(?:^|_)(?:API_?KEY|SECRET|TOKEN|PASSWORD|PRIVATE_?KEY|CLIENT_?SECRET|WEBHOOK)(?:_|$)/i + +const OS_PASSTHROUGH_KEYS = [ + 'PATH', + 'HOME', + 'USER', + 'SHELL', + 'TMPDIR', + 'TMP', + 'TEMP', + 'SYSTEMROOT', + 'CI', + 'GITHUB_ACTIONS', + 'PLAYWRIGHT_BROWSERS_PATH', +] as const + +export interface ChildEnvironment { + env: Record + discoveredKeys: string[] + shadowedKeys: string[] +} + +export interface BuildChildEnvironmentOptions { + values: Record + required: readonly string[] + allowedSensitiveKeys: ReadonlySet + envDirectory?: string +} + +export function discoverEnvFileKeys(directory = SIM_APP_DIR): string[] { + if (!existsSync(directory)) return [] + + const keys = new Set() + const files = readdirSync(directory) + .filter((name) => name === '.env' || name.startsWith('.env.')) + .sort() + + for (const file of files) { + const contents = readFileSync(path.join(directory, file), 'utf8') + for (const line of contents.split(/\r?\n/)) { + const match = ENV_KEY_PATTERN.exec(line) + if (match) keys.add(match[1]) + } + } + + return [...keys].sort() +} + +export function buildChildEnvironment({ + values, + required, + allowedSensitiveKeys, + envDirectory = SIM_APP_DIR, +}: BuildChildEnvironmentOptions): ChildEnvironment { + const missing = required.filter((key) => !values[key]?.trim()) + if (missing.length > 0) { + throw new Error(`Missing required E2E environment values: ${missing.join(', ')}`) + } + + const env: Record = {} + for (const key of OS_PASSTHROUGH_KEYS) { + const value = process.env[key] + if (value !== undefined) env[key] = value + } + + for (const [key, value] of Object.entries(values)) { + if (SENSITIVE_KEY_PATTERN.test(key) && !allowedSensitiveKeys.has(key)) { + throw new Error(`Sensitive E2E environment key is not explicitly allowed: ${key}`) + } + env[key] = value + } + + const discoveredKeys = discoverEnvFileKeys(envDirectory) + const shadowedKeys: string[] = [] + for (const key of discoveredKeys) { + if (key in env) continue + env[key] = '' + shadowedKeys.push(key) + } + + return { env, discoveredKeys, shadowedKeys } +} + +export function formatRedactedEnvironmentSummary( + profile: string, + childEnvironment: ChildEnvironment +): string { + return [ + `E2E profile: ${profile}`, + `Allowed keys: ${Object.keys(childEnvironment.env) + .filter((key) => !childEnvironment.shadowedKeys.includes(key)) + .sort() + .join(', ')}`, + `Shadowed local keys: ${childEnvironment.shadowedKeys.join(', ') || '(none)'}`, + ].join('\n') +} + +export const E2E_OS_PASSTHROUGH_KEYS = OS_PASSTHROUGH_KEYS diff --git a/apps/sim/e2e/support/hosts.ts b/apps/sim/e2e/support/hosts.ts new file mode 100644 index 00000000000..0b51910c931 --- /dev/null +++ b/apps/sim/e2e/support/hosts.ts @@ -0,0 +1,35 @@ +import { promises as dns } from 'node:dns' +import { isIP } from 'node:net' +import { E2E_HOST } from './deployment-profile' + +export function isLoopbackAddress(address: string): boolean { + if (address === '::1' || address === '0:0:0:0:0:0:0:1') return true + if (isIP(address) === 4) { + const first = Number(address.split('.')[0]) + return first === 127 + } + return false +} + +export async function assertE2eHostResolvesToLoopback(hostname = E2E_HOST): Promise { + let records: Array<{ address: string }> + try { + records = await dns.lookup(hostname, { all: true, verbatim: true }) + } catch { + throw new Error(getHostMappingError(hostname, [])) + } + + const addresses = [...new Set(records.map(({ address }) => address))] + if (addresses.length === 0 || addresses.some((address) => !isLoopbackAddress(address))) { + throw new Error(getHostMappingError(hostname, addresses)) + } + return addresses +} + +function getHostMappingError(hostname: string, addresses: string[]): string { + const observed = addresses.length > 0 ? addresses.join(', ') : 'no addresses' + return [ + `${hostname} must resolve only to loopback; observed ${observed}.`, + `Add it once with: echo "127.0.0.1 ${hostname}" | sudo tee -a /etc/hosts`, + ].join(' ') +} diff --git a/apps/sim/e2e/support/paths.ts b/apps/sim/e2e/support/paths.ts new file mode 100644 index 00000000000..44ec033e0e4 --- /dev/null +++ b/apps/sim/e2e/support/paths.ts @@ -0,0 +1,11 @@ +import path from 'node:path' + +export const SIM_APP_DIR = path.resolve(process.cwd()) +export const REPO_ROOT = path.resolve(SIM_APP_DIR, '../..') +export const DB_PACKAGE_DIR = path.join(REPO_ROOT, 'packages/db') +export const REALTIME_APP_DIR = path.join(REPO_ROOT, 'apps/realtime') +export const PLAYWRIGHT_CLI = path.join(REPO_ROOT, 'node_modules/@playwright/test/cli.js') + +export function getRunDirectory(runId: string): string { + return path.join(SIM_APP_DIR, 'e2e/.runs', runId) +} diff --git a/apps/sim/e2e/support/process.ts b/apps/sim/e2e/support/process.ts new file mode 100644 index 00000000000..11ad5e5088a --- /dev/null +++ b/apps/sim/e2e/support/process.ts @@ -0,0 +1,71 @@ +import { type ChildProcess, spawn } from 'node:child_process' +import { closeSync, mkdirSync, openSync } from 'node:fs' +import path from 'node:path' + +export interface CommandOptions { + name: string + command: string + args: string[] + cwd: string + env: Record + logsDirectory: string +} + +export interface ManagedProcess { + name: string + child: ChildProcess + logPath: string + stop(): Promise +} + +export function spawnManagedProcess(options: CommandOptions): ManagedProcess { + mkdirSync(options.logsDirectory, { recursive: true }) + const logPath = path.join(options.logsDirectory, `${options.name}.log`) + const logFd = openSync(logPath, 'a') + const child: ChildProcess = spawn(options.command, options.args, { + cwd: options.cwd, + env: options.env as NodeJS.ProcessEnv, + stdio: ['ignore', logFd, logFd], + }) + child.once('exit', () => closeSync(logFd)) + + return { + name: options.name, + child, + logPath, + stop: () => stopProcess(child), + } +} + +export async function runCommand(options: CommandOptions): Promise { + const managed = spawnManagedProcess(options) + const exitCode = await waitForExit(managed.child) + if (exitCode !== 0) { + throw new Error(`${options.name} exited with code ${exitCode}. See ${managed.logPath}`) + } +} + +export async function stopProcesses(processes: ManagedProcess[]): Promise { + await Promise.allSettled([...processes].reverse().map((process) => process.stop())) +} + +function waitForExit(child: ChildProcess): Promise { + return new Promise((resolve, reject) => { + child.once('error', reject) + child.once('exit', (code) => resolve(code)) + }) +} + +async function stopProcess(child: ChildProcess): Promise { + if (child.exitCode !== null || child.signalCode !== null) return + child.kill('SIGTERM') + + const stopped = await Promise.race([ + waitForExit(child).then(() => true), + new Promise((resolve) => setTimeout(() => resolve(false), 5_000)), + ]) + if (stopped) return + + child.kill('SIGKILL') + await waitForExit(child) +} diff --git a/apps/sim/e2e/support/readiness.ts b/apps/sim/e2e/support/readiness.ts new file mode 100644 index 00000000000..504bca72db1 --- /dev/null +++ b/apps/sim/e2e/support/readiness.ts @@ -0,0 +1,31 @@ +export interface ReadinessOptions { + name: string + url: string + timeoutMs?: number + intervalMs?: number + validate?: (response: Response) => Promise | boolean +} + +export async function waitForHttpReady({ + name, + url, + timeoutMs = 120_000, + intervalMs = 500, + validate = (response) => response.ok, +}: ReadinessOptions): Promise { + const deadline = Date.now() + timeoutMs + let lastError: unknown + + while (Date.now() < deadline) { + try { + const response = await fetch(url, { redirect: 'manual', signal: AbortSignal.timeout(5_000) }) + if (await validate(response)) return + lastError = new Error(`${name} returned ${response.status}`) + } catch (error) { + lastError = error + } + await new Promise((resolve) => setTimeout(resolve, intervalMs)) + } + + throw new Error(`${name} did not become ready at ${url}: ${String(lastError)}`) +} diff --git a/apps/sim/e2e/support/stack.ts b/apps/sim/e2e/support/stack.ts new file mode 100644 index 00000000000..7daf9f93fea --- /dev/null +++ b/apps/sim/e2e/support/stack.ts @@ -0,0 +1,119 @@ +import path from 'node:path' +import { DB_PACKAGE_DIR, PLAYWRIGHT_CLI, REALTIME_APP_DIR, REPO_ROOT, SIM_APP_DIR } from './paths' +import { type ManagedProcess, runCommand, spawnManagedProcess } from './process' +import { waitForHttpReady } from './readiness' + +export interface StackCommandOptions { + bunExecutable: string + nodeExecutable: string + env: Record + logsDirectory: string +} + +export async function runMigrations(options: StackCommandOptions): Promise { + await runCommand({ + name: 'migrate', + command: options.bunExecutable, + args: ['--no-env-file', path.join(DB_PACKAGE_DIR, 'scripts/migrate.ts')], + cwd: DB_PACKAGE_DIR, + env: options.env, + logsDirectory: options.logsDirectory, + }) +} + +export async function buildApp(options: StackCommandOptions): Promise { + await runCommand({ + name: 'sandbox-bundles', + command: options.bunExecutable, + args: ['--no-env-file', 'run', 'build:sandbox-bundles'], + cwd: SIM_APP_DIR, + env: options.env, + logsDirectory: options.logsDirectory, + }) + await runCommand({ + name: 'next-build', + command: options.nodeExecutable, + args: [path.join(REPO_ROOT, 'node_modules/next/dist/bin/next'), 'build'], + cwd: SIM_APP_DIR, + env: options.env, + logsDirectory: options.logsDirectory, + }) +} + +export async function startRealtime(options: StackCommandOptions): Promise { + const realtime = spawnManagedProcess({ + name: 'realtime', + command: options.bunExecutable, + args: ['--no-env-file', 'src/index.ts'], + cwd: REALTIME_APP_DIR, + env: { + ...options.env, + SIM_DB_ROLE: 'realtime', + DB_APP_NAME: 'sim-realtime', + PORT: '3002', + }, + logsDirectory: options.logsDirectory, + }) + try { + await waitForHttpReady({ + name: 'Realtime', + url: 'http://127.0.0.1:3002/health', + validate: async (response) => { + if (!response.ok) return false + const body = (await response.json()) as { status?: string } + return body.status === 'ok' + }, + }) + return realtime + } catch (error) { + await realtime.stop() + throw error + } +} + +export async function startApp(options: StackCommandOptions): Promise { + const app = spawnManagedProcess({ + name: 'app', + command: options.nodeExecutable, + args: [ + path.join(REPO_ROOT, 'node_modules/next/dist/bin/next'), + 'start', + '-p', + '3000', + '-H', + '0.0.0.0', + ], + cwd: SIM_APP_DIR, + env: { + ...options.env, + SIM_DB_ROLE: 'web', + DB_APP_NAME: 'sim-app', + PORT: '3000', + }, + logsDirectory: options.logsDirectory, + }) + try { + await waitForHttpReady({ + name: 'Next.js', + url: 'http://127.0.0.1:3000/api/health', + }) + return app + } catch (error) { + await app.stop() + throw error + } +} + +export async function runPlaywright( + options: Omit, + args: string[] +): Promise { + await runCommand({ + name: 'playwright', + command: options.nodeExecutable, + args: [PLAYWRIGHT_CLI, 'test', '--config=playwright.config.ts', ...args], + cwd: SIM_APP_DIR, + env: options.env, + logsDirectory: options.logsDirectory, + }) +} diff --git a/apps/sim/lib/auth/auth.ts b/apps/sim/lib/auth/auth.ts index ea2d1f0b227..b0dba3441aa 100644 --- a/apps/sim/lib/auth/auth.ts +++ b/apps/sim/lib/auth/auth.ts @@ -54,6 +54,7 @@ import { import { pauseProSubscriptionForOrgCoverage } from '@/lib/billing/organizations/membership' import { isPro, isTeam } from '@/lib/billing/plan-helpers' import { getPlans, resolvePlanFromStripeSubscription } from '@/lib/billing/plans' +import { buildStripeClientConfig } from '@/lib/billing/stripe-client-config' import { syncSeatsFromStripeQuantity } from '@/lib/billing/validation/seat-management' import { handleAbandonedCheckout } from '@/lib/billing/webhooks/checkout' import { handleChargeDispute, handleDisputeClosed } from '@/lib/billing/webhooks/disputes' @@ -188,9 +189,7 @@ const validStripeKey = env.STRIPE_SECRET_KEY let stripeClient = null if (validStripeKey) { - stripeClient = new Stripe(env.STRIPE_SECRET_KEY || '', { - apiVersion: '2025-08-27.basil', - }) + stripeClient = new Stripe(validStripeKey, buildStripeClientConfig(env)) } export const auth = betterAuth({ diff --git a/apps/sim/lib/billing/stripe-client-config.test.ts b/apps/sim/lib/billing/stripe-client-config.test.ts new file mode 100644 index 00000000000..e15d41f203b --- /dev/null +++ b/apps/sim/lib/billing/stripe-client-config.test.ts @@ -0,0 +1,123 @@ +/** @vitest-environment node */ + +import { describe, expect, it } from 'vitest' +import { + buildStripeClientConfig, + STRIPE_API_VERSION, + STRIPE_E2E_PROFILE, + type StripeClientConfigEnvironment, +} from '@/lib/billing/stripe-client-config' + +const guardedEnvironment: StripeClientConfigEnvironment = { + DATABASE_URL: 'postgresql://sim:sim@127.0.0.1:5432/sim_e2e_config_test', + E2E_PROFILE: STRIPE_E2E_PROFILE, + STRIPE_API_BASE_URL: 'http://127.0.0.1:12111', + STRIPE_SECRET_KEY: 'sk_test_e2e_config', +} + +describe('buildStripeClientConfig', () => { + it('preserves the normal Stripe configuration without an override', () => { + expect( + buildStripeClientConfig({ + DATABASE_URL: 'postgresql://db.example.com/sim', + STRIPE_SECRET_KEY: 'sk_live_normal_deployment', + }) + ).toEqual({ + apiVersion: STRIPE_API_VERSION, + }) + }) + + it('builds an HTTP transport for a guarded loopback fake', () => { + expect(buildStripeClientConfig(guardedEnvironment)).toEqual({ + apiVersion: STRIPE_API_VERSION, + host: '127.0.0.1', + port: '12111', + protocol: 'http', + }) + }) + + it('uses the HTTP default port when the override omits one', () => { + expect( + buildStripeClientConfig({ + ...guardedEnvironment, + STRIPE_API_BASE_URL: 'http://localhost', + }) + ).toMatchObject({ + host: 'localhost', + port: 80, + protocol: 'http', + }) + }) + + it.each(['sk_test_e2e_config', 'sk_live_e2e_config'])( + 'fails closed when the E2E profile configures %s without an override', + (stripeSecretKey) => { + expect(() => + buildStripeClientConfig({ + ...guardedEnvironment, + STRIPE_API_BASE_URL: undefined, + STRIPE_SECRET_KEY: stripeSecretKey, + }) + ).toThrow('STRIPE_API_BASE_URL is required') + } + ) + + it.each([ + { + name: 'a missing E2E profile', + environment: { E2E_PROFILE: undefined }, + message: 'requires E2E_PROFILE', + }, + { + name: 'a different E2E profile', + environment: { E2E_PROFILE: 'self-hosted-chromium' }, + message: 'requires E2E_PROFILE', + }, + { + name: 'a live Stripe key', + environment: { STRIPE_SECRET_KEY: 'sk_live_not_redirectable' }, + message: 'requires a Stripe test secret key', + }, + { + name: 'a public database', + environment: { DATABASE_URL: 'postgresql://db.example.com/sim' }, + message: 'requires a guarded sim_e2e_* database', + }, + { + name: 'an empty E2E database suffix', + environment: { DATABASE_URL: 'postgresql://127.0.0.1/sim_e2e_' }, + message: 'requires a guarded sim_e2e_* database', + }, + { + name: 'a malformed database URL', + environment: { DATABASE_URL: 'not-a-database-url' }, + message: 'requires a guarded sim_e2e_* database', + }, + ])('rejects an override with $name', ({ environment, message }) => { + expect(() => + buildStripeClientConfig({ + ...guardedEnvironment, + ...environment, + }) + ).toThrow(message) + }) + + it.each([ + 'https://127.0.0.1:12111', + 'http://stripe.example.com:12111', + 'http://0.0.0.0:12111', + 'http://user:password@127.0.0.1:12111', + 'http://127.0.0.1:12111/v1', + 'http://127.0.0.1:12111?mode=test', + 'http://127.0.0.1:12111#fragment', + ' http://127.0.0.1:12111', + 'not-a-url', + ])('rejects an unsafe or malformed override: %s', (stripeApiBaseUrl) => { + expect(() => + buildStripeClientConfig({ + ...guardedEnvironment, + STRIPE_API_BASE_URL: stripeApiBaseUrl, + }) + ).toThrow() + }) +}) diff --git a/apps/sim/lib/billing/stripe-client-config.ts b/apps/sim/lib/billing/stripe-client-config.ts new file mode 100644 index 00000000000..df6b13dddf7 --- /dev/null +++ b/apps/sim/lib/billing/stripe-client-config.ts @@ -0,0 +1,114 @@ +import type Stripe from 'stripe' + +export const STRIPE_API_VERSION = '2025-08-27.basil' as const +export const STRIPE_E2E_PROFILE = 'hosted-billing-chromium' as const + +export interface StripeClientConfigEnvironment { + DATABASE_URL?: string + E2E_PROFILE?: string + STRIPE_API_BASE_URL?: string + STRIPE_SECRET_KEY?: string +} + +const E2E_DATABASE_NAME_PATTERN = /^sim_e2e_[A-Za-z0-9_-]+$/ + +function getDatabaseName(databaseUrl: string | undefined): string | null { + if (!databaseUrl) return null + + try { + const parsed = new URL(databaseUrl) + if (parsed.protocol !== 'postgres:' && parsed.protocol !== 'postgresql:') return null + + const pathSegments = parsed.pathname.split('/').filter(Boolean) + if (pathSegments.length !== 1) return null + + return decodeURIComponent(pathSegments[0]) + } catch { + return null + } +} + +function isLoopbackHostname(hostname: string): boolean { + const normalized = hostname.toLowerCase() + if (normalized === 'localhost' || normalized === '[::1]') return true + + const octets = normalized.split('.') + return ( + octets.length === 4 && + octets.every((octet) => /^\d+$/.test(octet) && Number(octet) <= 255) && + Number(octets[0]) === 127 + ) +} + +function parseStripeApiBaseUrl(rawUrl: string): URL { + if (rawUrl.trim() !== rawUrl) { + throw new Error('STRIPE_API_BASE_URL must not contain surrounding whitespace') + } + + let parsed: URL + try { + parsed = new URL(rawUrl) + } catch { + throw new Error('STRIPE_API_BASE_URL must be a valid absolute URL') + } + + if (parsed.protocol !== 'http:') { + throw new Error('STRIPE_API_BASE_URL must use HTTP for the loopback E2E fake') + } + if (!isLoopbackHostname(parsed.hostname)) { + throw new Error('STRIPE_API_BASE_URL must use a loopback hostname') + } + if (parsed.username || parsed.password) { + throw new Error('STRIPE_API_BASE_URL must not contain credentials') + } + if (parsed.search || parsed.hash) { + throw new Error('STRIPE_API_BASE_URL must not contain a query or hash') + } + if (parsed.pathname !== '/') { + throw new Error('STRIPE_API_BASE_URL must not contain a non-root path') + } + + return parsed +} + +/** + * Builds the Stripe SDK configuration while keeping the E2E transport override + * unavailable to normal deployments and production credentials. + */ +export function buildStripeClientConfig( + environment: StripeClientConfigEnvironment +): Stripe.StripeConfig { + const baseConfig: Stripe.StripeConfig = { + apiVersion: STRIPE_API_VERSION, + } + const override = environment.STRIPE_API_BASE_URL + + if (override === undefined || override === '') { + if (environment.E2E_PROFILE === STRIPE_E2E_PROFILE && environment.STRIPE_SECRET_KEY) { + throw new Error( + 'STRIPE_API_BASE_URL is required when the hosted billing E2E profile configures Stripe' + ) + } + return baseConfig + } + + if (environment.E2E_PROFILE !== STRIPE_E2E_PROFILE) { + throw new Error(`STRIPE_API_BASE_URL requires E2E_PROFILE=${STRIPE_E2E_PROFILE}`) + } + if (!environment.STRIPE_SECRET_KEY?.startsWith('sk_test_')) { + throw new Error('STRIPE_API_BASE_URL requires a Stripe test secret key') + } + + const databaseName = getDatabaseName(environment.DATABASE_URL) + if (!databaseName || !E2E_DATABASE_NAME_PATTERN.test(databaseName)) { + throw new Error('STRIPE_API_BASE_URL requires a guarded sim_e2e_* database') + } + + const endpoint = parseStripeApiBaseUrl(override) + return { + ...baseConfig, + host: endpoint.hostname, + port: endpoint.port || 80, + protocol: 'http', + } +} diff --git a/apps/sim/lib/billing/stripe-client.ts b/apps/sim/lib/billing/stripe-client.ts index 13bb089845b..9f1d6edc808 100644 --- a/apps/sim/lib/billing/stripe-client.ts +++ b/apps/sim/lib/billing/stripe-client.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import Stripe from 'stripe' +import { buildStripeClientConfig } from '@/lib/billing/stripe-client-config' import { env } from '@/lib/core/config/env' const logger = createLogger('StripeClient') @@ -37,9 +38,7 @@ const createStripeClientSingleton = () => { try { isInitializing = true - stripeClient = new Stripe(env.STRIPE_SECRET_KEY || '', { - apiVersion: '2025-08-27.basil', - }) + stripeClient = new Stripe(env.STRIPE_SECRET_KEY || '', buildStripeClientConfig(env)) logger.info('Stripe client initialized successfully') return stripeClient diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index c86eb9e7ff4..0972bd75634 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -68,7 +68,9 @@ export const env = createEnv({ REDIS_TLS_SERVERNAME: z.string().min(1).optional(), // TLS SNI override; required when REDIS_URL targets an IP over rediss:// (e.g. trigger.dev PrivateLink VPCE IP) so cert hostname verification matches the ElastiCache cert's CN // Payment & Billing + E2E_PROFILE: z.string().min(1).optional(), // Server-only E2E deployment profile selector STRIPE_SECRET_KEY: z.string().min(1).optional(), // Stripe secret key for payment processing + STRIPE_API_BASE_URL: z.string().url().optional(), // Guarded server-only Stripe API override for E2E fakes STRIPE_WEBHOOK_SECRET: z.string().min(1).optional(), // General Stripe webhook secret STRIPE_FREE_PRICE_ID: z.string().min(1).optional(), // Stripe price ID for free tier FREE_TIER_COST_LIMIT: z.number().optional(), // Cost limit for free tier users diff --git a/apps/sim/package.json b/apps/sim/package.json index a7800a9f904..bb1c1a7b8fc 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -24,6 +24,9 @@ "test": "vitest run", "test:watch": "vitest", "test:coverage": "vitest run --coverage", + "test:e2e": "bun --no-env-file e2e/scripts/run.ts", + "test:e2e:playwright": "node ../../node_modules/@playwright/test/cli.js test --config=playwright.config.ts", + "test:e2e:install-browsers": "node ../../node_modules/@playwright/test/cli.js install chromium", "email:dev": "email dev --dir components/emails", "type-check": "NODE_OPTIONS='--max-old-space-size=8192' tsc --noEmit", "lint": "biome check --write --unsafe .", @@ -220,6 +223,7 @@ "zustand": "^5.0.13" }, "devDependencies": { + "@playwright/test": "1.61.1", "@sim/testing": "workspace:*", "@sim/tsconfig": "workspace:*", "@tailwindcss/typography": "0.5.19", diff --git a/apps/sim/playwright.config.ts b/apps/sim/playwright.config.ts new file mode 100644 index 00000000000..0160ca29411 --- /dev/null +++ b/apps/sim/playwright.config.ts @@ -0,0 +1,48 @@ +import { defineConfig, devices } from '@playwright/test' + +const isCI = process.env.CI === 'true' +const baseURL = process.env.E2E_BASE_URL ?? 'http://e2e.sim.ai:3000' + +export default defineConfig({ + testDir: './e2e', + testMatch: '**/*.spec.ts', + fullyParallel: true, + forbidOnly: isCI, + retries: 0, + timeout: 60_000, + expect: { + timeout: 10_000, + }, + reporter: [ + ['list'], + ['html', { open: 'never', outputFolder: 'playwright-report' }], + ...(isCI ? ([['github']] as const) : []), + ], + outputDir: 'test-results', + use: { + ...devices['Desktop Chrome'], + baseURL, + actionTimeout: 15_000, + navigationTimeout: 30_000, + trace: 'retain-on-failure', + screenshot: 'only-on-failure', + video: 'off', + }, + projects: [ + { + name: 'hosted-billing-chromium-navigation', + testMatch: [ + '**/foundation/**/*.spec.ts', + '**/settings/smoke/unauthenticated.spec.ts', + '**/settings/navigation/**/*.spec.ts', + ], + workers: isCI ? 2 : 1, + }, + { + name: 'hosted-billing-chromium-workflows', + testMatch: ['**/settings/smoke/authenticated.spec.ts', '**/settings/workflows/**/*.spec.ts'], + fullyParallel: false, + workers: 1, + }, + ], +}) diff --git a/apps/sim/vitest.config.ts b/apps/sim/vitest.config.ts index a966ad5233e..02425c6f712 100644 --- a/apps/sim/vitest.config.ts +++ b/apps/sim/vitest.config.ts @@ -16,7 +16,7 @@ export default defineConfig({ globals: true, environment: 'node', include: ['**/*.test.{ts,tsx}'], - exclude: [...configDefaults.exclude, '**/node_modules/**', '**/dist/**'], + exclude: [...configDefaults.exclude, '**/node_modules/**', '**/dist/**', '**/e2e/**'], setupFiles: ['./vitest.setup.ts'], pool: 'threads', isolate: true, diff --git a/biome.json b/biome.json index 271e701c8b4..3271ed0dde1 100644 --- a/biome.json +++ b/biome.json @@ -22,6 +22,9 @@ "!**/.env", "!**/.vercel", "!**/coverage", + "!**/playwright-report", + "!**/test-results", + "!**/e2e/.runs", "!**/public/sw.js", "!**/public/workbox-*.js", "!**/public/worker-*.js", diff --git a/bun.lock b/bun.lock index f3ad75a78dd..51ef90840cb 100644 --- a/bun.lock +++ b/bun.lock @@ -288,6 +288,7 @@ "zustand": "^5.0.13", }, "devDependencies": { + "@playwright/test": "1.61.1", "@sim/testing": "workspace:*", "@sim/tsconfig": "workspace:*", "@tailwindcss/typography": "0.5.19", @@ -1331,6 +1332,8 @@ "@pkgjs/parseargs": ["@pkgjs/parseargs@0.11.0", "", {}, "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg=="], + "@playwright/test": ["@playwright/test@1.61.1", "", { "dependencies": { "playwright": "1.61.1" }, "bin": { "playwright": "cli.js" } }, "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig=="], + "@posthog/core": ["@posthog/core@1.24.4", "", { "dependencies": { "cross-spawn": "^7.0.6" } }, "sha512-S+TolwBHSSJz7WWtgaELQWQqXviSm3uf1e+qorWUts0bZcgPwWzhnmhCUZAhvn0NVpTQHDJ3epv+hHbPLl5dHg=="], "@posthog/types": ["@posthog/types@1.364.4", "", {}, "sha512-U7NpIy9XWrzz1q/66xyDu8Wm12a7avNRKRn5ISPT5kuCJQRaeAaHuf+dpgrFnuqjCCgxg+oIY/ReJdlZ+8/z4Q=="], @@ -3467,6 +3470,10 @@ "platform": ["platform@1.3.6", "", {}, "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg=="], + "playwright": ["playwright@1.61.1", "", { "dependencies": { "playwright-core": "1.61.1" }, "optionalDependencies": { "fsevents": "2.3.2" }, "bin": { "playwright": "cli.js" } }, "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ=="], + + "playwright-core": ["playwright-core@1.61.1", "", { "bin": { "playwright-core": "cli.js" } }, "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg=="], + "points-on-curve": ["points-on-curve@0.2.0", "", {}, "sha512-0mYKnYYe9ZcqMCWhUjItv/oHjvgEsfKvnUTg8sAtnHr3GVy7rGkXCb6d5cSyqrWqL4k81b9CPg3urd+T7aop3A=="], "points-on-path": ["points-on-path@0.2.1", "", { "dependencies": { "path-data-parser": "0.1.0", "points-on-curve": "0.2.0" } }, "sha512-25ClnWWuw7JbWZcgqY/gJ4FQWadKxGWk+3kR/7kD0tCaDtPPMj7oHu2ToLaVhfpnHrZzYby2w6tUA0eOIuUg8g=="], @@ -4781,6 +4788,8 @@ "pino-pretty/pino-abstract-transport": ["pino-abstract-transport@3.0.0", "", { "dependencies": { "split2": "^4.0.0" } }, "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg=="], + "playwright/fsevents": ["fsevents@2.3.2", "", { "os": "darwin" }, "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA=="], + "postcss-nested/postcss-selector-parser": ["postcss-selector-parser@6.1.4", "", { "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" } }, "sha512-bIoJLOmjCO1S9XdY/DcnR5hJxvrDir1PbGChrzXG3vw0/FOliy/fA3dmdhQ441kah4gKv+TwckGzex6wNS5cnQ=="], "posthog-js/@opentelemetry/api-logs": ["@opentelemetry/api-logs@0.208.0", "", { "dependencies": { "@opentelemetry/api": "^1.3.0" } }, "sha512-CjruKY9V6NMssL/T1kAFgzosF1v9o6oeN+aX5JB/C/xPNtmgIJqcXHG7fA82Ou1zCpWGl4lROQUKwUNE1pMCyg=="], From 84c92e34aa5c1d2ee107f79055ed5e7d3ed999ac Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Mon, 20 Jul 2026 16:42:03 -0700 Subject: [PATCH 02/10] test(e2e): harden runner lifecycle and artifacts Fail closed on stale processes, preserve cleanup during interruption, and keep credentials and personal fixture data out of uploaded diagnostics. --- .github/workflows/test-build.yml | 3 +- apps/realtime/src/routes/http.ts | 1 + apps/sim/app/api/health/route.test.ts | 15 +- apps/sim/app/api/health/route.ts | 3 + apps/sim/e2e/README.md | 6 +- apps/sim/e2e/fakes/stripe/server.ts | 60 +----- apps/sim/e2e/foundation/admin-api.spec.ts | 17 -- .../e2e/foundation/provider-isolation.spec.ts | 7 + apps/sim/e2e/foundation/safety.spec.ts | 24 ++- apps/sim/e2e/scripts/cleanup-database.ts | 16 ++ apps/sim/e2e/scripts/options.ts | 11 +- apps/sim/e2e/scripts/run.ts | 175 ++++++++++++++---- .../e2e/settings/smoke/authenticated.spec.ts | 54 +----- apps/sim/e2e/support/deployment-profile.ts | 3 - apps/sim/e2e/support/diagnostics.ts | 30 +++ apps/sim/e2e/support/probes.ts | 47 +++++ apps/sim/e2e/support/process.ts | 67 ++++++- apps/sim/e2e/support/stack.ts | 46 +++-- apps/sim/package.json | 3 +- apps/sim/playwright.config.ts | 9 +- 20 files changed, 404 insertions(+), 193 deletions(-) delete mode 100644 apps/sim/e2e/foundation/admin-api.spec.ts create mode 100644 apps/sim/e2e/foundation/provider-isolation.spec.ts create mode 100644 apps/sim/e2e/scripts/cleanup-database.ts create mode 100644 apps/sim/e2e/support/diagnostics.ts create mode 100644 apps/sim/e2e/support/probes.ts diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 0e25b9056c5..3c23831b572 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -333,7 +333,7 @@ jobs: run: bun run test:e2e - name: Upload E2E diagnostics - if: always() + if: failure() || cancelled() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: settings-e2e-${{ github.run_id }} @@ -341,5 +341,6 @@ jobs: apps/sim/playwright-report/ apps/sim/test-results/ apps/sim/e2e/.runs/ + !apps/sim/e2e/.runs/**/auth/** if-no-files-found: ignore retention-days: 7 \ No newline at end of file diff --git a/apps/realtime/src/routes/http.ts b/apps/realtime/src/routes/http.ts index 0f8ed73cc52..0bfef282fcf 100644 --- a/apps/realtime/src/routes/http.ts +++ b/apps/realtime/src/routes/http.ts @@ -69,6 +69,7 @@ export function createHttpHandler(roomManager: IRoomManager, logger: Logger) { status: 'ok', timestamp: new Date().toISOString(), connections, + ...(process.env.E2E_RUN_ID ? { runId: process.env.E2E_RUN_ID } : {}), }) ) } catch (error) { diff --git a/apps/sim/app/api/health/route.test.ts b/apps/sim/app/api/health/route.test.ts index 6abca827579..21d4d9d6a91 100644 --- a/apps/sim/app/api/health/route.test.ts +++ b/apps/sim/app/api/health/route.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { GET } from '@/app/api/health/route' describe('GET /api/health', () => { @@ -14,4 +14,17 @@ describe('GET /api/health', () => { timestamp: expect.any(String), }) }) + + it('returns the E2E run identity when configured', async () => { + vi.stubEnv('E2E_RUN_ID', 'run-health-check') + try { + const response = await GET() + await expect(response.json()).resolves.toMatchObject({ + status: 'ok', + runId: 'run-health-check', + }) + } finally { + vi.unstubAllEnvs() + } + }) }) diff --git a/apps/sim/app/api/health/route.ts b/apps/sim/app/api/health/route.ts index 5486272998c..517f9251724 100644 --- a/apps/sim/app/api/health/route.ts +++ b/apps/sim/app/api/health/route.ts @@ -1,11 +1,14 @@ /** * Health check endpoint for deployment platforms and container probes. */ +export const dynamic = 'force-dynamic' + export async function GET(): Promise { return Response.json( { status: 'ok', timestamp: new Date().toISOString(), + ...(process.env.E2E_RUN_ID ? { runId: process.env.E2E_RUN_ID } : {}), }, { status: 200 } ) diff --git a/apps/sim/e2e/README.md b/apps/sim/e2e/README.md index c026698f89a..1eb02cc9285 100644 --- a/apps/sim/e2e/README.md +++ b/apps/sim/e2e/README.md @@ -51,8 +51,10 @@ bun run test:e2e -- --project=hosted-billing-chromium-navigation bun run test:e2e -- --grep "unauthenticated" ``` -For a local rerun after a successful build with the same hosted/public profile, -use `--skip-build`. CI and clean validation runs must always build. +Do not invoke `playwright test` directly. Raw Playwright bypasses environment, +database, process, sharding, and teardown guards; the config rejects runs that +were not launched by the orchestrator. Report and trace viewer commands remain +safe because they do not execute tests. Sharding is supported only for the navigation project. The runner rejects `--shard` for `hosted-billing-chromium-workflows`. diff --git a/apps/sim/e2e/fakes/stripe/server.ts b/apps/sim/e2e/fakes/stripe/server.ts index 8d9bcd2bba4..3aec1ce800c 100644 --- a/apps/sim/e2e/fakes/stripe/server.ts +++ b/apps/sim/e2e/fakes/stripe/server.ts @@ -1,11 +1,5 @@ import { createHash, timingSafeEqual } from 'node:crypto' -import { - createServer, - type IncomingHttpHeaders, - type IncomingMessage, - type Server, - type ServerResponse, -} from 'node:http' +import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http' import type { AddressInfo } from 'node:net' export const STRIPE_FAKE_ENDPOINTS = { @@ -14,20 +8,13 @@ export const STRIPE_FAKE_ENDPOINTS = { reset: '/__control/reset', } as const -const REDACTED = '[REDACTED]' const DEFAULT_MAX_BODY_BYTES = 64 * 1024 const FORM_CONTENT_TYPE = 'application/x-www-form-urlencoded' -const SENSITIVE_FIELD_PATTERN = /authorization|api.?key|secret|token|password|card|source/i - -type RecordedParameters = Record export interface StripeFakeRequestRecord { sequence: number method: string path: string - query: RecordedParameters - headers: RecordedParameters - body: RecordedParameters | null unexpected: boolean } @@ -90,41 +77,6 @@ function secureEqual(actual: string | undefined, expected: string): boolean { ) } -function appendParameter( - target: RecordedParameters, - key: string, - value: string -): RecordedParameters { - const recordedValue = SENSITIVE_FIELD_PATTERN.test(key) ? REDACTED : value - const existing = target[key] - if (existing === undefined) { - target[key] = recordedValue - } else if (Array.isArray(existing)) { - existing.push(recordedValue) - } else { - target[key] = [existing, recordedValue] - } - return target -} - -function redactParameters(parameters: URLSearchParams): RecordedParameters { - const redacted: RecordedParameters = {} - for (const [key, value] of parameters) { - appendParameter(redacted, key, value) - } - return redacted -} - -function redactHeaders(headers: IncomingHttpHeaders): RecordedParameters { - const redacted: RecordedParameters = {} - for (const key of ['authorization', 'content-type', 'idempotency-key', 'stripe-version']) { - const value = headers[key] - if (value === undefined) continue - redacted[key] = key === 'authorization' ? REDACTED : value - } - return redacted -} - function cloneRequestLog(records: StripeFakeRequestRecord[]): StripeFakeRequestRecord[] { return structuredClone(records) } @@ -335,7 +287,6 @@ export function createStripeFakeServer(options: StripeFakeServerOptions): Stripe const requestId = `req_e2e_${String(sequence).padStart(6, '0')}` const expected = isExpectedStripeRequest(method, url.pathname) let formBody: URLSearchParams | null = null - let recordedBody: RecordedParameters | null = null try { if (method !== 'GET' && method !== 'HEAD') { @@ -343,9 +294,6 @@ export function createStripeFakeServer(options: StripeFakeServerOptions): Stripe if (rawBody) { if (request.headers['content-type']?.startsWith(FORM_CONTENT_TYPE)) { formBody = new URLSearchParams(rawBody) - recordedBody = redactParameters(formBody) - } else { - recordedBody = { content: REDACTED } } } } @@ -354,9 +302,6 @@ export function createStripeFakeServer(options: StripeFakeServerOptions): Stripe sequence, method, path: url.pathname, - query: redactParameters(url.searchParams), - headers: redactHeaders(request.headers), - body: recordedBody, unexpected: !expected, }) const bodyError = @@ -380,9 +325,6 @@ export function createStripeFakeServer(options: StripeFakeServerOptions): Stripe sequence, method, path: url.pathname, - query: redactParameters(url.searchParams), - headers: redactHeaders(request.headers), - body: recordedBody, unexpected: !expected, }) diff --git a/apps/sim/e2e/foundation/admin-api.spec.ts b/apps/sim/e2e/foundation/admin-api.spec.ts deleted file mode 100644 index 44c9911c23f..00000000000 --- a/apps/sim/e2e/foundation/admin-api.spec.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { expect, test } from '@playwright/test' - -test('Admin API is configured and rejects missing credentials', async ({ request }) => { - const adminKey = process.env.E2E_ADMIN_API_KEY - expect(adminKey, 'E2E_ADMIN_API_KEY must be provided to the Playwright worker').toBeTruthy() - - const unauthorized = await request.get('/api/v1/admin/users?limit=1&offset=0') - expect(unauthorized.status()).toBe(401) - - const authorized = await request.get('/api/v1/admin/users?limit=1&offset=0', { - headers: { 'x-admin-key': adminKey! }, - }) - const responseText = await authorized.text() - expect(authorized.status(), responseText).toBe(200) - const body = JSON.parse(responseText) as { data?: unknown[] } - expect(Array.isArray(body.data)).toBe(true) -}) diff --git a/apps/sim/e2e/foundation/provider-isolation.spec.ts b/apps/sim/e2e/foundation/provider-isolation.spec.ts new file mode 100644 index 00000000000..f31c7a62644 --- /dev/null +++ b/apps/sim/e2e/foundation/provider-isolation.spec.ts @@ -0,0 +1,7 @@ +import { expect, test } from '@playwright/test' + +test('blacklisted local providers skip outbound model discovery', async ({ request }) => { + const response = await request.get('/api/providers/ollama/models') + expect(response.status()).toBe(200) + expect(await response.json()).toEqual({ models: [] }) +}) diff --git a/apps/sim/e2e/foundation/safety.spec.ts b/apps/sim/e2e/foundation/safety.spec.ts index 92f2447545d..b97b7cf3aa3 100644 --- a/apps/sim/e2e/foundation/safety.spec.ts +++ b/apps/sim/e2e/foundation/safety.spec.ts @@ -1,4 +1,5 @@ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { createServer } from 'node:net' import os from 'node:os' import path from 'node:path' import { loadEnvConfig } from '@next/env' @@ -11,6 +12,7 @@ import { } from '../support/database' import { buildChildEnvironment, discoverEnvFileKeys } from '../support/env' import { isLoopbackAddress } from '../support/hosts' +import { assertPortAvailable } from '../support/process' test.describe('foundation safety guards', () => { test('discovers env keys without leaking values and shadows unknown keys', () => { @@ -37,16 +39,20 @@ test.describe('foundation safety guards', () => { } }) - test('@next/env preserves a pre-set process value over an env file', () => { + test('@next/env preserves non-empty and empty shadow values over an env file', () => { const directory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-next-env-')) const key = `SIM_E2E_CANARY_${Date.now()}` + const emptyKey = `${key}_EMPTY` try { - writeFileSync(path.join(directory, '.env'), `${key}=file-value\n`) + writeFileSync(path.join(directory, '.env'), `${key}=file-value\n${emptyKey}=must-not-load\n`) process.env[key] = 'shadowed-value' + process.env[emptyKey] = '' loadEnvConfig(directory, false, console, true) expect(process.env[key]).toBe('shadowed-value') + expect(process.env[emptyKey]).toBe('') } finally { delete process.env[key] + delete process.env[emptyKey] rmSync(directory, { recursive: true, force: true }) } }) @@ -80,4 +86,18 @@ test.describe('foundation safety guards', () => { parseRunOptions(['--project=hosted-billing-chromium-workflows', '--shard=1/2']) ).toThrow(/coupled workflows must remain unsharded/) }) + + test('port preflight rejects an existing listener', async () => { + const server = createServer() + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + try { + const address = server.address() + if (!address || typeof address === 'string') throw new Error('Expected TCP listener') + await expect(assertPortAvailable(address.port)).rejects.toThrow(/to be free/) + } finally { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())) + ) + } + }) }) diff --git a/apps/sim/e2e/scripts/cleanup-database.ts b/apps/sim/e2e/scripts/cleanup-database.ts new file mode 100644 index 00000000000..353138250eb --- /dev/null +++ b/apps/sim/e2e/scripts/cleanup-database.ts @@ -0,0 +1,16 @@ +import { dropRunDatabase } from '../support/database' + +const adminUrl = process.env.E2E_PG_ADMIN_URL +const databaseName = process.env.E2E_DATABASE_NAME + +if (!adminUrl || !databaseName) { + console.error('Synchronous E2E cleanup requires E2E_PG_ADMIN_URL and E2E_DATABASE_NAME') + process.exit(1) +} + +try { + await dropRunDatabase(adminUrl, databaseName) +} catch (error) { + console.error(error) + process.exit(1) +} diff --git a/apps/sim/e2e/scripts/options.ts b/apps/sim/e2e/scripts/options.ts index 71a895fb1fa..785ac3cd81f 100644 --- a/apps/sim/e2e/scripts/options.ts +++ b/apps/sim/e2e/scripts/options.ts @@ -3,13 +3,16 @@ const WORKFLOWS_PROJECT = 'hosted-billing-chromium-workflows' export interface E2eRunOptions { playwrightArgs: string[] - skipBuild: boolean } export function parseRunOptions(argv: string[]): E2eRunOptions { const normalizedArgs = argv[0] === '--' ? argv.slice(1) : [...argv] - const skipBuild = normalizedArgs.includes('--skip-build') - const playwrightArgs = normalizedArgs.filter((arg) => arg !== '--skip-build') + if (normalizedArgs.includes('--skip-build')) { + throw new Error( + '--skip-build is not supported because E2E builds contain run-specific hosted configuration' + ) + } + const playwrightArgs = [...normalizedArgs] const projects = getOptionValues(playwrightArgs, '--project') const hasShard = hasOption(playwrightArgs, '--shard') @@ -24,7 +27,7 @@ export function parseRunOptions(argv: string[]): E2eRunOptions { ) } - return { playwrightArgs, skipBuild } + return { playwrightArgs } } function hasOption(args: string[], name: string): boolean { diff --git a/apps/sim/e2e/scripts/run.ts b/apps/sim/e2e/scripts/run.ts index daade957034..7769b7ddb56 100644 --- a/apps/sim/e2e/scripts/run.ts +++ b/apps/sim/e2e/scripts/run.ts @@ -4,16 +4,28 @@ import { mkdirSync, writeFileSync } from 'node:fs' import path from 'node:path' import { type StripeFakeServer, startStripeFakeServer } from '../fakes/stripe/server' import { + buildRunDatabaseUrl, createRunDatabase, createRunDatabaseName, dropRunDatabase, type RunDatabase, } from '../support/database' import { createHostedBillingProfile, E2E_ORIGIN, E2E_PROFILE } from '../support/deployment-profile' +import { assertNoForbiddenProviderInitialization } from '../support/diagnostics' import { formatRedactedEnvironmentSummary } from '../support/env' import { assertE2eHostResolvesToLoopback } from '../support/hosts' -import { getRunDirectory } from '../support/paths' -import { type ManagedProcess, stopProcesses } from '../support/process' +import { getRunDirectory, SIM_APP_DIR } from '../support/paths' +import { + assertAdminApiBoundary, + type FoundationProvisioningResult, + inspectFoundationUsers, +} from '../support/probes' +import { + assertPortAvailable, + type ManagedProcess, + signalAllManagedProcesses, + stopAllManagedProcesses, +} from '../support/process' import { buildApp, runMigrations, runPlaywright, startApp, startRealtime } from '../support/stack' import { parseRunOptions } from './options' @@ -25,9 +37,9 @@ async function main(): Promise { const runId = createRunId() const runDirectory = getRunDirectory(runId) const logsDirectory = path.join(runDirectory, 'logs') - const storageStatePath = path.join(runDirectory, 'auth', 'foundation.json') + const storageStateDirectory = path.join(runDirectory, 'auth') mkdirSync(logsDirectory, { recursive: true }) - mkdirSync(path.dirname(storageStatePath), { recursive: true }) + mkdirSync(storageStateDirectory, { recursive: true }) const nodeExecutable = resolveNode22() const bunExecutable = resolveBunExecutable() @@ -35,14 +47,89 @@ async function main(): Promise { let runDatabase: RunDatabase | null = null let stripeFake: StripeFakeServer | null = null - const processes: ManagedProcess[] = [] + let realtime: ManagedProcess | null = null + let app: ManagedProcess | null = null + let cleanupPromise: Promise | null = null let failed = false + let interrupted = false + + const cleanup = (): Promise => { + if (cleanupPromise) return cleanupPromise + cleanupPromise = (async () => { + await stopAllManagedProcesses() + + if (stripeFake) { + writeFileSync( + path.join(logsDirectory, 'stripe-requests.json'), + JSON.stringify(stripeFake.requestLog, null, 2) + ) + await stripeFake.stop() + } + + if (runDatabase) { + try { + await dropRunDatabase(adminDatabaseUrl, runDatabase.name) + } catch (error) { + failed = true + process.exitCode = 1 + console.error(`Failed to drop ${runDatabase.name}:`, error) + } + } + })() + return cleanupPromise + } + + const handleSignal = (signal: NodeJS.Signals): void => { + if (interrupted) return + interrupted = true + failed = true + const exitCode = signal === 'SIGINT' ? 130 : 143 + process.exitCode = exitCode + console.error(`Received ${signal}; cleaning up the E2E run`) + signalAllManagedProcesses() + if (stripeFake) { + writeFileSync( + path.join(logsDirectory, 'stripe-requests.json'), + JSON.stringify(stripeFake.requestLog, null, 2) + ) + } + if (runDatabase) { + const cleanupResult = spawnSync( + bunExecutable, + ['--no-env-file', path.join(SIM_APP_DIR, 'e2e/scripts/cleanup-database.ts')], + { + cwd: SIM_APP_DIR, + encoding: 'utf8', + env: { + NODE_ENV: 'test', + PATH: process.env.PATH ?? '', + E2E_PG_ADMIN_URL: adminDatabaseUrl, + E2E_DATABASE_NAME: runDatabase.name, + }, + } + ) + if (cleanupResult.status !== 0) { + console.error(cleanupResult.stderr || cleanupResult.stdout) + } else { + runDatabase = null + } + } + process.exit(exitCode) + } + process.once('SIGINT', handleSignal) + process.once('SIGTERM', handleSignal) try { const hostAddresses = await assertE2eHostResolvesToLoopback() console.info(`E2E host resolved to loopback: ${hostAddresses.join(', ')}`) + await Promise.all([assertPortAvailable(3000), assertPortAvailable(3002)]) - runDatabase = await createRunDatabase(adminDatabaseUrl, createRunDatabaseName(runId)) + const runDatabaseName = createRunDatabaseName(runId) + runDatabase = { + name: runDatabaseName, + url: buildRunDatabaseUrl(adminDatabaseUrl, runDatabaseName), + } + await createRunDatabase(adminDatabaseUrl, runDatabaseName) stripeFake = await startStripeFakeServer({ apiKey: STRIPE_TEST_KEY, hostname: '127.0.0.1', @@ -66,15 +153,16 @@ async function main(): Promise { } await runMigrations(stackOptions) - if (!options.skipBuild) await buildApp(stackOptions) - processes.push(await startRealtime(stackOptions)) - processes.push(await startApp(stackOptions)) + await buildApp(stackOptions) + realtime = await startRealtime(stackOptions) + app = await startApp(stackOptions) + await assertAdminApiBoundary(E2E_ORIGIN, profile.childEnvironment.env.ADMIN_API_KEY) + assertNoForbiddenProviderInitialization([app.logPath, realtime.logPath]) const playwrightEnvironment = createPlaywrightEnvironment( profile.childEnvironment.env, runId, - stripeFake.baseUrl, - storageStatePath + storageStateDirectory ) await runPlaywright( { @@ -84,31 +172,16 @@ async function main(): Promise { }, options.playwrightArgs ) + const provisioning = await inspectFoundationUsers(runDatabase.url, runId) + assertAuthenticatedSmokeEffectsIfPresent(stripeFake, provisioning) } catch (error) { failed = true console.error(error) process.exitCode = 1 } finally { - await stopProcesses(processes) - - if (stripeFake) { - writeFileSync( - path.join(logsDirectory, 'stripe-requests.json'), - JSON.stringify(stripeFake.requestLog, null, 2) - ) - await stripeFake.stop() - } - - if (runDatabase) { - try { - await dropRunDatabase(adminDatabaseUrl, runDatabase.name) - } catch (error) { - failed = true - process.exitCode = 1 - console.error(`Failed to drop ${runDatabase.name}:`, error) - } - } - + await cleanup() + process.off('SIGINT', handleSignal) + process.off('SIGTERM', handleSignal) console.info(`E2E ${failed ? 'failed' : 'completed'}; diagnostics: ${runDirectory}`) } } @@ -145,8 +218,7 @@ function resolveNode22(): string { function createPlaywrightEnvironment( stackEnvironment: Record, runId: string, - stripeFakeUrl: string, - storageStatePath: string + storageStateDirectory: string ): Record { const keys = [ 'PATH', @@ -169,12 +241,41 @@ function createPlaywrightEnvironment( return { ...env, E2E_PROFILE, + E2E_ORCHESTRATED: '1', E2E_RUN_ID: runId, E2E_BASE_URL: E2E_ORIGIN, - E2E_ADMIN_API_KEY: stackEnvironment.ADMIN_API_KEY, - E2E_STRIPE_FAKE_URL: stripeFakeUrl, - E2E_STRIPE_FAKE_KEY: stackEnvironment.STRIPE_SECRET_KEY, - E2E_STORAGE_STATE_PATH: storageStatePath, + E2E_STORAGE_STATE_DIR: storageStateDirectory, + } +} + +function assertAuthenticatedSmokeEffectsIfPresent( + stripeFake: StripeFakeServer, + provisioning: FoundationProvisioningResult +): void { + const created = stripeFake.requestLog.some( + ({ method, path }) => method === 'POST' && path === '/v1/customers' + ) + const unexpected = stripeFake.requestLog.filter((request) => request.unexpected) + if (unexpected.length > 0) { + throw new Error( + `Stripe fake received unsupported requests: ${unexpected + .map(({ method, path }) => `${method} ${path}`) + .join(', ')}` + ) + } + if (!created && provisioning.count === 0) { + console.info('Authenticated foundation smoke was filtered out; skipping its post-run probes') + return + } + if (!created) throw new Error('Billing-enabled signup did not create a fake Stripe customer') + if (provisioning.count === 0) { + throw new Error('Authenticated smoke did not create a foundation user') + } + if (!provisioning.allHaveStripeCustomers) { + throw new Error('A foundation user was not persisted with its fake Stripe customer') + } + if (!provisioning.allHaveStats) { + throw new Error('A foundation user was not initialized with user_stats') } } diff --git a/apps/sim/e2e/settings/smoke/authenticated.spec.ts b/apps/sim/e2e/settings/smoke/authenticated.spec.ts index a4b3f5fd8ed..1363dd5f836 100644 --- a/apps/sim/e2e/settings/smoke/authenticated.spec.ts +++ b/apps/sim/e2e/settings/smoke/authenticated.spec.ts @@ -1,28 +1,21 @@ +import { createHash } from 'node:crypto' +import path from 'node:path' import { expect, test } from '@playwright/test' -interface StripeRequestLog { - requests: Array<{ - method: string - path: string - unexpected: boolean - }> -} - test('billing-enabled signup, login, and settings use real Sim boundaries', async ({ browser, page, - request, }, testInfo) => { test.slow() const runId = process.env.E2E_RUN_ID ?? `${Date.now()}` - const email = `e2e-foundation-${runId}@example.com` + const testIdentity = createHash('sha256') + .update(`${runId}:${testInfo.project.name}:${testInfo.workerIndex}:${testInfo.repeatEachIndex}`) + .digest('hex') + .slice(0, 16) + const email = `e2e-foundation-${runId}-${testIdentity}@example.com` const password = 'E2eFoundation1!' - const fakeUrl = requiredEnv('E2E_STRIPE_FAKE_URL') - const fakeKey = requiredEnv('E2E_STRIPE_FAKE_KEY') - const adminKey = requiredEnv('E2E_ADMIN_API_KEY') - const storageStatePath = - process.env.E2E_STORAGE_STATE_PATH ?? testInfo.outputPath('foundation-auth.json') + const storageStatePath = path.join(requiredEnv('E2E_STORAGE_STATE_DIR'), `${testIdentity}.json`) await page.goto('/signup') await expect(page.getByRole('heading', { name: 'Create an account' })).toBeVisible() @@ -37,37 +30,6 @@ test('billing-enabled signup, login, and settings use real Sim boundaries', asyn const workspaceId = new URL(page.url()).pathname.split('/')[2] expect(workspaceId).toBeTruthy() - await expect - .poll(async () => { - const response = await request.get(`${fakeUrl}/__control/requests`, { - headers: { authorization: `Bearer ${fakeKey}` }, - }) - expect(response.ok()).toBe(true) - const log = (await response.json()) as StripeRequestLog - return { - created: log.requests.some( - ({ method, path }) => method === 'POST' && path === '/v1/customers' - ), - unexpected: log.requests.filter(({ unexpected }) => unexpected).length, - } - }) - .toEqual({ created: true, unexpected: 0 }) - - const sessionResponse = await page.request.get('/api/auth/get-session') - expect(sessionResponse.ok()).toBe(true) - const session = (await sessionResponse.json()) as { user?: { id?: string } } - const userId = session.user?.id - expect(userId).toBeTruthy() - - const billingResponse = await request.get(`/api/v1/admin/users/${userId}/billing`, { - headers: { 'x-admin-key': adminKey }, - }) - expect(billingResponse.ok()).toBe(true) - const billing = (await billingResponse.json()) as { - data?: { stripeCustomerId?: string | null } - } - expect(billing.data?.stripeCustomerId).toMatch(/^cus_e2e_/) - const settingsPath = `/workspace/${workspaceId}/settings/general` await page.goto(settingsPath) await assertGeneralSettings(page) diff --git a/apps/sim/e2e/support/deployment-profile.ts b/apps/sim/e2e/support/deployment-profile.ts index 0419c801eab..c2348269fda 100644 --- a/apps/sim/e2e/support/deployment-profile.ts +++ b/apps/sim/e2e/support/deployment-profile.ts @@ -30,7 +30,6 @@ const ALLOWED_SENSITIVE_KEYS = new Set([ 'API_ENCRYPTION_KEY', 'INTERNAL_API_SECRET', 'ADMIN_API_KEY', - 'E2E_ADMIN_API_KEY', 'EMAIL_PASSWORD_SIGNUP_ENABLED', 'NEXT_PUBLIC_EMAIL_PASSWORD_SIGNUP_ENABLED', 'STRIPE_SECRET_KEY', @@ -72,7 +71,6 @@ export function createHostedBillingProfile({ API_ENCRYPTION_KEY: '22'.repeat(32), INTERNAL_API_SECRET: 'e2e-internal-api-secret-at-least-32-characters', ADMIN_API_KEY: 'e2e-admin-api-key-at-least-32-characters-long', - E2E_ADMIN_API_KEY: 'e2e-admin-api-key-at-least-32-characters-long', BILLING_ENABLED: 'true', NEXT_PUBLIC_BILLING_ENABLED: 'true', EMAIL_VERIFICATION_ENABLED: 'false', @@ -87,7 +85,6 @@ export function createHostedBillingProfile({ STRIPE_WEBHOOK_SECRET: 'whsec_sim_e2e_foundation', STRIPE_FREE_PRICE_ID: 'price_e2e_free', STRIPE_API_BASE_URL: stripeApiBaseUrl, - E2E_STRIPE_FAKE_URL: stripeApiBaseUrl, SOCKET_SERVER_URL: 'http://127.0.0.1:3002', NEXT_PUBLIC_SOCKET_URL: E2E_SOCKET_ORIGIN, CI: ci ? 'true' : 'false', diff --git a/apps/sim/e2e/support/diagnostics.ts b/apps/sim/e2e/support/diagnostics.ts new file mode 100644 index 00000000000..003aa3e5bcc --- /dev/null +++ b/apps/sim/e2e/support/diagnostics.ts @@ -0,0 +1,30 @@ +import { existsSync, readFileSync } from 'node:fs' + +const FORBIDDEN_PROVIDER_LOG_PATTERNS = [ + /api\.stripe\.com/i, + /api\.agentmail\.to/i, + /telemetry\.simstudio\.ai/i, + /Failed to fetch Ollama models/i, + /Failed to initialize .*provider/i, + /Failed to initialize .*client/i, +] + +/** + * Scan only service startup logs. Later settings tests intentionally exercise + * URL-validation denials whose error copy must not be treated as provider boot. + */ +export function assertNoForbiddenProviderInitialization(logPaths: string[]): void { + const violations: string[] = [] + for (const logPath of logPaths) { + if (!existsSync(logPath)) continue + const contents = readFileSync(logPath, 'utf8') + for (const pattern of FORBIDDEN_PROVIDER_LOG_PATTERNS) { + if (pattern.test(contents)) violations.push(`${logPath}: ${pattern}`) + } + } + if (violations.length > 0) { + throw new Error( + `Shadowed or forbidden provider initialization was detected:\n${violations.join('\n')}` + ) + } +} diff --git a/apps/sim/e2e/support/probes.ts b/apps/sim/e2e/support/probes.ts new file mode 100644 index 00000000000..3e4241be574 --- /dev/null +++ b/apps/sim/e2e/support/probes.ts @@ -0,0 +1,47 @@ +import postgres from 'postgres' + +export async function assertAdminApiBoundary(origin: string, adminKey: string): Promise { + const endpoint = `${origin}/api/v1/admin/users?limit=1&offset=0` + const unauthorized = await fetch(endpoint) + if (unauthorized.status !== 401) { + throw new Error(`Admin API missing-key probe returned ${unauthorized.status}, expected 401`) + } + + const authorized = await fetch(endpoint, { + headers: { 'x-admin-key': adminKey }, + }) + if (!authorized.ok) { + throw new Error(`Configured Admin API probe returned ${authorized.status}`) + } +} + +export interface FoundationProvisioningResult { + count: number + allHaveStripeCustomers: boolean + allHaveStats: boolean +} + +export async function inspectFoundationUsers( + databaseUrl: string, + runId: string +): Promise { + const sql = postgres(databaseUrl, { max: 1, connect_timeout: 10 }) + try { + const pattern = `e2e-foundation-${runId}-%@example.com` + const rows = await sql>` + SELECT + u.stripe_customer_id AS "stripeCustomerId", + (s.user_id IS NOT NULL) AS "hasStats" + FROM "user" u + LEFT JOIN user_stats s ON s.user_id = u.id + WHERE u.email LIKE ${pattern} + ` + return { + count: rows.length, + allHaveStripeCustomers: rows.every((row) => row.stripeCustomerId?.startsWith('cus_e2e_')), + allHaveStats: rows.every((row) => row.hasStats), + } + } finally { + await sql.end() + } +} diff --git a/apps/sim/e2e/support/process.ts b/apps/sim/e2e/support/process.ts index 11ad5e5088a..6f175425dfc 100644 --- a/apps/sim/e2e/support/process.ts +++ b/apps/sim/e2e/support/process.ts @@ -1,5 +1,6 @@ import { type ChildProcess, spawn } from 'node:child_process' import { closeSync, mkdirSync, openSync } from 'node:fs' +import { createServer } from 'node:net' import path from 'node:path' export interface CommandOptions { @@ -18,6 +19,8 @@ export interface ManagedProcess { stop(): Promise } +const activeProcesses = new Set() + export function spawnManagedProcess(options: CommandOptions): ManagedProcess { mkdirSync(options.logsDirectory, { recursive: true }) const logPath = path.join(options.logsDirectory, `${options.name}.log`) @@ -27,14 +30,18 @@ export function spawnManagedProcess(options: CommandOptions): ManagedProcess { env: options.env as NodeJS.ProcessEnv, stdio: ['ignore', logFd, logFd], }) - child.once('exit', () => closeSync(logFd)) - - return { + const managed: ManagedProcess = { name: options.name, child, logPath, stop: () => stopProcess(child), } + activeProcesses.add(managed) + child.once('exit', () => { + activeProcesses.delete(managed) + closeSync(logFd) + }) + return managed } export async function runCommand(options: CommandOptions): Promise { @@ -49,6 +56,60 @@ export async function stopProcesses(processes: ManagedProcess[]): Promise await Promise.allSettled([...processes].reverse().map((process) => process.stop())) } +export async function stopAllManagedProcesses(): Promise { + await stopProcesses([...activeProcesses]) +} + +export function signalAllManagedProcesses(): void { + for (const managed of activeProcesses) { + if (managed.child.exitCode === null && managed.child.signalCode === null) { + managed.child.kill('SIGTERM') + } + } +} + +export async function waitForManagedProcessReady( + process: ManagedProcess, + readiness: Promise +): Promise { + if (process.child.exitCode !== null || process.child.signalCode !== null) { + throw new Error(`${process.name} exited before readiness. See ${process.logPath}`) + } + + let onExit: ((code: number | null, signal: NodeJS.Signals | null) => void) | undefined + const exited = new Promise((_, reject) => { + onExit = (code, signal) => { + reject( + new Error( + `${process.name} exited before readiness with ${code ?? signal ?? 'unknown status'}. See ${process.logPath}` + ) + ) + } + process.child.once('exit', onExit) + }) + + try { + await Promise.race([readiness, exited]) + } finally { + if (onExit) process.child.off('exit', onExit) + } +} + +export async function assertPortAvailable(port: number, host = '127.0.0.1'): Promise { + await new Promise((resolve, reject) => { + const server = createServer() + server.once('error', (error) => { + reject(new Error(`E2E requires ${host}:${port} to be free: ${String(error)}`)) + }) + server.listen({ host, port, exclusive: true }, () => { + server.close((error) => { + if (error) reject(error) + else resolve() + }) + }) + }) +} + function waitForExit(child: ChildProcess): Promise { return new Promise((resolve, reject) => { child.once('error', reject) diff --git a/apps/sim/e2e/support/stack.ts b/apps/sim/e2e/support/stack.ts index 7daf9f93fea..1b7b890c574 100644 --- a/apps/sim/e2e/support/stack.ts +++ b/apps/sim/e2e/support/stack.ts @@ -1,6 +1,11 @@ import path from 'node:path' import { DB_PACKAGE_DIR, PLAYWRIGHT_CLI, REALTIME_APP_DIR, REPO_ROOT, SIM_APP_DIR } from './paths' -import { type ManagedProcess, runCommand, spawnManagedProcess } from './process' +import { + type ManagedProcess, + runCommand, + spawnManagedProcess, + waitForManagedProcessReady, +} from './process' import { waitForHttpReady } from './readiness' export interface StackCommandOptions { @@ -55,15 +60,18 @@ export async function startRealtime(options: StackCommandOptions): Promise { - if (!response.ok) return false - const body = (await response.json()) as { status?: string } - return body.status === 'ok' - }, - }) + await waitForManagedProcessReady( + realtime, + waitForHttpReady({ + name: 'Realtime', + url: 'http://127.0.0.1:3002/health', + validate: async (response) => { + if (!response.ok) return false + const body = (await response.json()) as { status?: string; runId?: string } + return body.status === 'ok' && body.runId === options.env.E2E_RUN_ID + }, + }) + ) return realtime } catch (error) { await realtime.stop() @@ -81,7 +89,7 @@ export async function startApp(options: StackCommandOptions): Promise { + if (!response.ok) return false + const body = (await response.json()) as { status?: string; runId?: string } + return body.status === 'ok' && body.runId === options.env.E2E_RUN_ID + }, + }) + ) return app } catch (error) { await app.stop() diff --git a/apps/sim/package.json b/apps/sim/package.json index bb1c1a7b8fc..288bf38b3c7 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -24,8 +24,7 @@ "test": "vitest run", "test:watch": "vitest", "test:coverage": "vitest run --coverage", - "test:e2e": "bun --no-env-file e2e/scripts/run.ts", - "test:e2e:playwright": "node ../../node_modules/@playwright/test/cli.js test --config=playwright.config.ts", + "test:e2e": "exec bun --no-env-file e2e/scripts/run.ts", "test:e2e:install-browsers": "node ../../node_modules/@playwright/test/cli.js install chromium", "email:dev": "email dev --dir components/emails", "type-check": "NODE_OPTIONS='--max-old-space-size=8192' tsc --noEmit", diff --git a/apps/sim/playwright.config.ts b/apps/sim/playwright.config.ts index 0160ca29411..37d1a66fe74 100644 --- a/apps/sim/playwright.config.ts +++ b/apps/sim/playwright.config.ts @@ -1,5 +1,11 @@ import { defineConfig, devices } from '@playwright/test' +if (process.env.E2E_ORCHESTRATED !== '1') { + throw new Error( + 'Playwright tests must run through `bun run test:e2e` so database, environment, and teardown guards are active' + ) +} + const isCI = process.env.CI === 'true' const baseURL = process.env.E2E_BASE_URL ?? 'http://e2e.sim.ai:3000' @@ -9,6 +15,7 @@ export default defineConfig({ fullyParallel: true, forbidOnly: isCI, retries: 0, + workers: 1, timeout: 60_000, expect: { timeout: 10_000, @@ -36,7 +43,7 @@ export default defineConfig({ '**/settings/smoke/unauthenticated.spec.ts', '**/settings/navigation/**/*.spec.ts', ], - workers: isCI ? 2 : 1, + workers: 1, }, { name: 'hosted-billing-chromium-workflows', From d3d084532d09821e67ed05608353bf544a5c2b65 Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Mon, 20 Jul 2026 17:18:14 -0700 Subject: [PATCH 03/10] test(e2e): tighten isolation and shutdown guards Make destructive database operations and service bindings strictly local while ensuring interrupted runs fully stop managed children before cleanup. --- .github/workflows/test-build.yml | 7 --- .gitignore | 1 + apps/realtime/src/env.ts | 1 + apps/realtime/src/index.ts | 7 +-- apps/sim/e2e/foundation/safety.spec.ts | 10 +++- apps/sim/e2e/scripts/run.ts | 21 ++++++-- apps/sim/e2e/support/database.ts | 8 ++- apps/sim/e2e/support/diagnostics.ts | 32 +++++++++-- apps/sim/e2e/support/hosts.ts | 12 ++++- apps/sim/e2e/support/process.ts | 74 +++++++++++++++++++++++--- apps/sim/e2e/support/stack.ts | 1 + 11 files changed, 144 insertions(+), 30 deletions(-) diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 3c23831b572..5c46cf66b2e 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -294,12 +294,6 @@ jobs: key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} path: ./node_modules - - name: Mount E2E Turbo cache (Sticky Disk) - uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 - with: - key: ${{ github.repository }}-turbo-cache-e2e-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} - path: ./.turbo - - name: Mount Playwright browsers (Sticky Disk) uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1 with: @@ -329,7 +323,6 @@ jobs: env: CI: 'true' E2E_PG_ADMIN_URL: 'postgresql://postgres:postgres@127.0.0.1:5432/postgres' - TURBO_CACHE_DIR: .turbo run: bun run test:e2e - name: Upload E2E diagnostics diff --git a/.gitignore b/.gitignore index 52fd13db03f..dc42d1c8e3f 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ package-lock.json **/dist/ **/standalone/ sim-standalone.tar.gz +/.artifacts/ # redis dump.rdb diff --git a/apps/realtime/src/env.ts b/apps/realtime/src/env.ts index 5afdf3a20f3..2f75c4a595f 100644 --- a/apps/realtime/src/env.ts +++ b/apps/realtime/src/env.ts @@ -14,6 +14,7 @@ const EnvSchema = z.object({ INTERNAL_API_SECRET: z.string().min(32), NEXT_PUBLIC_APP_URL: z.string().url(), ALLOWED_ORIGINS: z.string().optional(), + REALTIME_HOST: z.string().min(1).default('0.0.0.0'), PORT: z.coerce.number().int().positive().default(3002), SIM_DB_ROLE: z.enum(['web', 'trigger', 'realtime']).optional(), DISABLE_AUTH: z diff --git a/apps/realtime/src/index.ts b/apps/realtime/src/index.ts index e43184c1f02..0c84867156b 100644 --- a/apps/realtime/src/index.ts +++ b/apps/realtime/src/index.ts @@ -31,6 +31,7 @@ async function createRoomManager(io: SocketIOServer): Promise { async function main() { const httpServer = createServer() const PORT = env.PORT + const HOST = env.REALTIME_HOST logger.info('Starting Socket.IO server...', { port: PORT, @@ -96,9 +97,9 @@ async function main() { await assertSchemaCompatibility() - httpServer.listen(PORT, '0.0.0.0', () => { - logger.info(`Socket.IO server running on port ${PORT}`) - logger.info(`Health check available at: http://localhost:${PORT}/health`) + httpServer.listen(PORT, HOST, () => { + logger.info(`Socket.IO server running on ${HOST}:${PORT}`) + logger.info(`Health check available at: http://${HOST}:${PORT}/health`) }) const shutdown = async () => { diff --git a/apps/sim/e2e/foundation/safety.spec.ts b/apps/sim/e2e/foundation/safety.spec.ts index b97b7cf3aa3..129eff3365d 100644 --- a/apps/sim/e2e/foundation/safety.spec.ts +++ b/apps/sim/e2e/foundation/safety.spec.ts @@ -11,7 +11,7 @@ import { buildRunDatabaseUrl, } from '../support/database' import { buildChildEnvironment, discoverEnvFileKeys } from '../support/env' -import { isLoopbackAddress } from '../support/hosts' +import { areValidE2eHostAddresses, isLoopbackAddress } from '../support/hosts' import { assertPortAvailable } from '../support/process' test.describe('foundation safety guards', () => { @@ -63,6 +63,12 @@ test.describe('foundation safety guards', () => { expect(() => assertLoopbackPostgresUrl('postgresql://postgres:postgres@example.com/postgres') ).toThrow() + expect(() => + assertLoopbackPostgresUrl('postgresql://postgres:postgres@localhost/postgres') + ).toThrow() + expect(() => + assertLoopbackPostgresUrl('postgresql://postgres:postgres@127.0.0.1/postgres?sslmode=disable') + ).toThrow() expect( buildRunDatabaseUrl( 'postgresql://postgres:postgres@127.0.0.1:5432/postgres', @@ -76,6 +82,8 @@ test.describe('foundation safety guards', () => { expect(isLoopbackAddress('127.10.20.30')).toBe(true) expect(isLoopbackAddress('::1')).toBe(true) expect(isLoopbackAddress('8.8.8.8')).toBe(false) + expect(areValidE2eHostAddresses(['127.0.0.1', '::1'])).toBe(true) + expect(areValidE2eHostAddresses(['::1'])).toBe(false) }) test('sharding is limited to the navigation project', () => { diff --git a/apps/sim/e2e/scripts/run.ts b/apps/sim/e2e/scripts/run.ts index 7769b7ddb56..69c66686baa 100644 --- a/apps/sim/e2e/scripts/run.ts +++ b/apps/sim/e2e/scripts/run.ts @@ -11,7 +11,10 @@ import { type RunDatabase, } from '../support/database' import { createHostedBillingProfile, E2E_ORIGIN, E2E_PROFILE } from '../support/deployment-profile' -import { assertNoForbiddenProviderInitialization } from '../support/diagnostics' +import { + assertNoForbiddenProviderInitialization, + assertNoForbiddenProviderTraffic, +} from '../support/diagnostics' import { formatRedactedEnvironmentSummary } from '../support/env' import { assertE2eHostResolvesToLoopback } from '../support/hosts' import { getRunDirectory, SIM_APP_DIR } from '../support/paths' @@ -23,8 +26,8 @@ import { import { assertPortAvailable, type ManagedProcess, - signalAllManagedProcesses, stopAllManagedProcesses, + stopAllManagedProcessesSync, } from '../support/process' import { buildApp, runMigrations, runPlaywright, startApp, startRealtime } from '../support/stack' import { parseRunOptions } from './options' @@ -86,7 +89,10 @@ async function main(): Promise { const exitCode = signal === 'SIGINT' ? 130 : 143 process.exitCode = exitCode console.error(`Received ${signal}; cleaning up the E2E run`) - signalAllManagedProcesses() + const survivingPids = stopAllManagedProcessesSync() + if (survivingPids.length > 0) { + console.error(`E2E child processes survived SIGKILL: ${survivingPids.join(', ')}`) + } if (stripeFake) { writeFileSync( path.join(logsDirectory, 'stripe-requests.json'), @@ -179,6 +185,15 @@ async function main(): Promise { console.error(error) process.exitCode = 1 } finally { + try { + assertNoForbiddenProviderTraffic( + [app?.logPath, realtime?.logPath].filter((value): value is string => Boolean(value)) + ) + } catch (error) { + failed = true + process.exitCode = 1 + console.error(error) + } await cleanup() process.off('SIGINT', handleSignal) process.off('SIGTERM', handleSignal) diff --git a/apps/sim/e2e/support/database.ts b/apps/sim/e2e/support/database.ts index 03e471b3ad0..6e2093dc3f0 100644 --- a/apps/sim/e2e/support/database.ts +++ b/apps/sim/e2e/support/database.ts @@ -28,7 +28,13 @@ export function assertSafeDatabaseName(name: string): void { export function assertLoopbackPostgresUrl(rawUrl: string): URL { const url = new URL(rawUrl) const hostname = url.hostname.replace(/^\[|\]$/g, '') - if (!(hostname === 'localhost' || hostname === '::1' || isLoopbackAddress(hostname))) { + if (!['postgres:', 'postgresql:'].includes(url.protocol)) { + throw new Error(`E2E PostgreSQL admin URL requires postgres protocol, received ${url.protocol}`) + } + if (url.search || url.hash) { + throw new Error('E2E PostgreSQL admin URL must not contain query parameters or a fragment') + } + if (!isLoopbackAddress(hostname)) { throw new Error(`E2E PostgreSQL admin URL must be loopback, received ${url.hostname}`) } return url diff --git a/apps/sim/e2e/support/diagnostics.ts b/apps/sim/e2e/support/diagnostics.ts index 003aa3e5bcc..370b3212153 100644 --- a/apps/sim/e2e/support/diagnostics.ts +++ b/apps/sim/e2e/support/diagnostics.ts @@ -1,10 +1,14 @@ import { existsSync, readFileSync } from 'node:fs' -const FORBIDDEN_PROVIDER_LOG_PATTERNS = [ +const FORBIDDEN_PROVIDER_TRAFFIC_PATTERNS = [ /api\.stripe\.com/i, /api\.agentmail\.to/i, /telemetry\.simstudio\.ai/i, /Failed to fetch Ollama models/i, +] + +const FORBIDDEN_PROVIDER_STARTUP_PATTERNS = [ + ...FORBIDDEN_PROVIDER_TRAFFIC_PATTERNS, /Failed to initialize .*provider/i, /Failed to initialize .*client/i, ] @@ -14,17 +18,35 @@ const FORBIDDEN_PROVIDER_LOG_PATTERNS = [ * URL-validation denials whose error copy must not be treated as provider boot. */ export function assertNoForbiddenProviderInitialization(logPaths: string[]): void { + assertLogsDoNotMatch( + logPaths, + FORBIDDEN_PROVIDER_STARTUP_PATTERNS, + 'Shadowed or forbidden provider initialization was detected' + ) +} + +/** + * Re-scan after browser activity using only concrete outbound signatures. + * Intentional URL-validation errors in later workflow suites remain allowed. + */ +export function assertNoForbiddenProviderTraffic(logPaths: string[]): void { + assertLogsDoNotMatch( + logPaths, + FORBIDDEN_PROVIDER_TRAFFIC_PATTERNS, + 'Forbidden provider traffic was detected' + ) +} + +function assertLogsDoNotMatch(logPaths: string[], patterns: RegExp[], message: string): void { const violations: string[] = [] for (const logPath of logPaths) { if (!existsSync(logPath)) continue const contents = readFileSync(logPath, 'utf8') - for (const pattern of FORBIDDEN_PROVIDER_LOG_PATTERNS) { + for (const pattern of patterns) { if (pattern.test(contents)) violations.push(`${logPath}: ${pattern}`) } } if (violations.length > 0) { - throw new Error( - `Shadowed or forbidden provider initialization was detected:\n${violations.join('\n')}` - ) + throw new Error(`${message}:\n${violations.join('\n')}`) } } diff --git a/apps/sim/e2e/support/hosts.ts b/apps/sim/e2e/support/hosts.ts index 0b51910c931..80a927f87b9 100644 --- a/apps/sim/e2e/support/hosts.ts +++ b/apps/sim/e2e/support/hosts.ts @@ -11,6 +11,14 @@ export function isLoopbackAddress(address: string): boolean { return false } +export function areValidE2eHostAddresses(addresses: string[]): boolean { + return ( + addresses.length > 0 && + addresses.every(isLoopbackAddress) && + addresses.some((address) => isIP(address) === 4 && isLoopbackAddress(address)) + ) +} + export async function assertE2eHostResolvesToLoopback(hostname = E2E_HOST): Promise { let records: Array<{ address: string }> try { @@ -20,7 +28,7 @@ export async function assertE2eHostResolvesToLoopback(hostname = E2E_HOST): Prom } const addresses = [...new Set(records.map(({ address }) => address))] - if (addresses.length === 0 || addresses.some((address) => !isLoopbackAddress(address))) { + if (!areValidE2eHostAddresses(addresses)) { throw new Error(getHostMappingError(hostname, addresses)) } return addresses @@ -29,7 +37,7 @@ export async function assertE2eHostResolvesToLoopback(hostname = E2E_HOST): Prom function getHostMappingError(hostname: string, addresses: string[]): string { const observed = addresses.length > 0 ? addresses.join(', ') : 'no addresses' return [ - `${hostname} must resolve only to loopback; observed ${observed}.`, + `${hostname} must resolve only to loopback and include IPv4 127.0.0.1; observed ${observed}.`, `Add it once with: echo "127.0.0.1 ${hostname}" | sudo tee -a /etc/hosts`, ].join(' ') } diff --git a/apps/sim/e2e/support/process.ts b/apps/sim/e2e/support/process.ts index 6f175425dfc..ac104f8f6dc 100644 --- a/apps/sim/e2e/support/process.ts +++ b/apps/sim/e2e/support/process.ts @@ -1,4 +1,4 @@ -import { type ChildProcess, spawn } from 'node:child_process' +import { type ChildProcess, spawn, spawnSync } from 'node:child_process' import { closeSync, mkdirSync, openSync } from 'node:fs' import { createServer } from 'node:net' import path from 'node:path' @@ -60,12 +60,33 @@ export async function stopAllManagedProcesses(): Promise { await stopProcesses([...activeProcesses]) } -export function signalAllManagedProcesses(): void { - for (const managed of activeProcesses) { - if (managed.child.exitCode === null && managed.child.signalCode === null) { - managed.child.kill('SIGTERM') - } +export function stopAllManagedProcessesSync( + termTimeoutMs = 5_000, + killTimeoutMs = 2_000 +): number[] { + const managedProcesses = [...activeProcesses] + for (const managed of managedProcesses) sendSignal(managed.child, 'SIGTERM') + + const deadline = Date.now() + termTimeoutMs + while (Date.now() < deadline && managedProcesses.some(({ child }) => isProcessRunning(child))) { + sleepSync(50) + } + + for (const { child } of managedProcesses) { + if (isProcessRunning(child)) sendSignal(child, 'SIGKILL') + } + + const killDeadline = Date.now() + killTimeoutMs + while ( + Date.now() < killDeadline && + managedProcesses.some(({ child }) => isProcessRunning(child)) + ) { + sleepSync(25) } + + return managedProcesses.flatMap(({ child }) => + child.pid && isProcessRunning(child) ? [child.pid] : [] + ) } export async function waitForManagedProcessReady( @@ -119,7 +140,7 @@ function waitForExit(child: ChildProcess): Promise { async function stopProcess(child: ChildProcess): Promise { if (child.exitCode !== null || child.signalCode !== null) return - child.kill('SIGTERM') + sendSignal(child, 'SIGTERM') const stopped = await Promise.race([ waitForExit(child).then(() => true), @@ -127,6 +148,43 @@ async function stopProcess(child: ChildProcess): Promise { ]) if (stopped) return - child.kill('SIGKILL') + sendSignal(child, 'SIGKILL') await waitForExit(child) } + +function sendSignal(child: ChildProcess, signal: NodeJS.Signals): void { + if (!child.pid) return + try { + child.kill(signal) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error + } +} + +/** + * Managed commands currently run their server/build work in the direct child. + * If a future command introduces durable grandchildren, process-tree cleanup + * should be added alongside that command rather than implied here. + */ +function isProcessRunning(child: ChildProcess): boolean { + if (!child.pid) return false + try { + process.kill(child.pid, 0) + if (process.platform !== 'win32') { + const status = spawnSync('ps', ['-o', 'stat=', '-p', String(child.pid)], { + encoding: 'utf8', + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + }) + if (status.status !== 0 || status.stdout.trim().startsWith('Z')) return false + } + return true + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ESRCH') return false + throw error + } +} + +function sleepSync(milliseconds: number): void { + const signal = new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT)) + Atomics.wait(signal, 0, 0, milliseconds) +} diff --git a/apps/sim/e2e/support/stack.ts b/apps/sim/e2e/support/stack.ts index 1b7b890c574..3e6c71a29cd 100644 --- a/apps/sim/e2e/support/stack.ts +++ b/apps/sim/e2e/support/stack.ts @@ -55,6 +55,7 @@ export async function startRealtime(options: StackCommandOptions): Promise Date: Mon, 20 Jul 2026 18:27:25 -0700 Subject: [PATCH 04/10] test(e2e): finalize harness safety boundaries Close the remaining lifecycle, credential-isolation, and CLI escape hatches so future persona and settings suites can build on a fail-closed foundation. --- .github/workflows/test-build.yml | 6 +- apps/sim/e2e/README.md | 11 + apps/sim/e2e/fakes/stripe/server.ts | 34 +-- apps/sim/e2e/foundation/safety.spec.ts | 30 ++- apps/sim/e2e/foundation/stripe-fake.spec.ts | 17 ++ apps/sim/e2e/scripts/cleanup-database.ts | 15 +- apps/sim/e2e/scripts/options.ts | 43 ++-- apps/sim/e2e/scripts/run.ts | 161 ++++++++++----- .../e2e/settings/smoke/authenticated.spec.ts | 5 + apps/sim/e2e/support/database.ts | 8 +- apps/sim/e2e/support/deployment-profile.ts | 18 +- apps/sim/e2e/support/env.ts | 2 - apps/sim/e2e/support/probes.ts | 4 +- apps/sim/e2e/support/process.ts | 194 ++++++++++-------- apps/sim/e2e/support/readiness.ts | 28 ++- apps/sim/e2e/support/stack.ts | 8 +- apps/sim/lib/billing/stripe-client-config.ts | 2 +- 17 files changed, 406 insertions(+), 180 deletions(-) diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 5c46cf66b2e..4d3fa2dbde4 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -304,9 +304,9 @@ jobs: uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: path: ./apps/sim/.next/cache - key: ${{ runner.os }}-nextjs-e2e-hosted-billing-${{ hashFiles('bun.lock') }} + key: ${{ runner.os }}-nextjs-e2e-hosted-billing-${{ hashFiles('bun.lock') }}-${{ github.sha }} restore-keys: | - ${{ runner.os }}-nextjs-e2e-hosted-billing- + ${{ runner.os }}-nextjs-e2e-hosted-billing-${{ hashFiles('bun.lock') }}- - name: Install dependencies run: bun install --frozen-lockfile @@ -319,6 +319,7 @@ jobs: run: bun run test:e2e:install-browsers -- --with-deps - name: Run settings E2E foundation + timeout-minutes: 40 working-directory: apps/sim env: CI: 'true' @@ -336,4 +337,5 @@ jobs: apps/sim/e2e/.runs/ !apps/sim/e2e/.runs/**/auth/** if-no-files-found: ignore + include-hidden-files: true retention-days: 7 \ No newline at end of file diff --git a/apps/sim/e2e/README.md b/apps/sim/e2e/README.md index 1eb02cc9285..f4e33d0f4af 100644 --- a/apps/sim/e2e/README.md +++ b/apps/sim/e2e/README.md @@ -6,6 +6,10 @@ per-run pgvector database. ## One-time setup +0. Install Node 22 and Bun. Playwright workers require Node 22; set + `E2E_NODE_BINARY` to an alternate Node 22 executable when `node` on `PATH` + points elsewhere. + 1. Map the hosted E2E origin to loopback: ```bash @@ -25,6 +29,9 @@ per-run pgvector database. pgvector/pgvector:pg17 ``` + Cleanup uses `DROP DATABASE ... WITH (FORCE)`, which requires PostgreSQL 13 + or newer and is supported by the pinned pgvector/PostgreSQL 17 image. + 3. Install Chromium from `apps/sim`: ```bash @@ -80,3 +87,7 @@ node ../../node_modules/@playwright/test/cli.js show-trace test-results//t The runner starts every child process from a fresh environment. It allowlists only deterministic E2E values and shadows keys found in local `.env*` files, so developer credentials are not used as test state or written to reports. + +Provider log scans are diagnostic tripwires, not proof of zero egress. The +primary boundaries are the default-deny child environment, provider disabling, +loopback-only service bindings, and guarded Stripe transport. diff --git a/apps/sim/e2e/fakes/stripe/server.ts b/apps/sim/e2e/fakes/stripe/server.ts index 3aec1ce800c..8f89c48102e 100644 --- a/apps/sim/e2e/fakes/stripe/server.ts +++ b/apps/sim/e2e/fakes/stripe/server.ts @@ -20,7 +20,7 @@ export interface StripeFakeRequestRecord { export interface StripeFakeServerOptions { apiKey: string - hostname?: '127.0.0.1' | '::1' + hostname?: '127.0.0.1' maxBodyBytes?: number port?: number } @@ -169,19 +169,17 @@ function unescapeSearchValue(value: string): string { return value.replace(/\\(["\\])/g, '$1') } -function customerMatchesSearch(customer: FakeCustomer, query: string): boolean { - const emailMatch = /(?:^|\s)email:"((?:\\.|[^"])*)"/.exec(query) - if (emailMatch && customer.email !== unescapeSearchValue(emailMatch[1])) return false - - const metadataPattern = /(-?)metadata\["([^"]+)"\]:"((?:\\.|[^"])*)"/g - for (const match of query.matchAll(metadataPattern)) { - const isNegative = match[1] === '-' - const actual = customer.metadata[match[2]] - const expected = unescapeSearchValue(match[3]) - if (isNegative ? actual === expected : actual !== expected) return false +function parseSupportedCustomerSearchEmail(query: string): string { + const match = /^email:"((?:\\.|[^"])*)" AND -metadata\["customerType"\]:"organization"$/.exec( + query + ) + if (!match) { + throw new RequestBodyError( + `Stripe fake does not implement customer search query: ${query}`, + 501 + ) } - - return true + return unescapeSearchValue(match[1]) } function parseLimit(parameters: URLSearchParams): number { @@ -362,9 +360,13 @@ export function createStripeFakeServer(options: StripeFakeServerOptions): Stripe method === 'GET' ? url.searchParams : (formBody ?? parseFormBody(request, '')) const query = parameters.get('query') if (!query) throw new RequestBodyError('Stripe fake customer search requires query', 400) + const email = parseSupportedCustomerSearchEmail(query) const data = [...customers.values()] - .filter((customer) => customerMatchesSearch(customer, query)) + .filter( + (customer) => + customer.email === email && customer.metadata.customerType !== 'organization' + ) .slice(0, parseLimit(parameters)) sendJson( response, @@ -430,6 +432,10 @@ export function createStripeFakeServer(options: StripeFakeServerOptions): Stripe error instanceof RequestBodyError ? error : new RequestBodyError('Stripe fake rejected malformed request parameters', 400) + if (bodyError.status === 501) { + const recorded = records.at(-1) + if (recorded) recorded.unexpected = true + } sendStripeError( response, bodyError.status, diff --git a/apps/sim/e2e/foundation/safety.spec.ts b/apps/sim/e2e/foundation/safety.spec.ts index 129eff3365d..529ac2d911f 100644 --- a/apps/sim/e2e/foundation/safety.spec.ts +++ b/apps/sim/e2e/foundation/safety.spec.ts @@ -12,7 +12,7 @@ import { } from '../support/database' import { buildChildEnvironment, discoverEnvFileKeys } from '../support/env' import { areValidE2eHostAddresses, isLoopbackAddress } from '../support/hosts' -import { assertPortAvailable } from '../support/process' +import { assertPortAvailable, spawnManagedProcess } from '../support/process' test.describe('foundation safety guards', () => { test('discovers env keys without leaking values and shadows unknown keys', () => { @@ -95,6 +95,15 @@ test.describe('foundation safety guards', () => { ).toThrow(/coupled workflows must remain unsharded/) }) + test('Playwright CLI arguments cannot override orchestration invariants', () => { + expect(() => parseRunOptions(['--workers=8'])).toThrow(/cannot override/) + expect(() => parseRunOptions(['--config=other.config.ts'])).toThrow(/cannot override/) + expect(() => parseRunOptions(['--project', 'hosted-billing-chromium-navigation'])).toThrow( + /canonical/ + ) + expect(() => parseRunOptions(['--pass-with-no-tests'])).toThrow(/cannot override/) + }) + test('port preflight rejects an existing listener', async () => { const server = createServer() await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) @@ -108,4 +117,23 @@ test.describe('foundation safety guards', () => { ) } }) + + test('spawn failures finalize without hanging cleanup', async () => { + const logsDirectory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-spawn-')) + try { + const managed = spawnManagedProcess({ + name: 'missing-command', + command: path.join(logsDirectory, 'does-not-exist'), + args: [], + cwd: logsDirectory, + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + logsDirectory, + }) + const completion = await managed.completion + expect(completion.error).toBeTruthy() + await expect(managed.stop()).resolves.toBeUndefined() + } finally { + rmSync(logsDirectory, { recursive: true, force: true }) + } + }) }) diff --git a/apps/sim/e2e/foundation/stripe-fake.spec.ts b/apps/sim/e2e/foundation/stripe-fake.spec.ts index 12187b57535..8eec71d6870 100644 --- a/apps/sim/e2e/foundation/stripe-fake.spec.ts +++ b/apps/sim/e2e/foundation/stripe-fake.spec.ts @@ -27,6 +27,23 @@ test('Stripe fake records allowlisted calls and rejects unknown routes', async ( id: expect.stringMatching(/^cus_e2e_/), }) + const search = await fetch( + `${baseUrl}/v1/customers/search?${new URLSearchParams({ + query: 'email:"foundation@example.com" AND -metadata["customerType"]:"organization"', + limit: '1', + })}`, + { headers: { authorization: `Bearer ${apiKey}` } } + ) + expect(search.status).toBe(200) + + const unsupportedSearch = await fetch( + `${baseUrl}/v1/customers/search?${new URLSearchParams({ + query: 'name:"Foundation"', + })}`, + { headers: { authorization: `Bearer ${apiKey}` } } + ) + expect(unsupportedSearch.status).toBe(501) + const unknown = await fetch(`${baseUrl}/v1/invoices`, { headers: { authorization: `Bearer ${apiKey}` }, }) diff --git a/apps/sim/e2e/scripts/cleanup-database.ts b/apps/sim/e2e/scripts/cleanup-database.ts index 353138250eb..72a05c2a281 100644 --- a/apps/sim/e2e/scripts/cleanup-database.ts +++ b/apps/sim/e2e/scripts/cleanup-database.ts @@ -9,7 +9,20 @@ if (!adminUrl || !databaseName) { } try { - await dropRunDatabase(adminUrl, databaseName) + // A signal may arrive while CREATE DATABASE is still completing. Repeating + // the forced drop catches that narrow race without weakening name/host guards. + const deadline = Date.now() + 5_000 + let lastError: unknown + do { + try { + await dropRunDatabase(adminUrl, databaseName) + lastError = undefined + } catch (error) { + lastError = error + } + await new Promise((resolve) => setTimeout(resolve, 100)) + } while (Date.now() < deadline) + if (lastError) throw lastError } catch (error) { console.error(error) process.exit(1) diff --git a/apps/sim/e2e/scripts/options.ts b/apps/sim/e2e/scripts/options.ts index 785ac3cd81f..fb8bdde3e0f 100644 --- a/apps/sim/e2e/scripts/options.ts +++ b/apps/sim/e2e/scripts/options.ts @@ -1,5 +1,15 @@ const NAVIGATION_PROJECT = 'hosted-billing-chromium-navigation' const WORKFLOWS_PROJECT = 'hosted-billing-chromium-workflows' +const FORBIDDEN_OPTIONS = [ + '--config', + '-c', + '--workers', + '-j', + '--retries', + '--fully-parallel', + '--pass-with-no-tests', + '--list', +] as const export interface E2eRunOptions { playwrightArgs: string[] @@ -9,11 +19,26 @@ export function parseRunOptions(argv: string[]): E2eRunOptions { const normalizedArgs = argv[0] === '--' ? argv.slice(1) : [...argv] if (normalizedArgs.includes('--skip-build')) { throw new Error( - '--skip-build is not supported because E2E builds contain run-specific hosted configuration' + '--skip-build remains disabled until the planned profile/source-keyed build reuse experiment proves it safe' ) } + for (const option of FORBIDDEN_OPTIONS) { + if (hasOption(normalizedArgs, option)) { + throw new Error(`${option} cannot override E2E orchestration invariants`) + } + } + if (normalizedArgs.includes('--project')) { + throw new Error('Use canonical --project= syntax') + } + if (normalizedArgs.includes('--shard')) { + throw new Error('Use canonical --shard= syntax') + } const playwrightArgs = [...normalizedArgs] - const projects = getOptionValues(playwrightArgs, '--project') + const projects = getEqualsOptionValues(playwrightArgs, '--project') + const unknownProject = projects.find( + (project) => project !== NAVIGATION_PROJECT && project !== WORKFLOWS_PROJECT + ) + if (unknownProject) throw new Error(`Unknown E2E Playwright project: ${unknownProject}`) const hasShard = hasOption(playwrightArgs, '--shard') if ( @@ -34,16 +59,6 @@ function hasOption(args: string[], name: string): boolean { return args.some((arg) => arg === name || arg.startsWith(`${name}=`)) } -function getOptionValues(args: string[], name: string): string[] { - const values: string[] = [] - for (let index = 0; index < args.length; index += 1) { - const arg = args[index] - if (arg.startsWith(`${name}=`)) { - values.push(arg.slice(name.length + 1)) - } else if (arg === name && args[index + 1]) { - values.push(args[index + 1]) - index += 1 - } - } - return values +function getEqualsOptionValues(args: string[], name: string): string[] { + return args.filter((arg) => arg.startsWith(`${name}=`)).map((arg) => arg.slice(name.length + 1)) } diff --git a/apps/sim/e2e/scripts/run.ts b/apps/sim/e2e/scripts/run.ts index 69c66686baa..6ca5f5fca16 100644 --- a/apps/sim/e2e/scripts/run.ts +++ b/apps/sim/e2e/scripts/run.ts @@ -1,6 +1,6 @@ import { spawnSync } from 'node:child_process' import { randomUUID } from 'node:crypto' -import { mkdirSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from 'node:fs' import path from 'node:path' import { type StripeFakeServer, startStripeFakeServer } from '../fakes/stripe/server' import { @@ -15,7 +15,7 @@ import { assertNoForbiddenProviderInitialization, assertNoForbiddenProviderTraffic, } from '../support/diagnostics' -import { formatRedactedEnvironmentSummary } from '../support/env' +import { E2E_OS_PASSTHROUGH_KEYS, formatRedactedEnvironmentSummary } from '../support/env' import { assertE2eHostResolvesToLoopback } from '../support/hosts' import { getRunDirectory, SIM_APP_DIR } from '../support/paths' import { @@ -41,8 +41,12 @@ async function main(): Promise { const runDirectory = getRunDirectory(runId) const logsDirectory = path.join(runDirectory, 'logs') const storageStateDirectory = path.join(runDirectory, 'auth') + const markerDirectory = path.join(runDirectory, 'markers') + const homeDirectory = path.join(runDirectory, 'home') mkdirSync(logsDirectory, { recursive: true }) mkdirSync(storageStateDirectory, { recursive: true }) + mkdirSync(markerDirectory, { recursive: true }) + mkdirSync(homeDirectory, { recursive: true }) const nodeExecutable = resolveNode22() const bunExecutable = resolveBunExecutable() @@ -54,58 +58,69 @@ async function main(): Promise { let app: ManagedProcess | null = null let cleanupPromise: Promise | null = null let failed = false - let interrupted = false const cleanup = (): Promise => { if (cleanupPromise) return cleanupPromise cleanupPromise = (async () => { - await stopAllManagedProcesses() + const failures: unknown[] = [] + try { + await stopAllManagedProcesses() + } catch (error) { + failures.push(error) + } if (stripeFake) { - writeFileSync( - path.join(logsDirectory, 'stripe-requests.json'), - JSON.stringify(stripeFake.requestLog, null, 2) - ) - await stripeFake.stop() + try { + writeFileSync( + path.join(logsDirectory, 'stripe-requests.json'), + JSON.stringify(stripeFake.requestLog, null, 2) + ) + } catch (error) { + failures.push(error) + } + try { + await stripeFake.stop() + } catch (error) { + failures.push(error) + } } - if (runDatabase) { + for (const sensitiveDirectory of [storageStateDirectory, homeDirectory]) { try { - await dropRunDatabase(adminDatabaseUrl, runDatabase.name) + rmSync(sensitiveDirectory, { recursive: true, force: true }) } catch (error) { - failed = true - process.exitCode = 1 - console.error(`Failed to drop ${runDatabase.name}:`, error) + failures.push(error) } } + + try { + if (runDatabase) await dropRunDatabase(adminDatabaseUrl, runDatabase.name) + } catch (error) { + failures.push(error) + } + + if (failures.length > 0) { + throw new AggregateError(failures, 'One or more E2E cleanup stages failed') + } })() return cleanupPromise } const handleSignal = (signal: NodeJS.Signals): void => { - if (interrupted) return - interrupted = true failed = true const exitCode = signal === 'SIGINT' ? 130 : 143 process.exitCode = exitCode console.error(`Received ${signal}; cleaning up the E2E run`) - const survivingPids = stopAllManagedProcessesSync() - if (survivingPids.length > 0) { - console.error(`E2E child processes survived SIGKILL: ${survivingPids.join(', ')}`) - } - if (stripeFake) { - writeFileSync( - path.join(logsDirectory, 'stripe-requests.json'), - JSON.stringify(stripeFake.requestLog, null, 2) - ) - } + const signalFailures: unknown[] = [] if (runDatabase) { + console.error(`Dropping guarded E2E database ${runDatabase.name}`) const cleanupResult = spawnSync( bunExecutable, ['--no-env-file', path.join(SIM_APP_DIR, 'e2e/scripts/cleanup-database.ts')], { cwd: SIM_APP_DIR, encoding: 'utf8', + timeout: 15_000, env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '', @@ -115,13 +130,47 @@ async function main(): Promise { } ) if (cleanupResult.status !== 0) { - console.error(cleanupResult.stderr || cleanupResult.stdout) + signalFailures.push( + cleanupResult.error ?? + new Error(cleanupResult.stderr || cleanupResult.stdout || 'Database cleanup failed') + ) } else { runDatabase = null + console.error('Guarded E2E database cleanup complete') + } + } + try { + const survivingPids = stopAllManagedProcessesSync(500, 250) + if (survivingPids.length > 0) { + signalFailures.push( + new Error(`E2E child processes survived SIGKILL: ${survivingPids.join(', ')}`) + ) + } + console.error('E2E child process cleanup complete') + } catch (error) { + signalFailures.push(error) + } + if (stripeFake) { + try { + writeFileSync( + path.join(logsDirectory, 'stripe-requests.json'), + JSON.stringify(stripeFake.requestLog, null, 2) + ) + } catch (error) { + signalFailures.push(error) } } + for (const sensitiveDirectory of [storageStateDirectory, homeDirectory]) { + try { + rmSync(sensitiveDirectory, { recursive: true, force: true }) + } catch (error) { + signalFailures.push(error) + } + } + for (const error of signalFailures) console.error(error) process.exit(exitCode) } + // `once` is intentional: a second signal uses the OS default force termination. process.once('SIGINT', handleSignal) process.once('SIGTERM', handleSignal) @@ -147,6 +196,8 @@ async function main(): Promise { runId, databaseUrl: runDatabase.url, stripeApiBaseUrl: stripeFake.baseUrl, + homeDirectory, + playwrightBrowsersPath: resolvePlaywrightBrowsersPath(), ci: process.env.CI === 'true', }) console.info(formatRedactedEnvironmentSummary(profile.id, profile.childEnvironment)) @@ -168,7 +219,8 @@ async function main(): Promise { const playwrightEnvironment = createPlaywrightEnvironment( profile.childEnvironment.env, runId, - storageStateDirectory + storageStateDirectory, + markerDirectory ) await runPlaywright( { @@ -179,7 +231,11 @@ async function main(): Promise { options.playwrightArgs ) const provisioning = await inspectFoundationUsers(runDatabase.url, runId) - assertAuthenticatedSmokeEffectsIfPresent(stripeFake, provisioning) + assertAuthenticatedSmokeEffectsIfPresent( + stripeFake, + provisioning, + hasFoundationCompletionMarker(markerDirectory) + ) } catch (error) { failed = true console.error(error) @@ -194,7 +250,13 @@ async function main(): Promise { process.exitCode = 1 console.error(error) } - await cleanup() + try { + await cleanup() + } catch (error) { + failed = true + process.exitCode = 1 + console.error(error) + } process.off('SIGINT', handleSignal) process.off('SIGTERM', handleSignal) console.info(`E2E ${failed ? 'failed' : 'completed'}; diagnostics: ${runDirectory}`) @@ -230,24 +292,22 @@ function resolveNode22(): string { return executable } +function resolvePlaywrightBrowsersPath(): string { + if (process.env.PLAYWRIGHT_BROWSERS_PATH) return process.env.PLAYWRIGHT_BROWSERS_PATH + const parentHome = process.env.HOME + if (!parentHome) throw new Error('HOME is required to locate installed Playwright browsers') + return process.platform === 'darwin' + ? path.join(parentHome, 'Library/Caches/ms-playwright') + : path.join(parentHome, '.cache/ms-playwright') +} + function createPlaywrightEnvironment( stackEnvironment: Record, runId: string, - storageStateDirectory: string + storageStateDirectory: string, + markerDirectory: string ): Record { - const keys = [ - 'PATH', - 'HOME', - 'USER', - 'SHELL', - 'TMPDIR', - 'TMP', - 'TEMP', - 'SYSTEMROOT', - 'CI', - 'GITHUB_ACTIONS', - 'PLAYWRIGHT_BROWSERS_PATH', - ] + const keys = [...E2E_OS_PASSTHROUGH_KEYS, 'HOME', 'PLAYWRIGHT_BROWSERS_PATH'] const env: Record = {} for (const key of keys) { const value = stackEnvironment[key] @@ -260,12 +320,14 @@ function createPlaywrightEnvironment( E2E_RUN_ID: runId, E2E_BASE_URL: E2E_ORIGIN, E2E_STORAGE_STATE_DIR: storageStateDirectory, + E2E_MARKER_DIR: markerDirectory, } } function assertAuthenticatedSmokeEffectsIfPresent( stripeFake: StripeFakeServer, - provisioning: FoundationProvisioningResult + provisioning: FoundationProvisioningResult, + completed: boolean ): void { const created = stripeFake.requestLog.some( ({ method, path }) => method === 'POST' && path === '/v1/customers' @@ -278,7 +340,7 @@ function assertAuthenticatedSmokeEffectsIfPresent( .join(', ')}` ) } - if (!created && provisioning.count === 0) { + if (!completed) { console.info('Authenticated foundation smoke was filtered out; skipping its post-run probes') return } @@ -294,4 +356,11 @@ function assertAuthenticatedSmokeEffectsIfPresent( } } +function hasFoundationCompletionMarker(markerDirectory: string): boolean { + return ( + existsSync(markerDirectory) && + readdirSync(markerDirectory).some((name) => name.startsWith('foundation-authenticated-')) + ) +} + await main() diff --git a/apps/sim/e2e/settings/smoke/authenticated.spec.ts b/apps/sim/e2e/settings/smoke/authenticated.spec.ts index 1363dd5f836..a64c7e9a1db 100644 --- a/apps/sim/e2e/settings/smoke/authenticated.spec.ts +++ b/apps/sim/e2e/settings/smoke/authenticated.spec.ts @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto' +import { writeFileSync } from 'node:fs' import path from 'node:path' import { expect, test } from '@playwright/test' @@ -51,6 +52,10 @@ test('billing-enabled signup, login, and settings use real Sim boundaries', asyn } finally { await restoredContext.close() } + writeFileSync( + path.join(requiredEnv('E2E_MARKER_DIR'), `foundation-authenticated-${testIdentity}.json`), + JSON.stringify({ runId, testIdentity }) + ) }) async function assertGeneralSettings(page: import('@playwright/test').Page): Promise { diff --git a/apps/sim/e2e/support/database.ts b/apps/sim/e2e/support/database.ts index 6e2093dc3f0..cfa5eaae0c5 100644 --- a/apps/sim/e2e/support/database.ts +++ b/apps/sim/e2e/support/database.ts @@ -67,13 +67,7 @@ export async function dropRunDatabase(adminUrl: string, databaseName: string): P assertLoopbackPostgresUrl(adminUrl) const sql = postgres(adminUrl, { max: 1, connect_timeout: 10 }) try { - await sql` - SELECT pg_terminate_backend(pid) - FROM pg_stat_activity - WHERE datname = ${databaseName} - AND pid <> pg_backend_pid() - ` - await sql.unsafe(`DROP DATABASE IF EXISTS "${databaseName}"`) + await sql.unsafe(`DROP DATABASE IF EXISTS "${databaseName}" WITH (FORCE)`) } finally { await sql.end() } diff --git a/apps/sim/e2e/support/deployment-profile.ts b/apps/sim/e2e/support/deployment-profile.ts index c2348269fda..a52ad6aca3d 100644 --- a/apps/sim/e2e/support/deployment-profile.ts +++ b/apps/sim/e2e/support/deployment-profile.ts @@ -22,6 +22,8 @@ const REQUIRED_KEYS = [ 'STRIPE_API_BASE_URL', 'E2E_PROFILE', 'E2E_RUN_ID', + 'HOME', + 'PLAYWRIGHT_BROWSERS_PATH', ] as const const ALLOWED_SENSITIVE_KEYS = new Set([ @@ -40,6 +42,8 @@ export interface HostedBillingProfileOptions { runId: string databaseUrl: string stripeApiBaseUrl: string + homeDirectory: string + playwrightBrowsersPath: string ci: boolean } @@ -53,12 +57,22 @@ export function createHostedBillingProfile({ runId, databaseUrl, stripeApiBaseUrl, + homeDirectory, + playwrightBrowsersPath, ci, }: HostedBillingProfileOptions): HostedBillingProfile { const values: Record = { NODE_ENV: 'production', NODE_OPTIONS: '--no-warnings --max-old-space-size=8192', NEXT_TELEMETRY_DISABLED: '1', + HOME: homeDirectory, + XDG_CONFIG_HOME: `${homeDirectory}/xdg`, + AWS_EC2_METADATA_DISABLED: 'true', + AWS_SHARED_CREDENTIALS_FILE: '/dev/null', + AWS_CONFIG_FILE: '/dev/null', + CLOUDSDK_CONFIG: `${homeDirectory}/gcloud`, + AZURE_CONFIG_DIR: `${homeDirectory}/azure`, + PLAYWRIGHT_BROWSERS_PATH: playwrightBrowsersPath, E2E_PROFILE, E2E_RUN_ID: runId, E2E_BASE_URL: E2E_ORIGIN, @@ -114,7 +128,7 @@ function validateProfileValues(values: Record): void { throw new Error('E2E profile requires a sim_e2e_ database') } const stripeUrl = new URL(values.STRIPE_API_BASE_URL) - if (!['127.0.0.1', 'localhost', '::1', '[::1]'].includes(stripeUrl.hostname)) { - throw new Error('E2E Stripe API must use a loopback hostname') + if (stripeUrl.hostname !== '127.0.0.1') { + throw new Error('E2E Stripe API must use numeric IPv4 loopback 127.0.0.1') } } diff --git a/apps/sim/e2e/support/env.ts b/apps/sim/e2e/support/env.ts index 66e09b6ec99..7527596b41e 100644 --- a/apps/sim/e2e/support/env.ts +++ b/apps/sim/e2e/support/env.ts @@ -8,7 +8,6 @@ const SENSITIVE_KEY_PATTERN = const OS_PASSTHROUGH_KEYS = [ 'PATH', - 'HOME', 'USER', 'SHELL', 'TMPDIR', @@ -17,7 +16,6 @@ const OS_PASSTHROUGH_KEYS = [ 'SYSTEMROOT', 'CI', 'GITHUB_ACTIONS', - 'PLAYWRIGHT_BROWSERS_PATH', ] as const export interface ChildEnvironment { diff --git a/apps/sim/e2e/support/probes.ts b/apps/sim/e2e/support/probes.ts index 3e4241be574..40f4e42bae1 100644 --- a/apps/sim/e2e/support/probes.ts +++ b/apps/sim/e2e/support/probes.ts @@ -27,14 +27,14 @@ export async function inspectFoundationUsers( ): Promise { const sql = postgres(databaseUrl, { max: 1, connect_timeout: 10 }) try { - const pattern = `e2e-foundation-${runId}-%@example.com` + const prefix = `e2e-foundation-${runId}-` const rows = await sql>` SELECT u.stripe_customer_id AS "stripeCustomerId", (s.user_id IS NOT NULL) AS "hasStats" FROM "user" u LEFT JOIN user_stats s ON s.user_id = u.id - WHERE u.email LIKE ${pattern} + WHERE starts_with(u.email, ${prefix}) ` return { count: rows.length, diff --git a/apps/sim/e2e/support/process.ts b/apps/sim/e2e/support/process.ts index ac104f8f6dc..01e63dd2ace 100644 --- a/apps/sim/e2e/support/process.ts +++ b/apps/sim/e2e/support/process.ts @@ -15,10 +15,17 @@ export interface CommandOptions { export interface ManagedProcess { name: string child: ChildProcess + completion: Promise logPath: string stop(): Promise } +interface ProcessCompletion { + code: number | null + signal: NodeJS.Signals | null + error?: Error +} + const activeProcesses = new Set() export function spawnManagedProcess(options: CommandOptions): ManagedProcess { @@ -30,30 +37,56 @@ export function spawnManagedProcess(options: CommandOptions): ManagedProcess { env: options.env as NodeJS.ProcessEnv, stdio: ['ignore', logFd, logFd], }) + let resolveCompletion!: (completion: ProcessCompletion) => void + const completion = new Promise((resolve) => { + resolveCompletion = resolve + }) + let finalized = false const managed: ManagedProcess = { name: options.name, child, + completion, logPath, - stop: () => stopProcess(child), + stop: () => stopProcess(managed), } - activeProcesses.add(managed) - child.once('exit', () => { + const finalize = (result: ProcessCompletion): void => { + if (finalized) return + finalized = true activeProcesses.delete(managed) closeSync(logFd) - }) + resolveCompletion(result) + } + activeProcesses.add(managed) + child.once('error', (error) => finalize({ code: null, signal: null, error })) + child.once('exit', (code, signal) => finalize({ code, signal })) return managed } export async function runCommand(options: CommandOptions): Promise { const managed = spawnManagedProcess(options) - const exitCode = await waitForExit(managed.child) - if (exitCode !== 0) { - throw new Error(`${options.name} exited with code ${exitCode}. See ${managed.logPath}`) + const result = await managed.completion + if (result.error) { + throw new Error(`${options.name} failed to spawn. See ${managed.logPath}`, { + cause: result.error, + }) + } + if (result.code !== 0) { + throw new Error( + `${options.name} exited with ${result.code ?? result.signal ?? 'unknown status'}. See ${managed.logPath}` + ) } } export async function stopProcesses(processes: ManagedProcess[]): Promise { - await Promise.allSettled([...processes].reverse().map((process) => process.stop())) + const results = await Promise.allSettled( + [...processes].reverse().map((process) => process.stop()) + ) + const failures = results.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [] + ) + if (failures.length > 0) { + throw new AggregateError(failures, 'Failed to stop one or more managed E2E processes') + } } export async function stopAllManagedProcesses(): Promise { @@ -64,55 +97,37 @@ export function stopAllManagedProcessesSync( termTimeoutMs = 5_000, killTimeoutMs = 2_000 ): number[] { - const managedProcesses = [...activeProcesses] - for (const managed of managedProcesses) sendSignal(managed.child, 'SIGTERM') - - const deadline = Date.now() + termTimeoutMs - while (Date.now() < deadline && managedProcesses.some(({ child }) => isProcessRunning(child))) { - sleepSync(50) - } - - for (const { child } of managedProcesses) { - if (isProcessRunning(child)) sendSignal(child, 'SIGKILL') - } - - const killDeadline = Date.now() + killTimeoutMs - while ( - Date.now() < killDeadline && - managedProcesses.some(({ child }) => isProcessRunning(child)) - ) { - sleepSync(25) - } - - return managedProcesses.flatMap(({ child }) => - child.pid && isProcessRunning(child) ? [child.pid] : [] - ) + const processIds = [ + ...new Set([...activeProcesses].flatMap(({ child }) => (child.pid ? [child.pid] : []))), + ] + sendSignalToPids(processIds, 'SIGTERM') + sleepSync(termTimeoutMs) + let runningPids = getRunningPids(processIds) + sendSignalToPids(runningPids, 'SIGKILL') + sleepSync(killTimeoutMs) + runningPids = getRunningPids(processIds) + return runningPids } export async function waitForManagedProcessReady( process: ManagedProcess, - readiness: Promise + readiness: (signal: AbortSignal) => Promise ): Promise { if (process.child.exitCode !== null || process.child.signalCode !== null) { throw new Error(`${process.name} exited before readiness. See ${process.logPath}`) } - let onExit: ((code: number | null, signal: NodeJS.Signals | null) => void) | undefined - const exited = new Promise((_, reject) => { - onExit = (code, signal) => { - reject( - new Error( - `${process.name} exited before readiness with ${code ?? signal ?? 'unknown status'}. See ${process.logPath}` - ) - ) - } - process.child.once('exit', onExit) + const controller = new AbortController() + const exited = process.completion.then((result) => { + throw new Error( + `${process.name} exited before readiness with ${result.error?.message ?? result.code ?? result.signal ?? 'unknown status'}. See ${process.logPath}` + ) }) try { - await Promise.race([readiness, exited]) + await Promise.race([readiness(controller.signal), exited]) } finally { - if (onExit) process.child.off('exit', onExit) + controller.abort(new Error(`${process.name} readiness polling cancelled`)) } } @@ -131,57 +146,72 @@ export async function assertPortAvailable(port: number, host = '127.0.0.1'): Pro }) } -function waitForExit(child: ChildProcess): Promise { - return new Promise((resolve, reject) => { - child.once('error', reject) - child.once('exit', (code) => resolve(code)) - }) -} - -async function stopProcess(child: ChildProcess): Promise { +async function stopProcess(managed: ManagedProcess): Promise { + const { child } = managed + if (!child.pid) { + await managed.completion + return + } if (child.exitCode !== null || child.signalCode !== null) return - sendSignal(child, 'SIGTERM') + const processIds = [child.pid] + sendSignalToPids(processIds, 'SIGTERM') const stopped = await Promise.race([ - waitForExit(child).then(() => true), + managed.completion.then(() => true), new Promise((resolve) => setTimeout(() => resolve(false), 5_000)), ]) if (stopped) return - sendSignal(child, 'SIGKILL') - await waitForExit(child) + sendSignalToPids(processIds.filter(isPidRunning), 'SIGKILL') + const killed = await Promise.race([ + managed.completion.then(() => true), + new Promise((resolve) => setTimeout(() => resolve(false), 2_000)), + ]) + if (!killed) { + throw new Error(`Managed process ${managed.name} did not exit after SIGKILL`) + } } -function sendSignal(child: ChildProcess, signal: NodeJS.Signals): void { - if (!child.pid) return - try { - child.kill(signal) - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error +function sendSignalToPids(processIds: number[], signal: NodeJS.Signals): void { + for (const processId of processIds) { + try { + process.kill(processId, signal) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error + } } } -/** - * Managed commands currently run their server/build work in the direct child. - * If a future command introduces durable grandchildren, process-tree cleanup - * should be added alongside that command rather than implied here. - */ -function isProcessRunning(child: ChildProcess): boolean { - if (!child.pid) return false - try { - process.kill(child.pid, 0) - if (process.platform !== 'win32') { - const status = spawnSync('ps', ['-o', 'stat=', '-p', String(child.pid)], { - encoding: 'utf8', - env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, - }) - if (status.status !== 0 || status.stdout.trim().startsWith('Z')) return false - } - return true - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ESRCH') return false - throw error +function isPidRunning(processId: number): boolean { + return getRunningPids([processId]).length > 0 +} + +function getRunningPids(processIds: number[]): number[] { + if (processIds.length === 0) return [] + if (process.platform === 'win32') { + return processIds.filter((processId) => { + try { + process.kill(processId, 0) + return true + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ESRCH') return false + throw error + } + }) } + + const status = spawnSync('ps', ['-o', 'pid=,stat=', '-p', processIds.join(',')], { + encoding: 'utf8', + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + }) + if (status.status !== 0 && status.status !== 1) { + throw new Error(`Unable to inspect managed E2E processes: ${status.stderr}`) + } + return status.stdout.split('\n').flatMap((line) => { + const [pidText, processStatus] = line.trim().split(/\s+/) + const pid = Number(pidText) + return Number.isInteger(pid) && !processStatus?.startsWith('Z') ? [pid] : [] + }) } function sleepSync(milliseconds: number): void { diff --git a/apps/sim/e2e/support/readiness.ts b/apps/sim/e2e/support/readiness.ts index 504bca72db1..628976bfa14 100644 --- a/apps/sim/e2e/support/readiness.ts +++ b/apps/sim/e2e/support/readiness.ts @@ -3,6 +3,7 @@ export interface ReadinessOptions { url: string timeoutMs?: number intervalMs?: number + signal?: AbortSignal validate?: (response: Response) => Promise | boolean } @@ -11,21 +12,44 @@ export async function waitForHttpReady({ url, timeoutMs = 120_000, intervalMs = 500, + signal, validate = (response) => response.ok, }: ReadinessOptions): Promise { const deadline = Date.now() + timeoutMs let lastError: unknown while (Date.now() < deadline) { + if (signal?.aborted) throw signal.reason try { - const response = await fetch(url, { redirect: 'manual', signal: AbortSignal.timeout(5_000) }) + const timeoutSignal = AbortSignal.timeout(5_000) + const response = await fetch(url, { + redirect: 'manual', + signal: signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal, + }) if (await validate(response)) return lastError = new Error(`${name} returned ${response.status}`) } catch (error) { + if (signal?.aborted) throw signal.reason lastError = error } - await new Promise((resolve) => setTimeout(resolve, intervalMs)) + await waitForInterval(intervalMs, signal) } throw new Error(`${name} did not become ready at ${url}: ${String(lastError)}`) } + +async function waitForInterval(milliseconds: number, signal?: AbortSignal): Promise { + await new Promise((resolve, reject) => { + const complete = () => { + signal?.removeEventListener('abort', abort) + resolve() + } + const timeout = setTimeout(complete, milliseconds) + const abort = () => { + clearTimeout(timeout) + reject(signal?.reason) + } + if (signal?.aborted) abort() + else signal?.addEventListener('abort', abort, { once: true }) + }) +} diff --git a/apps/sim/e2e/support/stack.ts b/apps/sim/e2e/support/stack.ts index 3e6c71a29cd..d1ce203d6b6 100644 --- a/apps/sim/e2e/support/stack.ts +++ b/apps/sim/e2e/support/stack.ts @@ -61,11 +61,11 @@ export async function startRealtime(options: StackCommandOptions): Promise waitForHttpReady({ name: 'Realtime', url: 'http://127.0.0.1:3002/health', + signal, validate: async (response) => { if (!response.ok) return false const body = (await response.json()) as { status?: string; runId?: string } @@ -102,11 +102,11 @@ export async function startApp(options: StackCommandOptions): Promise waitForHttpReady({ name: 'Next.js', url: 'http://127.0.0.1:3000/api/health', + signal, validate: async (response) => { if (!response.ok) return false const body = (await response.json()) as { status?: string; runId?: string } diff --git a/apps/sim/lib/billing/stripe-client-config.ts b/apps/sim/lib/billing/stripe-client-config.ts index df6b13dddf7..320cee74b77 100644 --- a/apps/sim/lib/billing/stripe-client-config.ts +++ b/apps/sim/lib/billing/stripe-client-config.ts @@ -30,7 +30,7 @@ function getDatabaseName(databaseUrl: string | undefined): string | null { function isLoopbackHostname(hostname: string): boolean { const normalized = hostname.toLowerCase() - if (normalized === 'localhost' || normalized === '[::1]') return true + if (normalized === 'localhost') return true const octets = normalized.split('.') return ( From 1723bd2685d274a7fad033f2e63a8dc847bf872b Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Mon, 20 Jul 2026 18:46:16 -0700 Subject: [PATCH 05/10] test(e2e): harden interrupted-run cleanup Use a detached cleanup supervisor so repeated signals cannot interrupt process-group shutdown or forced removal of the guarded run database. --- apps/sim/e2e/README.md | 5 ++ apps/sim/e2e/foundation/safety.spec.ts | 1 + apps/sim/e2e/scripts/cleanup-database.ts | 29 --------- apps/sim/e2e/scripts/run.ts | 77 +++++++++--------------- apps/sim/e2e/scripts/signal-cleanup.ts | 67 +++++++++++++++++++++ apps/sim/e2e/support/database.ts | 27 ++++++++- apps/sim/e2e/support/hosts.ts | 4 +- apps/sim/e2e/support/process.ts | 33 ++++------ 8 files changed, 142 insertions(+), 101 deletions(-) delete mode 100644 apps/sim/e2e/scripts/cleanup-database.ts create mode 100644 apps/sim/e2e/scripts/signal-cleanup.ts diff --git a/apps/sim/e2e/README.md b/apps/sim/e2e/README.md index f4e33d0f4af..8241e2431ff 100644 --- a/apps/sim/e2e/README.md +++ b/apps/sim/e2e/README.md @@ -51,6 +51,11 @@ The runner creates a unique `sim_e2e_` database, migrates it, starts the Stripe fake and realtime, builds and starts Next.js, runs Playwright on Node 22, then stops services and drops only that guarded database. +On interruption, the runner launches a detached cleanup supervisor before +exiting. It terminates managed process groups, force-drops the guarded database, +and removes temporary auth/cloud-config directories even if another Ctrl-C +terminates the foreground package runner. + Pass Playwright arguments after `--`: ```bash diff --git a/apps/sim/e2e/foundation/safety.spec.ts b/apps/sim/e2e/foundation/safety.spec.ts index 529ac2d911f..87ec3e3a0cf 100644 --- a/apps/sim/e2e/foundation/safety.spec.ts +++ b/apps/sim/e2e/foundation/safety.spec.ts @@ -84,6 +84,7 @@ test.describe('foundation safety guards', () => { expect(isLoopbackAddress('8.8.8.8')).toBe(false) expect(areValidE2eHostAddresses(['127.0.0.1', '::1'])).toBe(true) expect(areValidE2eHostAddresses(['::1'])).toBe(false) + expect(areValidE2eHostAddresses(['127.0.0.2'])).toBe(false) }) test('sharding is limited to the navigation project', () => { diff --git a/apps/sim/e2e/scripts/cleanup-database.ts b/apps/sim/e2e/scripts/cleanup-database.ts deleted file mode 100644 index 72a05c2a281..00000000000 --- a/apps/sim/e2e/scripts/cleanup-database.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { dropRunDatabase } from '../support/database' - -const adminUrl = process.env.E2E_PG_ADMIN_URL -const databaseName = process.env.E2E_DATABASE_NAME - -if (!adminUrl || !databaseName) { - console.error('Synchronous E2E cleanup requires E2E_PG_ADMIN_URL and E2E_DATABASE_NAME') - process.exit(1) -} - -try { - // A signal may arrive while CREATE DATABASE is still completing. Repeating - // the forced drop catches that narrow race without weakening name/host guards. - const deadline = Date.now() + 5_000 - let lastError: unknown - do { - try { - await dropRunDatabase(adminUrl, databaseName) - lastError = undefined - } catch (error) { - lastError = error - } - await new Promise((resolve) => setTimeout(resolve, 100)) - } while (Date.now() < deadline) - if (lastError) throw lastError -} catch (error) { - console.error(error) - process.exit(1) -} diff --git a/apps/sim/e2e/scripts/run.ts b/apps/sim/e2e/scripts/run.ts index 6ca5f5fca16..517fda0c6bf 100644 --- a/apps/sim/e2e/scripts/run.ts +++ b/apps/sim/e2e/scripts/run.ts @@ -1,6 +1,14 @@ -import { spawnSync } from 'node:child_process' +import { spawn, spawnSync } from 'node:child_process' import { randomUUID } from 'node:crypto' -import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from 'node:fs' +import { + closeSync, + existsSync, + mkdirSync, + openSync, + readdirSync, + rmSync, + writeFileSync, +} from 'node:fs' import path from 'node:path' import { type StripeFakeServer, startStripeFakeServer } from '../fakes/stripe/server' import { @@ -25,9 +33,9 @@ import { } from '../support/probes' import { assertPortAvailable, + getActiveManagedProcessGroupIds, type ManagedProcess, stopAllManagedProcesses, - stopAllManagedProcessesSync, } from '../support/process' import { buildApp, runMigrations, runPlaywright, startApp, startRealtime } from '../support/stack' import { parseRunOptions } from './options' @@ -111,63 +119,38 @@ async function main(): Promise { const exitCode = signal === 'SIGINT' ? 130 : 143 process.exitCode = exitCode console.error(`Received ${signal}; cleaning up the E2E run`) - const signalFailures: unknown[] = [] + if (stripeFake) { + try { + writeFileSync( + path.join(logsDirectory, 'stripe-requests.json'), + JSON.stringify(stripeFake.requestLog, null, 2) + ) + } catch {} + } if (runDatabase) { - console.error(`Dropping guarded E2E database ${runDatabase.name}`) - const cleanupResult = spawnSync( + const cleanupLogFd = openSync(path.join(logsDirectory, 'signal-cleanup.log'), 'a') + const cleanupProcess = spawn( bunExecutable, - ['--no-env-file', path.join(SIM_APP_DIR, 'e2e/scripts/cleanup-database.ts')], + ['--no-env-file', path.join(SIM_APP_DIR, 'e2e/scripts/signal-cleanup.ts')], { cwd: SIM_APP_DIR, - encoding: 'utf8', - timeout: 15_000, + detached: true, env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '', + HOME: homeDirectory, E2E_PG_ADMIN_URL: adminDatabaseUrl, E2E_DATABASE_NAME: runDatabase.name, + E2E_CLEANUP_PROCESS_GROUPS: getActiveManagedProcessGroupIds().join(','), + E2E_CLEANUP_DIRECTORIES: JSON.stringify([storageStateDirectory, homeDirectory]), }, + stdio: ['ignore', cleanupLogFd, cleanupLogFd], } ) - if (cleanupResult.status !== 0) { - signalFailures.push( - cleanupResult.error ?? - new Error(cleanupResult.stderr || cleanupResult.stdout || 'Database cleanup failed') - ) - } else { - runDatabase = null - console.error('Guarded E2E database cleanup complete') - } - } - try { - const survivingPids = stopAllManagedProcessesSync(500, 250) - if (survivingPids.length > 0) { - signalFailures.push( - new Error(`E2E child processes survived SIGKILL: ${survivingPids.join(', ')}`) - ) - } - console.error('E2E child process cleanup complete') - } catch (error) { - signalFailures.push(error) - } - if (stripeFake) { - try { - writeFileSync( - path.join(logsDirectory, 'stripe-requests.json'), - JSON.stringify(stripeFake.requestLog, null, 2) - ) - } catch (error) { - signalFailures.push(error) - } - } - for (const sensitiveDirectory of [storageStateDirectory, homeDirectory]) { - try { - rmSync(sensitiveDirectory, { recursive: true, force: true }) - } catch (error) { - signalFailures.push(error) - } + cleanupProcess.unref() + closeSync(cleanupLogFd) + console.error(`Detached E2E cleanup supervisor started as PID ${cleanupProcess.pid}`) } - for (const error of signalFailures) console.error(error) process.exit(exitCode) } // `once` is intentional: a second signal uses the OS default force termination. diff --git a/apps/sim/e2e/scripts/signal-cleanup.ts b/apps/sim/e2e/scripts/signal-cleanup.ts new file mode 100644 index 00000000000..0e282d0b7da --- /dev/null +++ b/apps/sim/e2e/scripts/signal-cleanup.ts @@ -0,0 +1,67 @@ +import { rmSync } from 'node:fs' +import { sleep } from '@sim/utils/helpers' +import { dropRunDatabaseWithRetries } from '../support/database' + +const adminUrl = process.env.E2E_PG_ADMIN_URL +const databaseName = process.env.E2E_DATABASE_NAME +const processGroupIds = (process.env.E2E_CLEANUP_PROCESS_GROUPS ?? '') + .split(',') + .map(Number) + .filter(Number.isInteger) +const sensitiveDirectories = JSON.parse(process.env.E2E_CLEANUP_DIRECTORIES ?? '[]') as string[] + +if (!adminUrl || !databaseName) { + console.error('Signal cleanup requires E2E_PG_ADMIN_URL and E2E_DATABASE_NAME') + process.exit(1) +} + +const failures: unknown[] = [] +signalProcessGroups(processGroupIds, 'SIGTERM', failures) +await sleep(500) + +try { + await dropRunDatabaseWithRetries(adminUrl, databaseName) +} catch (error) { + failures.push(error) +} + +signalProcessGroups(processGroupIds, 'SIGKILL', failures) +await sleep(250) + +for (const directory of sensitiveDirectories) { + try { + rmSync(directory, { recursive: true, force: true }) + } catch (error) { + failures.push(error) + } +} + +for (const error of failures) console.error(error) +process.exit(failures.length > 0 ? 1 : 0) + +function signalProcessGroups( + groupIds: number[], + signal: NodeJS.Signals, + failures: unknown[] +): void { + for (const groupId of groupIds) { + try { + if (process.platform !== 'win32') process.kill(-groupId, signal) + else process.kill(groupId, signal) + } catch (error) { + const code = (error as NodeJS.ErrnoException).code + if (code === 'ESRCH') continue + if (code === 'EPERM' && process.platform !== 'win32') { + try { + process.kill(groupId, signal) + } catch (fallbackError) { + if ((fallbackError as NodeJS.ErrnoException).code !== 'ESRCH') { + failures.push(fallbackError) + } + } + } else { + failures.push(error) + } + } + } +} diff --git a/apps/sim/e2e/support/database.ts b/apps/sim/e2e/support/database.ts index cfa5eaae0c5..d2e3747f2e8 100644 --- a/apps/sim/e2e/support/database.ts +++ b/apps/sim/e2e/support/database.ts @@ -1,3 +1,4 @@ +import { sleep } from '@sim/utils/helpers' import postgres from 'postgres' import { isLoopbackAddress } from './hosts' @@ -65,10 +66,34 @@ export async function createRunDatabase( export async function dropRunDatabase(adminUrl: string, databaseName: string): Promise { assertSafeDatabaseName(databaseName) assertLoopbackPostgresUrl(adminUrl) - const sql = postgres(adminUrl, { max: 1, connect_timeout: 10 }) + const sql = postgres(adminUrl, { + max: 1, + connect_timeout: 10, + onnotice: () => {}, + }) try { await sql.unsafe(`DROP DATABASE IF EXISTS "${databaseName}" WITH (FORCE)`) } finally { await sql.end() } } + +export async function dropRunDatabaseWithRetries( + adminUrl: string, + databaseName: string, + deadlineMs = 5_000, + intervalMs = 100 +): Promise { + const deadline = Date.now() + deadlineMs + let lastError: unknown + do { + try { + await dropRunDatabase(adminUrl, databaseName) + lastError = undefined + } catch (error) { + lastError = error + } + await sleep(intervalMs) + } while (Date.now() < deadline) + if (lastError) throw lastError +} diff --git a/apps/sim/e2e/support/hosts.ts b/apps/sim/e2e/support/hosts.ts index 80a927f87b9..81e6127755c 100644 --- a/apps/sim/e2e/support/hosts.ts +++ b/apps/sim/e2e/support/hosts.ts @@ -13,9 +13,7 @@ export function isLoopbackAddress(address: string): boolean { export function areValidE2eHostAddresses(addresses: string[]): boolean { return ( - addresses.length > 0 && - addresses.every(isLoopbackAddress) && - addresses.some((address) => isIP(address) === 4 && isLoopbackAddress(address)) + addresses.length > 0 && addresses.every(isLoopbackAddress) && addresses.includes('127.0.0.1') ) } diff --git a/apps/sim/e2e/support/process.ts b/apps/sim/e2e/support/process.ts index 01e63dd2ace..378b47afdd8 100644 --- a/apps/sim/e2e/support/process.ts +++ b/apps/sim/e2e/support/process.ts @@ -34,6 +34,7 @@ export function spawnManagedProcess(options: CommandOptions): ManagedProcess { const logFd = openSync(logPath, 'a') const child: ChildProcess = spawn(options.command, options.args, { cwd: options.cwd, + detached: process.platform !== 'win32', env: options.env as NodeJS.ProcessEnv, stdio: ['ignore', logFd, logFd], }) @@ -93,20 +94,8 @@ export async function stopAllManagedProcesses(): Promise { await stopProcesses([...activeProcesses]) } -export function stopAllManagedProcessesSync( - termTimeoutMs = 5_000, - killTimeoutMs = 2_000 -): number[] { - const processIds = [ - ...new Set([...activeProcesses].flatMap(({ child }) => (child.pid ? [child.pid] : []))), - ] - sendSignalToPids(processIds, 'SIGTERM') - sleepSync(termTimeoutMs) - let runningPids = getRunningPids(processIds) - sendSignalToPids(runningPids, 'SIGKILL') - sleepSync(killTimeoutMs) - runningPids = getRunningPids(processIds) - return runningPids +export function getActiveManagedProcessGroupIds(): number[] { + return [...new Set([...activeProcesses].flatMap(({ child }) => (child.pid ? [child.pid] : [])))] } export async function waitForManagedProcessReady( @@ -175,9 +164,16 @@ async function stopProcess(managed: ManagedProcess): Promise { function sendSignalToPids(processIds: number[], signal: NodeJS.Signals): void { for (const processId of processIds) { try { - process.kill(processId, signal) + if (process.platform !== 'win32') process.kill(-processId, signal) + else process.kill(processId, signal) } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ESRCH') throw error + const code = (error as NodeJS.ErrnoException).code + if (code === 'ESRCH') continue + if (code === 'EPERM' && process.platform !== 'win32') { + process.kill(processId, signal) + continue + } + throw error } } } @@ -213,8 +209,3 @@ function getRunningPids(processIds: number[]): number[] { return Number.isInteger(pid) && !processStatus?.startsWith('Z') ? [pid] : [] }) } - -function sleepSync(milliseconds: number): void { - const signal = new Int32Array(new SharedArrayBuffer(Int32Array.BYTES_PER_ELEMENT)) - Atomics.wait(signal, 0, 0, milliseconds) -} From 5a5ae8401849239f4919c6e5ac283455bf509a3d Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Mon, 20 Jul 2026 18:53:13 -0700 Subject: [PATCH 06/10] test(e2e): guard signal cleanup edge cases Prevent empty process-group targets and verify forced-drop retries against final database state so interrupted runs cannot self-signal or report false cleanup failures. --- apps/sim/e2e/foundation/safety.spec.ts | 7 +++++++ apps/sim/e2e/scripts/run.ts | 3 +++ apps/sim/e2e/scripts/signal-cleanup.ts | 16 ++++++++++------ apps/sim/e2e/support/database.ts | 23 ++++++++++++++++++++++- apps/sim/e2e/support/signal-cleanup.ts | 8 ++++++++ 5 files changed, 50 insertions(+), 7 deletions(-) create mode 100644 apps/sim/e2e/support/signal-cleanup.ts diff --git a/apps/sim/e2e/foundation/safety.spec.ts b/apps/sim/e2e/foundation/safety.spec.ts index 87ec3e3a0cf..57021dd43f7 100644 --- a/apps/sim/e2e/foundation/safety.spec.ts +++ b/apps/sim/e2e/foundation/safety.spec.ts @@ -13,6 +13,7 @@ import { import { buildChildEnvironment, discoverEnvFileKeys } from '../support/env' import { areValidE2eHostAddresses, isLoopbackAddress } from '../support/hosts' import { assertPortAvailable, spawnManagedProcess } from '../support/process' +import { parseProcessGroupIds } from '../support/signal-cleanup' test.describe('foundation safety guards', () => { test('discovers env keys without leaking values and shadows unknown keys', () => { @@ -105,6 +106,12 @@ test.describe('foundation safety guards', () => { expect(() => parseRunOptions(['--pass-with-no-tests'])).toThrow(/cannot override/) }) + test('empty signal cleanup groups never become PID zero', () => { + expect(parseProcessGroupIds(undefined)).toEqual([]) + expect(parseProcessGroupIds('')).toEqual([]) + expect(parseProcessGroupIds(' 123, 0, -1, nope, 456 ')).toEqual([123, 456]) + }) + test('port preflight rejects an existing listener', async () => { const server = createServer() await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) diff --git a/apps/sim/e2e/scripts/run.ts b/apps/sim/e2e/scripts/run.ts index 517fda0c6bf..02184512205 100644 --- a/apps/sim/e2e/scripts/run.ts +++ b/apps/sim/e2e/scripts/run.ts @@ -61,6 +61,7 @@ async function main(): Promise { const adminDatabaseUrl = process.env.E2E_PG_ADMIN_URL ?? DEFAULT_ADMIN_DATABASE_URL let runDatabase: RunDatabase | null = null + let databaseCreationComplete = false let stripeFake: StripeFakeServer | null = null let realtime: ManagedProcess | null = null let app: ManagedProcess | null = null @@ -141,6 +142,7 @@ async function main(): Promise { HOME: homeDirectory, E2E_PG_ADMIN_URL: adminDatabaseUrl, E2E_DATABASE_NAME: runDatabase.name, + E2E_DATABASE_CREATION_COMPLETE: String(databaseCreationComplete), E2E_CLEANUP_PROCESS_GROUPS: getActiveManagedProcessGroupIds().join(','), E2E_CLEANUP_DIRECTORIES: JSON.stringify([storageStateDirectory, homeDirectory]), }, @@ -168,6 +170,7 @@ async function main(): Promise { url: buildRunDatabaseUrl(adminDatabaseUrl, runDatabaseName), } await createRunDatabase(adminDatabaseUrl, runDatabaseName) + databaseCreationComplete = true stripeFake = await startStripeFakeServer({ apiKey: STRIPE_TEST_KEY, hostname: '127.0.0.1', diff --git a/apps/sim/e2e/scripts/signal-cleanup.ts b/apps/sim/e2e/scripts/signal-cleanup.ts index 0e282d0b7da..5d24f82df5a 100644 --- a/apps/sim/e2e/scripts/signal-cleanup.ts +++ b/apps/sim/e2e/scripts/signal-cleanup.ts @@ -1,14 +1,13 @@ import { rmSync } from 'node:fs' import { sleep } from '@sim/utils/helpers' -import { dropRunDatabaseWithRetries } from '../support/database' +import { dropRunDatabase, dropRunDatabaseWithRetries } from '../support/database' +import { parseProcessGroupIds } from '../support/signal-cleanup' const adminUrl = process.env.E2E_PG_ADMIN_URL const databaseName = process.env.E2E_DATABASE_NAME -const processGroupIds = (process.env.E2E_CLEANUP_PROCESS_GROUPS ?? '') - .split(',') - .map(Number) - .filter(Number.isInteger) +const processGroupIds = parseProcessGroupIds(process.env.E2E_CLEANUP_PROCESS_GROUPS) const sensitiveDirectories = JSON.parse(process.env.E2E_CLEANUP_DIRECTORIES ?? '[]') as string[] +const databaseCreationComplete = process.env.E2E_DATABASE_CREATION_COMPLETE === 'true' if (!adminUrl || !databaseName) { console.error('Signal cleanup requires E2E_PG_ADMIN_URL and E2E_DATABASE_NAME') @@ -20,7 +19,11 @@ signalProcessGroups(processGroupIds, 'SIGTERM', failures) await sleep(500) try { - await dropRunDatabaseWithRetries(adminUrl, databaseName) + if (databaseCreationComplete) { + await dropRunDatabase(adminUrl, databaseName) + } else { + await dropRunDatabaseWithRetries(adminUrl, databaseName) + } } catch (error) { failures.push(error) } @@ -45,6 +48,7 @@ function signalProcessGroups( failures: unknown[] ): void { for (const groupId of groupIds) { + if (!Number.isInteger(groupId) || groupId <= 0) continue try { if (process.platform !== 'win32') process.kill(-groupId, signal) else process.kill(groupId, signal) diff --git a/apps/sim/e2e/support/database.ts b/apps/sim/e2e/support/database.ts index d2e3747f2e8..8284e707791 100644 --- a/apps/sim/e2e/support/database.ts +++ b/apps/sim/e2e/support/database.ts @@ -95,5 +95,26 @@ export async function dropRunDatabaseWithRetries( } await sleep(intervalMs) } while (Date.now() < deadline) - if (lastError) throw lastError + if (await runDatabaseExists(adminUrl, databaseName)) { + throw ( + lastError ?? + new Error(`Guarded E2E database still exists after forced-drop retry window: ${databaseName}`) + ) + } +} + +async function runDatabaseExists(adminUrl: string, databaseName: string): Promise { + assertSafeDatabaseName(databaseName) + assertLoopbackPostgresUrl(adminUrl) + const sql = postgres(adminUrl, { max: 1, connect_timeout: 10, onnotice: () => {} }) + try { + const rows = await sql>` + SELECT EXISTS( + SELECT 1 FROM pg_database WHERE datname = ${databaseName} + ) AS "exists" + ` + return rows[0]?.exists ?? false + } finally { + await sql.end() + } } diff --git a/apps/sim/e2e/support/signal-cleanup.ts b/apps/sim/e2e/support/signal-cleanup.ts new file mode 100644 index 00000000000..18fcfe1e0d8 --- /dev/null +++ b/apps/sim/e2e/support/signal-cleanup.ts @@ -0,0 +1,8 @@ +export function parseProcessGroupIds(rawValue: string | undefined): number[] { + return (rawValue ?? '') + .split(',') + .map((value) => value.trim()) + .filter(Boolean) + .map(Number) + .filter((value) => Number.isInteger(value) && value > 0) +} From d3c370345aa2e6bacf348851b5904ce9b44c22c8 Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Mon, 20 Jul 2026 18:57:51 -0700 Subject: [PATCH 07/10] test(e2e): require exact host and race-safe signals Reject dual-stack host mappings that cannot reach IPv4-only services and tolerate child exit races during process-group signal fallback. --- apps/sim/e2e/foundation/safety.spec.ts | 3 ++- apps/sim/e2e/support/hosts.ts | 6 ++---- apps/sim/e2e/support/process.ts | 8 +++++++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/apps/sim/e2e/foundation/safety.spec.ts b/apps/sim/e2e/foundation/safety.spec.ts index 57021dd43f7..56ec19f27c0 100644 --- a/apps/sim/e2e/foundation/safety.spec.ts +++ b/apps/sim/e2e/foundation/safety.spec.ts @@ -83,7 +83,8 @@ test.describe('foundation safety guards', () => { expect(isLoopbackAddress('127.10.20.30')).toBe(true) expect(isLoopbackAddress('::1')).toBe(true) expect(isLoopbackAddress('8.8.8.8')).toBe(false) - expect(areValidE2eHostAddresses(['127.0.0.1', '::1'])).toBe(true) + expect(areValidE2eHostAddresses(['127.0.0.1'])).toBe(true) + expect(areValidE2eHostAddresses(['127.0.0.1', '::1'])).toBe(false) expect(areValidE2eHostAddresses(['::1'])).toBe(false) expect(areValidE2eHostAddresses(['127.0.0.2'])).toBe(false) }) diff --git a/apps/sim/e2e/support/hosts.ts b/apps/sim/e2e/support/hosts.ts index 81e6127755c..40588361a0f 100644 --- a/apps/sim/e2e/support/hosts.ts +++ b/apps/sim/e2e/support/hosts.ts @@ -12,9 +12,7 @@ export function isLoopbackAddress(address: string): boolean { } export function areValidE2eHostAddresses(addresses: string[]): boolean { - return ( - addresses.length > 0 && addresses.every(isLoopbackAddress) && addresses.includes('127.0.0.1') - ) + return addresses.length > 0 && addresses.every((address) => address === '127.0.0.1') } export async function assertE2eHostResolvesToLoopback(hostname = E2E_HOST): Promise { @@ -35,7 +33,7 @@ export async function assertE2eHostResolvesToLoopback(hostname = E2E_HOST): Prom function getHostMappingError(hostname: string, addresses: string[]): string { const observed = addresses.length > 0 ? addresses.join(', ') : 'no addresses' return [ - `${hostname} must resolve only to loopback and include IPv4 127.0.0.1; observed ${observed}.`, + `${hostname} must resolve exclusively to IPv4 127.0.0.1; observed ${observed}.`, `Add it once with: echo "127.0.0.1 ${hostname}" | sudo tee -a /etc/hosts`, ].join(' ') } diff --git a/apps/sim/e2e/support/process.ts b/apps/sim/e2e/support/process.ts index 378b47afdd8..5b1cd0d5538 100644 --- a/apps/sim/e2e/support/process.ts +++ b/apps/sim/e2e/support/process.ts @@ -170,7 +170,13 @@ function sendSignalToPids(processIds: number[], signal: NodeJS.Signals): void { const code = (error as NodeJS.ErrnoException).code if (code === 'ESRCH') continue if (code === 'EPERM' && process.platform !== 'win32') { - process.kill(processId, signal) + try { + process.kill(processId, signal) + } catch (fallbackError) { + if ((fallbackError as NodeJS.ErrnoException).code !== 'ESRCH') { + throw fallbackError + } + } continue } throw error From df04b3e78643b0f8bf70d06f4d85df8f03a2d0a2 Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Mon, 20 Jul 2026 19:07:36 -0700 Subject: [PATCH 08/10] test(e2e): force IPv4 for hosted test origin Accept safe dual-stack loopback resolution while pinning Chromium and Node traffic to 127.0.0.1 so CI reaches IPv4-only services reliably. --- apps/sim/e2e/README.md | 4 +++- apps/sim/e2e/foundation/safety.spec.ts | 2 +- apps/sim/e2e/support/deployment-profile.ts | 2 +- apps/sim/e2e/support/hosts.ts | 6 ++++-- apps/sim/playwright.config.ts | 3 +++ 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/apps/sim/e2e/README.md b/apps/sim/e2e/README.md index 8241e2431ff..49f108e8dd7 100644 --- a/apps/sim/e2e/README.md +++ b/apps/sim/e2e/README.md @@ -16,7 +16,9 @@ per-run pgvector database. echo "127.0.0.1 e2e.sim.ai" | sudo tee -a /etc/hosts ``` - The runner refuses to start unless every resolved address is loopback. + The runner refuses to start unless every resolved address is loopback and an + IPv4 `127.0.0.1` result is present. Chromium and Node are configured to prefer + that IPv4 mapping, so CI environments that also synthesize `::1` remain safe. 2. Start a local pgvector/Postgres admin instance: diff --git a/apps/sim/e2e/foundation/safety.spec.ts b/apps/sim/e2e/foundation/safety.spec.ts index 56ec19f27c0..fab6bc0118e 100644 --- a/apps/sim/e2e/foundation/safety.spec.ts +++ b/apps/sim/e2e/foundation/safety.spec.ts @@ -84,7 +84,7 @@ test.describe('foundation safety guards', () => { expect(isLoopbackAddress('::1')).toBe(true) expect(isLoopbackAddress('8.8.8.8')).toBe(false) expect(areValidE2eHostAddresses(['127.0.0.1'])).toBe(true) - expect(areValidE2eHostAddresses(['127.0.0.1', '::1'])).toBe(false) + expect(areValidE2eHostAddresses(['127.0.0.1', '::1'])).toBe(true) expect(areValidE2eHostAddresses(['::1'])).toBe(false) expect(areValidE2eHostAddresses(['127.0.0.2'])).toBe(false) }) diff --git a/apps/sim/e2e/support/deployment-profile.ts b/apps/sim/e2e/support/deployment-profile.ts index a52ad6aca3d..badcd7bb054 100644 --- a/apps/sim/e2e/support/deployment-profile.ts +++ b/apps/sim/e2e/support/deployment-profile.ts @@ -63,7 +63,7 @@ export function createHostedBillingProfile({ }: HostedBillingProfileOptions): HostedBillingProfile { const values: Record = { NODE_ENV: 'production', - NODE_OPTIONS: '--no-warnings --max-old-space-size=8192', + NODE_OPTIONS: '--no-warnings --max-old-space-size=8192 --dns-result-order=ipv4first', NEXT_TELEMETRY_DISABLED: '1', HOME: homeDirectory, XDG_CONFIG_HOME: `${homeDirectory}/xdg`, diff --git a/apps/sim/e2e/support/hosts.ts b/apps/sim/e2e/support/hosts.ts index 40588361a0f..81e6127755c 100644 --- a/apps/sim/e2e/support/hosts.ts +++ b/apps/sim/e2e/support/hosts.ts @@ -12,7 +12,9 @@ export function isLoopbackAddress(address: string): boolean { } export function areValidE2eHostAddresses(addresses: string[]): boolean { - return addresses.length > 0 && addresses.every((address) => address === '127.0.0.1') + return ( + addresses.length > 0 && addresses.every(isLoopbackAddress) && addresses.includes('127.0.0.1') + ) } export async function assertE2eHostResolvesToLoopback(hostname = E2E_HOST): Promise { @@ -33,7 +35,7 @@ export async function assertE2eHostResolvesToLoopback(hostname = E2E_HOST): Prom function getHostMappingError(hostname: string, addresses: string[]): string { const observed = addresses.length > 0 ? addresses.join(', ') : 'no addresses' return [ - `${hostname} must resolve exclusively to IPv4 127.0.0.1; observed ${observed}.`, + `${hostname} must resolve only to loopback and include IPv4 127.0.0.1; observed ${observed}.`, `Add it once with: echo "127.0.0.1 ${hostname}" | sudo tee -a /etc/hosts`, ].join(' ') } diff --git a/apps/sim/playwright.config.ts b/apps/sim/playwright.config.ts index 37d1a66fe74..70cb2641644 100644 --- a/apps/sim/playwright.config.ts +++ b/apps/sim/playwright.config.ts @@ -29,6 +29,9 @@ export default defineConfig({ use: { ...devices['Desktop Chrome'], baseURL, + launchOptions: { + args: ['--host-resolver-rules=MAP e2e.sim.ai 127.0.0.1'], + }, actionTimeout: 15_000, navigationTimeout: 30_000, trace: 'retain-on-failure', From 47661af932f27aaa942ed0d1d0533c0a3176849d Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Mon, 20 Jul 2026 19:32:40 -0700 Subject: [PATCH 09/10] test(e2e): settle readiness exit race Mark successful readiness before the managed process completion branch can reject, preventing later normal shutdown from surfacing an abandoned promise failure. --- apps/sim/e2e/foundation/safety.spec.ts | 32 +++++++++++++++++++++++++- apps/sim/e2e/support/process.ts | 7 +++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/apps/sim/e2e/foundation/safety.spec.ts b/apps/sim/e2e/foundation/safety.spec.ts index fab6bc0118e..261180d16ed 100644 --- a/apps/sim/e2e/foundation/safety.spec.ts +++ b/apps/sim/e2e/foundation/safety.spec.ts @@ -12,7 +12,11 @@ import { } from '../support/database' import { buildChildEnvironment, discoverEnvFileKeys } from '../support/env' import { areValidE2eHostAddresses, isLoopbackAddress } from '../support/hosts' -import { assertPortAvailable, spawnManagedProcess } from '../support/process' +import { + assertPortAvailable, + spawnManagedProcess, + waitForManagedProcessReady, +} from '../support/process' import { parseProcessGroupIds } from '../support/signal-cleanup' test.describe('foundation safety guards', () => { @@ -145,4 +149,30 @@ test.describe('foundation safety guards', () => { rmSync(logsDirectory, { recursive: true, force: true }) } }) + + test('process exit after readiness does not create an unhandled rejection', async () => { + const logsDirectory = mkdtempSync(path.join(os.tmpdir(), 'sim-e2e-ready-')) + let unhandled: unknown + const onUnhandled = (error: unknown) => { + unhandled = error + } + process.once('unhandledRejection', onUnhandled) + try { + const managed = spawnManagedProcess({ + name: 'ready-process', + command: 'sleep', + args: ['10'], + cwd: logsDirectory, + env: { NODE_ENV: 'test', PATH: process.env.PATH ?? '' }, + logsDirectory, + }) + await waitForManagedProcessReady(managed, async () => {}) + await managed.stop() + await new Promise((resolve) => setImmediate(resolve)) + expect(unhandled).toBeUndefined() + } finally { + process.off('unhandledRejection', onUnhandled) + rmSync(logsDirectory, { recursive: true, force: true }) + } + }) }) diff --git a/apps/sim/e2e/support/process.ts b/apps/sim/e2e/support/process.ts index 5b1cd0d5538..406502275af 100644 --- a/apps/sim/e2e/support/process.ts +++ b/apps/sim/e2e/support/process.ts @@ -107,14 +107,19 @@ export async function waitForManagedProcessReady( } const controller = new AbortController() + let ready = false const exited = process.completion.then((result) => { + if (ready) return throw new Error( `${process.name} exited before readiness with ${result.error?.message ?? result.code ?? result.signal ?? 'unknown status'}. See ${process.logPath}` ) }) + const becameReady = readiness(controller.signal).then(() => { + ready = true + }) try { - await Promise.race([readiness(controller.signal), exited]) + await Promise.race([becameReady, exited]) } finally { controller.abort(new Error(`${process.name} readiness polling cancelled`)) } From 84fe21102719e0a14eb4ad3cd2179ea36d09016a Mon Sep 17 00:00:00 2001 From: Bill Leoutsakos Date: Mon, 20 Jul 2026 19:37:51 -0700 Subject: [PATCH 10/10] test(e2e): propagate IPv4 preference to probes Keep Node-based Playwright requests on the same IPv4 path as Chromium and use direct loopback for orchestrator probes under dual-stack CI DNS. --- apps/sim/e2e/foundation/runtime.spec.ts | 1 + apps/sim/e2e/scripts/run.ts | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/sim/e2e/foundation/runtime.spec.ts b/apps/sim/e2e/foundation/runtime.spec.ts index a3a394d107e..0e49d712bb7 100644 --- a/apps/sim/e2e/foundation/runtime.spec.ts +++ b/apps/sim/e2e/foundation/runtime.spec.ts @@ -4,4 +4,5 @@ test('Playwright workers run on Node 22', () => { expect(process.release.name).toBe('node') expect(Number(process.versions.node.split('.')[0])).toBe(22) expect(process.execPath.toLowerCase()).not.toContain('bun') + expect(process.env.NODE_OPTIONS).toContain('--dns-result-order=ipv4first') }) diff --git a/apps/sim/e2e/scripts/run.ts b/apps/sim/e2e/scripts/run.ts index 02184512205..9a1f102e921 100644 --- a/apps/sim/e2e/scripts/run.ts +++ b/apps/sim/e2e/scripts/run.ts @@ -199,7 +199,10 @@ async function main(): Promise { await buildApp(stackOptions) realtime = await startRealtime(stackOptions) app = await startApp(stackOptions) - await assertAdminApiBoundary(E2E_ORIGIN, profile.childEnvironment.env.ADMIN_API_KEY) + await assertAdminApiBoundary( + 'http://127.0.0.1:3000', + profile.childEnvironment.env.ADMIN_API_KEY + ) assertNoForbiddenProviderInitialization([app.logPath, realtime.logPath]) const playwrightEnvironment = createPlaywrightEnvironment( @@ -293,7 +296,7 @@ function createPlaywrightEnvironment( storageStateDirectory: string, markerDirectory: string ): Record { - const keys = [...E2E_OS_PASSTHROUGH_KEYS, 'HOME', 'PLAYWRIGHT_BROWSERS_PATH'] + const keys = [...E2E_OS_PASSTHROUGH_KEYS, 'HOME', 'NODE_OPTIONS', 'PLAYWRIGHT_BROWSERS_PATH'] const env: Record = {} for (const key of keys) { const value = stackEnvironment[key]