diff --git a/.github/CI.md b/.github/CI.md index 5b8d9994ea9..15b06c5be45 100644 --- a/.github/CI.md +++ b/.github/CI.md @@ -36,6 +36,8 @@ Closing a same-repository pull request deletes only its GitHub merge-ref caches Memory-heavy builds and browser jobs retain appropriately sized Blacksmith runners. Electron uses 4-vCPU Blacksmith: both shards passed within 14 seconds of the 8-vCPU control; the free GitHub rest shard was slower and its lease run stalled. +WebKit acceptance on Blacksmith uses the matching, digest-pinned Playwright browser image for mobile and docs navigation. The app, database fixtures, native modules, and harness stay on the host and retain loopback URLs. Chromium and GitHub fallback keep native browser installation and launch. Update both workspace Playwright dependencies, the image index digest, and the expected client version/revision checks together; verify the installed release's native launch timeout and real WebKit acceptance before changing the pin. + Every HTTP and browser group consumes the same production build in an isolated job/database. Electron also uses that build and skips its development-route warm-up. Local HTTP and Electron development runs retain their dev-server behavior. Build inputs exclude test-only files so a harness edit can reuse an unchanged production build. Desktop inbox protocol, assistant actions UI, and organization search UI run as three parallel HTTP jobs with isolated databases, apps, and reports. Each harness owns its acceptance task, so an edit selects its own job. The inbox protocol task depends on server and realtime code; the browser tasks also depend on workspace UI. Desktop inbox needs no browser. The UI jobs and browser extension install Chromium while their app starts, waiting for both before executing assertions; browser-install failures still fail the job and preserve the install log. No acceptance assertions are removed by the split. diff --git a/.github/scripts/http-e2e.sh b/.github/scripts/http-e2e.sh index 4c11adbaad2..e05cfbf3d09 100755 --- a/.github/scripts/http-e2e.sh +++ b/.github/scripts/http-e2e.sh @@ -318,10 +318,19 @@ JS export INTERNAL_API_SECRET=mobile-http-ci-local-secret-at-least-32-characters export NEXT_PUBLIC_SOCKET_URL=http://127.0.0.1:3025 SOCKET_SERVER_URL=http://127.0.0.1:3025 read -ra browsers <<< "${MOBILE_E2E_BROWSER:-chromium webkit}" - start_browser_install mobile --only-shell "${browsers[@]}" + if [ "${PLAYWRIGHT_WEBKIT_WS_ENDPOINT+x}" = x ]; then + if [ "${MOBILE_E2E_BROWSER:-}" != webkit ]; then + echo "::error::A remote endpoint requires explicit WebKit selection." >&2 + exit 1 + fi + else + start_browser_install mobile --only-shell "${browsers[@]}" + fi start_app mobile 3024 'mobile browser' record-http-status start_realtime mobile 3025 - wait_for_browsers + if [ "${PLAYWRIGHT_WEBKIT_WS_ENDPOINT+x}" != x ]; then + wait_for_browsers + fi MOBILE_E2E_BASE_URL="$NEXT_PUBLIC_APP_URL" \ MOBILE_E2E_DATABASE_URL="$DATABASE_URL" \ MOBILE_E2E_AUTH_SECRET="$BETTER_AUTH_SECRET" \ diff --git a/.github/scripts/playwright-server.sh b/.github/scripts/playwright-server.sh new file mode 100755 index 00000000000..284ce77c63b --- /dev/null +++ b/.github/scripts/playwright-server.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# Owns one pinned WebKit server; start prints its loopback endpoint, stop preserves its logs. +set -euo pipefail +umask 077 + +action=${1:-} +state_argument=${2:-} +if [ "$#" -ne 2 ] || { [ "$action" != start ] && [ "$action" != stop ]; }; then + echo 'Usage: playwright-server.sh ' >&2 + exit 2 +fi +: "${RUNNER_TEMP:?RUNNER_TEMP must be set}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY must be set}" +: "${GITHUB_RUN_ID:?GITHUB_RUN_ID must be set}" +: "${GITHUB_RUN_ATTEMPT:?GITHUB_RUN_ATTEMPT must be set}" +: "${GITHUB_JOB:?GITHUB_JOB must be set}" +if ! [[ "$GITHUB_RUN_ID" =~ ^[1-9][0-9]*$ && "$GITHUB_RUN_ATTEMPT" =~ ^[1-9][0-9]*$ && "$GITHUB_JOB" =~ ^[a-zA-Z0-9_-]+$ ]]; then + echo 'Invalid GitHub job ownership metadata.' >&2 + exit 2 +fi +state=$(node - "$RUNNER_TEMP" "$state_argument" <<'JS' +const fs = require('node:fs') +const path = require('node:path') +const root = fs.realpathSync(process.argv[2]) +const state = path.resolve(process.argv[3]) +if (path.dirname(state) !== root || (fs.existsSync(state) && fs.realpathSync(state) !== state)) + throw new Error('State must be a direct, non-symlink child of RUNNER_TEMP') +process.stdout.write(state) +JS +) +state_key=$(node -e 'process.stdout.write(require("node:crypto").createHash("sha256").update(process.argv[1]).digest("hex"))' "$state") +image='mcr.microsoft.com/playwright@sha256:2f05f8a734509a7b3f26a912639488ee08f58a7b292d14d62271bcff9038768f' +starting=false + +stop_container() { + local cid inspection remaining status=0 + [ -f "$state/cid" ] || return 0 + cid=$(< "$state/cid") + if ! [[ "$cid" =~ ^[0-9a-f]{64}$ ]]; then + echo 'Refusing to stop an invalid container ID.' >&2 + return 1 + fi + if ! inspection=$(docker container inspect "$cid" 2> "$state/inspect-error.log"); then + remaining=$(docker container ls --all --no-trunc --filter "id=$cid" --format '{{.ID}}') || return 1 + [ -z "$remaining" ] && return 0 + cat "$state/inspect-error.log" >&2 + return 1 + fi + if ! jq -e --arg repository "$GITHUB_REPOSITORY" --arg run "$GITHUB_RUN_ID" \ + --arg attempt "$GITHUB_RUN_ATTEMPT" --arg job "$GITHUB_JOB" --arg state "$state_key" \ + '.[0].Config.Labels | .["com.sim.ci.playwright-server"] == "true" + and .["com.sim.ci.repository"] == $repository and .["com.sim.ci.run"] == $run + and .["com.sim.ci.attempt"] == $attempt and .["com.sim.ci.job"] == $job + and .["com.sim.ci.state"] == $state' <<< "$inspection" > /dev/null; then + echo 'Refusing to stop a container owned by another job or state directory.' >&2 + return 1 + fi + docker stop --timeout 10 "$cid" > "$state/stop.log" 2>&1 || status=1 + docker logs "$cid" > "$state/server.log" 2>&1 || status=1 + docker rm "$cid" > "$state/remove.log" 2>&1 || status=1 + return "$status" +} + +trap ' + status=$? + trap - EXIT + if [ "$status" -ne 0 ] && [ "$starting" = true ]; then + stop_container || echo "Owned Playwright server cleanup failed; preserve its state and logs." >&2 + fi + exit "$status" +' EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +if [ "$action" = stop ]; then + stop_container + exit 0 +fi +if [ -e "$state/cid" ]; then + echo 'Refusing to reuse an existing Playwright server state.' >&2 + exit 1 +fi +mkdir -p -- "$state" +repo_root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd) +driver=$(node - "$repo_root/apps/sim" <<'JS' +const fs = require('node:fs') +const path = require('node:path') +const packagePath = require.resolve('playwright-core/package.json', { paths: [process.argv[2]] }) +const root = fs.realpathSync(path.dirname(packagePath)) +const manifest = JSON.parse(fs.readFileSync(path.join(root, 'browsers.json'), 'utf8')) +if (JSON.parse(fs.readFileSync(packagePath, 'utf8')).version !== '1.64.1' || + manifest.browsers.find(browser => browser.name === 'webkit')?.revision !== '2370') + throw new Error('The pinned WebKit image requires Playwright 1.64.1, revision 2370') +process.stdout.write(root) +JS +) +docker pull "$image" > "$state/pull.log" 2>&1 +starting=true +docker run --detach --init --cidfile "$state/cid" \ + --name "sim-ci-playwright-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-$GITHUB_JOB-$$" \ + --label com.sim.ci.playwright-server=true --label "com.sim.ci.repository=$GITHUB_REPOSITORY" \ + --label "com.sim.ci.run=$GITHUB_RUN_ID" --label "com.sim.ci.attempt=$GITHUB_RUN_ATTEMPT" \ + --label "com.sim.ci.job=$GITHUB_JOB" --label "com.sim.ci.state=$state_key" \ + --network host --cpus 8 --memory 4g --memory-swap 4g --pids-limit 512 --shm-size 256m \ + --mount "type=bind,src=$driver,dst=/opt/playwright-core,readonly" \ + "$image" node /opt/playwright-core/cli.js run-server --host 127.0.0.1 --port 0 \ + > "$state/start.log" 2>&1 +cid=$(< "$state/cid") +if ! [[ "$cid" =~ ^[0-9a-f]{64}$ ]]; then + echo 'Docker did not return a complete container ID.' >&2 + exit 1 +fi +docker container inspect "$cid" > "$state/metadata.json" +deadline=$((SECONDS + 30)) +while [ "$SECONDS" -lt "$deadline" ]; do + live=$(docker container inspect --format '{{.State.Running}} {{.State.Pid}}' "$cid") + if ! [[ "$live" =~ ^true\ [1-9][0-9]*$ ]]; then + docker logs "$cid" > "$state/server.log" 2>&1 + echo 'Playwright server exited before readiness.' >&2 + exit 1 + fi + docker logs "$cid" > "$state/server.log" 2>&1 + endpoint=$(sed -n 's@^Listening on \(ws://127\.0\.0\.1:[1-9][0-9]*/\)$@\1@p' "$state/server.log") + if [ -n "$endpoint" ]; then + node - "$endpoint" <<'JS' +const endpoint = process.argv[2] +if (!/^ws:\/\/127\.0\.0\.1:[1-9]\d*\/$/.test(endpoint)) throw new Error('Invalid server endpoint') +new URL(endpoint) +process.stdout.write(`${endpoint}\n`) +JS + exit 0 + fi + sleep 0.1 +done +echo 'Playwright server did not report a live loopback endpoint within 30 seconds.' >&2 +exit 1 diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ac0541fbcd6..160d48097bc 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -376,11 +376,43 @@ jobs: - *extract-e2e-build - name: Run end-to-end suites + id: acceptance working-directory: apps/sim env: MOBILE_E2E_BROWSER: ${{ matrix.browser }} MOBILE_E2E_SHARD: ${{ matrix.shard }} - run: bash ../../.github/scripts/http-e2e.sh "${{ matrix.group }}" + run: | + unset PLAYWRIGHT_WEBKIT_WS_ENDPOINT + if [ "${{ matrix.browser == 'webkit' && (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') }}" = true ]; then + webkit_state="$RUNNER_TEMP/playwright-webkit-server" + trap ' + status=$? + trap - EXIT INT TERM + if ! bash ../../.github/scripts/playwright-server.sh stop "$webkit_state"; then + echo "::error::Owned WebKit server cleanup failed." >&2 + if [ "$status" -eq 0 ]; then status=1; fi + fi + exit "$status" + ' EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + PLAYWRIGHT_WEBKIT_WS_ENDPOINT=$(bash ../../.github/scripts/playwright-server.sh start "$webkit_state") + export PLAYWRIGHT_WEBKIT_WS_ENDPOINT + fi + bash ../../.github/scripts/http-e2e.sh "${{ matrix.group }}" + + - name: Stop the owned WebKit server + if: always() && matrix.browser == 'webkit' && (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && steps.acceptance.outcome != 'skipped' + run: bash .github/scripts/playwright-server.sh stop "$RUNNER_TEMP/playwright-webkit-server" + + - name: Upload WebKit server failure logs + if: failure() && matrix.browser == 'webkit' && (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: webkit-server-${{ matrix.artifact }} + path: ${{ runner.temp }}/playwright-webkit-server/ + if-no-files-found: ignore + retention-days: 7 - name: Upload end-to-end reports and server logs if: failure() diff --git a/.github/workflows/docs-negotiation.yml b/.github/workflows/docs-negotiation.yml index 7172496217e..0fcdf596276 100644 --- a/.github/workflows/docs-negotiation.yml +++ b/.github/workflows/docs-negotiation.yml @@ -19,6 +19,7 @@ on: - '.github/actions/cache/**' - '.github/actions/setup/**' - '.github/scripts/stop-session.sh' + - '.github/scripts/playwright-server.sh' push: branches: [staging] paths: @@ -38,6 +39,7 @@ on: - '.github/actions/cache/**' - '.github/actions/setup/**' - '.github/scripts/stop-session.sh' + - '.github/scripts/playwright-server.sh' workflow_dispatch: inputs: benchmark: @@ -370,14 +372,17 @@ jobs: go-version: '1.25.x' cache: false - name: Install the selected navigation browser - if: matrix.group == 'chromium' || matrix.group == 'webkit' + if: matrix.group == 'chromium' || (matrix.group == 'webkit' && vars.CI_PROVIDER != '' && vars.CI_PROVIDER != 'blacksmith') env: BROWSER: ${{ matrix.group }} run: bunx --no-install playwright install --with-deps --only-shell "$BROWSER" - name: Verify production documentation + id: acceptance env: DOCS_ACCEPTANCE_GROUP: ${{ matrix.group }} run: | + unset PLAYWRIGHT_WEBKIT_WS_ENDPOINT + webkit_state='' mkdir -p apps/docs/test-results FEEDBACK_TLS_DIR=$(mktemp -d) export DOCS_FEEDBACK_TEST_CERT="$FEEDBACK_TLS_DIR/cert.pem" @@ -392,16 +397,27 @@ jobs: DOCS_SERVER_PID=$! finish() { local status=$1 - trap - EXIT + trap - EXIT INT TERM if bash .github/scripts/stop-session.sh "$DOCS_SERVER_PID" "$APP_TAG"; then wait "$DOCS_SERVER_PID" 2>/dev/null || true else - status=1 + if [ "$status" -eq 0 ]; then status=1; fi + fi + if [ -n "$webkit_state" ] && ! bash .github/scripts/playwright-server.sh stop "$webkit_state"; then + echo '::error::Owned WebKit server cleanup failed.' >&2 + if [ "$status" -eq 0 ]; then status=1; fi fi rm -rf "$FEEDBACK_TLS_DIR" exit "$status" } trap 'finish "$?"' EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + if [ "${{ matrix.group == 'webkit' && (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') }}" = true ]; then + webkit_state="$RUNNER_TEMP/playwright-webkit-server" + PLAYWRIGHT_WEBKIT_WS_ENDPOINT=$(bash .github/scripts/playwright-server.sh start "$webkit_state") + export PLAYWRIGHT_WEBKIT_WS_ENDPOINT + fi for attempt in $(seq 1 60); do if ! kill -0 "$DOCS_SERVER_PID" 2>/dev/null; then echo '::error::Documentation server exited before becoming ready.' @@ -431,6 +447,17 @@ jobs: *) echo "::error::Unknown documentation acceptance group: $DOCS_ACCEPTANCE_GROUP"; STATUS=1 ;; esac finish "$STATUS" + - name: Stop the owned WebKit server + if: always() && matrix.group == 'webkit' && (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && steps.acceptance.outcome != 'skipped' + run: bash .github/scripts/playwright-server.sh stop "$RUNNER_TEMP/playwright-webkit-server" + - name: Upload WebKit server failure logs + if: failure() && matrix.group == 'webkit' && (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: webkit-server-docs + path: ${{ runner.temp }}/playwright-webkit-server/ + if-no-files-found: ignore + retention-days: 14 - name: Upload acceptance regression reports if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 diff --git a/apps/docs/scripts/test-navigation-e2e.ts b/apps/docs/scripts/test-navigation-e2e.ts index e73c1f29036..b817e80c227 100644 --- a/apps/docs/scripts/test-navigation-e2e.ts +++ b/apps/docs/scripts/test-navigation-e2e.ts @@ -1,4 +1,5 @@ import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { createRequire } from 'node:module' import { dirname, join } from 'node:path' import { type Browser, @@ -30,6 +31,16 @@ const canonicalOrigin = new URL(process.env.NEXT_PUBLIC_DOCS_URL ?? 'https://doc const engine = process.env.DOCS_NAVIGATION_BROWSER ?? 'chromium' if (engine !== 'chromium' && engine !== 'webkit') throw new Error('DOCS_NAVIGATION_BROWSER must be chromium or webkit') +const webkitEndpoint = process.env.PLAYWRIGHT_WEBKIT_WS_ENDPOINT +if (webkitEndpoint !== undefined) { + if (engine !== 'webkit' || !/^ws:\/\/127\.0\.0\.1:[1-9]\d*\/$/.test(webkitEndpoint)) + throw new Error('A remote endpoint requires explicit WebKit selection and a loopback URL') + new URL(webkitEndpoint) + if ( + toRecord(createRequire(import.meta.url)('@playwright/test/package.json')).version !== '1.64.1' + ) + throw new Error('The remote WebKit image requires Playwright 1.64.1') +} const results: Array<{ name: string status: 'passed' | 'failed' | 'unavailable' @@ -162,7 +173,10 @@ async function verifyClipboard( let browser: Browser | undefined try { - browser = await (engine === 'webkit' ? webkit : chromium).launch() + browser = + webkitEndpoint === undefined + ? await (engine === 'webkit' ? webkit : chromium).launch() + : await webkit.connect(webkitEndpoint, { timeout: 180_000 }) const feedbackBrowser = browser await check('Feedback forwards votes and preserves unchanged retry identity', async () => { const cert = process.env.DOCS_FEEDBACK_TEST_CERT diff --git a/apps/sim/scripts/test-mobile-e2e.ts b/apps/sim/scripts/test-mobile-e2e.ts index 698dd952e4c..0209b908728 100644 --- a/apps/sim/scripts/test-mobile-e2e.ts +++ b/apps/sim/scripts/test-mobile-e2e.ts @@ -1,5 +1,6 @@ import assert from 'node:assert/strict' import { mkdir, writeFile } from 'node:fs/promises' +import { createRequire } from 'node:module' import { dirname, resolve } from 'node:path' import { type Browser, @@ -13,13 +14,15 @@ import { import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateId, generateShortId } from '@sim/utils/id' +import { toRecord } from '@sim/utils/object' import { makeSignature } from 'better-auth/crypto' import postgres from 'postgres' /** * Real browser coverage against a running local app and disposable PostgreSQL. * Requires MOBILE_E2E_BASE_URL, MOBILE_E2E_DATABASE_URL, MOBILE_E2E_AUTH_SECRET, - * and MOBILE_E2E_REPORT_PATH. Install Chromium and WebKit with Playwright first. + * and MOBILE_E2E_REPORT_PATH. Native launches require installed Playwright browsers. + * PLAYWRIGHT_WEBKIT_WS_ENDPOINT uses the pinned loopback WebKit server instead. * MOBILE_E2E_BROWSER and MOBILE_E2E_VIEWPORT narrow a diagnostic rerun. * MOBILE_E2E_SHARD (index/count, one-based) partitions all viewports across CI jobs. */ @@ -32,6 +35,19 @@ function requiredEnvironment(name: string): string { return value } +const selectedBrowser = process.env.MOBILE_E2E_BROWSER +const webkitEndpoint = process.env.PLAYWRIGHT_WEBKIT_WS_ENDPOINT +if (webkitEndpoint !== undefined) { + assert.equal(selectedBrowser, 'webkit', 'A remote endpoint requires explicit WebKit selection') + assert(/^ws:\/\/127\.0\.0\.1:[1-9]\d*\/$/.test(webkitEndpoint), 'Invalid WebKit endpoint') + new URL(webkitEndpoint) + assert.equal( + toRecord(createRequire(import.meta.url)('@playwright/test/package.json')).version, + '1.64.1', + 'The remote WebKit image requires Playwright 1.64.1' + ) +} + const baseUrl = new URL(requiredEnvironment('MOBILE_E2E_BASE_URL')) const databaseUrl = new URL(requiredEnvironment('MOBILE_E2E_DATABASE_URL')) const authSecret = requiredEnvironment('MOBILE_E2E_AUTH_SECRET') @@ -98,7 +114,6 @@ const viewports = [ { name: 'wide-desktop', width: 1920, height: 1080, touch: false }, ] as const const routes = ['home', 'files', 'tables', 'knowledge', 'logs', 'integrations', 'settings/general'] -const selectedBrowser = process.env.MOBILE_E2E_BROWSER const selectedViewport = process.env.MOBILE_E2E_VIEWPORT assert(!selectedBrowser || ['chromium', 'webkit'].includes(selectedBrowser), 'Unknown browser') assert( @@ -1281,7 +1296,10 @@ try { if (selectedBrowser && selectedBrowser !== browserName) continue let browser: Browser | undefined await check(`${browserName}/browser starts`, undefined, async () => { - browser = await browserType.launch() + browser = + webkitEndpoint === undefined + ? await browserType.launch() + : await browserType.connect(webkitEndpoint, { timeout: 180_000 }) }) if (!browser) continue try {