diff --git a/.github/workflows/docker-build-editor-worker.yml b/.github/workflows/docker-build-editor-worker.yml new file mode 100644 index 00000000000..d9cfc1cfd20 --- /dev/null +++ b/.github/workflows/docker-build-editor-worker.yml @@ -0,0 +1,164 @@ +name: Docker Build Editor Worker + +on: + pull_request: + paths: + - "apps/media-server/**" + - "apps/web/lib/editor-caption-access.ts" + - "apps/web/lib/editor-caption-transport.ts" + - "apps/web/lib/editor-recording-title.ts" + - "apps/desktop/src/assets/music/**" + - "apps/desktop/src-tauri/assets/backgrounds/**" + - "crates/**" + - "packages/editor-cap-bundle/**" + - "packages/editor-solid-web/**" + - "vendor/**" + - "Cargo.toml" + - "Cargo.lock" + - "package.json" + - "bun.lock" + - "bunfig.toml" + - ".github/workflows/docker-build-editor-worker.yml" + push: + branches: + - main + paths: + - "apps/media-server/**" + - "apps/web/lib/editor-caption-access.ts" + - "apps/web/lib/editor-caption-transport.ts" + - "apps/web/lib/editor-recording-title.ts" + - "apps/desktop/src/assets/music/**" + - "apps/desktop/src-tauri/assets/backgrounds/**" + - "crates/**" + - "packages/editor-cap-bundle/**" + - "packages/editor-solid-web/**" + - "vendor/**" + - "Cargo.toml" + - "Cargo.lock" + - "package.json" + - "bun.lock" + - "bunfig.toml" + - ".github/workflows/docker-build-editor-worker.yml" + +permissions: {} + +jobs: + build: + name: Build and test editor worker (${{ matrix.platform }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - platform: amd64 + runner: ubuntu-24.04 + - platform: arm64 + runner: ubuntu-24.04-arm + permissions: + contents: read + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + - name: Set up Docker Buildx v3.12.0 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f + - name: Build editor worker image with Docker v6.19.2 + uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 + with: + context: . + file: apps/media-server/Dockerfile.editor + platforms: linux/${{ matrix.platform }} + load: true + tags: cap-editor-worker:verification + cache-from: type=gha,scope=editor-worker-${{ matrix.platform }} + cache-to: type=gha,mode=max,scope=editor-worker-${{ matrix.platform }} + - name: Check native worker readiness + run: | + docker run --rm --network none \ + -e CAP_WEB_EDITOR_PUBLIC_ORIGIN=http://127.0.0.1:3457 \ + -e MEDIA_SERVER_WEBHOOK_SECRET=test \ + --entrypoint bun cap-editor-worker:verification \ + -e 'import app from "./src/editor-worker-app.ts"; const response = await app.request("/health"); if (!response.ok) process.exit(1)' + - name: Exercise paired native editor routes + run: | + docker run --rm --network none --cpus 4 --memory 4g \ + -e MEDIA_SERVER_WEBHOOK_SECRET=test \ + --entrypoint bun cap-editor-worker:verification test \ + src/__tests__/editor-worker-app.test.ts \ + src/__tests__/lib/editor-assets.test.ts \ + src/__tests__/lib/editor-clip-thumbnails.test.ts \ + src/__tests__/lib/editor-config-paths.test.ts \ + src/__tests__/lib/editor-image-assets.test.ts \ + src/__tests__/lib/editor-security.test.ts \ + src/__tests__/lib/editor-wallpapers.test.ts \ + src/__tests__/routes/editor-clips.test.ts \ + src/__tests__/routes/editor.test.ts + - name: Verify editor subprocess credential isolation + run: | + docker run --rm --network none \ + -e MEDIA_SERVER_WEBHOOK_SECRET=test \ + -e AWS_SECRET_ACCESS_KEY=test \ + --entrypoint bun cap-editor-worker:verification \ + src/__tests__/lib/editor-process-env-replay.ts + - name: Verify cursor-only MOV export + run: | + docker run --rm --network none --cpus 4 --memory 4g \ + --entrypoint bun cap-editor-worker:verification \ + src/__tests__/routes/editor-cursor-mov-replay.ts + - name: Replay browser-only Studio and worker exports + if: matrix.platform == 'amd64' + run: | + docker run --rm --cpus 4 --memory 8g --ipc host \ + -v "${{ github.workspace }}:/source" \ + -w /source/apps/media-server \ + cap-editor-worker:verification \ + bash -e -u -o pipefail -c ' + cd /source + bun install --frozen-lockfile + bunx playwright install --with-deps chromium webkit firefox + apt-get install -y --no-install-recommends xauth + bun run --filter @cap/editor-solid-web build + cd /source/apps/media-server + export CAP_EDITOR_UI_PARITY_ARTIFACT_DIR=/source/cap-editor-parity-artifacts + run_browser_only_studio() { + case "$CAP_EDITOR_UI_BROWSER" in + firefox|webkit) CAP_EDITOR_UI_HEADED=1 xvfb-run -a bun src/__tests__/routes/editor-solid-browser-only-replay.ts ;; + *) bun src/__tests__/routes/editor-solid-browser-only-replay.ts ;; + esac + } + run_worker_studio() { + case "$CAP_EDITOR_UI_BROWSER" in + firefox|webkit) CAP_EDITOR_UI_HEADED=1 xvfb-run -a bun run test:native-editor-ui ;; + *) bun run test:native-editor-ui ;; + esac + } + browser_replay_failed=0 + for browser in chromium firefox webkit; do + if ! CAP_EDITOR_UI_PARITY_ARTIFACT_DIR=/source/cap-editor-parity-artifacts CAP_EDITOR_UI_BROWSER="$browser" run_browser_only_studio; then + browser_replay_failed=1 + fi + done + for browser in chromium webkit firefox; do + for pro in 0 1; do + CAP_EDITOR_UI_SHARE_REPLAY=1 CAP_EDITOR_UI_BROWSER="$browser" CAP_EDITOR_UI_PRO_CAPTIONS="$pro" run_worker_studio + done + done + for repeat in 2 3; do + printf "WebKit Free replay %s\n" "$repeat" + CAP_EDITOR_UI_SHARE_REPLAY=1 CAP_EDITOR_UI_BROWSER=webkit CAP_EDITOR_UI_PRO_CAPTIONS=0 run_worker_studio + done + CAP_EDITOR_UI_CURSOR_MOV=1 CAP_EDITOR_UI_BROWSER=chromium CAP_EDITOR_UI_PRO_CAPTIONS=0 run_worker_studio + CAP_EDITOR_UI_CURSOR_MOV=1 CAP_EDITOR_UI_BROWSER=webkit CAP_EDITOR_UI_PRO_CAPTIONS=1 run_worker_studio + CAP_EDITOR_UI_CANVAS_FALLBACK=1 CAP_EDITOR_UI_BROWSER=chromium CAP_EDITOR_UI_PRO_CAPTIONS=0 run_worker_studio + CAP_EDITOR_UI_CANVAS_FALLBACK=1 CAP_EDITOR_UI_BROWSER=webkit CAP_EDITOR_UI_PRO_CAPTIONS=1 run_worker_studio + bun src/__tests__/routes/editor-presets-browser-replay.ts + if [ "$browser_replay_failed" -ne 0 ]; then exit 1; fi + ' + - name: Upload Studio parity diagnostics + if: failure() && matrix.platform == 'amd64' + uses: actions/upload-artifact@v4 + with: + name: web-studio-parity-diagnostics + path: cap-editor-parity-artifacts + if-no-files-found: ignore diff --git a/.github/workflows/recording-reliability.yml b/.github/workflows/recording-reliability.yml index 74838a71b43..796ac4d5f36 100644 --- a/.github/workflows/recording-reliability.yml +++ b/.github/workflows/recording-reliability.yml @@ -4,21 +4,31 @@ on: workflow_dispatch: pull_request: paths: + - "apps/chrome-extension/**" - "apps/web/**" - "apps/media-server/**" + - "packages/editor-solid-web/**" + - "packages/editor-cap-bundle/**" + - "packages/recorder-core/**" - "packages/database/**" - "packages/web-*/**" - "crates/recording/**" + - "crates/rendering/src/shaders/**" - "bun.lock" - ".github/workflows/recording-reliability.yml" push: branches: [main] paths: + - "apps/chrome-extension/**" - "apps/web/**" - "apps/media-server/**" + - "packages/editor-solid-web/**" + - "packages/editor-cap-bundle/**" + - "packages/recorder-core/**" - "packages/database/**" - "packages/web-*/**" - "crates/recording/**" + - "crates/rendering/src/shaders/**" - "bun.lock" - ".github/workflows/recording-reliability.yml" @@ -30,17 +40,208 @@ concurrency: cancel-in-progress: true jobs: + browser-cap-import: + name: Web editor recording folder import (Chromium/WebKit) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - uses: ./.github/actions/setup-js + - name: Install Playwright browsers + run: bunx playwright install --with-deps chromium webkit + - name: Replay browser recording import + run: bun run --filter @cap/media-server test:browser-cap-import + - name: Replay long captions in Chromium and WebKit + run: bun apps/media-server/src/__tests__/routes/editor-caption-cache-browser-replay.ts + + browser-studio-compositor: + name: Web Studio local compositor (${{ matrix.browser }}) + strategy: + fail-fast: false + matrix: + include: + - browser: chromium + runner: ubuntu-24.04 + headed: "0" + - browser: firefox + runner: ubuntu-24.04 + headed: "1" + - browser: webkit + runner: macos-15 + headed: "1" + runs-on: ${{ matrix.runner }} + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - uses: ./.github/actions/setup-js + - name: Install browser on Ubuntu + if: matrix.browser != 'webkit' + run: bunx playwright install --with-deps ${{ matrix.browser }} + - name: Install WebKit on macOS + if: matrix.browser == 'webkit' + run: bunx playwright install webkit + - name: Build browser compositor replay + run: bunx vite build --config packages/editor-solid-web/browser-replay/vite.config.ts + - name: Replay separate screen and camera clips on the local GPU + if: matrix.browser != 'firefox' + run: bun packages/editor-solid-web/browser-replay/replay.cjs ${{ matrix.browser }} + env: + CAP_REPLAY_HEADED: ${{ matrix.headed }} + - name: Check screen and camera frame parity at 8x + if: matrix.browser != 'firefox' + run: bun packages/editor-solid-web/browser-replay/replay.cjs ${{ matrix.browser }} + env: + CAP_REPLAY_HEADED: ${{ matrix.headed }} + CAP_REPLAY_INDEXED: "1" + - name: Replay Firefox clips through software WebGL under Xvfb + if: matrix.browser == 'firefox' + run: xvfb-run -a bun packages/editor-solid-web/browser-replay/replay.cjs firefox + env: + CAP_REPLAY_HEADED: ${{ matrix.headed }} + - name: Check Firefox screen and camera frame parity at 8x + if: matrix.browser == 'firefox' + run: xvfb-run -a bun packages/editor-solid-web/browser-replay/replay.cjs firefox + env: + CAP_REPLAY_HEADED: ${{ matrix.headed }} + CAP_REPLAY_INDEXED: "1" + - name: Stress continuous HTML video playback on a slow decoder + run: | + if [ "${{ matrix.browser }}" = "firefox" ]; then + xvfb-run -a bun packages/editor-solid-web/browser-replay/replay.cjs firefox + else + bun packages/editor-solid-web/browser-replay/replay.cjs ${{ matrix.browser }} + fi + env: + CAP_REPLAY_HEADED: ${{ matrix.headed }} + CAP_REPLAY_FORCE_HTML: "1" + CAP_REPLAY_SLOW_HTML: "1" + + browser-studio-webgpu: + name: Web Studio GPU auto-selection (Chromium Intel macOS) + runs-on: macos-15-intel + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - uses: ./.github/actions/setup-js + - name: Install Chromium on macOS + run: bunx playwright install chromium + - name: Build browser compositor replay + run: bunx vite build --config packages/editor-solid-web/browser-replay/vite.config.ts + - name: Replay paired clips with visible GPU output + run: bun packages/editor-solid-web/browser-replay/replay.cjs chromium + env: + CAP_REPLAY_HEADED: "1" + CAP_REPLAY_REQUIRE_WEBGPU: "1" + CAP_REPLAY_ALLOW_AUTO_FALLBACK: "1" + - name: Check visible screen and camera frame parity at 8x + run: bun packages/editor-solid-web/browser-replay/replay.cjs chromium + env: + CAP_REPLAY_HEADED: "1" + CAP_REPLAY_INDEXED: "1" + CAP_REPLAY_REQUIRE_WEBGPU: "1" + CAP_REPLAY_ALLOW_AUTO_FALLBACK: "1" + + browser-studio-intel-webgl: + name: Web Studio WebGL2 fallback (Chromium Intel macOS) + runs-on: macos-15-intel + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - uses: ./.github/actions/setup-js + - name: Install Chromium on macOS + run: bunx playwright install chromium + - name: Build browser compositor replay + run: bunx vite build --config packages/editor-solid-web/browser-replay/vite.config.ts + - name: Check visible WebGL2 fallback and paired 8x frame parity + run: bun packages/editor-solid-web/browser-replay/replay.cjs chromium + env: + CAP_REPLAY_HEADED: "1" + CAP_REPLAY_INDEXED: "1" + CAP_REPLAY_ONLY_WEBGL: "1" + + browser-studio-arm-webgpu: + name: Web Studio hardware WebGPU and probe fallback (Chromium arm64 macOS) + runs-on: macos-15 + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - uses: ./.github/actions/setup-js + - name: Install Chromium on macOS + run: bunx playwright install chromium + - name: Build browser compositor replay + run: bunx vite build --config packages/editor-solid-web/browser-replay/vite.config.ts + - name: Require healthy WebGPU playback + run: bun packages/editor-solid-web/browser-replay/replay.cjs chromium + env: + CAP_REPLAY_HEADED: "1" + CAP_REPLAY_REQUIRE_WEBGPU: "1" + - name: Require WebGL2 after a failed presentation probe + run: bun packages/editor-solid-web/browser-replay/replay.cjs chromium + env: + CAP_REPLAY_HEADED: "1" + CAP_REPLAY_SIMULATE_FAILED_PROBE: "1" + + browser-paired-recording: + name: Web recorder paired capture (${{ matrix.browser }}) + strategy: + fail-fast: false + matrix: + include: + - browser: chromium + runner: ubuntu-24.04 + headed: false + - browser: webkit + runner: macos-15-intel + headed: true + runs-on: ${{ matrix.runner }} + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - uses: ./.github/actions/setup-js + - name: Install Chromium + if: matrix.browser == 'chromium' + run: bunx playwright install --with-deps chromium + - name: Install WebKit + if: matrix.browser == 'webkit' + run: bunx playwright install webkit + - name: Replay paired web recorder capture + working-directory: apps/web + run: bun __tests__/web-recorder-paired-browser-replay.mjs ${{ matrix.browser }} + env: + CAP_REPLAY_HEADED: ${{ matrix.headed }} + + extension-paired-recording: + name: Chrome extension paired capture and recovery (Chromium) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - uses: ./.github/actions/setup-js + - name: Install Chromium + run: bunx playwright install --with-deps chromium + - name: Build Chrome extension + run: bun run --filter @cap/chrome-extension build + - name: Verify private tab input is omitted + working-directory: apps/chrome-extension + run: bun src/content/input-events-browser-replay.ts + - name: Replay paired capture and recovery in the built extension + working-directory: apps/chrome-extension + run: | + bunx playwright test e2e/recording-upload.spec.ts \ + --grep 'records a separate camera sidecar|keeps screen-only Instant recordings working without a webcam|recovers an offscreen start response lost|offers separate clip downloads|recovers camera, microphone and screen spools|uploads a camera multipart part during a long recording on a constrained network|hides confirmed uploaded backups from the options recovery list' + recording-contract: name: Recording ownership, preservation, and recovery runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - uses: ./.github/actions/setup-js - name: Verify recording contracts working-directory: apps/web run: | bun run vitest run \ + __tests__/editor-image-file.test.ts \ __tests__/unit/audio-level-publication.test.ts \ __tests__/unit/recording-output.test.ts \ __tests__/unit/recording-storage-lifecycle.test.ts \ @@ -57,3 +258,5 @@ jobs: __tests__/unit/upload-progress-playback.test.ts \ __tests__/unit/recording-prepare.test.ts \ __tests__/unit/s3-bucket-connections.test.ts + - name: Verify browser Solid caption cache + run: bun --conditions=browser test packages/editor-solid-web/src/tauri-bridge.test.ts diff --git a/.github/workflows/windows-targets.yml b/.github/workflows/windows-targets.yml index aebfc965560..3f03e63f743 100644 --- a/.github/workflows/windows-targets.yml +++ b/.github/workflows/windows-targets.yml @@ -44,12 +44,23 @@ jobs: $source = "crates/scap-targets/src/platform/win.rs" $patchedSource = [System.IO.File]::ReadAllBytes($source) try { - $baseline = git show "$($env:BASE_SHA):$source" - if ($LASTEXITCODE -ne 0) { throw "Could not load base Windows implementation" } - $baseline | Set-Content -Encoding utf8 $source - cargo test --locked -p scap-targets --test windows_targets -- --nocapture - $baselineExit = $LASTEXITCODE - "Base implementation discovery test exit code: $baselineExit (0 means this fixture already passed on the base)." | Add-Content $env:GITHUB_STEP_SUMMARY + git cat-file -e "$($env:BASE_SHA)^{commit}" 2>$null + if ($LASTEXITCODE -ne 0) { + git fetch --no-tags --depth=1 origin $env:BASE_SHA + if ($LASTEXITCODE -ne 0) { throw "Could not fetch PR base commit" } + } + $baseSource = git ls-tree $env:BASE_SHA -- $source + if ($LASTEXITCODE -ne 0) { throw "Could not inspect base Windows implementation" } + if ($baseSource) { + $baseline = git show "$($env:BASE_SHA):$source" + if ($LASTEXITCODE -ne 0) { throw "Could not load base Windows implementation" } + $baseline | Set-Content -Encoding utf8 $source + cargo test --locked -p scap-targets --test windows_targets -- --nocapture + $baselineExit = $LASTEXITCODE + "Base implementation discovery test exit code: $baselineExit (0 means this fixture already passed on the base)." | Add-Content $env:GITHUB_STEP_SUMMARY + } else { + "Base implementation discovery test skipped because $source is absent from the PR base." | Add-Content $env:GITHUB_STEP_SUMMARY + } } finally { [System.IO.File]::WriteAllBytes($source, $patchedSource) } diff --git a/Cargo.lock b/Cargo.lock index ffb628ea4f4..882cb66c181 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1406,6 +1406,7 @@ dependencies = [ "ort", "tokio", "tracing", + "web-time", "wgpu", "workspace-hack", ] @@ -1657,6 +1658,7 @@ dependencies = [ "futures", "image 0.25.8", "lru", + "relative-path", "ringbuf", "sentry", "serde", @@ -1667,6 +1669,7 @@ dependencies = [ "tokio-util", "tracing", "tracing-subscriber", + "uuid", "windows 0.60.0", "workspace-hack", ] @@ -2027,6 +2030,7 @@ dependencies = [ "futures-intrusive", "glyphon", "image 0.25.8", + "js-sys", "libloading 0.8.8", "log", "metal 0.31.0", @@ -2044,6 +2048,10 @@ dependencies = [ "tiny-skia", "tokio", "tracing", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "web-time", "wgpu", "wgpu-core", "wgpu-hal", @@ -2073,6 +2081,7 @@ name = "cap-rnnoise" version = "0.1.0" dependencies = [ "cc", + "libm", "workspace-hack", ] diff --git a/Cargo.toml b/Cargo.toml index 8a9be0ffb64..e8d0146f07a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -109,6 +109,18 @@ opt-level = 2 [profile.dev.package.cap-audio] opt-level = 2 +[profile.dev.package.cap-editor] +opt-level = 3 + +[profile.dev.package.image] +opt-level = 3 + +[profile.dev.package.png] +opt-level = 3 + +[profile.dev.package.flate2] +opt-level = 3 + # Optimize for smaller binary size [profile.release] panic = "unwind" diff --git a/apps/chrome-extension/e2e/overlay-ui.spec.ts b/apps/chrome-extension/e2e/overlay-ui.spec.ts index 307609c637a..542dc6d7216 100644 --- a/apps/chrome-extension/e2e/overlay-ui.spec.ts +++ b/apps/chrome-extension/e2e/overlay-ui.spec.ts @@ -202,7 +202,7 @@ const launchExtensionContext = async () => { const userDataDir = await mkdtemp(path.join(tmpdir(), "cap-close-e2e-")); const context = await chromium.launchPersistentContext(userDataDir, { channel: "chromium", - headless: true, + headless: process.env.CAP_EXTENSION_E2E_HEADED !== "1", args: [ `--disable-extensions-except=${extensionPath}`, `--load-extension=${extensionPath}`, @@ -379,9 +379,9 @@ const getClosedShadowNodeId = async ( for (let index = 0; index < attributes.length; index += 2) { if (attributes[index] !== attribute) continue; if (!classToken) return true; - return ( - attributes[index + 1]?.split(/\s+/).includes(classToken) === true - ); + return attribute === "class" + ? attributes[index + 1]?.split(/\s+/).includes(classToken) === true + : attributes[index + 1] === classToken; } return false; })?.nodeId ?? null @@ -484,7 +484,7 @@ test("idle bootstrap and tab activation never open the camera preview", async () } }); -test("dismissing or closing the panel also closes the idle camera preview", async () => { +test("closing extension UI removes the ready bar and camera preview", async () => { test.setTimeout(120_000); const mockServer = await createMockCapServer(); const extension = await launchExtensionContext(); @@ -502,30 +502,63 @@ test("dismissing or closing the panel also closes the idle camera preview", asyn await targetPage.goto(`${mockServer.origin}/capture.html`); await targetPage.bringToFront(); - await sendServiceWorkerMessage(messengerPage, { + const openResponse = await sendServiceWorkerMessage(messengerPage, { target: "service-worker", type: "open-recorder-panel", }); - + expect(openResponse).toMatchObject({ ok: true }); + const activeTabUrl = await worker.evaluate(async () => { + const tabs = await chrome.tabs.query({ + active: true, + currentWindow: true, + }); + return tabs[0]?.url; + }); + expect(activeTabUrl).toBe(targetPage.url()); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") !== null, { - timeout: 15_000, - }) + .poll(() => + worker.evaluate(async () => { + const items = await chrome.storage.session.get( + "cap-extension-shared-ui-state", + ); + const state = items["cap-extension-shared-ui-state"] as + | { panelOpen?: boolean } + | undefined; + return state?.panelOpen ?? false; + }), + ) .toBe(true); + + const devtools = await extension.context.newCDPSession(targetPage); + await expect + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + ), + ) + .not.toBeNull(); await expect .poll(() => frameWithUrl(targetPage, "camera-preview.html") !== null, { timeout: 15_000, }) .toBe(true); - await targetPage.screenshot({ path: "test-results/close-ui-before.png" }); - await targetPage.mouse.click(400, 300); - + const closeResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "close-extension-ui", + }); + expect(closeResponse).toMatchObject({ ok: true }); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") === null, { - timeout: 10_000, - }) - .toBe(true); + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + ), + ) + .toBeNull(); await expect .poll(() => frameWithUrl(targetPage, "camera-preview.html") === null, { timeout: 10_000, @@ -537,36 +570,52 @@ test("dismissing or closing the panel also closes the idle camera preview", asyn type: "open-recorder-panel", }); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") !== null, { - timeout: 15_000, - }) - .toBe(true); + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + ), + ) + .not.toBeNull(); await expect .poll(() => frameWithUrl(targetPage, "camera-preview.html") !== null, { timeout: 15_000, }) .toBe(true); - const panelFrame = frameWithUrl(targetPage, "popup.html"); - if (!panelFrame) throw new Error("panel frame missing"); - await panelFrame - .locator('button[aria-label="Close Cap and hide all recorder UI"]') - .click(); + const hideButton = await getClosedShadowElementBox( + devtools, + "aria-label", + "Hide recording bar", + ); + if (!hideButton) throw new Error("Hide recording bar button missing"); + await targetPage.mouse.click( + hideButton.x + hideButton.width / 2, + hideButton.y + hideButton.height / 2, + ); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") === null, { - timeout: 10_000, - }) - .toBe(true); + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + ), + ) + .toBeNull(); + expect(frameWithUrl(targetPage, "camera-preview.html")).not.toBeNull(); + await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "close-extension-ui", + }); await expect .poll(() => frameWithUrl(targetPage, "camera-preview.html") === null, { timeout: 10_000, }) .toBe(true); - await targetPage.waitForTimeout(2_500); expect(frameWithUrl(targetPage, "camera-preview.html")).toBeNull(); - await targetPage.screenshot({ path: "test-results/close-ui-after.png" }); } finally { await extension.cleanup(); await mockServer.close(); @@ -594,13 +643,17 @@ test("an abandoned recording start clears without leaving controls behind", asyn target: "service-worker", type: "open-recorder-panel", }); + const devtools = await extension.context.newCDPSession(targetPage); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") !== null, { - timeout: 15_000, - }) - .toBe(true); + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + ), + ) + .not.toBeNull(); - const devtools = await extension.context.newCDPSession(targetPage); await writeStaleCreatingState(worker); await expect.poll(() => readSessionRecordingPhase(worker)).toBe("creating"); await targetPage.waitForTimeout(500); @@ -608,7 +661,7 @@ test("an abandoned recording start clears without leaving controls behind", asyn await getClosedShadowElementBox( devtools, "class", - "cap-extension-recording-rail", + "cap-extension-active-recording-container", ), ).toBeNull(); @@ -623,15 +676,19 @@ test("an abandoned recording start clears without leaving controls behind", asyn expect(statusResponse.status?.phase).toBe("idle"); await expect.poll(() => readSessionRecordingPhase(worker)).toBe("idle"); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") === null, { - timeout: 10_000, - }) - .toBe(true); + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + ), + ) + .toBeNull(); expect( await getClosedShadowElementBox( devtools, "class", - "cap-extension-recording-rail", + "cap-extension-active-recording-container", ), ).toBeNull(); } finally { @@ -640,7 +697,7 @@ test("an abandoned recording start clears without leaving controls behind", asyn } }); -test("a failed recording start reopens the panel with the error", async () => { +test("a failed recording start shows the error on the recorded tab", async () => { test.setTimeout(120_000); const mockServer = await createMockCapServer({ failInstantRecordings: true }); const extension = await launchExtensionContext(); @@ -672,26 +729,35 @@ test("a failed recording start reopens the panel with the error", async () => { expect(startResponse.ok).toBe(false); expect(startResponse.canceled).toBeFalsy(); - // The status should be a visible error, not a silent reset to idle. const statusResponse = (await sendServiceWorkerMessage(messengerPage, { target: "service-worker", type: "get-recording-status", - })) as { status?: { phase?: string } }; + })) as { status?: { phase?: string; message?: string } }; expect(statusResponse.status?.phase).toBe("error"); - // The panel should reopen in the page to show the failure. + const devtools = await extension.context.newCDPSession(targetPage); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") !== null, { - timeout: 10_000, - }) - .toBe(true); - const panelFrame = frameWithUrl(targetPage, "popup.html"); - if (!panelFrame) throw new Error("panel frame missing"); - await expect(panelFrame.getByText("Recording failed.")).toBeVisible({ - timeout: 10_000, - }); + .poll(() => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-recording-error-toast", + ), + ) + .not.toBeNull(); + const toastNodeId = await getClosedShadowNodeId( + devtools, + "class", + "cap-extension-recording-error-toast", + ); + if (!toastNodeId) throw new Error("recording error toast missing"); + const { outerHTML } = (await devtools.send("DOM.getOuterHTML", { + nodeId: toastNodeId, + })) as { outerHTML: string }; + expect(outerHTML).toContain("Recording failed."); + expect(outerHTML).toContain(statusResponse.status?.message); await targetPage.screenshot({ - path: "test-results/start-error-panel.png", + path: "test-results/start-error-toast.png", }); } finally { await extension.cleanup(); @@ -724,9 +790,7 @@ test("the countdown appears while recording setup is still pending", async () => type: "start-recording", mode: "fullscreen", }); - await expect - .poll(mockServer.isInstantRecordingRequested, { timeout: 10_000 }) - .toBe(true); + void startPromise.catch(() => undefined); const devtools = await extension.context.newCDPSession(targetPage); await expect @@ -737,9 +801,12 @@ test("the countdown appears while recording setup is still pending", async () => "class", "cap-extension-countdown", )) !== null, - { timeout: 1_000 }, + { timeout: 5_000 }, ) .toBe(true); + await expect + .poll(mockServer.isInstantRecordingRequested, { timeout: 10_000 }) + .toBe(true); await targetPage.screenshot({ path: "test-results/countdown-during-setup.png", }); @@ -869,11 +936,77 @@ test("recording controls stay stable and the camera resizes directly", async () target: "service-worker", type: "open-recorder-panel", }); + const devtools = await extension.context.newCDPSession(targetPage); await expect - .poll(() => frameWithUrl(targetPage, "popup.html") !== null, { - timeout: 15_000, + .poll( + async () => + (await getClosedShadowElementBox( + devtools, + "class", + "cap-extension-control-bar", + )) !== null, + { timeout: 15_000 }, + ) + .toBe(true); + await expect + .poll(() => frameWithUrl(targetPage, "camera-preview.html") !== null, { + timeout: 10_000, }) .toBe(true); + const initialCameraBox = await getClosedShadowElementBox( + devtools, + "data-camera-preview", + ); + if (!initialCameraBox) throw new Error("camera preview missing"); + await targetPage.mouse.move( + initialCameraBox.x + initialCameraBox.width / 2, + initialCameraBox.y + initialCameraBox.height / 2, + ); + await expect + .poll(() => + getClosedShadowComputedStyle( + devtools, + "data-camera-resize-ne", + "opacity", + ), + ) + .toBe("1"); + const cameraResizeHandle = await getClosedShadowElementBox( + devtools, + "data-camera-resize-ne", + ); + if (!cameraResizeHandle) throw new Error("camera resize handle missing"); + await targetPage.screenshot({ + path: "test-results/camera-resize-handles.png", + }); + await targetPage.mouse.move( + cameraResizeHandle.x + cameraResizeHandle.width / 2, + cameraResizeHandle.y + cameraResizeHandle.height / 2, + ); + await targetPage.mouse.down(); + await targetPage.mouse.move( + cameraResizeHandle.x + cameraResizeHandle.width / 2 + 80, + cameraResizeHandle.y + cameraResizeHandle.height / 2 - 80, + { steps: 8 }, + ); + await targetPage.mouse.up(); + await expect + .poll(async () => { + const resizedCamera = await getClosedShadowElementBox( + devtools, + "data-camera-preview", + ); + return ( + resizedCamera !== null && + resizedCamera.width - initialCameraBox.width > 50 && + resizedCamera.height - initialCameraBox.height > 50 + ); + }) + .toBe(true); + await expect.poll(() => readStoredWebcamSize(worker)).toBeGreaterThan(230); + await targetPage.screenshot({ + path: "test-results/camera-resized.png", + }); const startResponse = (await sendServiceWorkerMessage(messengerPage, { target: "service-worker", @@ -882,104 +1015,86 @@ test("recording controls stay stable and the camera resizes directly", async () })) as { ok: boolean }; expect(startResponse.ok).toBe(true); await expect - .poll(() => frameWithUrl(targetPage, "camera-preview.html") !== null, { + .poll(() => frameWithUrl(targetPage, "camera-preview.html") === null, { timeout: 10_000, }) .toBe(true); - const devtools = await extension.context.newCDPSession(targetPage); await expect .poll( async () => (await getClosedShadowElementBox( devtools, "class", - "cap-extension-recording-rail", + "cap-extension-active-recording-container", )) !== null, { timeout: 10_000 }, ) .toBe(true); + await expect + .poll(async () => { + const box = await getClosedShadowElementBox( + devtools, + "class", + "cap-extension-active-recording-container", + ); + return box?.height ?? 0; + }) + .toBeGreaterThanOrEqual(47); const initialBox = await getClosedShadowElementBox( devtools, "class", - "cap-extension-recording-rail", + "cap-extension-active-recording-container", ); if (!initialBox) throw new Error("recording bar missing"); expect(initialBox.x).toBeLessThanOrEqual(20); - expect(initialBox.width).toBeGreaterThan(100); - expect(initialBox.width).toBeLessThanOrEqual(128); - expect(initialBox.height).toBeLessThanOrEqual(44); - const handleBox = await getClosedShadowElementBox( - devtools, - "data-drag-handle", - ); - if (!handleBox) throw new Error("recording bar drag handle missing"); + expect(initialBox.width).toBeGreaterThan(70); + expect(initialBox.height).toBeGreaterThanOrEqual(47); const timeBox = await getClosedShadowElementBox( devtools, - "data-recording-time", + "class", + "cap-extension-recording-badge-time", ); - if (!timeBox || timeBox.width < 30) { + if (!timeBox || timeBox.width < 20) { throw new Error("recording time is not visible at rest"); } - expect( - await getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "opacity", - ), - ).toBe("0"); - await targetPage.mouse.move( - handleBox.x + handleBox.width / 2, - handleBox.y + handleBox.height / 2, + initialBox.x + initialBox.width / 2, + initialBox.y + initialBox.height / 2, ); await expect - .poll(() => - getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "opacity", - ), + .poll( + async () => + (await getClosedShadowElementBox( + devtools, + "class", + "cap-extension-vertical-dock", + )) !== null, ) - .toBe("1"); - const actionsBox = await getClosedShadowElementBox( - devtools, - "data-recording-actions", - ); - if (!actionsBox) throw new Error("recording action capsule missing"); - const actionGap = actionsBox.x - (initialBox.x + initialBox.width); - expect(actionGap).toBeGreaterThanOrEqual(4); - expect(actionGap).toBeLessThanOrEqual(8); - expect( - await getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "border-top-left-radius", - ), - ).toBe("14px"); + .toBe(true); const pauseBox = await getClosedShadowElementBox( devtools, - "data-recording-pause", + "aria-label", + "Pause recording", ); - if (!pauseBox) throw new Error("pause action missing on hover"); + if (!pauseBox) throw new Error("pause control missing on hover"); await targetPage.mouse.move( pauseBox.x + pauseBox.width / 2, pauseBox.y + pauseBox.height / 2, ); - await targetPage.waitForTimeout(300); - expect( - await getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "opacity", - ), - ).toBe("1"); await targetPage.screenshot({ path: "test-results/recording-bar-expanded.png", }); + const pauseClickBox = await getClosedShadowElementBox( + devtools, + "aria-label", + "Pause recording", + ); + if (!pauseClickBox) + throw new Error("pause control disappeared before click"); await targetPage.mouse.click( - pauseBox.x + pauseBox.width / 2, - pauseBox.y + pauseBox.height / 2, + pauseClickBox.x + pauseClickBox.width / 2, + pauseClickBox.y + pauseClickBox.height / 2, ); await expect .poll(async () => { @@ -990,16 +1105,25 @@ test("recording controls stay stable and the camera resizes directly", async () return response.status?.phase; }) .toBe("paused"); - expect( - await getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "opacity", - ), - ).toBe("1"); + await expect + .poll( + async () => + (await getClosedShadowElementBox( + devtools, + "aria-label", + "Resume recording", + )) !== null, + ) + .toBe(true); + const resumeBox = await getClosedShadowElementBox( + devtools, + "aria-label", + "Resume recording", + ); + if (!resumeBox) throw new Error("resume control missing"); await targetPage.mouse.click( - pauseBox.x + pauseBox.width / 2, - pauseBox.y + pauseBox.height / 2, + resumeBox.x + resumeBox.width / 2, + resumeBox.y + resumeBox.height / 2, ); await expect .poll(async () => { @@ -1011,13 +1135,13 @@ test("recording controls stay stable and the camera resizes directly", async () }) .toBe("recording"); await targetPage.mouse.move( - handleBox.x + handleBox.width / 2, - handleBox.y + handleBox.height / 2, + initialBox.x + initialBox.width / 2, + initialBox.y + initialBox.height / 2, ); await targetPage.mouse.down(); await targetPage.mouse.move( - handleBox.x + handleBox.width / 2 + 120, - handleBox.y + handleBox.height / 2 - 80, + initialBox.x + initialBox.width / 2 + 120, + initialBox.y + initialBox.height / 2 - 80, { steps: 8 }, ); await targetPage.mouse.up(); @@ -1027,7 +1151,7 @@ test("recording controls stay stable and the camera resizes directly", async () const movedBox = await getClosedShadowElementBox( devtools, "class", - "cap-extension-recording-rail", + "cap-extension-active-recording-container", ); return ( movedBox !== null && @@ -1066,72 +1190,17 @@ test("recording controls stay stable and the camera resizes directly", async () await targetPage.mouse.move(760, 40); await expect - .poll(() => - getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "opacity", - ), - ) - .toBe("0"); - await targetPage.waitForTimeout(500); - await targetPage.screenshot({ - path: "test-results/recording-bar-visible.png", - }); - - const initialCameraBox = await getClosedShadowElementBox( - devtools, - "data-camera-preview", - ); - if (!initialCameraBox) throw new Error("camera preview missing"); - await targetPage.mouse.move( - initialCameraBox.x + initialCameraBox.width / 2, - initialCameraBox.y + initialCameraBox.height / 2, - ); - await expect - .poll(() => - getClosedShadowComputedStyle( - devtools, - "data-camera-resize-ne", - "opacity", - ), + .poll( + async () => + (await getClosedShadowElementBox( + devtools, + "class", + "cap-extension-vertical-dock", + )) === null, ) - .toBe("1"); - const cameraResizeHandle = await getClosedShadowElementBox( - devtools, - "data-camera-resize-ne", - ); - if (!cameraResizeHandle) throw new Error("camera resize handle missing"); - await targetPage.screenshot({ - path: "test-results/camera-resize-handles.png", - }); - await targetPage.mouse.move( - cameraResizeHandle.x + cameraResizeHandle.width / 2, - cameraResizeHandle.y + cameraResizeHandle.height / 2, - ); - await targetPage.mouse.down(); - await targetPage.mouse.move( - cameraResizeHandle.x + cameraResizeHandle.width / 2 + 80, - cameraResizeHandle.y + cameraResizeHandle.height / 2 - 80, - { steps: 8 }, - ); - await targetPage.mouse.up(); - await expect - .poll(async () => { - const resizedCamera = await getClosedShadowElementBox( - devtools, - "data-camera-preview", - ); - return ( - resizedCamera !== null && - resizedCamera.width - initialCameraBox.width > 50 && - resizedCamera.height - initialCameraBox.height > 50 - ); - }) .toBe(true); - await expect.poll(() => readStoredWebcamSize(worker)).toBeGreaterThan(230); await targetPage.screenshot({ - path: "test-results/camera-resized.png", + path: "test-results/recording-bar-visible.png", }); const stopResponse = (await sendServiceWorkerMessage(messengerPage, { diff --git a/apps/chrome-extension/e2e/recording-upload.spec.ts b/apps/chrome-extension/e2e/recording-upload.spec.ts index fcf73c51481..240979ea1eb 100644 --- a/apps/chrome-extension/e2e/recording-upload.spec.ts +++ b/apps/chrome-extension/e2e/recording-upload.spec.ts @@ -1,4 +1,4 @@ -import { mkdtemp, rm } from "node:fs/promises"; +import { cp, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { createServer, type IncomingMessage, @@ -7,6 +7,7 @@ import { import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { STREAMING_MP4_MIME_TYPES } from "@cap/recorder-core/recorder-constants"; import { type BrowserContext, chromium, @@ -16,6 +17,10 @@ import { } from "@playwright/test"; import type { RecordingStatus } from "../src/shared/types"; +let videoExtension = "webm"; +let rawSubpath = "raw-upload.webm"; +let cameraSubpath = "camera-upload.webm"; + type ChromeRuntimeResponse = | { ok: true; @@ -27,13 +32,23 @@ type ChromeRuntimeResponse = }; type MockState = { + abortBodies: unknown[]; completeBodies: unknown[]; progressBodies: unknown[]; initiateBodies: unknown[]; presignBodies: unknown[]; uploadBytes: number[]; + uploadBytesBySubpath: Record; + inputParts: Record; + completedPartsBySubpath: Record; uploadHeaders: Record[]; videoId: string; + simulateSlowCameraUpload: boolean; + failCameraCompletion: boolean; + failAudioCompletion: boolean; + failInputCompletion: boolean; + failScreenCompletion: boolean; + cameraInitiateDelayMs: number; }; type ChromeGlobal = typeof globalThis & { @@ -55,7 +70,9 @@ type ChromeGlobal = typeof globalThis & { }; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const extensionPath = path.resolve(__dirname, "../dist"); +const extensionPath = process.env.CAP_EXTENSION_E2E_DIR + ? path.resolve(process.env.CAP_EXTENSION_E2E_DIR) + : path.resolve(__dirname, "../dist"); const SETTINGS_KEY = "cap-extension-settings"; const AUTH_KEY = "cap-extension-auth"; const BOOTSTRAP_CACHE_KEY = "cap-extension-bootstrap-cache"; @@ -103,62 +120,38 @@ const sendHtml = (response: ServerResponse, html: string) => { const animatedCapturePage = () => ` - - Cap E2E Capture Target - - + Cap E2E Capture Target - - +
+
+
+ +
Custom input
+
PIN widget
+ + `; const createMockCapServer = async () => { const state: MockState = { + abortBodies: [], completeBodies: [], progressBodies: [], initiateBodies: [], presignBodies: [], uploadBytes: [], + uploadBytesBySubpath: {}, + inputParts: {}, + completedPartsBySubpath: {}, uploadHeaders: [], videoId: `e2e-${Date.now()}`, + simulateSlowCameraUpload: false, + failCameraCompletion: false, + failAudioCompletion: false, + failInputCompletion: false, + failScreenCompletion: false, + cameraInitiateDelayMs: 0, }; const server = createServer(async (request, response) => { @@ -215,14 +208,35 @@ const createMockCapServer = async () => { request.method === "POST" && url.pathname === "/api/upload/multipart/initiate" ) { - state.initiateBodies.push(await parseJsonBody(request)); + const body = await parseJsonBody(request); + state.initiateBodies.push(body); + if ( + state.cameraInitiateDelayMs > 0 && + body && + typeof body === "object" && + "subpath" in body && + body.subpath === cameraSubpath + ) { + await new Promise((resolve) => + setTimeout(resolve, state.cameraInitiateDelayMs), + ); + } sendJson(response, 200, { - uploadId: "upload-e2e", + uploadId: `upload-e2e-${state.initiateBodies.length}`, provider: "s3", }); return; } + if ( + request.method === "POST" && + url.pathname === "/api/upload/multipart/abort" + ) { + state.abortBodies.push(await parseJsonBody(request)); + sendJson(response, 200, { success: true }); + return; + } + if ( request.method === "POST" && url.pathname === "/api/upload/multipart/presign-part" @@ -237,7 +251,7 @@ const createMockCapServer = async () => { ? body.partNumber : state.presignBodies.length; sendJson(response, 200, { - presignedUrl: `${baseUrl()}/mock-s3/part-${partNumber}`, + presignedUrl: `${baseUrl()}/mock-s3/part-${partNumber}?subpath=${encodeURIComponent(String(body && typeof body === "object" && "subpath" in body ? body.subpath : "unknown"))}`, provider: "s3", }); return; @@ -246,13 +260,30 @@ const createMockCapServer = async () => { if (request.method === "PUT" && url.pathname.startsWith("/mock-s3/")) { const body = await readRequestBody(request); state.uploadBytes.push(body.byteLength); + const subpath = url.searchParams.get("subpath") ?? "unknown"; + const partNumber = Number(url.pathname.slice("/mock-s3/part-".length)); + if ( + subpath === "input-events-upload.ndjson" && + Number.isSafeInteger(partNumber) + ) { + state.inputParts[partNumber] = body; + } + state.uploadBytesBySubpath[subpath] = + (state.uploadBytesBySubpath[subpath] ?? 0) + body.byteLength; state.uploadHeaders.push(request.headers); + if (state.simulateSlowCameraUpload && subpath === cameraSubpath) { + await new Promise((resolve) => + setTimeout(resolve, Math.ceil(body.byteLength / 1_250)), + ); + } response.writeHead(200, { "Access-Control-Allow-Origin": "*", "Access-Control-Expose-Headers": "ETag", ETag: `"etag-${state.uploadBytes.length}"`, }); response.end(); + state.completedPartsBySubpath[subpath] = + (state.completedPartsBySubpath[subpath] ?? 0) + 1; return; } @@ -260,10 +291,57 @@ const createMockCapServer = async () => { request.method === "POST" && url.pathname === "/api/upload/multipart/complete" ) { - state.completeBodies.push(await parseJsonBody(request)); + const body = await parseJsonBody(request); + state.completeBodies.push(body); + if ( + state.failCameraCompletion && + !!body && + typeof body === "object" && + "subpath" in body && + body.subpath === cameraSubpath + ) { + sendJson(response, 400, { error: "Camera completion rejected" }); + return; + } + if ( + state.failAudioCompletion && + !!body && + typeof body === "object" && + "subpath" in body && + body.subpath === "mic-upload.webm" + ) { + sendJson(response, 400, { error: "Microphone completion rejected" }); + return; + } + if ( + state.failInputCompletion && + !!body && + typeof body === "object" && + "subpath" in body && + body.subpath === "input-events-upload.ndjson" + ) { + sendJson(response, 400, { + error: "Input events completion rejected", + }); + return; + } + if ( + state.failScreenCompletion && + !!body && + typeof body === "object" && + "subpath" in body && + body.subpath === rawSubpath + ) { + sendJson(response, 400, { error: "Screen completion rejected" }); + return; + } sendJson(response, 200, { success: true, - processingStarted: true, + processingStarted: + !!body && + typeof body === "object" && + "subpath" in body && + body.subpath === rawSubpath, }); return; } @@ -333,19 +411,34 @@ const createMockCapServer = async () => { const launchExtensionContext = async () => { const userDataDir = await mkdtemp(path.join(tmpdir(), "cap-extension-e2e-")); + const testExtensionPath = path.join(userDataDir, "extension"); + await cp(extensionPath, testExtensionPath, { recursive: true }); + const manifestPath = path.join(testExtensionPath, "manifest.json"); + const manifest: Record = JSON.parse( + await readFile(manifestPath, "utf8"), + ); + manifest.commands = { + _execute_action: { + suggested_key: { + default: "Ctrl+Shift+Y", + mac: "Command+Shift+Y", + }, + }, + }; + await writeFile(manifestPath, JSON.stringify(manifest)); const context = await chromium.launchPersistentContext(userDataDir, { channel: "chromium", - headless: true, + headless: process.env.CAP_EXTENSION_E2E_HEADED !== "1", args: [ - `--disable-extensions-except=${extensionPath}`, - `--load-extension=${extensionPath}`, + `--disable-extensions-except=${testExtensionPath}`, + `--load-extension=${testExtensionPath}`, "--allow-http-screen-capture", "--auto-select-desktop-capture-source=Cap E2E Capture Target", "--auto-select-tab-capture-source-by-title=Cap E2E Capture Target", "--enable-usermedia-screen-capturing", "--autoplay-policy=no-user-gesture-required", "--use-fake-device-for-media-stream", - "--use-fake-ui-for-media-stream", + "--auto-accept-camera-and-microphone-capture", ], }); const cleanup = async () => { @@ -370,6 +463,48 @@ const getServiceWorker = async (context: BrowserContext) => { const getExtensionId = (worker: Awaited>) => new URL(worker.url()).host; +type DevToolsNode = { + nodeId: number; + attributes?: string[]; + children?: DevToolsNode[]; + shadowRoots?: DevToolsNode[]; +}; + +const findCameraPreview = (node: DevToolsNode): DevToolsNode | null => { + if (node.attributes?.includes("data-camera-preview")) return node; + for (const child of [...(node.children ?? []), ...(node.shadowRoots ?? [])]) { + const result = findCameraPreview(child); + if (result) return result; + } + return null; +}; + +const expectCameraPreviewOutOfCapture = async (page: Page) => { + await expect(page.locator("#cap-extension-recorder-overlay")).toHaveAttribute( + "data-cap-mounted", + "true", + ); + const session = await page.context().newCDPSession(page); + try { + await session.send("DOM.enable"); + await expect + .poll(async () => { + const document = (await session.send("DOM.getDocument", { + depth: -1, + pierce: true, + })) as { root: DevToolsNode }; + const node = findCameraPreview(document.root); + if (!node?.attributes) return true; + const styleIndex = node.attributes.indexOf("style"); + if (styleIndex < 0) return false; + return /visibility:\s*hidden/.test(node.attributes[styleIndex + 1]); + }) + .toBe(true); + } finally { + await session.detach(); + } +}; + const openExtensionMessengerPage = async ( context: BrowserContext, worker: Awaited>, @@ -382,6 +517,8 @@ const openExtensionMessengerPage = async ( const configureExtension = async ( worker: Awaited>, apiBaseUrl: string, + microphoneEnabled = false, + webcamEnabled = true, ) => { await worker.evaluate( async ({ @@ -390,6 +527,8 @@ const configureExtension = async ( recordingMode, settingsKey, apiBaseUrl, + microphoneEnabled, + webcamEnabled, }) => { const chromeApi = (globalThis as ChromeGlobal).chrome; await new Promise((resolve, reject) => { @@ -434,15 +573,15 @@ const configureExtension = async ( microphone: null, }, webcam: { - enabled: false, - deviceId: null, + enabled: webcamEnabled, + deviceId: "__cap_default_camera__", position: "bottom-left", size: 230, shape: "round", mirror: false, }, microphone: { - enabled: false, + enabled: microphoneEnabled, deviceId: null, }, systemAudio: { @@ -477,6 +616,8 @@ const configureExtension = async ( bootstrapKey: BOOTSTRAP_CACHE_KEY, recordingMode: RECORDING_MODE, settingsKey: SETTINGS_KEY, + microphoneEnabled, + webcamEnabled, }, ); }; @@ -499,7 +640,59 @@ const sendServiceWorkerMessage = async ( }); }, message); -const expectSuccessfulUpload = async (page: Page, state: MockState) => { +const injectUncertainOffscreenStartResponse = async ( + worker: Awaited>, + deliverRequest: boolean, +) => { + await worker.evaluate((deliverRequest) => { + const runtime = (globalThis as ChromeGlobal).chrome.runtime; + const sendMessage = runtime.sendMessage.bind(runtime) as ( + message: unknown, + callback: (response: unknown) => void, + ) => void; + let injected = false; + Object.defineProperty(runtime, "sendMessage", { + configurable: true, + value: (message: unknown, callback: (response: unknown) => void) => { + if ( + injected || + typeof message !== "object" || + message === null || + !("target" in message) || + message.target !== "offscreen" || + !("type" in message) || + message.type !== "start-recording" + ) { + sendMessage(message, callback); + return; + } + injected = true; + const loseResponse = () => { + Object.defineProperty(runtime, "lastError", { + configurable: true, + value: { + message: + "The message port closed before a response was received.", + }, + }); + try { + callback(undefined); + } finally { + Reflect.deleteProperty(runtime, "lastError"); + } + }; + if (deliverRequest) sendMessage(message, loseResponse); + else globalThis.setTimeout(loseResponse, 0); + }, + }); + }, deliverRequest); +}; + +const expectSuccessfulUpload = async ( + page: Page, + state: MockState, + inputSidecar = false, +) => { await expect .poll(async () => { const response = await sendServiceWorkerMessage(page, { @@ -511,20 +704,30 @@ const expectSuccessfulUpload = async (page: Page, state: MockState) => { }) .toBe("completed"); - expect(state.initiateBodies).toHaveLength(1); + expect(state.initiateBodies).toHaveLength(inputSidecar ? 3 : 2); + expect(state.initiateBodies[0]).toMatchObject({ subpath: rawSubpath }); + expect(state.initiateBodies[1]).toMatchObject({ + subpath: cameraSubpath, + }); + if (inputSidecar) { + expect(state.initiateBodies[2]).toMatchObject({ + subpath: "input-events-upload.ndjson", + }); + } expect(state.presignBodies.length).toBeGreaterThanOrEqual(1); expect(state.uploadBytes.length).toBeGreaterThanOrEqual(1); expect( state.uploadBytes.reduce((total, bytes) => total + bytes, 0), ).toBeGreaterThan(0); - expect(state.completeBodies).toHaveLength(1); + expect(state.completeBodies).toHaveLength(inputSidecar ? 3 : 2); expect(state.progressBodies.length).toBeGreaterThanOrEqual(1); const completeBody = state.completeBodies[0]; expect(completeBody).toMatchObject({ videoId: state.videoId, - uploadId: "upload-e2e", - subpath: "raw-upload.webm", + uploadId: "upload-e2e-2", + subpath: cameraSubpath, + screenSubpath: rawSubpath, }); expect( completeBody && @@ -534,26 +737,243 @@ const expectSuccessfulUpload = async (page: Page, state: MockState) => { ? completeBody.parts.length : 0, ).toBeGreaterThanOrEqual(1); + const cameraOffsetMs = + completeBody && + typeof completeBody === "object" && + "cameraOffsetMs" in completeBody + ? completeBody.cameraOffsetMs + : null; + expect( + typeof cameraOffsetMs === "number" && Number.isFinite(cameraOffsetMs), + ).toBe(true); + if (inputSidecar) { + expect(state.completeBodies[1]).toMatchObject({ + subpath: "input-events-upload.ndjson", + screenSubpath: rawSubpath, + }); + expect( + state.uploadBytesBySubpath["input-events-upload.ndjson"], + ).toBeGreaterThan(0); + } + expect(state.completeBodies[inputSidecar ? 2 : 1]).toMatchObject({ + subpath: rawSubpath, + uploadId: "upload-e2e-1", + }); + expect(state.uploadBytesBySubpath[cameraSubpath]).toBeGreaterThan(0); + expect(state.uploadBytesBySubpath[rawSubpath]).toBeGreaterThan(0); }; +const readLiveCameraOffset = async ( + worker: Awaited>, +) => + worker.evaluate(async () => { + const key = "cap-extension-live-recordings"; + const items = await new Promise>((resolve) => + chrome.storage.local.get([key], (result) => resolve(result)), + ); + const manifests = items[key]; + if (!Array.isArray(manifests)) { + throw new Error(`Live manifest storage is ${typeof manifests}`); + } + const offset = (manifests[0] as { cameraOffsetMs?: unknown } | undefined) + ?.cameraOffsetMs; + if (typeof offset !== "number") { + throw new Error( + `Live manifests: ${manifests.length}; first camera offset: ${typeof offset}`, + ); + } + return offset; + }); + +const readLiveMicrophoneOffset = async ( + worker: Awaited>, +) => + worker.evaluate(async () => { + const key = "cap-extension-live-recordings"; + const items = await new Promise>((resolve) => + chrome.storage.local.get([key], (result) => resolve(result)), + ); + const manifests = items[key]; + const sources = Array.isArray(manifests) + ? ( + manifests[0] as { + audioSources?: Array<{ kind: string; offsetMs: number }>; + } + )?.audioSources + : undefined; + const offset = sources?.find((source) => source.kind === "mic")?.offsetMs; + if (typeof offset !== "number") { + throw new Error("The live microphone offset is unavailable"); + } + return offset; + }); + +const readLiveCameraSessionId = async ( + worker: Awaited>, +) => + worker.evaluate(async () => { + const key = "cap-extension-live-recordings"; + const items = await new Promise>((resolve) => + chrome.storage.local.get([key], (result) => resolve(result)), + ); + const manifests = items[key]; + const sessionId = Array.isArray(manifests) + ? (manifests[0] as { cameraSessionId?: unknown } | undefined) + ?.cameraSessionId + : null; + if (typeof sessionId !== "string") { + throw new Error("The live camera spool session is unavailable"); + } + return sessionId; + }); + +const deleteCameraSpoolSession = async ( + worker: Awaited>, + sessionId: string, +) => + worker.evaluate(async (sessionId) => { + const request = indexedDB.open("cap-recording-spool", 1); + const database = await new Promise((resolve, reject) => { + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); + const read = database.transaction("chunks", "readonly"); + const keysRequest = read + .objectStore("chunks") + .index("by-session") + .getAllKeys(IDBKeyRange.only(sessionId)); + const keys = await new Promise((resolve, reject) => { + keysRequest.onsuccess = () => resolve(keysRequest.result); + keysRequest.onerror = () => reject(keysRequest.error); + }); + const write = database.transaction(["sessions", "chunks"], "readwrite"); + const finished = new Promise((resolve, reject) => { + write.oncomplete = () => resolve(); + write.onerror = () => reject(write.error); + write.onabort = () => reject(write.error); + }); + write.objectStore("sessions").delete(sessionId); + const chunks = write.objectStore("chunks"); + for (const key of keys) chunks.delete(key); + await finished; + database.close(); + }, sessionId); + +const readFailedCameraRecovery = async ( + worker: Awaited>, +) => + worker.evaluate(async () => { + const key = "cap-extension-failed-recordings"; + const items = await new Promise>((resolve) => + chrome.storage.local.get([key], (result) => resolve(result)), + ); + const recordings = items[key]; + if (!Array.isArray(recordings)) return null; + const failed = recordings[0] as + | { + videoId?: unknown; + cameraSessionId?: unknown; + cameraSubpath?: unknown; + cameraOffsetMs?: unknown; + cameraRetryUnavailable?: unknown; + } + | undefined; + return failed + ? { + videoId: failed.videoId, + cameraSessionId: failed.cameraSessionId, + cameraSubpath: failed.cameraSubpath, + cameraOffsetMs: failed.cameraOffsetMs, + cameraRetryUnavailable: failed.cameraRetryUnavailable, + } + : null; + }); + +const hasLiveRecordingManifest = async ( + worker: Awaited>, +) => + worker.evaluate(async () => { + const key = "cap-extension-live-recordings"; + const items = await new Promise>((resolve) => + chrome.storage.local.get([key], (result) => resolve(result)), + ); + return Array.isArray(items[key]) && items[key].length > 0; + }); + const startRecording = async ( context: BrowserContext, worker: Awaited>, apiBaseUrl: string, + mode: "fullscreen" | "tab" = RECORDING_MODE, + recordingMs = RECORDING_MS, + microphoneEnabled = false, + webcamEnabled = true, ) => { - await configureExtension(worker, apiBaseUrl); + await configureExtension( + worker, + apiBaseUrl, + microphoneEnabled, + webcamEnabled, + ); const messengerPage = await openExtensionMessengerPage(context, worker); const capturePage = await context.newPage(); await capturePage.goto(`${apiBaseUrl}/capture.html`); await capturePage.bringToFront(); + await expect + .poll(async () => { + const tabs = await worker.evaluate(() => + chrome.tabs.query({ active: true, lastFocusedWindow: true }), + ); + return tabs[0]?.url; + }) + .toBe(`${apiBaseUrl}/capture.html`); + if (mode === "tab") { + await worker.evaluate(() => { + const target = globalThis as typeof globalThis & { + capE2eActionClicks?: number; + }; + target.capE2eActionClicks = 0; + chrome.action.onClicked.addListener(() => { + target.capE2eActionClicks = (target.capE2eActionClicks ?? 0) + 1; + }); + }); + } + if (mode === "tab") { + await expect + .poll( + () => + worker.evaluate( + () => + ( + globalThis as typeof globalThis & { + capE2eActionClicks?: number; + } + ).capE2eActionClicks ?? 0, + ), + { + message: + "Click the Cap extension action icon in the test Chromium window", + timeout: 60_000, + }, + ) + .toBeGreaterThan(0); + } const startResponse = await sendServiceWorkerMessage(messengerPage, { target: "service-worker", type: "start-recording", - mode: RECORDING_MODE, + mode, }); if (!startResponse.ok) { - throw new Error(startResponse.error); + const captures = await worker.evaluate( + () => + new Promise((resolve) => + chrome.tabCapture.getCapturedTabs(resolve), + ), + ); + throw new Error( + `${startResponse.error}; captured tabs: ${JSON.stringify(captures)}`, + ); } await expect @@ -567,7 +987,7 @@ const startRecording = async ( }) .toBe("recording"); - await capturePage.waitForTimeout(RECORDING_MS); + await capturePage.waitForTimeout(recordingMs); return { capturePage, messengerPage, @@ -575,6 +995,28 @@ const startRecording = async ( }; test.describe("extension recording upload", () => { + test.beforeAll(async () => { + const browser = await chromium.launch({ channel: "chromium" }); + try { + const page = await browser.newPage(); + const streamsMp4 = await page.evaluate( + (candidates) => + candidates.some((candidate) => + MediaRecorder.isTypeSupported(candidate), + ), + [ + ...STREAMING_MP4_MIME_TYPES.videoOnly, + ...STREAMING_MP4_MIME_TYPES.withAudio, + ], + ); + videoExtension = streamsMp4 ? "mp4" : "webm"; + rawSubpath = `raw-upload.${videoExtension}`; + cameraSubpath = `camera-upload.${videoExtension}`; + } finally { + await browser.close(); + } + }); + let mockServer: Awaited> | null = null; let extension: Awaited> | null = null; @@ -589,7 +1031,98 @@ test.describe("extension recording upload", () => { await mockServer?.close(); }); - test("records the selected display surface, uploads non-empty multipart data, and completes", async () => { + for (const deliverRequest of [false, true]) { + test(`recovers an offscreen start response lost ${deliverRequest ? "after" : "before"} capture starts`, async () => { + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + await injectUncertainOffscreenStartResponse(worker, deliverRequest); + const { messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + ); + expect(mockServer.state.initiateBodies).toHaveLength(2); + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expectSuccessfulUpload(messengerPage, mockServer.state); + }); + } + + test("hides confirmed uploaded backups from the options recovery list", async () => { + if (!extension) throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + const optionsPage = await extension.context.newPage(); + await optionsPage.goto( + `chrome-extension://${getExtensionId(worker)}/options.html`, + ); + await optionsPage.evaluate(async () => { + const sessionId = "uploaded-options-backup-e2e"; + const request = indexedDB.open("cap-recording-spool", 1); + request.onupgradeneeded = () => { + const database = request.result; + if (!database.objectStoreNames.contains("sessions")) { + database.createObjectStore("sessions", { keyPath: "sessionId" }); + } + if (!database.objectStoreNames.contains("chunks")) { + const chunks = database.createObjectStore("chunks", { + keyPath: ["sessionId", "index"], + }); + chunks.createIndex("by-session", "sessionId", { unique: false }); + } + }; + const database = await new Promise((resolve, reject) => { + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error); + }); + const transaction = database.transaction( + ["sessions", "chunks"], + "readwrite", + ); + const oldTime = Date.now() - 120_000; + transaction.objectStore("sessions").put({ + sessionId, + mimeType: "video/webm", + totalBytes: 5, + chunkCount: 1, + createdAt: oldTime, + updatedAt: oldTime, + }); + transaction.objectStore("chunks").put({ + sessionId, + index: 0, + blob: new Blob(["hello"], { type: "video/webm" }), + }); + await new Promise((resolve, reject) => { + transaction.oncomplete = () => resolve(); + transaction.onerror = () => reject(transaction.error); + transaction.onabort = () => reject(transaction.error); + }); + database.close(); + localStorage.setItem(`cap-recording-spool-uploaded:${sessionId}`, "1"); + }); + await optionsPage.reload(); + await expect(optionsPage.locator(".recovery-item")).toHaveCount(0); + expect( + await optionsPage.evaluate(() => + localStorage.getItem( + "cap-recording-spool-uploaded:uploaded-options-backup-e2e", + ), + ), + ).toBe("1"); + await optionsPage.evaluate(() => + localStorage.removeItem( + "cap-recording-spool-uploaded:uploaded-options-backup-e2e", + ), + ); + await optionsPage.reload(); + await expect(optionsPage.locator(".recovery-item")).toHaveCount(1); + }); + + test("keeps screen-only Instant recordings working without a webcam", async () => { if (!extension || !mockServer) throw new Error("Test harness did not start"); const worker = await getServiceWorker(extension.context); @@ -597,8 +1130,54 @@ test.describe("extension recording upload", () => { extension.context, worker, mockServer.origin, + RECORDING_MODE, + RECORDING_MS, + false, + false, + ); + expect(mockServer.state.initiateBodies).toEqual([ + expect.objectContaining({ subpath: rawSubpath }), + ]); + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + if (!response.ok) return response.error; + return response.status?.phase; + }) + .toBe("completed"); + expect(mockServer.state.completeBodies).toEqual([ + expect.objectContaining({ subpath: rawSubpath }), + ]); + expect(mockServer.state.uploadBytesBySubpath[rawSubpath]).toBeGreaterThan( + 0, + ); + expect( + mockServer.state.uploadBytesBySubpath[cameraSubpath], + ).toBeUndefined(); + }); + + test("records a separate camera sidecar and hides its page overlay during display capture", async () => { + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + const { capturePage, messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, ); + await expectCameraPreviewOutOfCapture(capturePage); + const persistedCameraOffsetMs = await readLiveCameraOffset(worker); + expect(typeof persistedCameraOffsetMs).toBe("number"); + expect(mockServer.state.initiateBodies).toHaveLength(2); const stopResponse = await sendServiceWorkerMessage(messengerPage, { target: "service-worker", type: "stop-recording", @@ -606,6 +1185,852 @@ test.describe("extension recording upload", () => { expect(stopResponse).toMatchObject({ ok: true }); await expectSuccessfulUpload(messengerPage, mockServer.state); + expect(mockServer.state.completeBodies[0]).toMatchObject({ + cameraOffsetMs: persistedCameraOffsetMs, + }); + }); + + test("uploads microphone audio independently of screen and camera in Chromium", async () => { + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + const { messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + "fullscreen", + RECORDING_MS, + true, + ); + const persistedAudioOffsetMs = await readLiveMicrophoneOffset(worker); + expect(mockServer.state.initiateBodies).toHaveLength(3); + expect(mockServer.state.initiateBodies).toContainEqual( + expect.objectContaining({ + subpath: "mic-upload.webm", + contentType: "audio/webm", + }), + ); + + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("completed"); + expect(mockServer.state.completeBodies).toHaveLength(3); + const audioCompletion = mockServer.state.completeBodies.find( + (body) => + !!body && + typeof body === "object" && + "subpath" in body && + body.subpath === "mic-upload.webm", + ); + expect(audioCompletion).toMatchObject({ + videoId: mockServer.state.videoId, + subpath: "mic-upload.webm", + screenSubpath: rawSubpath, + audioOffsetMs: persistedAudioOffsetMs, + }); + expect( + mockServer.state.uploadBytesBySubpath["mic-upload.webm"], + ).toBeGreaterThan(0); + expect( + mockServer.state.uploadBytesBySubpath[cameraSubpath], + ).toBeGreaterThan(0); + expect(mockServer.state.uploadBytesBySubpath[rawSubpath]).toBeGreaterThan( + 0, + ); + }); + + test("keeps the microphone spool downloadable and retries all sources after audio upload failure", async () => { + test.setTimeout(120_000); + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + mockServer.state.failAudioCompletion = true; + const worker = await getServiceWorker(extension.context); + const { messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + "fullscreen", + RECORDING_MS, + true, + ); + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("error"); + + const uploadPage = await extension.context.newPage(); + await uploadPage.goto( + `chrome-extension://${getExtensionId(worker)}/uploading.html?videoId=${mockServer.state.videoId}`, + ); + await expect( + uploadPage.getByRole("button", { name: "Download microphone" }), + ).toBeVisible(); + const micDownload = uploadPage.waitForEvent("download"); + await uploadPage + .getByRole("button", { name: "Download microphone" }) + .click(); + expect((await micDownload).suggestedFilename()).toContain("-mic.webm"); + const screenDownload = uploadPage.waitForEvent("download"); + await uploadPage.getByRole("button", { name: "Download screen" }).click(); + expect((await screenDownload).suggestedFilename()).toContain( + `-screen.${videoExtension}`, + ); + + mockServer.state.failAudioCompletion = false; + const retryResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "retry-upload", + videoId: mockServer.state.videoId, + }); + expect(retryResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("completed"); + expect( + mockServer.state.completeBodies.filter( + (body) => + !!body && + typeof body === "object" && + "subpath" in body && + body.subpath === "mic-upload.webm", + ), + ).toHaveLength(2); + }); + + test("keeps uploaded microphone audio downloadable when screen completion fails", async () => { + test.setTimeout(120_000); + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + mockServer.state.failScreenCompletion = true; + const worker = await getServiceWorker(extension.context); + const { messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + "fullscreen", + RECORDING_MS, + true, + ); + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("error"); + expect(mockServer.state.completeBodies).toContainEqual( + expect.objectContaining({ subpath: "mic-upload.webm" }), + ); + const uploadPage = await extension.context.newPage(); + await uploadPage.goto( + `chrome-extension://${getExtensionId(worker)}/uploading.html?videoId=${mockServer.state.videoId}`, + ); + const downloadButton = uploadPage.getByRole("button", { + name: "Download microphone", + }); + await expect(downloadButton).toBeVisible(); + const micDownload = uploadPage.waitForEvent("download"); + await downloadButton.click(); + expect((await micDownload).suggestedFilename()).toContain("-mic.webm"); + + mockServer.state.failScreenCompletion = false; + const retryResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "retry-upload", + videoId: mockServer.state.videoId, + }); + expect(retryResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("completed"); + expect( + mockServer.state.completeBodies.filter( + (body) => + !!body && + typeof body === "object" && + "subpath" in body && + body.subpath === "mic-upload.webm", + ), + ).toHaveLength(2); + }); + + test("uploads a camera multipart part during a long recording on a constrained network", async () => { + test.setTimeout(330_000); + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + mockServer.state.simulateSlowCameraUpload = true; + const worker = await getServiceWorker(extension.context); + const { messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + "fullscreen", + 0, + ); + const persistedCameraOffsetMs = await readLiveCameraOffset(worker); + expect(mockServer.state.initiateBodies).toHaveLength(2); + await expect + .poll( + () => mockServer?.state.completedPartsBySubpath[cameraSubpath] ?? 0, + { timeout: 240_000 }, + ) + .toBeGreaterThan(0); + expect( + mockServer.state.uploadBytesBySubpath[cameraSubpath], + ).toBeGreaterThanOrEqual(5 * 1024 * 1024); + + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expectSuccessfulUpload(messengerPage, mockServer.state); + expect(mockServer.state.completeBodies[0]).toMatchObject({ + cameraOffsetMs: persistedCameraOffsetMs, + }); + }); + + test("offers separate clip downloads after a camera upload failure", async () => { + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + mockServer.state.failCameraCompletion = true; + const worker = await getServiceWorker(extension.context); + const { messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + ); + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("error"); + await expect + .poll(() => readFailedCameraRecovery(worker)) + .toMatchObject({ videoId: mockServer.state.videoId }); + const uploadPage = await extension.context.newPage(); + await uploadPage.goto( + `chrome-extension://${getExtensionId(worker)}/uploading.html?videoId=${mockServer.state.videoId}`, + ); + await expect( + uploadPage.getByRole("button", { name: "Try again" }), + ).toBeVisible(); + const screenDownload = uploadPage.waitForEvent("download"); + await uploadPage.getByRole("button", { name: "Download screen" }).click(); + expect((await screenDownload).suggestedFilename()).toContain( + `-screen.${videoExtension}`, + ); + const cameraDownload = uploadPage.waitForEvent("download"); + await uploadPage.getByRole("button", { name: "Download camera" }).click(); + expect((await cameraDownload).suggestedFilename()).toContain( + `-camera.${videoExtension}`, + ); + await worker.evaluate(async () => { + const key = "cap-extension-failed-recordings"; + const data = await new Promise>((resolve) => + chrome.storage.local.get([key], (items) => resolve(items)), + ); + const recordings = data[key]; + if (!Array.isArray(recordings) || !recordings[0]) { + throw new Error("The failed paired recording is unavailable"); + } + const failed = { ...(recordings[0] as Record) }; + delete failed.cameraSessionId; + failed.cameraRetryUnavailable = true; + await new Promise((resolve) => + chrome.storage.local.set({ [key]: [failed] }, () => resolve()), + ); + }); + await uploadPage.reload(); + await expect( + uploadPage.getByRole("button", { name: "Try again" }), + ).toHaveCount(0); + await expect( + uploadPage.getByRole("button", { name: "Download camera" }), + ).toHaveCount(0); + await expect( + uploadPage.getByRole("button", { name: "Download screen" }), + ).toBeVisible(); + expect(mockServer.state.completeBodies).toEqual([ + expect.objectContaining({ subpath: cameraSubpath }), + ]); + }); + + test("recovers camera, microphone and screen spools after the offscreen recorder closes", async () => { + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + const { messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + "fullscreen", + RECORDING_MS, + true, + ); + const persistedCameraOffsetMs = await readLiveCameraOffset(worker); + const persistedAudioOffsetMs = await readLiveMicrophoneOffset(worker); + await worker.evaluate(() => chrome.offscreen.closeDocument()); + + const refreshResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-media-devices", + }); + expect(refreshResponse).toMatchObject({ ok: true }); + await expect + .poll(() => readFailedCameraRecovery(worker)) + .toMatchObject({ + videoId: mockServer.state.videoId, + cameraSubpath, + cameraOffsetMs: persistedCameraOffsetMs, + }); + const optionsPage = await extension.context.newPage(); + await optionsPage.addInitScript(() => { + const target = globalThis as typeof globalThis & { + capE2eChunkReads?: number; + }; + target.capE2eChunkReads = 0; + const originalGetAll = IDBIndex.prototype.getAll; + IDBIndex.prototype.getAll = function ( + ...args: Parameters + ) { + if (this.name === "by-session") { + target.capE2eChunkReads = (target.capE2eChunkReads ?? 0) + 1; + } + return originalGetAll.apply(this, args); + }; + }); + await optionsPage.goto( + `chrome-extension://${getExtensionId(worker)}/options.html`, + ); + await expect(optionsPage.locator(".recovery-item")).toHaveCount(1); + expect( + await optionsPage.evaluate( + () => + (globalThis as typeof globalThis & { capE2eChunkReads?: number }) + .capE2eChunkReads ?? 0, + ), + ).toBe(0); + const screenDownload = optionsPage.waitForEvent("download"); + await optionsPage.getByRole("button", { name: "Download screen" }).click(); + expect((await screenDownload).suggestedFilename()).toContain( + `-screen.${videoExtension}`, + ); + const cameraDownload = optionsPage.waitForEvent("download"); + await optionsPage.getByRole("button", { name: "Download camera" }).click(); + expect((await cameraDownload).suggestedFilename()).toContain( + `-camera.${videoExtension}`, + ); + const micDownload = optionsPage.waitForEvent("download"); + await optionsPage + .getByRole("button", { name: "Download microphone" }) + .click(); + expect((await micDownload).suggestedFilename()).toContain("-mic.webm"); + expect( + await optionsPage.evaluate( + () => + (globalThis as typeof globalThis & { capE2eChunkReads?: number }) + .capE2eChunkReads ?? 0, + ), + ).toBeGreaterThanOrEqual(3); + + const retryResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "retry-upload", + videoId: mockServer.state.videoId, + }); + expect(retryResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("completed"); + expect(mockServer.state.completeBodies).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + subpath: cameraSubpath, + cameraOffsetMs: persistedCameraOffsetMs, + }), + expect.objectContaining({ + subpath: "mic-upload.webm", + audioOffsetMs: persistedAudioOffsetMs, + }), + expect.objectContaining({ subpath: rawSubpath }), + ]), + ); + expect( + mockServer.state.uploadBytesBySubpath[cameraSubpath], + ).toBeGreaterThan(0); + expect( + mockServer.state.uploadBytesBySubpath["mic-upload.webm"], + ).toBeGreaterThan(0); + expect(mockServer.state.uploadBytesBySubpath[rawSubpath]).toBeGreaterThan( + 0, + ); + expect(await readFailedCameraRecovery(worker)).toBeNull(); + expect(await hasLiveRecordingManifest(worker)).toBe(false); + }); + + test("never retries a screen-only upload after a crash loses the camera spool", async () => { + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + const { messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + ); + const cameraSessionId = await readLiveCameraSessionId(worker); + await worker.evaluate(() => chrome.offscreen.closeDocument()); + await deleteCameraSpoolSession(worker, cameraSessionId); + const refreshResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-media-devices", + }); + expect(refreshResponse).toMatchObject({ ok: true }); + await expect + .poll(() => readFailedCameraRecovery(worker)) + .toMatchObject({ + videoId: mockServer.state.videoId, + cameraRetryUnavailable: true, + }); + const optionsPage = await extension.context.newPage(); + await optionsPage.goto( + `chrome-extension://${getExtensionId(worker)}/options.html`, + ); + await expect(optionsPage.locator(".recovery-item")).toHaveCount(1); + await expect( + optionsPage.getByRole("button", { name: "Retry upload" }), + ).toHaveCount(0); + await expect( + optionsPage.getByRole("button", { name: "Download screen" }), + ).toBeVisible(); + const retryResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "retry-upload", + videoId: mockServer.state.videoId, + }); + expect(retryResponse).toMatchObject({ + ok: false, + error: expect.stringContaining( + "camera recording is unavailable to retry", + ), + }); + expect(mockServer.state.completeBodies).toHaveLength(0); + }); + + test("aborts both multipart sessions when Stop cancels camera upload setup", async () => { + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + mockServer.state.cameraInitiateDelayMs = 1_500; + const worker = await getServiceWorker(extension.context); + await configureExtension(worker, mockServer.origin); + const startPage = await openExtensionMessengerPage( + extension.context, + worker, + ); + const stopPage = await openExtensionMessengerPage( + extension.context, + worker, + ); + const capturePage = await extension.context.newPage(); + await capturePage.goto(`${mockServer.origin}/capture.html`); + await capturePage.bringToFront(); + const startPromise = sendServiceWorkerMessage(startPage, { + target: "service-worker", + type: "start-recording", + mode: "fullscreen", + }); + await expect.poll(() => mockServer?.state.initiateBodies.length).toBe(2); + const stopResponse = await sendServiceWorkerMessage(stopPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + const startResponse = await startPromise; + expect(startResponse).toMatchObject({ ok: false }); + expect(mockServer.state.abortBodies).toEqual( + expect.arrayContaining([ + expect.objectContaining({ subpath: rawSubpath }), + expect.objectContaining({ subpath: cameraSubpath }), + ]), + ); + expect(mockServer.state.completeBodies).toHaveLength(0); + }); + + test("keeps camera sidecar separate in current-tab capture", async () => { + test.skip( + process.env.CAP_EXTENSION_E2E_HEADED !== "1" || + process.platform !== "darwin", + "Headed current-tab capture is available on macOS", + ); + test.setTimeout(120_000); + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + const { capturePage, messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + "tab", + 20_000, + ); + await expectCameraPreviewOutOfCapture(capturePage); + await capturePage.locator("input[type='password']").click(); + await capturePage.keyboard.type("secret"); + await capturePage + .getByRole("textbox", { name: "Custom private field" }) + .click(); + await capturePage.keyboard.type("private"); + await capturePage.locator("[aria-label='Custom PIN widget']").click(); + await capturePage.keyboard.type("1234"); + await capturePage.getByRole("button", { name: "Secret button" }).click(); + await capturePage.keyboard.type("secret"); + await capturePage.locator("[aria-label='Canvas secret widget']").click(); + await capturePage.keyboard.type("pin"); + await capturePage.mouse.click(500, 400); + await capturePage.keyboard.press("Escape"); + await capturePage.goto(`${mockServer.origin}/capture.html?step=2`); + await capturePage.bringToFront(); + await capturePage.mouse.click(600, 500); + await capturePage.keyboard.press("ArrowRight"); + await capturePage.waitForTimeout(500); + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expectSuccessfulUpload(messengerPage, mockServer.state, true); + const inputText = Buffer.concat( + Object.entries(mockServer.state.inputParts) + .sort(([a], [b]) => Number(a) - Number(b)) + .map(([, bytes]) => bytes), + ).toString("utf8"); + const inputLines = inputText + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); + expect(inputLines[0]).toMatchObject({ version: 1 }); + const recordedKeys = inputLines + .filter((line) => line.kind === "keyDown") + .map((line) => line.key); + expect( + recordedKeys.every((key) => typeof key === "string" && key.length > 1), + ).toBe(true); + expect( + inputLines + .filter((line) => line.kind === "keyDown") + .every((line) => + typeof line.code === "string" + ? !/^(Key[A-Z]|Digit[0-9]|Numpad[0-9])$/.test(line.code) + : false, + ), + ).toBe(true); + expect(inputLines).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: "down" }), + expect.objectContaining({ kind: "keyDown", key: "Escape" }), + expect.objectContaining({ kind: "keyDown", key: "ArrowRight" }), + ]), + ); + }); + + test("completes screen and camera when the tab input sidecar fails", async () => { + test.skip( + process.env.CAP_EXTENSION_E2E_HEADED !== "1" || + process.platform !== "darwin", + "Headed current-tab capture is available on macOS", + ); + test.setTimeout(120_000); + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + const { capturePage, messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + "tab", + 8_000, + ); + await capturePage.mouse.click(400, 300); + await capturePage.keyboard.press("Escape"); + mockServer.state.failInputCompletion = true; + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("completed"); + expect(mockServer.state.completeBodies).toEqual( + expect.arrayContaining([ + expect.objectContaining({ subpath: cameraSubpath }), + expect.objectContaining({ subpath: "input-events-upload.ndjson" }), + expect.objectContaining({ subpath: rawSubpath }), + ]), + ); + expect(mockServer.state.completeBodies.at(-1)).toMatchObject({ + subpath: rawSubpath, + }); + const failed = await worker.evaluate(async () => { + const items = await chrome.storage.local.get([ + "cap-extension-failed-recordings", + ]); + return items["cap-extension-failed-recordings"]; + }); + expect(Array.isArray(failed) ? failed : []).toEqual([]); + const tabInputSession = await worker.evaluate(async () => { + const key = "cap-extension-tab-input-capture"; + const saved = await chrome.storage.session.get([key]); + return saved[key] ?? null; + }); + expect(tabInputSession).toBeNull(); + }); + + test("recovers a tab input sidecar before retrying the screen upload", async () => { + test.skip( + process.env.CAP_EXTENSION_E2E_HEADED !== "1" || + process.platform !== "darwin", + "Headed current-tab capture is available on macOS", + ); + test.setTimeout(120_000); + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + const { capturePage, messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + "tab", + 8_000, + ); + await capturePage.mouse.click(400, 300); + await capturePage.keyboard.press("Escape"); + mockServer.state.failInputCompletion = true; + mockServer.state.failScreenCompletion = true; + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("error"); + const recovery = await worker.evaluate(async () => { + const items = await chrome.storage.local.get([ + "cap-extension-failed-recordings", + ]); + const failed = items["cap-extension-failed-recordings"]; + return Array.isArray(failed) ? failed[0] : null; + }); + if (!recovery || typeof recovery !== "object") + throw new Error("The input sidecar recovery record is missing"); + const failed = recovery as Record; + expect(failed.videoId).toBe(mockServer.state.videoId); + expect(failed.inputEventsRetryUnavailable).not.toBe(true); + expect(failed.inputEventsSessionId).toEqual(expect.any(String)); + expect(failed.inputEventsTotalBytes).toBeGreaterThan(0); + mockServer.state.failInputCompletion = false; + mockServer.state.failScreenCompletion = false; + const retryResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "retry-upload", + videoId: mockServer.state.videoId, + }); + expect(retryResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("completed"); + expect( + mockServer.state.completeBodies.filter( + (body) => + body && + typeof body === "object" && + "subpath" in body && + body.subpath === "input-events-upload.ndjson", + ).length, + ).toBeGreaterThanOrEqual(2); + expect(mockServer.state.completeBodies.at(-1)).toMatchObject({ + subpath: rawSubpath, + }); + }); + + test("retries intact screen media when tab input metadata is unavailable", async () => { + test.skip( + process.env.CAP_EXTENSION_E2E_HEADED !== "1" || + process.platform !== "darwin", + "Headed current-tab capture is available on macOS", + ); + test.setTimeout(120_000); + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const worker = await getServiceWorker(extension.context); + const { capturePage, messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + "tab", + 8_000, + ); + await capturePage.mouse.click(400, 300); + await capturePage.keyboard.press("Escape"); + mockServer.state.failInputCompletion = true; + mockServer.state.failScreenCompletion = true; + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("error"); + const recovery = await worker.evaluate(async () => { + const items = await chrome.storage.local.get([ + "cap-extension-failed-recordings", + ]); + const failed = items["cap-extension-failed-recordings"]; + return Array.isArray(failed) ? failed[0] : null; + }); + if (!recovery || typeof recovery !== "object") + throw new Error("The failed screen recording is missing"); + expect(recovery).toMatchObject({ + videoId: mockServer.state.videoId, + inputEventsSessionId: expect.any(String), + }); + expect( + await worker.evaluate(async (videoId) => { + const key = "cap-extension-failed-recordings"; + const items = await chrome.storage.local.get([key]); + const failed: unknown = items[key]; + if (!Array.isArray(failed)) return false; + const updated = failed.map((entry: unknown) => { + if ( + !entry || + typeof entry !== "object" || + !("videoId" in entry) || + entry.videoId !== videoId + ) + return entry; + return { ...entry, inputEventsRetryUnavailable: true }; + }); + await chrome.storage.local.set({ [key]: updated }); + return true; + }, mockServer.state.videoId), + ).toBe(true); + const inputAttempts = mockServer.state.completeBodies.filter( + (body) => + body && + typeof body === "object" && + "subpath" in body && + body.subpath === "input-events-upload.ndjson", + ).length; + mockServer.state.failInputCompletion = false; + mockServer.state.failScreenCompletion = false; + const retryResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "retry-upload", + videoId: mockServer.state.videoId, + }); + expect(retryResponse).toMatchObject({ ok: true }); + await expect + .poll(async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }) + .toBe("completed"); + expect(mockServer.state.completeBodies.at(-1)).toMatchObject({ + subpath: rawSubpath, + }); + expect( + mockServer.state.completeBodies.filter( + (body) => + body && + typeof body === "object" && + "subpath" in body && + body.subpath === "input-events-upload.ndjson", + ).length, + ).toBe(inputAttempts); }); test("can complete two consecutive recording uploads without stale state", async () => { @@ -638,6 +2063,8 @@ test.describe("extension recording upload", () => { mockServer.state.initiateBodies = []; mockServer.state.presignBodies = []; mockServer.state.uploadBytes = []; + mockServer.state.uploadBytesBySubpath = {}; + mockServer.state.completedPartsBySubpath = {}; mockServer.state.uploadHeaders = []; mockServer.state.videoId = `e2e-${Date.now()}-second`; diff --git a/apps/chrome-extension/e2e/webcam-start.spec.ts b/apps/chrome-extension/e2e/webcam-start.spec.ts index dc1344b00f4..c66ade47f3d 100644 --- a/apps/chrome-extension/e2e/webcam-start.spec.ts +++ b/apps/chrome-extension/e2e/webcam-start.spec.ts @@ -301,9 +301,45 @@ test("repro: start recording with webcam preview enabled and live", async () => type: "bootstrap", }); logs.push(`bootstrap: ${JSON.stringify(bootstrapResponse).slice(0, 200)}`); + const openResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "open-recorder-panel", + }); + logs.push(`open panel: ${JSON.stringify(openResponse)}`); + await expect + .poll(() => + targetPage + .frames() + .some((frame) => frame.url().includes("camera-preview.html")), + ) + .toBe(true); // Give the camera preview time to go live (frames flowing over WebRTC). await targetPage.waitForTimeout(6_000); + const previewFrame = targetPage + .frames() + .find((frame) => frame.url().includes("camera-preview.html")); + if (!previewFrame) throw new Error("Camera preview frame was not mounted"); + await expect + .poll(() => + previewFrame.evaluate( + () => document.querySelector("video")?.videoWidth ?? 0, + ), + ) + .toBeGreaterThan(0); + const pictureInPictureSupported = await previewFrame.evaluate( + () => document.pictureInPictureEnabled, + ); + if (pictureInPictureSupported) { + await previewFrame.locator("[data-pip-control]").click(); + await expect + .poll(() => + previewFrame.evaluate( + () => document.pictureInPictureElement !== null, + ), + ) + .toBe(true); + } await targetPage.screenshot({ path: "test-results/repro-before-start.png", }); @@ -342,6 +378,13 @@ test("repro: start recording with webcam preview enabled and live", async () => type: "get-recording-status", })) as { status?: { phase?: string } }; expect(finalStatus.status?.phase).toBe("recording"); + await expect + .poll(() => + targetPage + .frames() + .every((frame) => !frame.url().includes("camera-preview.html")), + ) + .toBe(true); // Content scripts read chrome.storage.session; without the service // worker widening the access level every call fails with this error. diff --git a/apps/chrome-extension/src/background/service-worker.ts b/apps/chrome-extension/src/background/service-worker.ts index 166072e3050..82a6172c5e0 100644 --- a/apps/chrome-extension/src/background/service-worker.ts +++ b/apps/chrome-extension/src/background/service-worker.ts @@ -23,6 +23,7 @@ import { loadSettings, loadSharedRecordingState, loadSharedUiState, + loadTabInputCaptureSession, loadUploadProgressTabId, loadWebcamPreviewDismissed, registerOverlayToken, @@ -32,10 +33,16 @@ import { savePendingAuth, saveSettings, saveSharedRecordingState, + saveTabInputCaptureSession, saveUploadProgressTabId, saveWebcamPreviewDismissed, updateSharedUiState, } from "../shared/storage"; +import { + executeStorageBridgeRequest, + isStorageBridgeRequest, + isTrustedOffscreenStorageSender, +} from "../shared/storage-bridge"; import type { BootstrapData, CameraDevice, @@ -83,7 +90,6 @@ let activePreviewTabId: number | null = null; let pendingPreviewTabId: number | null = null; let offscreenDocumentCreation: Promise | null = null; let browserWindowFocused = true; -let externalCaptureAutoPipPending = false; let recordingStartInFlight: Promise | null = null; // Content scripts read the webcam "dismissed" flag and the cached preview @@ -254,6 +260,10 @@ const isTransientOffscreenMessageError = (error: unknown) => { ); }; +const isUncertainOffscreenStartError = (error: unknown) => + error instanceof Error && + error.message.toLowerCase().includes("message port closed before a response"); + const sendOffscreenRuntimeMessage = (message: OffscreenRequest) => new Promise((resolve, reject) => { chrome.runtime.sendMessage(message, (response) => { @@ -265,6 +275,37 @@ const sendOffscreenRuntimeMessage = (message: OffscreenRequest) => }); }); +const reconcileUncertainOffscreenStart = async () => { + let lastError: unknown; + for (let attempt = 1; attempt <= OFFSCREEN_MESSAGE_ATTEMPTS; attempt += 1) { + await wait(OFFSCREEN_MESSAGE_RETRY_DELAY_MS); + await ensureOffscreenDocument(); + try { + const response = await sendOffscreenRuntimeMessage({ + target: "offscreen", + type: "get-recording-status", + }); + if (!response.ok) return response; + const status = response.status; + if (!status) throw new Error("Offscreen recorder status is unavailable"); + if (status.phase === "idle") return null; + if (status.phase === "error") { + return { ok: false, error: status.message } satisfies OffscreenResponse; + } + return response; + } catch (error) { + lastError = error; + if ( + !isTransientOffscreenMessageError(error) && + !isUncertainOffscreenStartError(error) + ) { + throw error; + } + } + } + throw lastError instanceof Error ? lastError : new Error(String(lastError)); +}; + const sendOffscreen = async ( message: OffscreenRequest, options: { createIfMissing?: boolean } = {}, @@ -284,6 +325,14 @@ const sendOffscreen = async ( return await sendOffscreenRuntimeMessage(message); } catch (error) { lastError = error; + if ( + message.type === "start-recording" && + isUncertainOffscreenStartError(error) + ) { + const recovered = await reconcileUncertainOffscreenStart(); + if (recovered) return recovered; + if (attempt < OFFSCREEN_MESSAGE_ATTEMPTS) continue; + } if ( options.createIfMissing === false || attempt === OFFSCREEN_MESSAGE_ATTEMPTS || @@ -314,10 +363,22 @@ const getTabStreamId = (tabId: number) => }); }); +const canUseDisplayPickerForTabCapture = (error: unknown) => + error instanceof Error && + /has not been invoked|activeTab permission/i.test(error.message); + const sendOverlayMessage = (tabId: number, message: OverlayMessage) => new Promise((resolve) => { - chrome.tabs.sendMessage(tabId, message, () => { - resolve(!chrome.runtime.lastError); + chrome.tabs.sendMessage(tabId, message, (response: unknown) => { + resolve( + !chrome.runtime.lastError && + !( + response && + typeof response === "object" && + "ok" in response && + response.ok === false + ), + ); }); }); @@ -485,21 +546,6 @@ const isBrowserCaptureSource = (source: RecordingCaptureSource) => source.displaySurface === "browser" || source.displaySurface === "tab"; -const isWindowCaptureSource = (source: RecordingCaptureSource) => - source.detectedMode === "window" || - source.displaySurface === "window" || - source.displaySurface === "application"; - -const isLikelyBrowserWindow = (source: RecordingCaptureSource) => { - if (!source.label) return false; - return /\b(google chrome|chrome|chromium|microsoft edge|edge|brave|arc|opera|vivaldi)\b/i.test( - source.label, - ); -}; - -const shouldAutoPipCaptureSource = (source: RecordingCaptureSource) => - isWindowCaptureSource(source) && !isLikelyBrowserWindow(source); - const isWebcamPreviewEnabled = (settings: ExtensionSettings) => Boolean(settings.webcam.enabled); @@ -777,6 +823,7 @@ const getPreviewTabIdForPip = async () => { }; const enterActivePreviewAutoPip = async () => { + if (isRecordingPreviewStatus(recordingStatus)) return false; const tabId = await getPreviewTabIdForPip(); if (tabId === null) return false; return sendOverlay(tabId, { type: "overlay-enter-auto-pip" }, false).catch( @@ -827,15 +874,11 @@ const broadcastOverlayCountdown = async ( if (!canInjectIntoTab(tab) || tab.id === undefined) { return undefined; } - return sendOverlay( - tab.id, - { - type: "overlay-countdown", - seconds, - durationMs, - }, - false, - ).catch(() => undefined); + return sendOverlay(tab.id, { + type: "overlay-countdown", + seconds, + durationMs, + }).catch(() => undefined); }), ); }; @@ -1182,7 +1225,6 @@ const resolveMicWarning = async ( const performRecordingStart = async (mode: RecordingMode) => { const { settings, auth, bootstrap } = await requireSignedInState(); - externalCaptureAutoPipPending = false; const recordingSettings = settings.capture.recordingMode === mode ? settings @@ -1206,7 +1248,6 @@ const performRecordingStart = async (mode: RecordingMode) => { if (micWarning) { const confirmed = await requestRecordingConfirmation(tabId, micWarning); if (!confirmed) { - externalCaptureAutoPipPending = false; return { ok: false, canceled: true, @@ -1215,20 +1256,61 @@ const performRecordingStart = async (mode: RecordingMode) => { } } - const tabStreamId = - mode === "tab" && tabId !== undefined - ? await getTabStreamId(tabId) - : undefined; - await showOverlayInTab(tab ?? null, recordingSettings, true); + if (activePreviewTabId !== null && activePreviewTabId !== tabId) { + const previousPreviewTab = await getTab(activePreviewTabId); + if (previousPreviewTab && canInjectIntoTab(previousPreviewTab)) { + const previousPrepared = await sendOverlay( + activePreviewTabId, + { + type: "overlay-settings", + settings: recordingSettings.webcam, + recording: true, + }, + false, + ); + if (!previousPrepared) { + return { + ok: false, + error: "The camera preview could not be closed before recording", + } satisfies OffscreenResponse; + } + } + } + const previewPrepared = await showOverlayInTab( + tab ?? null, + recordingSettings, + true, + ); + if ( + isWebcamPreviewEnabled(recordingSettings) && + tab && + canInjectIntoTab(tab) && + !previewPrepared + ) { + return { + ok: false, + error: "The camera preview could not be closed before recording", + } satisfies OffscreenResponse; + } const creatingStatus = { phase: "creating" } satisfies RecordingStatus; setRecordingStatusAndBroadcast(creatingStatus); + let tabStreamId: string | undefined; + if (mode === "tab" && tabId !== undefined) { + await ensureOffscreenDocument(); + try { + tabStreamId = await getTabStreamId(tabId); + } catch (error) { + console.warn("Tab capture stream ID unavailable", error); + if (!canUseDisplayPickerForTabCapture(error)) throw error; + } + } // The manifest injects the bootstrap content script into every page at // document_idle and onInstalled covers tabs that predate the extension, so // no blanket re-injection is needed here; sendOverlay still injects // per-tab on demand and the bootstrap lazy-loads the overlay UI. try { - return await sendOffscreen({ + const request: OffscreenRequest = { target: "offscreen", type: "start-recording", mode, @@ -1237,7 +1319,31 @@ const performRecordingStart = async (mode: RecordingMode) => { bootstrap, tabId, tabStreamId, - }); + }; + const response = await (tabStreamId + ? sendOffscreenRuntimeMessage(request) + : sendOffscreen(request)); + if ( + tabStreamId && + tabId !== undefined && + response.ok && + response.status?.phase === "recording" && + response.status.videoId + ) { + await saveTabInputCaptureSession({ + tabId, + recordingId: response.status.videoId, + }); + await chrome.tabs + .sendMessage(tabId, { + type: "input-capture-start", + recordingId: response.status.videoId, + }) + .catch((error: unknown) => { + console.warn("Tab input capture could not start", error); + }); + } + return response; } catch (error) { // The recorder panel closes as soon as the status leaves "idle", so a // silent reset would leave the user with no feedback at all. Broadcast @@ -1246,7 +1352,6 @@ const performRecordingStart = async (mode: RecordingMode) => { phase: "error", message: error instanceof Error ? error.message : String(error), }); - externalCaptureAutoPipPending = false; throw error; } }; @@ -1317,7 +1422,6 @@ const stopRecordingAndOpenDestination = async () => { if (response.ok && response.status) { setRecordingStatus(response.status); if (!isCapturingRecordingStatus(response.status)) { - externalCaptureAutoPipPending = false; await saveWebcamPreviewDismissed(true); await broadcastOverlayHide(); } @@ -1431,7 +1535,7 @@ const handlePreviewReady = async (tabId?: number) => { activePreviewTabId = tabId; pendingPreviewTabId = null; await hidePreviewTabsExcept(tabId); - if (!browserWindowFocused || externalCaptureAutoPipPending) { + if (!browserWindowFocused) { await enterActivePreviewAutoPip(); } }; @@ -1458,7 +1562,6 @@ const handlePreviewError = async ( const handleCaptureSource = async (source: RecordingCaptureSource) => { if (isBrowserCaptureSource(source)) { - externalCaptureAutoPipPending = false; const tabId = await findCapturedBrowserTabId(source); if (tabId !== null) { await focusTab(tabId); @@ -1467,14 +1570,6 @@ const handleCaptureSource = async (source: RecordingCaptureSource) => { await exitActivePreviewAutoPip(); return; } - - if (!shouldAutoPipCaptureSource(source)) { - externalCaptureAutoPipPending = false; - return; - } - - externalCaptureAutoPipPending = true; - await enterActivePreviewAutoPip(); }; const handleRequest = async ( @@ -1791,12 +1886,76 @@ const handleRequest = async ( return { ok: false, error: "Unknown request" }; }; -chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => { +chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { + if ( + message?.target === "service-worker" && + message.type === "input-capture-stop" + ) { + if ( + !isTrustedOffscreenStorageSender( + sender, + chrome.runtime.id, + chrome.runtime.getURL(OFFSCREEN_URL), + ) || + !Number.isSafeInteger(message.tabId) || + typeof message.recordingId !== "string" + ) { + sendResponse({ ok: false, error: "Tab input stop denied" }); + return false; + } + void Promise.all([syncRecordingStatus(), loadTabInputCaptureSession()]) + .then(async ([currentStatus, session]) => { + if ( + !session || + session.tabId !== message.tabId || + session.recordingId !== message.recordingId || + !("videoId" in currentStatus) || + currentStatus.videoId !== message.recordingId + ) { + return { ok: false, error: "Tab input stop denied" }; + } + try { + return await chrome.tabs.sendMessage(message.tabId, { + type: "input-capture-stop", + }); + } finally { + await saveTabInputCaptureSession(null); + } + }) + .then(sendResponse) + .catch((error: unknown) => + sendResponse({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }), + ); + return true; + } + if (isStorageBridgeRequest(message)) { + if ( + !isTrustedOffscreenStorageSender( + sender, + chrome.runtime.id, + chrome.runtime.getURL(OFFSCREEN_URL), + ) + ) { + sendResponse({ ok: false, error: "Storage request denied" }); + return false; + } + executeStorageBridgeRequest(message) + .then(sendResponse) + .catch((error: unknown) => + sendResponse({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }), + ); + return true; + } if (isRecordingStatusBroadcast(message)) { setRecordingStatusAndBroadcast(message.status); if (isCapturingRecordingStatus(message.status)) return false; - externalCaptureAutoPipPending = false; void (async () => { await saveWebcamPreviewDismissed(true); await broadcastOverlayHide(); @@ -1813,7 +1972,7 @@ chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => { if (!isServiceWorkerRequest(message)) return false; - handleRequest(message, _sender) + handleRequest(message, sender) .then(sendResponse) .catch((error: unknown) => { sendResponse({ @@ -1858,13 +2017,37 @@ chrome.windows.onFocusChanged.addListener((windowId) => { return; } browserWindowFocused = true; - externalCaptureAutoPipPending = false; void exitActivePreviewAutoPip() .then(() => syncActivePreview()) .catch(() => undefined); }); chrome.tabs.onUpdated.addListener((tabId, changeInfo, tab) => { + if (changeInfo.status === "complete") { + void loadTabInputCaptureSession() + .then(async (session) => { + if (!session || session.tabId !== tabId) return; + const currentStatus = await syncRecordingStatus(); + if ( + !isCapturingRecordingStatus(currentStatus) || + !("videoId" in currentStatus) || + currentStatus.videoId !== session.recordingId + ) { + if (!isActiveRecordingStatus(currentStatus)) { + await saveTabInputCaptureSession(null); + } + return; + } + if (!canInjectIntoTab(tab)) return; + await chrome.tabs.sendMessage(tabId, { + type: "input-capture-start", + recordingId: session.recordingId, + }); + }) + .catch((error: unknown) => { + console.warn("Tab input capture could not resume", error); + }); + } if (changeInfo.status === "complete" && tab.active) { void syncActivePreview(tabId).catch(() => undefined); } @@ -1884,6 +2067,11 @@ chrome.tabs.onRemoved.addListener((tabId) => { if (activePreviewTabId === tabId) { activePreviewTabId = null; } + void loadTabInputCaptureSession() + .then((session) => + session?.tabId === tabId ? saveTabInputCaptureSession(null) : undefined, + ) + .catch(() => undefined); if (pendingPreviewTabId === tabId) { pendingPreviewTabId = null; } diff --git a/apps/chrome-extension/src/content/bootstrap.ts b/apps/chrome-extension/src/content/bootstrap.ts index 60cf019e170..6183046cf8b 100644 --- a/apps/chrome-extension/src/content/bootstrap.ts +++ b/apps/chrome-extension/src/content/bootstrap.ts @@ -6,6 +6,7 @@ import { RECORDING_STATE_KEY, SHARED_UI_STATE_KEY, } from "../shared/storage-keys"; +import { initTabInputCapture } from "./input-events"; // The manifest injects only this bootstrap into every page: a few KB of // vanilla code that decides whether the page actually needs the recorder UI. @@ -186,5 +187,6 @@ const bootstrap = () => { const globalScope = globalThis as Record; if (globalScope[BOOTSTRAP_FLAG] !== true) { globalScope[BOOTSTRAP_FLAG] = true; + initTabInputCapture(); bootstrap(); } diff --git a/apps/chrome-extension/src/content/countdown-overlay.tsx b/apps/chrome-extension/src/content/countdown-overlay.tsx index 11cd3bb5180..7c3fc711c35 100644 --- a/apps/chrome-extension/src/content/countdown-overlay.tsx +++ b/apps/chrome-extension/src/content/countdown-overlay.tsx @@ -1,6 +1,7 @@ import { type CSSProperties, useCallback, useEffect, useState } from "react"; import { isOverlayMessage } from "../shared/messages"; import { sendServiceWorkerMessage } from "../shared/runtime"; +import { replayStartupMessages } from "./startup-messages"; type ActiveCountdown = { seconds: number; @@ -69,6 +70,7 @@ export function CountdownOverlay() { }; chrome.runtime.onMessage.addListener(handleMessage); + replayStartupMessages(handleMessage); return () => chrome.runtime.onMessage.removeListener(handleMessage); }, [dismiss]); diff --git a/apps/chrome-extension/src/content/input-events-browser-replay.ts b/apps/chrome-extension/src/content/input-events-browser-replay.ts new file mode 100644 index 00000000000..fcf9cceb88d --- /dev/null +++ b/apps/chrome-extension/src/content/input-events-browser-replay.ts @@ -0,0 +1,257 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { chromium } from "@playwright/test"; +import { parseCapturedTabInputBatch } from "../shared/input-events"; + +const html = ` + + + + + + +
Private text
+
Custom text
+
PIN
+ + + + +
Custom code field
+ + +
Unlabelled private widget
+
+ + +
+
+
+ +
+ +
+ + +
+ + +`; + +const tempDirectory = await mkdtemp(join(tmpdir(), "cap-input-privacy-")); +const entry = join(tempDirectory, "entry.ts"); +await writeFile( + entry, + `import { initTabInputCapture } from ${JSON.stringify(resolve(import.meta.dir, "input-events.ts"))};\ninitTabInputCapture();\n`, +); + +const bundled = await Bun.build({ + entrypoints: [entry], + target: "browser", + format: "iife", + splitting: false, +}); +assert.ok(bundled.success, bundled.logs.map((log) => log.message).join("\n")); +const contentScript = await bundled.outputs[0]?.text(); +assert.ok(contentScript); + +const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch() { + return new Response(html, { + headers: { "Content-Type": "text/html; charset=utf-8" }, + }); + }, +}); +const browser = await chromium.launch({ headless: true }); + +try { + const page = await browser.newPage({ + viewport: { width: 900, height: 800 }, + }); + const pageErrors: string[] = []; + page.on("pageerror", (error) => pageErrors.push(error.message)); + await page.addInitScript(() => { + type ContentListener = ( + message: { type: string; recordingId?: string }, + sender: unknown, + respond: (value: unknown) => void, + ) => boolean; + (window as Window & { capInputBatches?: unknown[] }).capInputBatches = []; + Object.defineProperty(window, "chrome", { + configurable: true, + value: { + runtime: { + lastError: null, + onMessage: { + addListener: (listener: ContentListener) => { + ( + window as Window & { + capInputListener?: ContentListener; + } + ).capInputListener = listener; + }, + }, + sendMessage: ( + message: unknown, + callback: (value: unknown) => void, + ) => { + ( + window as Window & { capInputBatches?: unknown[] } + ).capInputBatches?.push(message); + callback({ ok: true }); + }, + }, + storage: { + onChanged: { addListener: () => undefined }, + }, + }, + }); + }); + await page.goto(`http://127.0.0.1:${server.port}/capture.html`); + await page.addScriptTag({ content: contentScript }); + assert.deepEqual(pageErrors, []); + assert.equal( + await page.evaluate( + () => + typeof ( + window as Window & { + capInputListener?: (...args: unknown[]) => unknown; + } + ).capInputListener, + ), + "function", + ); + const requestCapture = (type: string) => + page.evaluate((requestType) => { + const listener = ( + window as Window & { + capInputListener?: ( + message: { type: string; recordingId: string }, + sender: unknown, + respond: (value: unknown) => void, + ) => boolean; + } + ).capInputListener; + if (!listener) throw new Error("Content listener did not load"); + return new Promise((resolve) => { + listener( + { type: requestType, recordingId: "privacy-replay" }, + {}, + resolve, + ); + }); + }, type); + assert.deepEqual(await requestCapture("input-capture-start"), { ok: true }); + + const privateSelectors = [ + "#password", + "#editable", + "#textbox", + "#pin", + "#secret", + "#email-label", + "#email", + "#otp-code", + "#custom-code", + "#svg-pin", + "#closed-widget", + "#focusable-secret", + "#card", + "#pay", + "#shadow-host input", + "#overlay-button", + ]; + const privateBounds = await Promise.all( + privateSelectors.map(async (selector) => { + const bounds = await page.locator(selector).boundingBox(); + assert.ok(bounds, selector); + return bounds; + }), + ); + for (const selector of privateSelectors) await page.locator(selector).click(); + const publicBounds = await page.locator("#public").boundingBox(); + assert.ok(publicBounds); + await page.locator("#public").click(); + const shippingBounds = await page.locator("#shipping").boundingBox(); + assert.ok(shippingBounds); + await page.locator("#shipping").click(); + await page.keyboard.press("Escape"); + assert.deepEqual(await requestCapture("input-capture-stop"), { ok: true }); + + const rawBatches = await page.evaluate( + () => + (window as Window & { capInputBatches?: unknown[] }).capInputBatches ?? + [], + ); + const batches = rawBatches.map(parseCapturedTabInputBatch); + assert.ok(batches.length > 0); + assert.ok(batches.every((batch) => batch !== null)); + const pointerSamples = batches.flatMap( + (batch) => + batch?.events.flatMap((event) => + event.kind === "move" || event.kind === "down" || event.kind === "up" + ? [ + { + kind: event.kind, + x: event.x * batch.viewportWidth, + y: event.y * batch.viewportHeight, + }, + ] + : [], + ) ?? [], + ); + const inside = ( + point: { x: number; y: number }, + bounds: { x: number; y: number; width: number; height: number }, + ) => + point.x >= bounds.x && + point.x <= bounds.x + bounds.width && + point.y >= bounds.y && + point.y <= bounds.y + bounds.height; + assert.ok( + pointerSamples.some( + (point) => point.kind === "down" && inside(point, publicBounds), + ), + ); + assert.ok( + pointerSamples.some( + (point) => point.kind === "down" && inside(point, shippingBounds), + ), + ); + for (const bounds of privateBounds) { + assert.equal( + pointerSamples.some((point) => inside(point, bounds)), + false, + ); + } + assert.ok( + batches.some((batch) => + batch?.events.some( + (event) => event.kind === "keyDown" && event.key === "Escape", + ), + ), + ); + assert.deepEqual(pageErrors, []); + process.stdout.write( + `${JSON.stringify({ privateTargets: privateSelectors.length, publicPointerEvents: pointerSamples.length, batches: batches.length, pageErrors })}\n`, + ); +} finally { + await browser.close(); + server.stop(true); + await rm(tempDirectory, { recursive: true, force: true }); +} diff --git a/apps/chrome-extension/src/content/input-events.test.ts b/apps/chrome-extension/src/content/input-events.test.ts new file mode 100644 index 00000000000..6aa1124baf7 --- /dev/null +++ b/apps/chrome-extension/src/content/input-events.test.ts @@ -0,0 +1,55 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { parseCapturedTabInputBatch } from "../shared/input-events"; +import { createTabInputCollectorId } from "./input-events"; + +afterEach(() => vi.unstubAllGlobals()); + +describe("tab input collector IDs", () => { + it("uses native secure-context UUIDs when available", () => { + const id = "67f7620d-a814-4329-acec-044e6aab944c"; + vi.stubGlobal("crypto", { randomUUID: () => id }); + expect(createTabInputCollectorId()).toBe(id); + }); + + it("creates valid, unique v4 IDs without randomUUID", () => { + let seed = 0; + vi.stubGlobal("crypto", { + getRandomValues: (bytes: Uint8Array) => { + for (let index = 0; index < bytes.length; index++) { + bytes[index] = seed % 256; + seed++; + } + return bytes; + }, + }); + const first = createTabInputCollectorId(); + const second = createTabInputCollectorId(); + expect(first).toBe("00010203-0405-4607-8809-0a0b0c0d0e0f"); + expect(second).not.toBe(first); + for (const collectorId of [first, second]) { + expect( + parseCapturedTabInputBatch({ + target: "offscreen", + type: "input-events-batch", + recordingId: "video-1", + collectorId, + sequence: 0, + platform: "MacIntel", + viewportWidth: 1273, + viewportHeight: 716, + events: [ + { + kind: "down", + epochMs: 1_000, + x: 0.5, + y: 0.5, + cursor: "pointer", + button: 0, + modifiers: [], + }, + ], + }), + ).not.toBeNull(); + } + }); +}); diff --git a/apps/chrome-extension/src/content/input-events.ts b/apps/chrome-extension/src/content/input-events.ts new file mode 100644 index 00000000000..8c8b2e94421 --- /dev/null +++ b/apps/chrome-extension/src/content/input-events.ts @@ -0,0 +1,345 @@ +import { + type CapturedTabInputEvent, + isSafeTabKeyboardEvent, +} from "../shared/input-events"; +import { RECORDING_STATE_KEY } from "../shared/storage-keys"; + +const MOVE_INTERVAL_MS = 16; +const BATCH_INTERVAL_MS = 250; +const MAX_BATCH_EVENTS = 128; +const MAX_PENDING_BATCHES = 8; +const OVERLAY_ROOT_ID = "cap-extension-recorder-overlay"; +const EDITABLE_ROLES = new Set([ + "textbox", + "searchbox", + "combobox", + "spinbutton", +]); +const SENSITIVE_CONTROL_NAME = + /\b(?:password|passcode|pin|otp|one[- ]time(?:[- ]code)?|cvv|cvc|credit card|card number|security code|secret)\b/i; +const SENSITIVE_FORM_FIELDS = `input[type="password"], input[autocomplete*="cc-"], input[autocomplete*="one-time-code"], input[name*="password" i], input[name="pin" i], input[name$="-pin" i], input[name$="_pin" i], input[name*="cvv" i], input[name*="cvc" i]`; +const ALLOWED_CURSORS = new Set([ + "auto", + "default", + "pointer", + "text", + "crosshair", + "grab", + "grabbing", + "not-allowed", + "ew-resize", + "ns-resize", +]); + +export const createTabInputCollectorId = () => { + const secureRandom = globalThis.crypto; + if (typeof secureRandom?.randomUUID === "function") { + return secureRandom.randomUUID(); + } + if (typeof secureRandom?.getRandomValues !== "function") { + throw new Error("Secure random source is unavailable"); + } + const bytes = secureRandom.getRandomValues(new Uint8Array(16)); + bytes[6] = ((bytes[6] ?? 0) & 0x0f) | 0x40; + bytes[8] = ((bytes[8] ?? 0) & 0x3f) | 0x80; + const hex = Array.from(bytes, (byte) => + byte.toString(16).padStart(2, "0"), + ).join(""); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +}; + +const modifiersOf = (event: MouseEvent | KeyboardEvent) => { + const modifiers: string[] = []; + if (event.metaKey) modifiers.push("Meta"); + if (event.ctrlKey) modifiers.push("LControl"); + if (event.altKey) modifiers.push("LAlt"); + if (event.shiftKey) modifiers.push("LShift"); + return modifiers; +}; + +const isEditableTarget = (event: Event) => + event.composedPath().some((target) => { + if (!(target instanceof HTMLElement)) return false; + return ( + target.isContentEditable || + target instanceof HTMLInputElement || + target instanceof HTMLTextAreaElement || + target instanceof HTMLSelectElement || + EDITABLE_ROLES.has(target.getAttribute("role") ?? "") || + target.hasAttribute("aria-multiline") + ); + }); + +const isSensitivePointerTarget = (event: PointerEvent) => + isEditableTarget(event) || + event.composedPath().some((target) => { + if (!(target instanceof Element)) return false; + if ( + target instanceof HTMLIFrameElement || + target.hasAttribute("inputmode") || + target.hasAttribute("data-private") || + target.hasAttribute("data-sensitive") || + SENSITIVE_CONTROL_NAME.test( + `${target.getAttribute("aria-label") ?? ""} ${target.getAttribute("title") ?? ""} ${target.getAttribute("name") ?? ""} ${target.localName.replaceAll("-", " ")}`, + ) + ) + return true; + if ( + target instanceof HTMLElement && + target.tabIndex >= 0 && + !(target instanceof HTMLButtonElement) && + !(target instanceof HTMLAnchorElement) + ) + return true; + if (target instanceof HTMLLabelElement && target.control) { + return ( + target.control instanceof HTMLInputElement || + target.control instanceof HTMLTextAreaElement || + target.control instanceof HTMLSelectElement + ); + } + return ( + target instanceof HTMLFormElement && + target.querySelector(SENSITIVE_FORM_FIELDS) !== null + ); + }); + +const isExtensionUiEvent = (event: Event) => + event + .composedPath() + .some( + (target) => target instanceof Element && target.id === OVERLAY_ROOT_ID, + ); + +export function initTabInputCapture() { + let recordingId: string | null = null; + let batch: CapturedTabInputEvent[] = []; + let pendingBatches = 0; + let sendChain: Promise = Promise.resolve(); + let sendFailed: Error | null = null; + let sequence = 0; + let collectorId = ""; + let timer: number | null = null; + let lastMoveAt = -Infinity; + let lastTarget: EventTarget | null = null; + let lastCursor = "default"; + let viewportWidth = window.innerWidth; + let viewportHeight = window.innerHeight; + + const stopListeners = () => { + recordingId = null; + if (timer !== null) window.clearInterval(timer); + timer = null; + document.removeEventListener("pointermove", onMove, true); + document.removeEventListener("pointerdown", onDown, true); + document.removeEventListener("pointerup", onUp, true); + document.removeEventListener("keydown", onKeyDown, true); + document.removeEventListener("keyup", onKeyUp, true); + }; + + const stop = async () => { + flush(); + stopListeners(); + await sendChain; + return sendFailed ? { ok: false, error: sendFailed.message } : { ok: true }; + }; + + const flush = () => { + if (!recordingId || batch.length === 0 || sendFailed) return; + if (pendingBatches >= MAX_PENDING_BATCHES) { + sendFailed = new Error("Tab input event queue exceeded its limit"); + stopListeners(); + return; + } + const events = batch; + batch = []; + pendingBatches += 1; + const message = { + target: "offscreen", + type: "input-events-batch", + recordingId, + collectorId, + sequence: sequence++, + platform: navigator.platform, + viewportWidth, + viewportHeight, + events, + }; + const pending = sendChain.then(() => { + if (sendFailed) throw sendFailed; + return new Promise((resolve, reject) => { + chrome.runtime.sendMessage(message, (response: unknown) => { + const error = chrome.runtime.lastError; + if (error) { + reject(new Error(error.message ?? "Tab input upload failed")); + return; + } + if ( + !response || + typeof response !== "object" || + !("ok" in response) || + response.ok !== true + ) { + reject(new Error("Tab input batch was not accepted")); + return; + } + resolve(); + }); + }); + }); + sendChain = pending.then( + () => { + pendingBatches -= 1; + }, + (error: unknown) => { + pendingBatches -= 1; + sendFailed = error instanceof Error ? error : new Error(String(error)); + stopListeners(); + }, + ); + }; + + const push = (event: CapturedTabInputEvent) => { + if (!recordingId) return; + if ( + window.innerWidth !== viewportWidth || + window.innerHeight !== viewportHeight + ) { + flush(); + viewportWidth = window.innerWidth; + viewportHeight = window.innerHeight; + } + batch.push(event); + if (batch.length >= MAX_BATCH_EVENTS) flush(); + }; + + const pointer = ( + kind: "move" | "down" | "up", + event: PointerEvent, + forceMove = false, + ) => { + if ( + !recordingId || + !event.isTrusted || + isExtensionUiEvent(event) || + isSensitivePointerTarget(event) || + window.innerWidth <= 0 || + window.innerHeight <= 0 + ) + return; + const epochMs = performance.timeOrigin + event.timeStamp; + if ( + kind === "move" && + !forceMove && + epochMs - lastMoveAt < MOVE_INTERVAL_MS + ) + return; + if (kind === "move") lastMoveAt = epochMs; + if (event.target !== lastTarget) { + lastTarget = event.target; + const cursor = + event.target instanceof Element + ? getComputedStyle(event.target).cursor + : "default"; + lastCursor = ALLOWED_CURSORS.has(cursor) ? cursor : "default"; + } + push({ + kind, + epochMs, + x: event.clientX / window.innerWidth, + y: event.clientY / window.innerHeight, + cursor: lastCursor, + button: event.button < 0 ? 0 : event.button, + modifiers: modifiersOf(event), + }); + }; + const onMove = (event: PointerEvent) => pointer("move", event); + const onDown = (event: PointerEvent) => { + pointer("move", event, true); + pointer("down", event); + }; + const onUp = (event: PointerEvent) => pointer("up", event); + const key = (kind: "keyDown" | "keyUp", event: KeyboardEvent) => { + if ( + !recordingId || + !event.isTrusted || + isExtensionUiEvent(event) || + isEditableTarget(event) || + !isSafeTabKeyboardEvent(event.key, event.code) + ) + return; + push({ + kind, + epochMs: performance.timeOrigin + event.timeStamp, + key: event.key, + code: event.code, + modifiers: modifiersOf(event), + }); + }; + const onKeyDown = (event: KeyboardEvent) => key("keyDown", event); + const onKeyUp = (event: KeyboardEvent) => key("keyUp", event); + + const start = async (nextRecordingId: string) => { + if (recordingId === nextRecordingId) return { ok: true }; + if (recordingId) await stop(); + recordingId = nextRecordingId; + collectorId = createTabInputCollectorId(); + batch = []; + pendingBatches = 0; + sendChain = Promise.resolve(); + sendFailed = null; + sequence = 0; + lastMoveAt = -Infinity; + lastTarget = null; + viewportWidth = window.innerWidth; + viewportHeight = window.innerHeight; + document.addEventListener("pointermove", onMove, { + capture: true, + passive: true, + }); + document.addEventListener("pointerdown", onDown, { + capture: true, + passive: true, + }); + document.addEventListener("pointerup", onUp, { + capture: true, + passive: true, + }); + document.addEventListener("keydown", onKeyDown, { + capture: true, + passive: true, + }); + document.addEventListener("keyup", onKeyUp, { + capture: true, + passive: true, + }); + timer = window.setInterval(flush, BATCH_INTERVAL_MS); + return { ok: true }; + }; + + chrome.runtime.onMessage.addListener((message: unknown, _sender, respond) => { + if (!message || typeof message !== "object" || !("type" in message)) + return false; + if ( + message.type === "input-capture-start" && + "recordingId" in message && + typeof message.recordingId === "string" + ) { + void start(message.recordingId).then(respond); + } else if (message.type === "input-capture-stop") { + void stop().then(respond); + } else { + return false; + } + return true; + }); + chrome.storage.onChanged.addListener((changes, area) => { + if (area !== "session" || !recordingId) return; + const state = changes[RECORDING_STATE_KEY]?.newValue; + if (!state || typeof state !== "object" || !("status" in state)) return; + const status = state.status; + if (!status || typeof status !== "object" || !("phase" in status)) return; + if (status.phase !== "recording" && status.phase !== "paused") void stop(); + }); + window.addEventListener("pagehide", () => void stop()); +} diff --git a/apps/chrome-extension/src/content/overlay.tsx b/apps/chrome-extension/src/content/overlay.tsx index f7f2946cef4..335db134cf6 100644 --- a/apps/chrome-extension/src/content/overlay.tsx +++ b/apps/chrome-extension/src/content/overlay.tsx @@ -17,6 +17,7 @@ import { loadSharedUiState, loadWebcamPreviewDismissed, OVERLAY_UI_STATE_KEY, + RECORDING_STATE_KEY, SETTINGS_KEY, SHARED_UI_STATE_KEY, saveLastWebcamPreviewFrame, @@ -30,6 +31,7 @@ import type { ExtensionSettings, OverlayPosition, RecordingStatus, + SharedRecordingState, WebcamPosition, WebcamPreviewFrame, WebcamSettings, @@ -43,6 +45,7 @@ import { ConfirmOverlay } from "./confirm-overlay"; import { CountdownOverlay } from "./countdown-overlay"; import overlayCss from "./overlay.css?inline"; import { RecordingBarOverlay } from "./recording-bar"; +import recordingErrorCss from "./recording-error-toast.css?inline"; import { replayStartupMessages, setStartupMessages } from "./startup-messages"; const ROOT_ID = "cap-extension-recorder-overlay"; @@ -157,6 +160,11 @@ type PreviewParentMessage = source: "cap-extension-overlay"; token: string; type: "stop"; + } + | { + source: "cap-extension-overlay"; + token: string; + type: "stop-for-capture"; }; const classNames = (...values: Array) => @@ -339,6 +347,7 @@ function OverlayApp() { const [previewPointerInside, setPreviewPointerInside] = useState(false); const [recordingPreviewActive, setRecordingPreviewActive] = useState(false); const [recorderPanelOpen, setRecorderPanelOpen] = useState(false); + const [recordingError, setRecordingError] = useState(null); const [previewTokenReady, setPreviewTokenReady] = useState(false); const iframeRef = useRef(null); const windowRef = useRef(null); @@ -348,6 +357,8 @@ function OverlayApp() { const pipPeerRef = useRef(null); const pipStreamRef = useRef(null); const pipSettingsRef = useRef(null); + const parentPictureInPictureRequestRef = + useRef | null>(null); const previewSessionIdRef = useRef(null); const webcamRef = useRef(null); const settingsRef = useRef(null); @@ -383,6 +394,28 @@ function OverlayApp() { }); }, []); + const requestParentPictureInPicture = useCallback( + async (video: HTMLVideoElement) => { + if (recordingPreviewActiveRef.current) return; + const request = video.requestPictureInPicture(); + parentPictureInPictureRequestRef.current = request; + try { + await request; + if ( + recordingPreviewActiveRef.current && + document.pictureInPictureElement === video + ) { + await document.exitPictureInPicture(); + } + } finally { + if (parentPictureInPictureRequestRef.current === request) { + parentPictureInPictureRequestRef.current = null; + } + } + }, + [], + ); + const persistPreviewFrame = useCallback( (frame: WebcamPreviewFrame | null, force: boolean) => { if (!frame) return; @@ -453,6 +486,67 @@ function OverlayApp() { postPreviewMessage, ]); + const stopFramePictureInPictureForCapture = useCallback( + () => + new Promise((resolve) => { + if (!iframeRef.current || !previewOpenRef.current || !iframeReady) { + resolve(true); + return; + } + + let settled = false; + const finish = (ok: boolean) => { + if (settled) return; + settled = true; + window.clearTimeout(timeout); + resolve(ok); + }; + const timeout = window.setTimeout(() => finish(false), 3000); + chrome.runtime.sendMessage( + { + source: "cap-extension-overlay", + token: PREVIEW_TOKEN, + type: "stop-for-capture", + } satisfies PreviewParentMessage, + (response: unknown) => { + const responseOk = + !chrome.runtime.lastError && + response !== null && + typeof response === "object" && + "ok" in response && + response.ok === true; + finish(responseOk); + }, + ); + }), + [iframeReady], + ); + + const prepareCameraForCapture = useCallback(async () => { + recordingPreviewActiveRef.current = true; + setRecordingPreviewActive(true); + const video = pipVideoRef.current; + const pendingParentRequest = parentPictureInPictureRequestRef.current; + const [frameResult, parentResult] = await Promise.allSettled([ + stopFramePictureInPictureForCapture(), + (async () => { + if (pendingParentRequest) { + await pendingParentRequest.catch(() => undefined); + } + if (video && document.pictureInPictureElement === video) { + await document.exitPictureInPicture(); + } + })(), + ]); + stopLocalPreview(); + return ( + frameResult.status === "fulfilled" && + frameResult.value && + parentResult.status === "fulfilled" && + (!video || document.pictureInPictureElement !== video) + ); + }, [stopFramePictureInPictureForCapture, stopLocalPreview]); + const applyWebcamSettings = useCallback( (getNext: (current: WebcamSettings) => WebcamSettings) => { setExtensionSettings((current) => { @@ -603,6 +697,19 @@ function OverlayApp() { }; }, [stopLocalPreview]); + useEffect(() => { + if (!recordingPreviewActive) return; + const video = pipVideoRef.current; + if (video && document.pictureInPictureElement === video) { + void document.exitPictureInPicture().catch(() => undefined); + } + postPreviewMessage({ + source: "cap-extension-overlay", + token: PREVIEW_TOKEN, + type: "exit-auto-pip", + }); + }, [postPreviewMessage, recordingPreviewActive]); + useEffect(() => { let disposed = false; Promise.all([ @@ -639,7 +746,19 @@ function OverlayApp() { }) .catch(() => undefined); }; + const syncRecordingError = () => { + loadSharedRecordingState() + .then((state) => { + if (!disposed) { + setRecordingError( + state?.status.phase === "error" ? state.status.message : null, + ); + } + }) + .catch(() => undefined); + }; syncSharedUi(); + syncRecordingError(); const handleStorageChange = ( changes: Record, areaName: string, @@ -647,6 +766,14 @@ function OverlayApp() { if (areaName === "session" && changes[SHARED_UI_STATE_KEY]) { syncSharedUi(); } + if (areaName === "session" && changes[RECORDING_STATE_KEY]) { + const state = changes[RECORDING_STATE_KEY].newValue as + | SharedRecordingState + | undefined; + setRecordingError( + state?.status.phase === "error" ? state.status.message : null, + ); + } }; chrome.storage.onChanged.addListener(handleStorageChange); return () => { @@ -708,12 +835,66 @@ function OverlayApp() { }, [stopLocalPreview]); useEffect(() => { + let disposed = false; + const applyPreviewSettings = (message: { + settings: WebcamSettings; + recording: boolean; + }) => { + if (disposed) return; + const webcamSettings = message.settings; + const sameLivePreview = + previewOpenRef.current && + livePreviewReadyRef.current && + isSameWebcamSettings(webcamRef.current, webcamSettings); + if (!sameLivePreview) { + setLivePreviewReady(false); + } else { + // A restarted service worker loses which tab owns the live preview, + // so re-announce readiness whenever it pushes the same settings. + void sendServiceWorkerMessage({ + target: "service-worker", + type: "webcam-preview-ready", + }).catch(() => undefined); + } + setPreviewError(null); + setShowPreviewError(false); + setPreviewOpen(true); + recordingPreviewActiveRef.current = message.recording; + setRecordingPreviewActive(message.recording); + if (settingsRef.current) { + setExtensionSettings({ + ...settingsRef.current, + webcam: webcamSettings, + }); + } else { + sendServiceWorkerMessage({ + target: "service-worker", + type: "get-overlay-settings", + }) + .then((response) => { + if (!disposed && response.ok && response.settings) { + setExtensionSettings({ + ...response.settings, + webcam: webcamSettings, + }); + } + }) + .catch(() => undefined); + } + }; + const handleMessage = ( message: unknown, _sender: chrome.runtime.MessageSender, sendResponse: (response?: unknown) => void, ) => { if (!isOverlayMessage(message)) return false; + if (message.type === "overlay-settings" && message.recording) { + void prepareCameraForCapture() + .then((ok) => sendResponse({ ok })) + .catch(() => sendResponse({ ok: false })); + return true; + } sendResponse({ ok: true }); @@ -750,6 +931,7 @@ function OverlayApp() { } if (message.type === "overlay-enter-auto-pip") { + if (recordingPreviewActiveRef.current) return false; // Only one Picture in Picture surface may drive at a time; racing // the parent fallback video against the preview iframe flips PiP // on and off and leaves the badge state inconsistent. @@ -770,7 +952,7 @@ function OverlayApp() { if (video && parentPipSupported) { void video .play() - .then(() => video.requestPictureInPicture()) + .then(() => requestParentPictureInPicture(video)) .catch(() => undefined); } return false; @@ -791,48 +973,13 @@ function OverlayApp() { if (message.type !== "overlay-settings") return false; - if (previewDismissedRef.current) return false; - - const webcamSettings = message.settings; - const sameLivePreview = - previewOpenRef.current && - livePreviewReadyRef.current && - isSameWebcamSettings(webcamRef.current, webcamSettings); - if (!sameLivePreview) { - setLivePreviewReady(false); - } else { - // A restarted service worker loses which tab owns the live preview, - // so re-announce readiness whenever it pushes the same settings. - void sendServiceWorkerMessage({ - target: "service-worker", - type: "webcam-preview-ready", - }).catch(() => undefined); - } - setPreviewError(null); - setShowPreviewError(false); - setPreviewOpen(true); - recordingPreviewActiveRef.current = message.recording; - setRecordingPreviewActive(message.recording); - if (settingsRef.current) { - setExtensionSettings({ - ...settingsRef.current, - webcam: webcamSettings, - }); - } else { - sendServiceWorkerMessage({ - target: "service-worker", - type: "get-overlay-settings", + void loadWebcamPreviewDismissed() + .then((dismissed) => { + if (disposed) return; + previewDismissedRef.current = dismissed; + if (!dismissed) applyPreviewSettings(message); }) - .then((response) => { - if (response.ok && response.settings) { - setExtensionSettings({ - ...response.settings, - webcam: webcamSettings, - }); - } - }) - .catch(() => undefined); - } + .catch(() => undefined); return false; }; @@ -841,8 +988,17 @@ function OverlayApp() { startupReplayedRef.current = true; replayStartupMessages(handleMessage); } - return () => chrome.runtime.onMessage.removeListener(handleMessage); - }, [parentPipSupported, postPreviewMessage, stopLocalPreview]); + return () => { + disposed = true; + chrome.runtime.onMessage.removeListener(handleMessage); + }; + }, [ + parentPipSupported, + postPreviewMessage, + prepareCameraForCapture, + requestParentPictureInPicture, + stopLocalPreview, + ]); const beginDrag = useCallback((clientX: number, clientY: number) => { isDraggingRef.current = true; @@ -1082,7 +1238,11 @@ function OverlayApp() { if (event.type === "ready") { setIframeReady(true); const current = webcamRef.current; - if (current?.enabled && current.deviceId) { + if ( + !recordingPreviewActiveRef.current && + current?.enabled && + current.deviceId + ) { postPreviewMessage({ source: "cap-extension-overlay", token: PREVIEW_TOKEN, @@ -1430,6 +1590,7 @@ function OverlayApp() { }, [applyWebcamSettings]); const handleTogglePictureInPicture = useCallback(() => { + if (recordingPreviewActiveRef.current) return; const video = pipVideoRef.current; if (video && document.pictureInPictureElement === video) { void document.exitPictureInPicture().catch(() => undefined); @@ -1454,10 +1615,10 @@ function OverlayApp() { void (async () => { try { await video.play().catch(() => undefined); - await video.requestPictureInPicture(); + await requestParentPictureInPicture(video); } catch {} })(); - }, [parentPipSupported, postPreviewMessage]); + }, [parentPipSupported, postPreviewMessage, requestParentPictureInPicture]); const handlePreviewLoad = useCallback(() => { const current = webcamRef.current; @@ -1511,6 +1672,7 @@ function OverlayApp() { width: `${metrics.width}px`, height: `${totalHeight}px`, borderRadius, + visibility: recordingPreviewActive ? "hidden" : "visible", }} onPointerDown={handlePointerDown} > @@ -1671,6 +1833,12 @@ function OverlayApp() { }).catch(() => undefined); }} /> + {recordingError && recorderPanelOpen ? ( +
+ Recording failed. + {recordingError} +
+ ) : null} {isDragging ? ( @@ -1726,7 +1894,7 @@ const mountOverlay = () => { root.dataset.capMounted = "true"; const shadow = root.attachShadow({ mode: "closed" }); const style = document.createElement("style"); - style.textContent = overlayCss; + style.textContent = `${overlayCss}\n${recordingErrorCss}`; const app = document.createElement("div"); shadow.append(style, app); document.documentElement.append(root); diff --git a/apps/chrome-extension/src/content/recording-error-toast.css b/apps/chrome-extension/src/content/recording-error-toast.css new file mode 100644 index 00000000000..b3b665e54a1 --- /dev/null +++ b/apps/chrome-extension/src/content/recording-error-toast.css @@ -0,0 +1,27 @@ +.cap-extension-recording-error-toast { + position: fixed; + top: 20px; + left: 50%; + z-index: 2147483647; + display: flex; + max-width: min(440px, calc(100vw - 40px)); + flex-direction: column; + gap: 5px; + padding: 12px 16px; + border: 1px solid #f1c1c4; + border-radius: 12px; + background: #fff; + box-shadow: 0 12px 40px rgba(0, 0, 0, 0.16); + color: #3f171b; + font-family: Inter, ui-sans-serif, system-ui, sans-serif; + font-size: 12px; + line-height: 1.35; + pointer-events: auto; + transform: translateX(-50%); +} + +.cap-extension-recording-error-toast strong { + color: #c9343e; + font-size: 13px; + font-weight: 600; +} diff --git a/apps/chrome-extension/src/offscreen/display-capture.test.ts b/apps/chrome-extension/src/offscreen/display-capture.test.ts index a1f860e5826..36b3654587e 100644 --- a/apps/chrome-extension/src/offscreen/display-capture.test.ts +++ b/apps/chrome-extension/src/offscreen/display-capture.test.ts @@ -4,6 +4,21 @@ import { captureDisplayStream } from "./display-capture"; const stream = {} as MediaStream; describe("captureDisplayStream", () => { + it("prefers the current tab when tab capture needs the display picker", async () => { + const request = vi.fn().mockResolvedValue(stream); + + await expect(captureDisplayStream("tab", false, request)).resolves.toBe( + stream, + ); + expect(request).toHaveBeenCalledTimes(1); + expect(request.mock.calls[0]?.[0]).toMatchObject({ + preferCurrentTab: true, + selfBrowserSurface: "include", + monitorTypeSurfaces: "exclude", + audio: false, + }); + }); + it("falls back when advanced options are rejected before a picker can open", async () => { const request = vi .fn() diff --git a/apps/chrome-extension/src/offscreen/display-capture.ts b/apps/chrome-extension/src/offscreen/display-capture.ts index af5ee9547da..846c62a626d 100644 --- a/apps/chrome-extension/src/offscreen/display-capture.ts +++ b/apps/chrome-extension/src/offscreen/display-capture.ts @@ -44,7 +44,7 @@ const canRetryWithoutShowingAnotherPicker = (attempt: CaptureAttempt) => shouldRetryDisplayMediaWithoutPreferences(attempt.error); export const captureDisplayStream = async ( - mode: Exclude, + mode: Exclude, includeAudio: boolean, requestDisplayMedia: DisplayMediaRequester, ) => { diff --git a/apps/chrome-extension/src/offscreen/input-coordinates.test.ts b/apps/chrome-extension/src/offscreen/input-coordinates.test.ts new file mode 100644 index 00000000000..bcb8450ecbd --- /dev/null +++ b/apps/chrome-extension/src/offscreen/input-coordinates.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vitest"; +import { mapTabPointerToVideo } from "./input-coordinates"; + +describe("tab input coordinates", () => { + it("places viewport corners inside Chrome tab-capture padding", () => { + const topLeft = mapTabPointerToVideo(0, 0, 1273, 716, 3024, 1964); + const center = mapTabPointerToVideo(0.5, 0.5, 1273, 716, 3024, 1964); + const bottomRight = mapTabPointerToVideo(1, 1, 1273, 716, 3024, 1964); + expect(topLeft?.x).toBe(0); + expect(topLeft?.y).toBeCloseTo(131 / 1964, 2); + expect(center).toEqual({ x: 0.5, y: 0.5 }); + expect(bottomRight?.x).toBe(1); + expect(bottomRight?.y).toBeCloseTo(1832 / 1964, 2); + }); + + it("maps portrait tabs with side padding", () => { + const pointer = mapTabPointerToVideo(0, 0, 716, 1273, 1920, 1080); + expect(pointer?.x).toBeGreaterThan(0); + expect(pointer?.y).toBe(0); + }); + + it("rejects impossible source dimensions", () => { + expect(mapTabPointerToVideo(0.5, 0.5, 0, 100, 1920, 1080)).toBeNull(); + expect( + mapTabPointerToVideo(Number.NaN, 0.5, 100, 100, 1920, 1080), + ).toBeNull(); + }); +}); diff --git a/apps/chrome-extension/src/offscreen/input-coordinates.ts b/apps/chrome-extension/src/offscreen/input-coordinates.ts new file mode 100644 index 00000000000..60e6499fcbb --- /dev/null +++ b/apps/chrome-extension/src/offscreen/input-coordinates.ts @@ -0,0 +1,34 @@ +export function mapTabPointerToVideo( + x: number, + y: number, + viewportWidth: number, + viewportHeight: number, + videoWidth: number, + videoHeight: number, +) { + if ( + ![x, y, viewportWidth, viewportHeight, videoWidth, videoHeight].every( + Number.isFinite, + ) || + viewportWidth <= 0 || + viewportHeight <= 0 || + videoWidth <= 0 || + videoHeight <= 0 + ) { + return null; + } + const scale = Math.min( + videoWidth / viewportWidth, + videoHeight / viewportHeight, + ); + const contentWidth = viewportWidth * scale; + const contentHeight = viewportHeight * scale; + return { + x: + (videoWidth - contentWidth) / (2 * videoWidth) + + (x * contentWidth) / videoWidth, + y: + (videoHeight - contentHeight) / (2 * videoHeight) + + (y * contentHeight) / videoHeight, + }; +} diff --git a/apps/chrome-extension/src/offscreen/input-event-sidecar.ts b/apps/chrome-extension/src/offscreen/input-event-sidecar.ts new file mode 100644 index 00000000000..71d36942727 --- /dev/null +++ b/apps/chrome-extension/src/offscreen/input-event-sidecar.ts @@ -0,0 +1,244 @@ +import { + InstantRecordingUploader, + initiateMultipartUpload, + type RecorderApiOptions, + RecordingSpool, + type VideoId, +} from "@cap/recorder-core"; +import { + type CapturedTabInputEvent, + parseCapturedTabInputBatch, +} from "../shared/input-events"; +import { mapTabPointerToVideo } from "./input-coordinates"; +import { InputTimeline } from "./input-timeline"; + +const INPUT_SUBPATH = "input-events-upload.ndjson"; +const INPUT_MIME_TYPE = "application/x-ndjson"; +const MAX_INPUT_BYTES = 64 * 1024 * 1024; +const MAX_COLLECTORS = 1_000; + +export class InputEventSidecar { + private timeline: InputTimeline | null = null; + private readonly nextSequenceByCollector = new Map(); + private appendChain = Promise.resolve(); + private platform: string | null = null; + private totalBytes = 0; + private failed: Error | null = null; + private uploadCompleted = false; + + private constructor( + readonly spool: RecordingSpool, + readonly recordingId: string, + readonly tabId: number, + readonly videoWidth: number, + readonly videoHeight: number, + ) {} + + static async create(input: { + recordingId: string; + tabId: number; + videoWidth: number; + videoHeight: number; + }) { + const spool = await RecordingSpool.create({ mimeType: INPUT_MIME_TYPE }); + return new InputEventSidecar( + spool, + input.recordingId, + input.tabId, + input.videoWidth, + input.videoHeight, + ); + } + + get recordedBytes() { + return this.totalBytes; + } + + get isUploadCompleted() { + return this.uploadCompleted; + } + + start(epochMs: number) { + this.timeline = new InputTimeline(epochMs); + } + + pause(epochMs: number) { + this.timeline?.pause(epochMs); + } + + resume(epochMs: number) { + this.timeline?.resume(epochMs); + } + + stop(epochMs: number) { + this.timeline?.stop(epochMs); + } + + accept(raw: unknown, senderTabId: number | undefined) { + const batch = parseCapturedTabInputBatch(raw); + if ( + !batch || + senderTabId !== this.tabId || + batch.recordingId !== this.recordingId + ) { + return Promise.reject(new Error("Tab input batch is unauthorized")); + } + const pending = this.appendChain.then(async () => { + if (this.failed) throw this.failed; + const nextSequence = + this.nextSequenceByCollector.get(batch.collectorId) ?? 0; + if (batch.sequence < nextSequence) return; + if (batch.sequence !== nextSequence) { + throw new Error("Tab input batch sequence is incomplete"); + } + if ( + nextSequence === 0 && + this.nextSequenceByCollector.size >= MAX_COLLECTORS + ) { + throw new Error("Too many tab input capture sessions"); + } + if (this.platform && this.platform !== batch.platform) { + throw new Error("Tab input platform changed during recording"); + } + const mapped = batch.events.flatMap((event) => { + const timeMs = this.timeline?.timeFor(event.epochMs) ?? null; + if (timeMs === null) return []; + return [ + this.mapEvent( + event, + timeMs, + batch.viewportWidth, + batch.viewportHeight, + ), + ]; + }); + const lines = [ + ...(!this.platform + ? [JSON.stringify({ version: 1, platform: batch.platform })] + : []), + ...mapped.map((event) => JSON.stringify(event)), + ]; + if (lines.length > 0) { + const blob = new Blob([`${lines.join("\n")}\n`], { + type: INPUT_MIME_TYPE, + }); + if (this.totalBytes + blob.size > MAX_INPUT_BYTES) { + throw new Error("Tab input events exceeded the supported size"); + } + await this.spool.appendChunk(blob); + this.totalBytes += blob.size; + } + this.platform ??= batch.platform; + this.nextSequenceByCollector.set(batch.collectorId, nextSequence + 1); + }); + this.appendChain = pending.then( + () => undefined, + (error: unknown) => { + this.failed = error instanceof Error ? error : new Error(String(error)); + }, + ); + return pending; + } + + private mapEvent( + event: CapturedTabInputEvent, + timeMs: number, + viewportWidth: number, + viewportHeight: number, + ) { + if (!("x" in event)) { + return { + kind: event.kind, + timeMs, + key: event.key, + code: event.code, + modifiers: event.modifiers, + }; + } + const position = mapTabPointerToVideo( + event.x, + event.y, + viewportWidth, + viewportHeight, + this.videoWidth, + this.videoHeight, + ); + if (!position) throw new Error("Tab input viewport is invalid"); + return { + kind: event.kind, + timeMs, + x: position.x, + y: position.y, + cursor: event.cursor, + button: event.button, + modifiers: event.modifiers, + }; + } + + async recoverBlob() { + await this.appendChain; + if (this.failed) throw this.failed; + return this.spool.recoverBlob(); + } + + async upload(input: { + videoId: VideoId; + screenSubpath: string; + durationSeconds: number; + api: RecorderApiOptions; + }) { + const blob = await this.recoverBlob(); + if (!blob || blob.size === 0) return false; + await uploadInputEventBlob({ ...input, blob }); + this.uploadCompleted = true; + return true; + } + + async dispose() { + await this.spool.dispose(); + } +} + +export async function uploadInputEventBlob(input: { + videoId: VideoId; + screenSubpath: string; + durationSeconds: number; + api: RecorderApiOptions; + blob: Blob; +}) { + if ( + input.blob.size < 1 || + input.blob.size > MAX_INPUT_BYTES || + input.blob.type !== INPUT_MIME_TYPE + ) { + throw new Error("Invalid tab input event sidecar"); + } + const api: RecorderApiOptions = { + ...input.api, + extraBody: { + ...input.api.extraBody, + screenSubpath: input.screenSubpath, + }, + }; + const session = await initiateMultipartUpload({ + videoId: input.videoId, + contentType: INPUT_MIME_TYPE, + subpath: INPUT_SUBPATH, + api, + }); + const uploader = new InstantRecordingUploader({ + videoId: input.videoId, + uploadId: session.uploadId, + provider: session.provider, + mimeType: INPUT_MIME_TYPE, + subpath: INPUT_SUBPATH, + api, + setUploadStatus: () => undefined, + sendProgressUpdate: async () => undefined, + }); + await uploader.finalize({ + finalBlob: input.blob, + durationSeconds: input.durationSeconds, + subpath: INPUT_SUBPATH, + }); +} diff --git a/apps/chrome-extension/src/offscreen/input-timeline.test.ts b/apps/chrome-extension/src/offscreen/input-timeline.test.ts new file mode 100644 index 00000000000..eb82cab6dfc --- /dev/null +++ b/apps/chrome-extension/src/offscreen/input-timeline.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { InputTimeline } from "./input-timeline"; + +describe("tab input timeline", () => { + it("maps delayed batches across pause and resume", () => { + const timeline = new InputTimeline(1_000); + expect(timeline.timeFor(1_100)).toBe(100); + timeline.pause(1_500); + timeline.resume(3_000); + expect(timeline.timeFor(1_250)).toBe(250); + expect(timeline.timeFor(2_000)).toBeNull(); + expect(timeline.timeFor(3_050)).toBe(550); + timeline.stop(3_500); + expect(timeline.timeFor(3_500)).toBeNull(); + }); + + it("keeps consecutive active spans continuous", () => { + const timeline = new InputTimeline(1_000); + timeline.pause(1_500); + timeline.resume(3_000); + timeline.pause(3_500); + timeline.resume(4_000); + expect(timeline.timeFor(4_050)).toBe(1_050); + expect(timeline.timeFor(500)).toBeNull(); + }); +}); diff --git a/apps/chrome-extension/src/offscreen/input-timeline.ts b/apps/chrome-extension/src/offscreen/input-timeline.ts new file mode 100644 index 00000000000..214500e311b --- /dev/null +++ b/apps/chrome-extension/src/offscreen/input-timeline.ts @@ -0,0 +1,57 @@ +type ActiveSpan = { + startEpochMs: number; + startClipMs: number; + endEpochMs: number | null; +}; + +export class InputTimeline { + private spans: ActiveSpan[]; + private clipDurationMs = 0; + + constructor(startEpochMs: number) { + this.spans = [{ startEpochMs, startClipMs: 0, endEpochMs: null }]; + } + + pause(epochMs: number) { + const span = this.spans.at(-1); + if (!span || span.endEpochMs !== null) return; + span.endEpochMs = Math.max(span.startEpochMs, epochMs); + this.clipDurationMs = + span.startClipMs + span.endEpochMs - span.startEpochMs; + } + + resume(epochMs: number) { + const span = this.spans.at(-1); + if (!span || span.endEpochMs === null) return; + this.spans.push({ + startEpochMs: Math.max(span.endEpochMs, epochMs), + startClipMs: this.clipDurationMs, + endEpochMs: null, + }); + } + + stop(epochMs: number) { + this.pause(epochMs); + } + + timeFor(epochMs: number) { + if (!Number.isFinite(epochMs)) return null; + let low = 0; + let high = this.spans.length - 1; + let candidate = -1; + while (low <= high) { + const middle = (low + high) >>> 1; + const span = this.spans[middle]; + if (epochMs < span.startEpochMs) { + high = middle - 1; + } else { + candidate = middle; + low = middle + 1; + } + } + if (candidate < 0) return null; + const span = this.spans[candidate]; + if (span.endEpochMs !== null && epochMs >= span.endEpochMs) return null; + return Math.max(0, span.startClipMs + epochMs - span.startEpochMs); + } +} diff --git a/apps/chrome-extension/src/offscreen/recorder.ts b/apps/chrome-extension/src/offscreen/recorder.ts index 6173df09647..ad3ba7df303 100644 --- a/apps/chrome-extension/src/offscreen/recorder.ts +++ b/apps/chrome-extension/src/offscreen/recorder.ts @@ -1,4 +1,5 @@ import { + AudioRecordingSidecar, appendLocalRecordingChunk, type ChunkUploadState, DEFAULT_API_REQUEST_TIMEOUT_MS, @@ -9,16 +10,20 @@ import { InstantRecordingUploader, initialLocalRecordingState, initiateMultipartUpload, + isRecordingSpoolUploaded, isUserCancellationError, type LocalRecordingState, listRecordingSpoolSessions, MultipartCompletionUncertainError, RECORDING_SPOOL_LIVE_MIN_IDLE_MS, + type RecorderApiOptions, RecordingSpool, recoverRecordingSpoolSession, selectRecordingPipeline, + uploadRecoveredAudioSidecar, type VideoId, } from "@cap/recorder-core"; +import { recorderOptions } from "@cap/recorder-core/recorder-encoding"; import { createInstantRecording, @@ -26,6 +31,7 @@ import { updateUploadProgress, } from "../shared/api"; import { toCameraDevices, toMicrophoneDevices } from "../shared/devices"; +import { parseCapturedTabInputBatch } from "../shared/input-events"; import { isOffscreenRequest } from "../shared/messages"; import { loadAuth, @@ -66,6 +72,7 @@ import { waitForIceGatheringComplete, } from "../shared/webrtc"; import { captureDisplayStream } from "./display-capture"; +import { InputEventSidecar, uploadInputEventBlob } from "./input-event-sidecar"; const RECORDING_TIMESLICE_MS = 1000; const RECORDING_TIMESLICE_GUARD_MS = RECORDING_TIMESLICE_MS * 3; @@ -93,13 +100,25 @@ type RecordingSound = "start-recording" | "stop-recording"; type ActiveRecording = { recorder: MediaRecorder; + cameraRecorder: MediaRecorder | null; + audioSidecars: AudioRecordingSidecar[]; + inputSidecar: InputEventSidecar | null; + inputCaptureTabId: number | null; + inputUploadApi: RecorderApiOptions; + inputStopPromise: Promise | null; + inputCaptureIncomplete: boolean; stopPromise: Promise; + cameraStopPromise: Promise | null; streams: MediaStream[]; + microphoneStream: MediaStream | null; recordingStream: MediaStream; statusTimer: number | null; spool: RecordingSpool; + cameraSpool: RecordingSpool | null; uploader: InstantRecordingUploader; + cameraUploader: InstantRecordingUploader | null; startedAt: number; + cameraOffsetMs: number | null; durationMs: number; lastResumedAt: number | null; videoId: VideoId; @@ -107,16 +126,25 @@ type ActiveRecording = { width: number; height: number; fps: number; + cameraWidth: number | null; + cameraHeight: number | null; + cameraFps: number | null; subpath: string; mimeType: string; + cameraSubpath: string | null; + cameraMimeType: string | null; maxDurationMs: number | null; audioContext?: AudioContext; chunkChain: Promise; + cameraChunkChain: Promise; dataRequestInterval: number | null; + cameraDataRequestInterval: number | null; chunkStartGuard: number | null; chunkingMode: ChunkingMode | null; lastChunkAt: number | null; recordedBytes: number; + cameraRecordedBytes: number; + cameraUploadCompleted: boolean; finalizePromise: Promise | null; cleanedUp: boolean; // A spool write failure (IndexedDB quota, backpressure) must not end an @@ -125,6 +153,8 @@ type ActiveRecording = { // streaming upload continues untouched. spoolFailed: boolean; memoryBackup: LocalRecordingState; + cameraSpoolFailed: boolean; + cameraMemoryBackup: LocalRecordingState; }; let activeRecording: ActiveRecording | null = null; @@ -437,12 +467,21 @@ const tabCaptureConstraints = (streamId: string, includeAudio: boolean) => const getMainStream = async (request: StartRecordingRequest) => { if (request.mode === "tab") { - if (!request.tabStreamId) throw new Error("Tab stream id is missing"); - return navigator.mediaDevices.getUserMedia( - tabCaptureConstraints( - request.tabStreamId, - request.settings.systemAudio.enabled, - ), + if (request.tabStreamId) { + return navigator.mediaDevices.getUserMedia( + tabCaptureConstraints( + request.tabStreamId, + request.settings.systemAudio.enabled, + ), + ); + } + return captureDisplayStream( + "tab", + request.settings.systemAudio.enabled, + (options) => + navigator.mediaDevices.getDisplayMedia( + options as DisplayMediaStreamOptions, + ), ); } @@ -635,6 +674,23 @@ const cleanupActiveRecording = async (recording: ActiveRecording) => { if (recording.chunkStartGuard !== null) { window.clearTimeout(recording.chunkStartGuard); } + if (recording.cameraDataRequestInterval !== null) { + window.clearInterval(recording.cameraDataRequestInterval); + } + if (recording.recorder.state !== "inactive") { + recording.recorder.stop(); + } + if ( + recording.cameraRecorder && + recording.cameraRecorder.state !== "inactive" + ) { + recording.cameraRecorder.stop(); + } + await Promise.all( + recording.audioSidecars.map((sidecar) => + sidecar.stop().catch(() => undefined), + ), + ); for (const stream of recording.streams) { stopTracks(stream); } @@ -656,19 +712,68 @@ const sweepOrphanedRecordingSpools = async () => { loadLiveRecordingManifests(), ]); const now = Date.now(); - const knownSessions = new Set(failed.map((entry) => entry.sessionId)); + const knownSessions = new Set( + failed.flatMap((entry) => [ + entry.sessionId, + ...(entry.cameraSessionId ? [entry.cameraSessionId] : []), + ...(entry.inputEventsSessionId ? [entry.inputEventsSessionId] : []), + ...(entry.audioSources ?? []).map((source) => source.sessionId), + ]), + ); + const sessionsById = new Map( + sessions.map((session) => [session.sessionId, session]), + ); const manifestsBySession = new Map( manifests.map((manifest) => [manifest.sessionId, manifest]), ); + const manifestsByCameraSession = new Map( + manifests.flatMap((manifest) => { + const sessionId = manifest.cameraSessionId; + return sessionId ? [[sessionId, manifest] as const] : []; + }), + ); + const manifestsByAudioSession = new Map( + manifests.flatMap((manifest) => + (manifest.audioSources ?? []).map( + (source) => [source.sessionId, manifest] as const, + ), + ), + ); + const manifestsByInputSession = new Map( + manifests.flatMap((manifest) => { + const sessionId = manifest.inputEventsSessionId; + return sessionId ? [[sessionId, manifest] as const] : []; + }), + ); const remainingSessions = new Set(); let entries = [...failed]; for (const orphan of sessions) { - if (activeRecording?.spool.sessionId === orphan.sessionId) { + if ( + activeRecording?.spool.sessionId === orphan.sessionId || + activeRecording?.cameraSpool?.sessionId === orphan.sessionId || + activeRecording?.inputSidecar?.spool.sessionId === orphan.sessionId || + activeRecording?.audioSidecars.some( + (sidecar) => sidecar.metadata.sessionId === orphan.sessionId, + ) + ) { remainingSessions.add(orphan.sessionId); continue; } - if (now - orphan.updatedAt < RECORDING_SPOOL_LIVE_MIN_IDLE_MS) { + if (isRecordingSpoolUploaded(orphan.sessionId)) { + void deleteRecoveredRecordingSpool(orphan.sessionId).catch( + () => undefined, + ); + continue; + } + const manifest = manifestsBySession.get(orphan.sessionId); + // A surviving manifest belongs to a previous offscreen document. This + // document is the extension's only spool writer, so the shared live + // window must not delay recovery after that writer has gone away. + if ( + now - orphan.updatedAt < RECORDING_SPOOL_LIVE_MIN_IDLE_MS && + !manifest + ) { remainingSessions.add(orphan.sessionId); continue; } @@ -682,20 +787,95 @@ const sweepOrphanedRecordingSpools = async () => { () => undefined, ); entries = entries.filter( - (entry) => entry.sessionId !== orphan.sessionId, + (entry) => + entry.sessionId !== orphan.sessionId && + entry.cameraSessionId !== orphan.sessionId, + ); + entries = entries.map((entry) => { + const audioSources = (entry.audioSources ?? []).filter( + (source) => source.sessionId !== orphan.sessionId, + ); + return audioSources.length === (entry.audioSources ?? []).length + ? entry + : { ...entry, audioSources, audioRetryUnavailable: true }; + }); + entries = entries.map((entry) => + entry.inputEventsSessionId === orphan.sessionId + ? { + ...entry, + inputEventsSessionId: undefined, + inputEventsTotalBytes: undefined, + inputEventsRetryUnavailable: true, + } + : entry, ); continue; } remainingSessions.add(orphan.sessionId); + const cameraOwner = manifestsByCameraSession.get(orphan.sessionId); + const audioOwner = manifestsByAudioSession.get(orphan.sessionId); + const inputOwner = manifestsByInputSession.get(orphan.sessionId); + const sourceOwner = cameraOwner ?? audioOwner ?? inputOwner; + if ( + sourceOwner && + (sessionsById.has(sourceOwner.sessionId) || + knownSessions.has(sourceOwner.sessionId)) + ) { + continue; + } if (!knownSessions.has(orphan.sessionId) && orphan.totalBytes > 0) { // A crash-stranded session whose live manifest survived keeps its // videoId/subpath so the entry stays retryable, not download-only. // The duration is a wall-clock estimate (it includes pauses); it // only feeds the completion metadata on retry. - const manifest = manifestsBySession.get(orphan.sessionId); + const cameraSession = manifest?.cameraSessionId + ? sessionsById.get(manifest.cameraSessionId) + : undefined; + const cameraMetadata = + manifest?.cameraSessionId && cameraSession?.totalBytes + ? { + cameraSessionId: manifest.cameraSessionId, + cameraMimeType: + manifest.cameraMimeType ?? cameraSession.mimeType, + cameraSubpath: manifest.cameraSubpath, + cameraOffsetMs: manifest.cameraOffsetMs ?? 0, + cameraWidth: manifest.cameraWidth, + cameraHeight: manifest.cameraHeight, + cameraFps: manifest.cameraFps, + cameraTotalBytes: cameraSession.totalBytes, + } + : {}; + const audioSources = (manifest?.audioSources ?? []).flatMap( + (source) => { + const session = sessionsById.get(source.sessionId); + return session?.totalBytes + ? [{ ...source, recordedBytes: session.totalBytes }] + : []; + }, + ); + const inputSession = manifest?.inputEventsSessionId + ? sessionsById.get(manifest.inputEventsSessionId) + : undefined; entries.push({ sessionId: orphan.sessionId, + ...cameraMetadata, + ...(manifest?.inputEventsSessionId && inputSession?.totalBytes + ? { + inputEventsSessionId: manifest.inputEventsSessionId, + inputEventsTotalBytes: inputSession.totalBytes, + } + : {}), + ...(manifest?.inputEventsSessionId && !inputSession + ? { inputEventsRetryUnavailable: true } + : {}), + ...(audioSources.length > 0 ? { audioSources } : {}), + ...(audioSources.length !== (manifest?.audioSources ?? []).length + ? { audioRetryUnavailable: true } + : {}), + ...(manifest?.cameraSessionId && !cameraSession?.totalBytes + ? { cameraRetryUnavailable: true } + : {}), videoId: manifest?.videoId ?? null, shareUrl: manifest?.shareUrl ?? null, mimeType: orphan.mimeType, @@ -710,6 +890,18 @@ const sweepOrphanedRecordingSpools = async () => { createdAt: orphan.updatedAt, message: "The recording was interrupted before its upload finished.", }); + if (manifest?.cameraSessionId && cameraSession?.totalBytes) { + knownSessions.add(manifest.cameraSessionId); + remainingSessions.add(manifest.cameraSessionId); + } + if (manifest?.inputEventsSessionId && inputSession?.totalBytes) { + knownSessions.add(manifest.inputEventsSessionId); + remainingSessions.add(manifest.inputEventsSessionId); + } + for (const source of audioSources) { + knownSessions.add(source.sessionId); + remainingSessions.add(source.sessionId); + } } } @@ -724,6 +916,37 @@ const sweepOrphanedRecordingSpools = async () => { await deleteRecoveredRecordingSpool(entry.sessionId).catch( () => undefined, ); + if (entry.cameraSessionId) { + survivingSessions.delete(entry.cameraSessionId); + await deleteRecoveredRecordingSpool(entry.cameraSessionId).catch( + () => undefined, + ); + } + if (entry.inputEventsSessionId) { + survivingSessions.delete(entry.inputEventsSessionId); + await deleteRecoveredRecordingSpool(entry.inputEventsSessionId).catch( + () => undefined, + ); + } + for (const source of entry.audioSources ?? []) { + survivingSessions.delete(source.sessionId); + await deleteRecoveredRecordingSpool(source.sessionId).catch( + () => undefined, + ); + } + } + if (startInProgress && !activeRecording) return; + if (activeRecording) { + survivingSessions.add(activeRecording.spool.sessionId); + if (activeRecording.cameraSpool) { + survivingSessions.add(activeRecording.cameraSpool.sessionId); + } + if (activeRecording.inputSidecar) { + survivingSessions.add(activeRecording.inputSidecar.spool.sessionId); + } + for (const sidecar of activeRecording.audioSidecars) { + survivingSessions.add(sidecar.metadata.sessionId); + } } await pruneLiveRecordingManifests(survivingSessions).catch(() => undefined); } catch { @@ -736,7 +959,10 @@ function stopRecorderAfterError(recorder: MediaRecorder) { recorder.stop(); } window.setTimeout(() => { - if (activeRecording?.recorder === recorder) { + if ( + activeRecording?.recorder === recorder || + activeRecording?.cameraRecorder === recorder + ) { void stopRecording(); } }, 0); @@ -761,6 +987,20 @@ const stopManualChunking = (recording: ActiveRecording) => { } }; +const requestCameraRecorderData = (recording: ActiveRecording) => { + if (recording.cameraRecorder?.state !== "recording") return; + try { + recording.cameraRecorder.requestData(); + } catch {} +}; + +const stopCameraChunking = (recording: ActiveRecording) => { + if (recording.cameraDataRequestInterval !== null) { + window.clearInterval(recording.cameraDataRequestInterval); + recording.cameraDataRequestInterval = null; + } +}; + const beginManualChunking = (recording: ActiveRecording) => { recording.chunkingMode = "manual"; recording.lastChunkAt = null; @@ -811,13 +1051,23 @@ const startRecording = async (request: StartRecordingRequest) => { const ownedStreams: MediaStream[] = []; let ownedVideoId: string | null = null; let ownedSpool: RecordingSpool | null = null; + let ownedCameraSpool: RecordingSpool | null = null; + let ownedCameraUploader: InstantRecordingUploader | null = null; + let ownedInputSidecar: InputEventSidecar | null = null; + const ownedAudioSidecars: AudioRecordingSidecar[] = []; + let ownedUploader: InstantRecordingUploader | null = null; let ownedRecording: ActiveRecording | null = null; let countdownPromise: Promise | null = null; try { status = { phase: "creating" }; - const mainStream = await getMainStream(request); + const mainStream = await getMainStream(request).catch((error: unknown) => { + throw new Error( + `Unable to capture ${request.mode} source: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, + ); + }); ownedStreams.push(mainStream); throwIfStartCanceled(); const captureSource = getCaptureSource(request, mainStream); @@ -847,6 +1097,34 @@ const startRecording = async (request: StartRecordingRequest) => { streams, routeFirstStreamToSpeakers: request.mode === "tab", }); + const cameraStream = + request.mode !== "camera" && request.settings.webcam.enabled + ? await getCameraPreviewStream(request.settings.webcam) + : null; + const cameraTrack = cameraStream?.getVideoTracks()[0]?.clone() ?? null; + if ( + request.settings.webcam.enabled && + request.mode !== "camera" && + !cameraTrack + ) { + throw new Error("The selected camera is not available for recording"); + } + const cameraRecordingStream = cameraTrack + ? new MediaStream([cameraTrack]) + : null; + if (cameraRecordingStream) { + ownedStreams.push(cameraRecordingStream); + streams.push(cameraRecordingStream); + } + const cameraDimensions = cameraRecordingStream + ? getStreamSize(cameraRecordingStream) + : null; + const cameraPipeline = cameraRecordingStream + ? selectRecordingPipeline(false) + : null; + if (cameraRecordingStream && !cameraPipeline) { + throw new Error("No supported camera recorder format is available"); + } const hasAudio = recordingStream.getAudioTracks().length > 0; const pipeline = selectRecordingPipeline(hasAudio); if (!pipeline) throw new Error("No supported recorder format is available"); @@ -883,10 +1161,6 @@ const startRecording = async (request: StartRecordingRequest) => { subpath, api, }); - throwIfStartCanceled(); - const spool = await RecordingSpool.create({ mimeType: pipeline.mimeType }); - ownedSpool = spool; - throwIfStartCanceled(); const uploader = new InstantRecordingUploader({ videoId: creation.id, uploadId: uploadSession.uploadId, @@ -940,15 +1214,144 @@ const startRecording = async (request: StartRecordingRequest) => { } }, }); + ownedUploader = uploader; + throwIfStartCanceled(); + const spool = await RecordingSpool.create({ mimeType: pipeline.mimeType }); + ownedSpool = spool; + throwIfStartCanceled(); + const cameraSpool = cameraPipeline + ? await RecordingSpool.create({ mimeType: cameraPipeline.mimeType }) + : null; + ownedCameraSpool = cameraSpool; + throwIfStartCanceled(); + const cameraSubpath = cameraPipeline + ? `camera-upload.${cameraPipeline.fileExtension}` + : null; + const cameraUploadApi = cameraSubpath + ? { + ...api, + extraBody: { screenSubpath: subpath, cameraOffsetMs: 0 }, + } + : null; + const cameraUploadSession = + cameraPipeline && cameraSubpath && cameraUploadApi + ? await initiateMultipartUpload({ + videoId: creation.id, + contentType: cameraPipeline.mimeType, + subpath: cameraSubpath, + api: cameraUploadApi, + }) + : null; + const cameraUploader = + cameraPipeline && cameraSubpath && cameraUploadApi && cameraUploadSession + ? new InstantRecordingUploader({ + videoId: creation.id, + uploadId: cameraUploadSession.uploadId, + provider: cameraUploadSession.provider, + mimeType: cameraPipeline.mimeType, + subpath: cameraSubpath, + api: cameraUploadApi, + setUploadStatus: () => undefined, + sendProgressUpdate: async () => undefined, + onFatalError: (error) => { + status = { + phase: "error", + message: error.message, + videoId: creation.id, + }; + broadcastStatus(); + void stopRecording(); + }, + }) + : null; + ownedCameraUploader = cameraUploader; + throwIfStartCanceled(); + if (microphoneStream?.getAudioTracks().length) { + ownedAudioSidecars.push( + await AudioRecordingSidecar.create({ + kind: "mic", + stream: new MediaStream(microphoneStream.getAudioTracks()), + videoId: creation.id, + screenSubpath: subpath, + api, + onFatalError: (error) => { + status = { + phase: "error", + message: error.message, + videoId: creation.id, + }; + broadcastStatus(); + void stopRecording(); + }, + onBackupFallback: (error) => { + console.warn("Microphone backup moved to memory", error); + }, + }), + ); + } + if (mainStream.getAudioTracks().length > 0) { + ownedAudioSidecars.push( + await AudioRecordingSidecar.create({ + kind: "systemAudio", + stream: new MediaStream(mainStream.getAudioTracks()), + videoId: creation.id, + screenSubpath: subpath, + api, + onFatalError: (error) => { + status = { + phase: "error", + message: error.message, + videoId: creation.id, + }; + broadcastStatus(); + void stopRecording(); + }, + onBackupFallback: (error) => { + console.warn("System audio backup moved to memory", error); + }, + }), + ); + } + throwIfStartCanceled(); - const recorder = new MediaRecorder(recordingStream, { - mimeType: pipeline.mimeType, - }); + const isSupported = (type: string) => MediaRecorder.isTypeSupported(type); + const recorder = new MediaRecorder( + recordingStream, + recorderOptions( + pipeline.mimeType, + recordingStream.getVideoTracks()[0], + isSupported, + ), + ); + const cameraRecorder = + cameraRecordingStream && cameraPipeline + ? new MediaRecorder( + cameraRecordingStream, + recorderOptions( + cameraPipeline.mimeType, + cameraRecordingStream.getVideoTracks()[0], + isSupported, + ), + ) + : null; await countdownPromise; throwIfStartCanceled(); const startedAt = Date.now(); + const inputSidecar = + request.tabStreamId && request.tabId !== undefined + ? await InputEventSidecar.create({ + recordingId: creation.id, + tabId: request.tabId, + videoWidth: width, + videoHeight: height, + }).catch((error: unknown) => { + console.warn("Tab input capture could not start", error); + return null; + }) + : null; + ownedInputSidecar = inputSidecar; const plan = request.bootstrap.plan; const maxDurationMs = !plan.isPro && plan.maxRecordingSeconds !== null @@ -957,13 +1360,28 @@ const startRecording = async (request: StartRecordingRequest) => { const recording: ActiveRecording = { recorder, + cameraRecorder, + audioSidecars: ownedAudioSidecars, + inputSidecar, + inputCaptureTabId: + request.tabStreamId && request.tabId !== undefined + ? request.tabId + : null, + inputUploadApi: api, + inputStopPromise: null, + inputCaptureIncomplete: false, stopPromise: Promise.resolve(), + cameraStopPromise: null, streams, + microphoneStream, recordingStream, statusTimer: null, spool, + cameraSpool, uploader, + cameraUploader, startedAt, + cameraOffsetMs: null, durationMs: 0, lastResumedAt: startedAt, videoId: creation.id, @@ -971,20 +1389,31 @@ const startRecording = async (request: StartRecordingRequest) => { width, height, fps, + cameraWidth: cameraDimensions?.width ?? null, + cameraHeight: cameraDimensions?.height ?? null, + cameraFps: cameraDimensions?.fps ?? null, subpath, mimeType: pipeline.mimeType, + cameraSubpath, + cameraMimeType: cameraPipeline?.mimeType ?? null, maxDurationMs, audioContext, chunkChain: Promise.resolve(), + cameraChunkChain: Promise.resolve(), dataRequestInterval: null, + cameraDataRequestInterval: null, chunkStartGuard: null, chunkingMode: null, lastChunkAt: null, recordedBytes: 0, + cameraRecordedBytes: 0, + cameraUploadCompleted: false, finalizePromise: null, cleanedUp: false, spoolFailed: false, memoryBackup: initialLocalRecordingState(), + cameraSpoolFailed: false, + cameraMemoryBackup: initialLocalRecordingState(), }; ownedRecording = recording; @@ -999,6 +1428,19 @@ const startRecording = async (request: StartRecordingRequest) => { // failed-recording entry (videoId, subpath) instead of download-only. await saveLiveRecordingManifest({ sessionId: spool.sessionId, + inputEventsSessionId: inputSidecar?.spool.sessionId, + audioSources: ownedAudioSidecars.map((sidecar) => sidecar.metadata), + ...(cameraSpool + ? { + cameraSessionId: cameraSpool.sessionId, + cameraMimeType: cameraPipeline?.mimeType, + cameraSubpath: recording.cameraSubpath ?? undefined, + cameraOffsetMs: 0, + cameraWidth: recording.cameraWidth ?? undefined, + cameraHeight: recording.cameraHeight ?? undefined, + cameraFps: recording.cameraFps ?? undefined, + } + : {}), videoId: creation.id, shareUrl: creation.shareUrl, mimeType: pipeline.mimeType, @@ -1013,6 +1455,13 @@ const startRecording = async (request: StartRecordingRequest) => { recorder.onstop = () => resolve(); recorder.onerror = () => reject(new Error("MediaRecorder failed")); }); + if (cameraRecorder) { + recording.cameraStopPromise = new Promise((resolve, reject) => { + cameraRecorder.onstop = () => resolve(); + cameraRecorder.onerror = () => + reject(new Error("Camera MediaRecorder failed")); + }); + } // A mid-recording recorder failure must stop the session right away; // without this the rejection sits unhandled while the timer keeps // ticking over a recorder that no longer produces chunks, and nothing @@ -1073,6 +1522,51 @@ const startRecording = async (request: StartRecordingRequest) => { stopRecorderAfterError(recorder); } }; + if (cameraRecorder && cameraSpool) { + cameraRecorder.ondataavailable = (event) => { + if (event.data.size === 0) return; + recording.cameraRecordedBytes += event.data.size; + const cameraRecordedBytes = recording.cameraRecordedBytes; + recording.cameraChunkChain = recording.cameraChunkChain.then( + async () => { + if (recording.cameraSpoolFailed) { + appendCameraMemoryBackupChunk(recording, event.data); + return; + } + try { + await cameraSpool.appendChunk(event.data); + } catch (error) { + recording.cameraSpoolFailed = true; + console.warn( + "Camera recording spool failed; keeping the local backup in memory", + error, + ); + } + }, + ); + try { + cameraUploader?.handleChunk(event.data, cameraRecordedBytes); + } catch (error) { + status = { + phase: "error", + message: error instanceof Error ? error.message : String(error), + videoId: creation.id, + }; + broadcastStatus(); + void stopRecording(); + } + }; + recording.cameraStopPromise?.catch(() => { + if (activeRecording !== recording || recording.finalizePromise) return; + status = { + phase: "error", + message: "Camera recording stopped unexpectedly", + videoId: creation.id, + }; + broadcastStatus(); + void stopRecording(); + }); + } status = { phase: "recording", @@ -1081,15 +1575,63 @@ const startRecording = async (request: StartRecordingRequest) => { durationMs: 0, updatedAt: startedAt, }; + const saveSourceManifest = () => + saveLiveRecordingManifest({ + sessionId: spool.sessionId, + inputEventsSessionId: inputSidecar?.spool.sessionId, + audioSources: ownedAudioSidecars.map((sidecar) => sidecar.metadata), + cameraSessionId: cameraSpool?.sessionId, + cameraMimeType: recording.cameraMimeType ?? undefined, + cameraSubpath: recording.cameraSubpath ?? undefined, + cameraOffsetMs: recording.cameraOffsetMs ?? 0, + cameraWidth: recording.cameraWidth ?? undefined, + cameraHeight: recording.cameraHeight ?? undefined, + cameraFps: recording.cameraFps ?? undefined, + videoId: creation.id, + shareUrl: creation.shareUrl, + mimeType: pipeline.mimeType, + subpath, + width, + height, + fps, + startedAt, + }); + let screenStartRequestedAt = performance.now(); try { recorder.start(RECORDING_TIMESLICE_MS); recording.chunkingMode = "timeslice"; scheduleTimesliceGuard(recording); } catch { + screenStartRequestedAt = performance.now(); recorder.start(); beginManualChunking(recording); } + inputSidecar?.start(performance.timeOrigin + screenStartRequestedAt); + if (cameraRecorder) { + let cameraStartRequestedAt = performance.now(); + try { + cameraRecorder.start(RECORDING_TIMESLICE_MS); + } catch { + cameraStartRequestedAt = performance.now(); + cameraRecorder.start(); + recording.cameraDataRequestInterval = window.setInterval(() => { + requestCameraRecorderData(recording); + }, RECORDING_TIMESLICE_MS); + } + recording.cameraOffsetMs = Math.round( + cameraStartRequestedAt - screenStartRequestedAt, + ); + if (cameraUploadApi) { + cameraUploadApi.extraBody.cameraOffsetMs = recording.cameraOffsetMs; + } + } + for (const audioSidecar of ownedAudioSidecars) { + audioSidecar.start(screenStartRequestedAt); + } + if (cameraRecorder || ownedAudioSidecars.length > 0) { + await saveSourceManifest().catch(() => undefined); + } playRecordingSound("start-recording", request.settings); // The service worker that sent start-recording may have been killed // while the capture picker was open, which destroys the response @@ -1109,12 +1651,38 @@ const startRecording = async (request: StartRecordingRequest) => { ); countdownResolve?.(); await countdownPromise?.catch(() => undefined); + if (ownedRecording) { + if (ownedRecording.recorder.state !== "inactive") { + ownedRecording.recorder.stop(); + await ownedRecording.stopPromise.catch(() => undefined); + } + if ( + ownedRecording.cameraRecorder && + ownedRecording.cameraRecorder.state !== "inactive" + ) { + ownedRecording.cameraRecorder.stop(); + await ownedRecording.cameraStopPromise?.catch(() => undefined); + } + await Promise.all([ + ownedRecording.chunkChain, + ownedRecording.cameraChunkChain, + ]); + } + await Promise.all( + ownedAudioSidecars.map((sidecar) => + sidecar.cancel().catch(() => undefined), + ), + ); // Release only what this attempt acquired. No chunk can have been // captured yet — chunks only flow once recorder.start() succeeds, after // which nothing here throws — so the spool holds no recoverable data. for (const stream of ownedStreams) { stopTracks(stream); } + await Promise.all([ + ownedUploader?.cancel().catch(() => undefined), + ownedCameraUploader?.cancel().catch(() => undefined), + ]); if (ownedVideoId) { await deleteInstantRecording( request.settings, @@ -1134,15 +1702,23 @@ const startRecording = async (request: StartRecordingRequest) => { ); await ownedSpool.dispose().catch(() => undefined); } + await ownedCameraSpool?.dispose().catch(() => undefined); + await ownedInputSidecar?.dispose().catch(() => undefined); // Reset the "creating" status so later status syncs do not report a // phantom in-progress recording. - if (status.phase === "creating") { + if ( + status.phase === "creating" || + (ownedRecording && + "videoId" in status && + status.videoId === ownedRecording.videoId) + ) { status = isUserCancellationError(error) ? { phase: "idle" } : { phase: "error", message: error instanceof Error ? error.message : String(error), }; + broadcastStatus(); } throw error; } finally { @@ -1163,6 +1739,22 @@ const appendMemoryBackupChunk = (recording: ActiveRecording, chunk: Blob) => { } }; +const appendCameraMemoryBackupChunk = ( + recording: ActiveRecording, + chunk: Blob, +) => { + const previous = recording.cameraMemoryBackup; + recording.cameraMemoryBackup = appendLocalRecordingChunk(previous, chunk, { + mode: "capped", + maxBytes: MEMORY_BACKUP_MAX_BYTES, + }); + if (recording.cameraMemoryBackup.overflowed && !previous.overflowed) { + console.warn( + "In-memory camera backup exceeded its cap; dropping the local copy", + ); + } +}; + // The complete recording: the spool when it stayed healthy, otherwise the // spooled prefix plus the in-memory continuation. Null once the capped // memory backup has overflowed — a truncated local copy must not masquerade @@ -1180,6 +1772,68 @@ const recoverRecordingBlob = async (recording: ActiveRecording) => { return new Blob(parts, { type: recording.mimeType }); }; +const recoverCameraRecordingBlob = async (recording: ActiveRecording) => { + const spool = recording.cameraSpool; + if (!spool) return null; + if (!recording.cameraSpoolFailed) return spool.recoverBlob(); + if (recording.cameraMemoryBackup.overflowed) return null; + const spooledBlob = await spool.recoverBlob().catch(() => null); + const parts = spooledBlob + ? [spooledBlob, ...recording.cameraMemoryBackup.chunks] + : recording.cameraMemoryBackup.chunks; + if (parts.length === 0) return null; + return new Blob(parts, { type: recording.cameraMimeType ?? "" }); +}; + +const uploadCameraSidecar = async (options: { + videoId: VideoId; + settings: ExtensionSettings; + authToken: string; + blob: Blob; + mimeType: string; + subpath: string; + screenSubpath: string; + cameraOffsetMs: number; + durationMs: number; + width: number; + height: number; + fps: number; +}) => { + const api = { + baseUrl: options.settings.apiBaseUrl, + authToken: options.authToken, + requestTimeoutMs: DEFAULT_API_REQUEST_TIMEOUT_MS, + extraBody: { + screenSubpath: options.screenSubpath, + cameraOffsetMs: options.cameraOffsetMs, + }, + }; + const uploadSession = await initiateMultipartUpload({ + videoId: options.videoId, + contentType: options.mimeType, + subpath: options.subpath, + api, + }); + const uploader = new InstantRecordingUploader({ + videoId: options.videoId, + uploadId: uploadSession.uploadId, + provider: uploadSession.provider, + mimeType: options.mimeType, + subpath: options.subpath, + api, + setUploadStatus: () => undefined, + sendProgressUpdate: async () => undefined, + }); + await uploader.finalize({ + finalBlob: options.blob, + durationSeconds: Math.max(1, Math.round(options.durationMs / 1000)), + width: options.width, + height: options.height, + fps: options.fps, + subpath: options.subpath, + }); +}; + // Returns whether the recording bytes are persisted and retryable from the // upload page. const rememberFailedRecording = async ( @@ -1189,6 +1843,7 @@ const rememberFailedRecording = async ( if (recording.recordedBytes === 0) return false; let sessionId = recording.spool.sessionId; + let cameraSessionId = recording.cameraSpool?.sessionId; if (recording.spoolFailed) { // The original spool is missing the tail that went to memory; a retry // reading it would upload a truncated file. Persist the full recording @@ -1214,9 +1869,85 @@ const rememberFailedRecording = async ( if (!replacement) return false; sessionId = replacement.sessionId; } + if (recording.cameraSpoolFailed && recording.cameraSpool) { + const fullBlob = await recoverCameraRecordingBlob(recording).catch( + () => null, + ); + const replacement = + fullBlob && fullBlob.size >= recording.cameraRecordedBytes + ? await RecordingSpool.create({ + mimeType: recording.cameraMimeType ?? fullBlob.type, + maxPendingChunkBytes: fullBlob.size, + }) + .then(async (spool) => { + await spool.appendChunk(fullBlob); + await spool.flush(); + return spool; + }) + .catch(() => null) + : null; + await recording.cameraSpool.dispose().catch(() => undefined); + cameraSessionId = replacement?.sessionId; + } + const audioSources = await Promise.all( + recording.audioSidecars.map((sidecar) => + sidecar.prepareRetryMetadata().catch(() => null), + ), + ); + const retryableAudioSources = audioSources.filter( + (source) => source !== null, + ); + const audioRetryUnavailable = audioSources.some( + (source, index) => + !recording.audioSidecars[index]?.isUploadCompleted && source === null, + ); + const inputSidecar = recording.inputSidecar; + const inputBlob = + inputSidecar && inputSidecar.recordedBytes > 0 + ? await inputSidecar.recoverBlob().catch(() => null) + : null; + const inputRetryUnavailable = + !!inputSidecar && + inputSidecar.recordedBytes > 0 && + !inputSidecar.isUploadCompleted && + (recording.inputCaptureIncomplete || + !inputBlob || + inputBlob.size !== inputSidecar.recordedBytes); const saved = await upsertFailedRecording({ sessionId, + ...(retryableAudioSources.length > 0 + ? { audioSources: retryableAudioSources } + : {}), + ...(audioRetryUnavailable ? { audioRetryUnavailable: true } : {}), + ...(inputRetryUnavailable ? { inputEventsRetryUnavailable: true } : {}), + ...(inputSidecar && + inputSidecar.recordedBytes > 0 && + inputBlob && + inputBlob.size === inputSidecar.recordedBytes && + !recording.inputCaptureIncomplete + ? { + inputEventsSessionId: inputSidecar.spool.sessionId, + inputEventsTotalBytes: inputSidecar.recordedBytes, + } + : {}), + ...(recording.cameraRecorder && + !recording.cameraUploadCompleted && + !(cameraSessionId && recording.cameraRecordedBytes > 0) + ? { cameraRetryUnavailable: true } + : {}), + ...(cameraSessionId && recording.cameraRecordedBytes > 0 + ? { + cameraSessionId, + cameraMimeType: recording.cameraMimeType ?? undefined, + cameraSubpath: recording.cameraSubpath ?? undefined, + cameraOffsetMs: recording.cameraOffsetMs ?? 0, + cameraWidth: recording.cameraWidth ?? undefined, + cameraHeight: recording.cameraHeight ?? undefined, + cameraFps: recording.cameraFps ?? undefined, + cameraTotalBytes: recording.cameraRecordedBytes, + } + : {}), videoId: recording.videoId, shareUrl: recording.shareUrl, mimeType: recording.mimeType, @@ -1237,6 +1968,21 @@ const rememberFailedRecording = async ( await deleteRecoveredRecordingSpool(dropped.sessionId).catch( () => undefined, ); + if (dropped.cameraSessionId) { + await deleteRecoveredRecordingSpool(dropped.cameraSessionId).catch( + () => undefined, + ); + } + if (dropped.inputEventsSessionId) { + await deleteRecoveredRecordingSpool(dropped.inputEventsSessionId).catch( + () => undefined, + ); + } + for (const source of dropped.audioSources ?? []) { + await deleteRecoveredRecordingSpool(source.sessionId).catch( + () => undefined, + ); + } } return saved.kept.some((entry) => entry.sessionId === sessionId); @@ -1244,9 +1990,13 @@ const rememberFailedRecording = async ( const finalizeRecording = async (recording: ActiveRecording) => { try { - await recording.stopPromise; + await Promise.all([ + recording.stopPromise, + recording.cameraStopPromise ?? Promise.resolve(), + ]); + await recording.inputStopPromise; await cleanupActiveRecording(recording); - await recording.chunkChain; + await Promise.all([recording.chunkChain, recording.cameraChunkChain]); // finalBlob is null when the capped memory backup overflowed. The // streamed parts still carry every byte; the uploader just falls back // to its recorded-bytes counter instead of the local blob's size. @@ -1254,6 +2004,50 @@ const finalizeRecording = async (recording: ActiveRecording) => { if ((!finalBlob || finalBlob.size === 0) && recording.recordedBytes === 0) { throw new Error("No recording data was captured"); } + if (recording.cameraRecorder && recording.cameraRecordedBytes === 0) { + throw new Error("The separate camera recording captured no data"); + } + if (recording.cameraRecordedBytes > 0) { + const cameraUploader = recording.cameraUploader; + const cameraSubpath = recording.cameraSubpath; + if ( + !recording.cameraSpool || + !cameraUploader || + !cameraSubpath || + !recording.cameraMimeType || + recording.cameraOffsetMs === null + ) { + throw new Error("The separate camera recording metadata is incomplete"); + } + await cameraUploader.finalize({ + finalBlob: null, + durationSeconds: Math.max(1, Math.round(recording.durationMs / 1000)), + width: recording.cameraWidth ?? DEFAULT_WIDTH, + height: recording.cameraHeight ?? DEFAULT_HEIGHT, + fps: recording.cameraFps ?? DEFAULT_FPS, + subpath: cameraSubpath, + }); + recording.cameraUploadCompleted = true; + } else { + await recording.cameraUploader?.cancel(); + } + for (const audioSidecar of recording.audioSidecars) { + await audioSidecar.finalize( + Math.max(1, Math.round(recording.durationMs / 1000)), + ); + } + if (recording.inputSidecar && !recording.inputCaptureIncomplete) { + await recording.inputSidecar + .upload({ + videoId: recording.videoId, + screenSubpath: recording.subpath, + durationSeconds: Math.max(1, Math.round(recording.durationMs / 1000)), + api: recording.inputUploadApi, + }) + .catch((error: unknown) => { + console.warn("Tab input events could not upload", error); + }); + } await recording.uploader.finalize({ finalBlob: finalBlob && finalBlob.size > 0 ? finalBlob : null, durationSeconds: Math.max(1, Math.round(recording.durationMs / 1000)), @@ -1262,7 +2056,30 @@ const finalizeRecording = async (recording: ActiveRecording) => { fps: recording.fps, subpath: recording.subpath, }); - await recording.spool.dispose(); + recording.spool.markUploaded(); + recording.cameraSpool?.markUploaded(); + if (recording.inputSidecar?.isUploadCompleted) { + recording.inputSidecar.spool.markUploaded(); + } + for (const sidecar of recording.audioSidecars) { + sidecar.markUploadedBackup(); + } + void Promise.all([ + recording.spool.dispose().catch((error) => { + console.error("Failed to remove uploaded screen backup", error); + }), + recording.cameraSpool?.dispose().catch((error) => { + console.error("Failed to remove uploaded camera backup", error); + }), + recording.inputSidecar?.dispose().catch((error) => { + console.error("Failed to remove uploaded input backup", error); + }), + ...recording.audioSidecars.map((sidecar) => + sidecar.disposeBackup().catch((error) => { + console.error("Failed to remove uploaded audio backup", error); + }), + ), + ]); await removeFailedRecording(recording.spool.sessionId).catch( () => undefined, ); @@ -1331,6 +2148,24 @@ const getCurrentUploadSnapshot = () => uploadStatus: undefined, }; +const stopTabInputCapture = async (recording: ActiveRecording) => { + if (recording.inputCaptureTabId === null) return; + const response: unknown = await chrome.runtime.sendMessage({ + target: "service-worker", + type: "input-capture-stop", + tabId: recording.inputCaptureTabId, + recordingId: recording.videoId, + }); + if ( + !response || + typeof response !== "object" || + !("ok" in response) || + response.ok !== true + ) { + throw new Error("Tab input events could not be flushed"); + } +}; + async function stopRecording() { // A stop during the pre-roll countdown cancels the start before any frame is // captured. Resolving the countdown wait lets startRecording's @@ -1354,13 +2189,31 @@ async function stopRecording() { } stopManualChunking(recording); + stopCameraChunking(recording); const now = Date.now(); recording.durationMs = getRecordingDuration(recording, now); recording.lastResumedAt = null; + recording.inputSidecar?.stop(performance.timeOrigin + performance.now()); + recording.inputStopPromise = + recording.inputCaptureTabId !== null + ? stopTabInputCapture(recording).catch((error: unknown) => { + recording.inputCaptureIncomplete = true; + console.warn("Tab input events could not flush", error); + }) + : null; if (recording.recorder.state !== "inactive") { recording.recorder.stop(); } + if ( + recording.cameraRecorder && + recording.cameraRecorder.state !== "inactive" + ) { + recording.cameraRecorder.stop(); + } + for (const audioSidecar of recording.audioSidecars) { + void audioSidecar.stop().catch(() => undefined); + } if (status.phase !== "error") { const snapshot = getCurrentUploadSnapshot(); @@ -1393,9 +2246,22 @@ const pauseRecording = () => { return status; } const now = Date.now(); - recording.durationMs = getRecordingDuration(recording, now); + const durationMs = getRecordingDuration(recording, now); + try { + recording.recorder.pause(); + recording.inputSidecar?.pause(performance.timeOrigin + performance.now()); + if (recording.cameraRecorder?.state === "recording") { + recording.cameraRecorder.pause(); + } + for (const audioSidecar of recording.audioSidecars) { + audioSidecar.pause(); + } + } catch (error) { + void stopRecording(); + throw error; + } + recording.durationMs = durationMs; recording.lastResumedAt = null; - recording.recorder.pause(); if (status.phase === "recording") { status = { ...status, @@ -1413,7 +2279,19 @@ const resumeRecording = () => { return status; } const now = Date.now(); - recording.recorder.resume(); + try { + recording.recorder.resume(); + recording.inputSidecar?.resume(performance.timeOrigin + performance.now()); + if (recording.cameraRecorder?.state === "paused") { + recording.cameraRecorder.resume(); + } + for (const audioSidecar of recording.audioSidecars) { + audioSidecar.resume(); + } + } catch (error) { + void stopRecording(); + throw error; + } recording.lastResumedAt = now; if (status.phase === "paused") { status = { @@ -1450,10 +2328,24 @@ const runFailedUploadRetry = async ( if (!failed?.videoId) { throw new Error("This recording is no longer available to retry."); } + if (failed.cameraRetryUnavailable) { + throw new Error( + "The separate camera recording is unavailable to retry. Download the screen recording or check your Cap.", + ); + } + if (failed.audioRetryUnavailable) { + throw new Error( + "A separate audio source is unavailable to retry. Download the saved sources or check your Cap.", + ); + } + if (failed.inputEventsRetryUnavailable) { + console.warn("Tab input events are unavailable during recording retry"); + } const orphan = await recoverRecordingSpoolSession(failed.sessionId); if (!orphan || orphan.blob.size === 0) { await removeFailedRecording(failed.sessionId).catch(() => undefined); + await removeLiveRecordingManifest(failed.sessionId).catch(() => undefined); throw new Error("The recorded data is no longer available."); } @@ -1497,6 +2389,78 @@ const runFailedUploadRetry = async ( }; try { + if (failed.cameraSessionId) { + const cameraOrphan = await recoverRecordingSpoolSession( + failed.cameraSessionId, + ); + if ( + !cameraOrphan || + cameraOrphan.blob.size === 0 || + !failed.cameraMimeType || + !failed.cameraSubpath || + typeof failed.cameraOffsetMs !== "number" || + !Number.isFinite(failed.cameraOffsetMs) + ) { + throw new Error("The separate camera recording is no longer available"); + } + await uploadCameraSidecar({ + videoId: typedVideoId, + settings, + authToken: auth.authApiKey, + blob: cameraOrphan.blob, + mimeType: failed.cameraMimeType, + subpath: failed.cameraSubpath, + screenSubpath: subpath, + cameraOffsetMs: failed.cameraOffsetMs, + durationMs: failed.durationMs, + width: failed.cameraWidth ?? failed.width ?? DEFAULT_WIDTH, + height: failed.cameraHeight ?? failed.height ?? DEFAULT_HEIGHT, + fps: failed.cameraFps ?? failed.fps ?? DEFAULT_FPS, + }); + } + for (const source of failed.audioSources ?? []) { + const audioOrphan = await recoverRecordingSpoolSession(source.sessionId); + if ( + !audioOrphan || + audioOrphan.blob.size !== source.recordedBytes || + audioOrphan.blob.size === 0 + ) { + throw new Error(`The separate ${source.kind} audio is unavailable`); + } + await uploadRecoveredAudioSidecar({ + videoId: typedVideoId, + source, + blob: audioOrphan.blob, + screenSubpath: subpath, + durationSeconds: Math.max(1, Math.round(failed.durationMs / 1000)), + api, + }); + } + if (failed.inputEventsSessionId && !failed.inputEventsRetryUnavailable) { + const inputOrphan = await recoverRecordingSpoolSession( + failed.inputEventsSessionId, + ).catch((error: unknown) => { + console.warn("Tab input events could not be recovered", error); + return null; + }); + if ( + inputOrphan && + inputOrphan.blob.size === failed.inputEventsTotalBytes && + inputOrphan.blob.size > 0 + ) { + await uploadInputEventBlob({ + videoId: typedVideoId, + screenSubpath: subpath, + durationSeconds: Math.max(1, Math.round(failed.durationMs / 1000)), + api, + blob: inputOrphan.blob, + }).catch((error: unknown) => { + console.warn("Tab input events could not upload during retry", error); + }); + } else { + console.warn("Tab input events are unavailable during recording retry"); + } + } const uploadSession = await initiateMultipartUpload({ videoId: typedVideoId, contentType: failed.mimeType, @@ -1543,7 +2507,23 @@ const runFailedUploadRetry = async ( await deleteRecoveredRecordingSpool(failed.sessionId).catch( () => undefined, ); + if (failed.cameraSessionId) { + await deleteRecoveredRecordingSpool(failed.cameraSessionId).catch( + () => undefined, + ); + } + if (failed.inputEventsSessionId) { + await deleteRecoveredRecordingSpool(failed.inputEventsSessionId).catch( + () => undefined, + ); + } + for (const source of failed.audioSources ?? []) { + await deleteRecoveredRecordingSpool(source.sessionId).catch( + () => undefined, + ); + } await removeFailedRecording(failed.sessionId).catch(() => undefined); + await removeLiveRecordingManifest(failed.sessionId).catch(() => undefined); setRetryStatus({ phase: "completed", videoId: typedVideoId, @@ -1708,12 +2688,7 @@ const handleRequest = async ( if (message.type === "toggle-microphone-mute") { const recording = activeRecording; if (recording) { - for (const stream of recording.streams) { - for (const track of stream.getAudioTracks()) { - track.enabled = !message.muted; - } - } - for (const track of recording.recordingStream.getAudioTracks()) { + for (const track of recording.microphoneStream?.getAudioTracks() ?? []) { track.enabled = !message.muted; } } @@ -1724,6 +2699,24 @@ const handleRequest = async ( }; chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => { + if (message?.type === "input-events-batch") { + const batch = parseCapturedTabInputBatch(message); + const recording = activeRecording; + if (!batch || !recording?.inputSidecar) { + sendResponse({ ok: false, error: "Tab input recording is unavailable" }); + return false; + } + recording.inputSidecar + .accept(batch, _sender.tab?.id) + .then(() => sendResponse({ ok: true })) + .catch((error: unknown) => + sendResponse({ + ok: false, + error: error instanceof Error ? error.message : String(error), + }), + ); + return true; + } if (!isOffscreenRequest(message)) return false; handleRequest(message) diff --git a/apps/chrome-extension/src/options/main.tsx b/apps/chrome-extension/src/options/main.tsx index 05a5d1ad081..7f4aa4c77e3 100644 --- a/apps/chrome-extension/src/options/main.tsx +++ b/apps/chrome-extension/src/options/main.tsx @@ -1,6 +1,8 @@ import { deleteRecoveredRecordingSpool, - recoverOrphanedRecordingSpools, + isRecordingSpoolUploaded, + listRecordingSpoolSessions, + recoverRecordingSpoolSession, } from "@cap/recorder-core"; import { useCallback, useEffect, useState } from "react"; import { createRoot } from "react-dom/client"; @@ -15,9 +17,11 @@ import { type FailedRecording, loadAuth, loadFailedRecordings, + loadLiveRecordingManifests, loadSettings, loadSharedRecordingState, removeFailedRecording, + removeLiveRecordingManifest, saveSettings, } from "../shared/storage"; import type { ExtensionAuth, ExtensionSettings } from "../shared/types"; @@ -51,13 +55,37 @@ const isActiveRecordingPhase = (phase: string | undefined) => // hold bytes, and also surfaces spools stranded by a crash before the // offscreen sweep could record them (those have no metadata yet). const loadRecoveredRecordings = async (): Promise => { - const [failed, spools, recordingState] = await Promise.all([ + const [failed, storedSpools, manifests, recordingState] = await Promise.all([ loadFailedRecordings(), - recoverOrphanedRecordingSpools(), + listRecordingSpoolSessions(), + loadLiveRecordingManifests(), loadSharedRecordingState().catch(() => null), ]); - const spoolSessions = new Set(spools.map((spool) => spool.sessionId)); - const knownSessions = new Set(failed.map((entry) => entry.sessionId)); + const spools = storedSpools.filter( + (spool) => !isRecordingSpoolUploaded(spool.sessionId), + ); + const spoolSessions = new Set( + spools + .filter((spool) => spool.totalBytes > 0) + .map((spool) => spool.sessionId), + ); + const knownSessions = new Set( + failed.flatMap((entry) => [ + entry.sessionId, + ...(entry.cameraSessionId ? [entry.cameraSessionId] : []), + ...(entry.inputEventsSessionId ? [entry.inputEventsSessionId] : []), + ...(entry.audioSources ?? []).map((source) => source.sessionId), + ]), + ); + for (const manifest of manifests) { + if (!spoolSessions.has(manifest.sessionId)) continue; + if (manifest.cameraSessionId) knownSessions.add(manifest.cameraSessionId); + if (manifest.inputEventsSessionId) + knownSessions.add(manifest.inputEventsSessionId); + for (const source of manifest.audioSources ?? []) { + knownSessions.add(source.sessionId); + } + } const entries = failed.filter((entry) => spoolSessions.has(entry.sessionId)); // Skip unknown spools while a recording is live anywhere: an in-flight @@ -139,12 +167,29 @@ function RecoveredRecordingsSection() { } }; - const download = (entry: FailedRecording) => + const download = ( + entry: FailedRecording, + kind: "screen" | "camera" | "mic" | "systemAudio", + ) => runAction(entry.sessionId, async () => { - const spools = await recoverOrphanedRecordingSpools(); - const spool = spools.find( - (candidate) => candidate.sessionId === entry.sessionId, + const audioSource = entry.audioSources?.find( + (source) => source.kind === kind, ); + const sessionId = + kind === "screen" + ? entry.sessionId + : kind === "camera" + ? entry.cameraSessionId + : audioSource?.sessionId; + const mimeType = + kind === "screen" + ? entry.mimeType + : kind === "camera" + ? entry.cameraMimeType + : audioSource?.mimeType; + const spool = sessionId + ? await recoverRecordingSpoolSession(sessionId) + : null; if (!spool || spool.blob.size === 0) { refresh(); throw new Error("The recorded data is no longer available."); @@ -152,10 +197,21 @@ function RecoveredRecordingsSection() { const url = URL.createObjectURL(spool.blob); const anchor = document.createElement("a"); anchor.href = url; - anchor.download = `cap-recording-${ + const recordingName = entry.videoId ?? - new Date(entry.createdAt).toISOString().replace(/[:.]/g, "-") - }.${fileExtensionForMimeType(entry.mimeType)}`; + new Date(entry.createdAt).toISOString().replace(/[:.]/g, "-"); + const separateSources = + !!entry.cameraSessionId || + entry.cameraRetryUnavailable === true || + (entry.audioSources?.length ?? 0) > 0 || + entry.audioRetryUnavailable === true; + const label = + kind === "screen" + ? separateSources + ? "-screen" + : "" + : `-${kind === "systemAudio" ? "system-audio" : kind}`; + anchor.download = `cap-recording-${recordingName}${label}.${fileExtensionForMimeType(mimeType ?? spool.mimeType)}`; anchor.click(); window.setTimeout(() => URL.revokeObjectURL(url), 10_000); return null; @@ -163,8 +219,16 @@ function RecoveredRecordingsSection() { const remove = (entry: FailedRecording) => runAction(entry.sessionId, async () => { - await deleteRecoveredRecordingSpool(entry.sessionId); + await Promise.all( + [ + entry.sessionId, + ...(entry.cameraSessionId ? [entry.cameraSessionId] : []), + ...(entry.inputEventsSessionId ? [entry.inputEventsSessionId] : []), + ...(entry.audioSources ?? []).map((source) => source.sessionId), + ].map((sessionId) => deleteRecoveredRecordingSpool(sessionId)), + ); await removeFailedRecording(entry.sessionId).catch(() => undefined); + await removeLiveRecordingManifest(entry.sessionId).catch(() => undefined); refresh(); return null; }); @@ -222,26 +286,56 @@ function RecoveredRecordingsSection() {
- {entry.videoId && ( - + {entry.videoId && + !entry.cameraRetryUnavailable && + !entry.audioRetryUnavailable && ( + + )} + {(entry.cameraRetryUnavailable || + entry.audioRetryUnavailable) && ( + + A separate source is missing. Download available sources. + )} + {entry.cameraSessionId && !entry.cameraRetryUnavailable && ( + + )} + {entry.audioSources?.map((source) => ( + + ))} +

diff --git a/apps/cli/src/selftest/playback.rs b/apps/cli/src/selftest/playback.rs index 57e997d9471..07cc2328639 100644 --- a/apps/cli/src/selftest/playback.rs +++ b/apps/cli/src/selftest/playback.rs @@ -917,6 +917,7 @@ mod fixture { keyboard_segments: Vec::new(), audio_segments: Vec::new(), camera3d_segments: Vec::new(), + waveform_segments: Vec::new(), }), clips: vec![ClipConfiguration { index: 0, diff --git a/apps/desktop-gpui/Cargo.lock b/apps/desktop-gpui/Cargo.lock index 635d63bb1bc..c657cb0279f 100644 --- a/apps/desktop-gpui/Cargo.lock +++ b/apps/desktop-gpui/Cargo.lock @@ -1445,6 +1445,7 @@ dependencies = [ "ndarray 0.16.1", "ort", "tracing", + "web-time", "wgpu 25.0.2", "workspace-hack", ] @@ -1600,6 +1601,7 @@ version = "0.1.0" dependencies = [ "axum", "cap-audio", + "cap-enc-ffmpeg", "cap-media", "cap-media-info", "cap-project", @@ -1611,6 +1613,7 @@ dependencies = [ "futures", "image 0.25.10", "lru", + "relative-path", "ringbuf", "sentry", "serde", @@ -1621,6 +1624,7 @@ dependencies = [ "tokio-util", "tracing", "tracing-subscriber", + "uuid", "windows 0.60.0", "workspace-hack", ] @@ -1922,6 +1926,7 @@ dependencies = [ "futures-intrusive", "glyphon", "image 0.25.10", + "js-sys", "libloading 0.8.9", "log", "metal 0.31.0", @@ -1936,6 +1941,10 @@ dependencies = [ "tiny-skia", "tokio", "tracing", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "web-time", "wgpu 25.0.2", "wgpu-core 25.0.2", "wgpu-hal 25.0.2", @@ -1949,6 +1958,7 @@ name = "cap-rnnoise" version = "0.1.0" dependencies = [ "cc", + "libm", "workspace-hack", ] diff --git a/apps/desktop-gpui/src/assets.rs b/apps/desktop-gpui/src/assets.rs index 65eabb37589..9c814a2e8ad 100644 --- a/apps/desktop-gpui/src/assets.rs +++ b/apps/desktop-gpui/src/assets.rs @@ -85,7 +85,6 @@ const ICONS: &[(&str, &[u8])] = assets!("icons": "laptop.svg", "shuffle.svg", "gift.svg", - "history.svg", "hotkeys.svg", "image.svg", "info.svg", @@ -99,7 +98,6 @@ const ICONS: &[(&str, &[u8])] = assets!("icons": "message-square-plus.svg", "mic-off.svg", "microphone.svg", - "minimize.svg", "minus.svg", "monitor.svg", "more-vertical.svg", @@ -189,7 +187,6 @@ const ICONS: &[(&str, &[u8])] = assets!("icons": // The main window's hand-drawn traffic lights: the x and expand glyphs // `CaptionControlsMacOS.tsx` inlines, shown while the group is hovered. "traffic-close.svg", - "traffic-zoom.svg", // The remaining settings pages (`settings_pages.rs`). `circle-check` is // Cap's own (`packages/ui-solid/icons/circle-check.svg`, hotkeys.tsx's // IconCapCircleCheck); the rest are the Lucide 24x24 originals the pages' @@ -335,6 +332,7 @@ mod tests { // `ui::SelectionHeader` names the check and the trash itself. include_str!("ui/selection_header.rs"), include_str!("ui/radio_cards.rs"), + include_str!("ui/windows_caption.rs"), // The onboarding window's welcome cards and permissions surface; the // per-permission row glyphs are named on `OSPermission::icon`. include_str!("onboarding_window.rs"), diff --git a/apps/desktop-gpui/src/editor_canvas.rs b/apps/desktop-gpui/src/editor_canvas.rs index 14618f7bea7..c3c118016ba 100644 --- a/apps/desktop-gpui/src/editor_canvas.rs +++ b/apps/desktop-gpui/src/editor_canvas.rs @@ -1420,6 +1420,7 @@ impl EditorWindow { } } } + OverlayTrackKind::Waveform => {} } } } diff --git a/apps/desktop-gpui/src/editor_clips.rs b/apps/desktop-gpui/src/editor_clips.rs index 72f90015b6e..ee967a2fd2a 100644 --- a/apps/desktop-gpui/src/editor_clips.rs +++ b/apps/desktop-gpui/src/editor_clips.rs @@ -2472,6 +2472,7 @@ fn ensure_project_timeline<'a>( keyboard_segments: Vec::new(), audio_segments: Vec::new(), camera3d_segments: Vec::new(), + waveform_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), }); diff --git a/apps/desktop-gpui/src/editor_edits.rs b/apps/desktop-gpui/src/editor_edits.rs index 93e1889d773..a3a17bf69eb 100644 --- a/apps/desktop-gpui/src/editor_edits.rs +++ b/apps/desktop-gpui/src/editor_edits.rs @@ -1822,6 +1822,7 @@ pub fn ensure_timeline(project: &mut ProjectConfiguration, clip_display_duration keyboard_segments: Vec::new(), audio_segments: Vec::new(), camera3d_segments: Vec::new(), + waveform_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), }); diff --git a/apps/desktop-gpui/src/editor_timeline.rs b/apps/desktop-gpui/src/editor_timeline.rs index 987bb776ee2..e82f1539a7d 100644 --- a/apps/desktop-gpui/src/editor_timeline.rs +++ b/apps/desktop-gpui/src/editor_timeline.rs @@ -1023,16 +1023,17 @@ pub struct TrackRow { } impl TrackRow { - pub fn from_overlay_track(track: OverlayTrack) -> Self { + pub fn from_overlay_track(track: OverlayTrack) -> Option { let kind = match track.kind { OverlayTrackKind::Mask => TrackKind::Mask, OverlayTrackKind::Image => TrackKind::Image, OverlayTrackKind::Text => TrackKind::Text, + OverlayTrackKind::Waveform => return None, }; - Self { + Some(Self { kind, lane: track.track, - } + }) } } @@ -1369,7 +1370,7 @@ fn build_rows( config .resolved_overlay_order(&overlay_tracks) .into_iter() - .map(TrackRow::from_overlay_track), + .filter_map(TrackRow::from_overlay_track), ); for lane in (0..lane_count(&model.audio).max(lanes.audio)).rev() { rows.push(TrackRow { diff --git a/apps/desktop-gpui/src/editor_window.rs b/apps/desktop-gpui/src/editor_window.rs index 5cbec1e496e..2bb1c804d39 100644 --- a/apps/desktop-gpui/src/editor_window.rs +++ b/apps/desktop-gpui/src/editor_window.rs @@ -445,6 +445,7 @@ pub fn preflight(path: &std::path::Path) -> Result { keyboard_segments: Vec::new(), audio_segments: Vec::new(), camera3d_segments: Vec::new(), + waveform_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), }); @@ -11983,6 +11984,7 @@ mod tests { keyboard_segments: Vec::new(), audio_segments: Vec::new(), camera3d_segments: Vec::new(), + waveform_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), }), @@ -12049,6 +12051,7 @@ mod tests { keyboard_segments: Vec::new(), audio_segments: Vec::new(), camera3d_segments: Vec::new(), + waveform_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), }), @@ -12293,6 +12296,7 @@ mod tests { edits::default_camera3d_segment(0.0, 2.0), edits::default_camera3d_segment(2.0, 4.0), ], + waveform_segments: Vec::new(), }), ..Default::default() }; diff --git a/apps/desktop-gpui/src/presets.rs b/apps/desktop-gpui/src/presets.rs index 1ca45d8150e..0c79ed549d1 100644 --- a/apps/desktop-gpui/src/presets.rs +++ b/apps/desktop-gpui/src/presets.rs @@ -245,6 +245,7 @@ mod tests { keyboard_segments: Vec::new(), audio_segments: Vec::new(), camera3d_segments: Vec::new(), + waveform_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), }), diff --git a/apps/desktop-gpui/src/transcription.rs b/apps/desktop-gpui/src/transcription.rs index ce2ed9dc1e6..4fb69b13675 100644 --- a/apps/desktop-gpui/src/transcription.rs +++ b/apps/desktop-gpui/src/transcription.rs @@ -2083,6 +2083,7 @@ pub fn apply_caption_result( keyboard_segments: Vec::new(), audio_segments: Vec::new(), camera3d_segments: Vec::new(), + waveform_segments: Vec::new(), style_segments: Vec::new(), image_segments: Vec::new(), }); diff --git a/apps/desktop/src-tauri/examples/desktop-display-transport-benchmark.rs b/apps/desktop/src-tauri/examples/desktop-display-transport-benchmark.rs index f7449e2f940..d6eeec71c9f 100644 --- a/apps/desktop/src-tauri/examples/desktop-display-transport-benchmark.rs +++ b/apps/desktop/src-tauri/examples/desktop-display-transport-benchmark.rs @@ -175,6 +175,7 @@ async fn load_recording( keyboard_segments: Vec::new(), audio_segments: Vec::new(), camera3d_segments: Vec::new(), + waveform_segments: Vec::new(), }); } } diff --git a/apps/desktop/src-tauri/src/clip_thumbnails.rs b/apps/desktop/src-tauri/src/clip_thumbnails.rs index 65860a5ad13..e4e0ac657d1 100644 --- a/apps/desktop/src-tauri/src/clip_thumbnails.rs +++ b/apps/desktop/src-tauri/src/clip_thumbnails.rs @@ -1,14 +1,19 @@ -use std::{path::Path, sync::LazyLock}; +use std::sync::LazyLock; use cap_project::{RecordingMetaInner, StudioRecordingMeta}; use tokio::sync::Semaphore; use crate::editor_window::WindowEditorInstance; -const THUMB_MAX_WIDTH: u32 = 240; -const THUMB_JPEG_QUALITY: u8 = 70; -const SEEK_DECODE_PACKET_LIMIT: usize = 4096; -const SEEK_DECODE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); +#[path = "../../../../crates/editor/src/clip_thumbnails_shared.rs"] +mod clip_thumbnails_shared; + +use clip_thumbnails_shared::decode_clip_thumbnail; +#[cfg(test)] +use clip_thumbnails_shared::{ + SEEK_DECODE_PACKET_LIMIT, decode_clip_thumbnail_with_budget, persist_clip_thumbnail, + thumbnail_frame_reaches_target, +}; static THUMBNAIL_SEMAPHORE: LazyLock = LazyLock::new(|| Semaphore::new(4)); @@ -71,222 +76,6 @@ pub async fn get_clip_thumbnail( Ok(cache_path.to_string_lossy().into_owned()) } -fn decode_clip_thumbnail(input: &Path, time: f64, output: &Path) -> Result<(), String> { - decode_clip_thumbnail_with_budget( - input, - time, - output, - SEEK_DECODE_PACKET_LIMIT, - SEEK_DECODE_TIMEOUT, - ) -} - -fn decode_clip_thumbnail_with_budget( - input: &Path, - time: f64, - output: &Path, - packet_limit: usize, - timeout: std::time::Duration, -) -> Result<(), String> { - use ffmpeg::rescale::{Rescale, TIME_BASE}; - - let mut ictx = - ffmpeg::format::input(input).map_err(|e| format!("Failed to open video: {e}"))?; - - let stream = ictx - .streams() - .best(ffmpeg::media::Type::Video) - .ok_or("No video stream found")?; - let stream_index = stream.index(); - let stream_time_base = stream.time_base(); - let stream_start = match stream.start_time() { - ffmpeg::ffi::AV_NOPTS_VALUE => 0, - timestamp => timestamp, - }; - let target_timestamp = ((time * 1_000_000.0) as i64) - .rescale((1, 1_000_000), stream_time_base) - .saturating_add(stream_start); - - let mut decoder = ffmpeg::codec::context::Context::from_parameters(stream.parameters()) - .map_err(|e| e.to_string())? - .decoder() - .video() - .map_err(|e| e.to_string())?; - - let src_width = decoder.width(); - let src_height = decoder.height(); - if src_width == 0 || src_height == 0 { - return Err("Invalid video dimensions".to_string()); - } - - let scale = (THUMB_MAX_WIDTH as f32 / src_width as f32).min(1.0); - let target_width = ((src_width as f32 * scale).round() as u32).max(1); - let target_height = ((src_height as f32 * scale).round() as u32).max(1); - - let mut scaler = ffmpeg::software::scaling::context::Context::get( - decoder.format(), - src_width, - src_height, - ffmpeg::format::Pixel::RGB24, - target_width, - target_height, - ffmpeg::software::scaling::flag::Flags::BILINEAR, - ) - .map_err(|e| e.to_string())?; - - if time > 0.0 { - let position_us = (time * 1_000_000.0) as i64; - let seek_target = target_timestamp.rescale(stream_time_base, TIME_BASE); - decoder.flush(); - ictx.seek(seek_target, ..seek_target) - .map_err(|e| format!("Failed to seek to {position_us}us: {e}"))?; - } - - let mut frame = ffmpeg::frame::Video::empty(); - let mut decoded = ffmpeg::frame::Video::empty(); - let mut got_frame = false; - let mut reached_target = false; - let mut decoder_finished = false; - let mut packets_tried = 0usize; - let decode_started = std::time::Instant::now(); - - 'outer: for (packet_stream, packet) in ictx.packets() { - if decode_started.elapsed() >= timeout { - return Err("Thumbnail decode time budget exhausted".to_string()); - } - if packet_stream.index() != stream_index { - continue; - } - - packets_tried += 1; - - if decoder.send_packet(&packet).is_err() { - if packets_tried >= packet_limit { - return Err("Thumbnail decode packet budget exhausted".to_string()); - } - continue; - } - - loop { - if decode_started.elapsed() >= timeout { - return Err("Thumbnail decode time budget exhausted".to_string()); - } - match decoder.receive_frame(&mut decoded) { - Ok(()) => { - std::mem::swap(&mut frame, &mut decoded); - got_frame = true; - if thumbnail_frame_reaches_target(&frame, target_timestamp) { - reached_target = true; - break 'outer; - } - } - Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::ffi::EAGAIN => break, - Err(ffmpeg::Error::Eof) => { - decoder_finished = true; - break 'outer; - } - Err(e) => { - if packets_tried >= packet_limit { - return Err(format!("Failed to decode frame: {e}")); - } - break; - } - } - } - - if packets_tried >= packet_limit { - return Err("Thumbnail decode packet budget exhausted".to_string()); - } - } - - if !reached_target && !decoder_finished { - decoder - .send_eof() - .map_err(|e| format!("Failed to flush decoder: {e}"))?; - loop { - if decode_started.elapsed() >= timeout { - return Err("Thumbnail decode time budget exhausted".to_string()); - } - match decoder.receive_frame(&mut decoded) { - Ok(()) => { - std::mem::swap(&mut frame, &mut decoded); - got_frame = true; - if thumbnail_frame_reaches_target(&frame, target_timestamp) { - break; - } - } - Err(ffmpeg::Error::Eof) => break, - Err(ffmpeg::Error::Other { errno }) if errno == ffmpeg::ffi::EAGAIN => break, - Err(e) => return Err(format!("Failed to flush decoder: {e}")), - } - } - } - - if !got_frame { - return Err("No decodable frames found".to_string()); - } - - let mut rgb_frame = ffmpeg::frame::Video::empty(); - scaler - .run(&frame, &mut rgb_frame) - .map_err(|e| e.to_string())?; - - let width = rgb_frame.width() as usize; - let height = rgb_frame.height() as usize; - let src_stride = rgb_frame.stride(0); - let dst_stride = width * 3; - if src_stride < dst_stride { - return Err(format!( - "Unexpected RGB stride: src_stride={src_stride}, expected >= {dst_stride}" - )); - } - let mut img_buffer = vec![0u8; height * dst_stride]; - for y in 0..height { - let src_slice = &rgb_frame.data(0)[y * src_stride..y * src_stride + dst_stride]; - img_buffer[y * dst_stride..(y + 1) * dst_stride].copy_from_slice(src_slice); - } - - use image::ImageEncoder; - - let mut jpeg_bytes = Vec::new(); - image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg_bytes, THUMB_JPEG_QUALITY) - .write_image( - &img_buffer, - width as u32, - height as u32, - image::ExtendedColorType::Rgb8, - ) - .map_err(|e| format!("Failed to encode thumbnail: {e}"))?; - - persist_clip_thumbnail(output, &jpeg_bytes) -} - -fn thumbnail_frame_reaches_target(frame: &ffmpeg::frame::Video, target_timestamp: i64) -> bool { - frame - .timestamp() - .or_else(|| frame.pts()) - .is_none_or(|timestamp| timestamp >= target_timestamp) -} - -fn persist_clip_thumbnail(output: &Path, jpeg_bytes: &[u8]) -> Result<(), String> { - use std::io::Write; - - let parent = output.parent().unwrap_or_else(|| Path::new(".")); - std::fs::create_dir_all(parent) - .map_err(|e| format!("Failed to create thumbnail directory: {e}"))?; - - let mut staged = tempfile::NamedTempFile::new_in(parent) - .map_err(|e| format!("Failed to stage thumbnail: {e}"))?; - staged - .write_all(jpeg_bytes) - .map_err(|e| format!("Failed to write thumbnail: {e}"))?; - staged - .persist(output) - .map_err(|e| format!("Failed to persist thumbnail: {e}"))?; - - Ok(()) -} - #[cfg(test)] mod persistence_tests { use super::*; diff --git a/apps/desktop/src-tauri/src/import.rs b/apps/desktop/src-tauri/src/import.rs index cebe7d9c065..4eedc6b8b4f 100644 --- a/apps/desktop/src-tauri/src/import.rs +++ b/apps/desktop/src-tauri/src/import.rs @@ -6,10 +6,9 @@ use cap_enc_ffmpeg::{ }; use cap_media_info::{AudioInfo, FFRational, Pixel, VideoInfo, ensure_even}; use cap_project::{ - AudioMeta, ClipConfiguration, CursorEvents, CursorMeta, Cursors, InstantRecordingMeta, - MultipleSegment, MultipleSegments, Platform, ProjectConfiguration, RecordingMeta, - RecordingMetaInner, SingleSegment, StudioRecordingMeta, StudioRecordingStatus, - TimelineConfiguration, TimelineSegment, VideoMeta, XY, + AudioMeta, ClipConfiguration, Cursors, InstantRecordingMeta, MultipleSegment, MultipleSegments, + Platform, ProjectConfiguration, RecordingMeta, RecordingMetaInner, SingleSegment, + StudioRecordingMeta, StudioRecordingStatus, TimelineConfiguration, TimelineSegment, VideoMeta, }; use ffmpeg::{ ChannelLayout, @@ -17,11 +16,10 @@ use ffmpeg::{ format::{self as avformat}, }; use image::ImageEncoder; -use relative_path::{Component as RelativeComponent, RelativePathBuf}; +use relative_path::RelativePathBuf; use serde::{Deserialize, Serialize}; use specta::Type; use std::{ - collections::HashMap, path::{Path, PathBuf}, str::FromStr, }; @@ -38,9 +36,6 @@ use crate::{ const VIDEO_IMPORT_EXTENSIONS: &[&str] = &["mp4", "mov", "avi", "mkv", "webm", "wmv", "m4v", "flv"]; const IMAGE_IMPORT_EXTENSIONS: &[&str] = &["png", "jpg", "jpeg", "webp", "gif", "bmp", "tif", "tiff"]; -const AUDIO_IMPORT_EXTENSIONS: &[&str] = &["ogg", "m4a", "mp3", "wav", "aac", "flac"]; -const KEYBOARD_IMPORT_EXTENSIONS: &[&str] = &["bin", "json"]; -const CURSOR_EVENTS_IMPORT_EXTENSIONS: &[&str] = &["json"]; const MAX_IMAGE_DIMENSION: u32 = 16_384; #[derive(Serialize, Deserialize, Type, Clone, Debug)] @@ -161,89 +156,6 @@ fn is_cap_project_path(path: &Path) -> bool { path.is_dir() && path.join("recording-meta.json").is_file() } -fn normalized_metadata_relative_path( - path: &RelativePathBuf, - asset_kind: &str, -) -> Result { - let normalized = path.as_str().replace('\\', "/"); - let path = RelativePathBuf::from(normalized); - let raw = path.as_str(); - if raw.is_empty() - || raw.starts_with('/') - || raw.contains(':') - || path - .components() - .any(|component| matches!(component, RelativeComponent::ParentDir)) - { - return Err(format!( - "Invalid {asset_kind} path in recording metadata: {raw}" - )); - } - - Ok(path) -} - -fn source_asset_path( - source_project_path: &Path, - source_relative_path: &RelativePathBuf, - asset_kind: &str, - allowed_extensions: &[&str], -) -> Result, String> { - let source_relative_path = normalized_metadata_relative_path(source_relative_path, asset_kind)?; - - if !has_supported_extension(Path::new(source_relative_path.as_str()), allowed_extensions) { - return Err(format!( - "Unsupported {asset_kind} file type: {}", - source_relative_path.as_str() - )); - } - - let source_path = source_relative_path.to_path(source_project_path); - if !source_path.is_file() { - return Ok(None); - } - - let source_root = source_project_path - .canonicalize() - .map_err(|e| format!("Failed to resolve source project path: {e}"))?; - let canonical_source_path = source_path - .canonicalize() - .map_err(|e| format!("Failed to resolve {asset_kind} path: {e}"))?; - - if !canonical_source_path.starts_with(&source_root) { - return Err(format!( - "{asset_kind} path escapes source project: {}", - source_relative_path.as_str() - )); - } - - Ok(Some(canonical_source_path)) -} - -fn required_source_asset_path( - source_project_path: &Path, - source_relative_path: &RelativePathBuf, - asset_kind: &str, - allowed_extensions: &[&str], -) -> Result { - source_asset_path( - source_project_path, - source_relative_path, - asset_kind, - allowed_extensions, - )? - .ok_or_else(|| { - format!( - "Missing {asset_kind} file: {}", - source_relative_path.to_path(source_project_path).display() - ) - }) -} - -fn legacy_cursor_relative_path(path: &str) -> Result { - normalized_metadata_relative_path(&RelativePathBuf::from(path), "cursor image") -} - fn editor_project_path_from_window(window: &Window) -> Result { let CapWindowId::Editor { id } = CapWindowId::from_str(window.label()).map_err(|e| e.to_string())? @@ -331,42 +243,6 @@ fn full_timeline_for_segments( .collect() } -fn get_source_video_duration_secs( - source_meta: &RecordingMeta, - video: &VideoMeta, -) -> Result { - let source_path = required_source_asset_path( - &source_meta.project_path, - &video.path, - "video", - VIDEO_IMPORT_EXTENSIONS, - )?; - get_video_duration_secs(&source_path) -} - -fn full_timeline_for_source_segments( - source_meta: &RecordingMeta, - segments: &[MultipleSegment], -) -> Result, String> { - segments - .iter() - .enumerate() - .map(|(index, segment)| { - let duration = get_source_video_duration_secs(source_meta, &segment.display)?; - Ok(TimelineSegment { - recording_clip: index as u32, - timescale: 1.0, - start: 0.0, - end: duration, - name: None, - speed_audio_mode: None, - hide_cursor: None, - volume: None, - }) - }) - .collect() -} - fn ensure_project_timeline<'a>( config: &'a mut ProjectConfiguration, project_path: &Path, @@ -386,6 +262,7 @@ fn ensure_project_timeline<'a>( keyboard_segments: Vec::new(), audio_segments: Vec::new(), camera3d_segments: Vec::new(), + waveform_segments: Vec::new(), }); } @@ -440,600 +317,6 @@ fn unique_segment_dir( } } -fn relative_file_extension(path: &RelativePathBuf, fallback: &str) -> String { - Path::new(path.as_str()) - .extension() - .and_then(|ext| ext.to_str()) - .filter(|ext| !ext.is_empty()) - .unwrap_or(fallback) - .to_string() -} - -fn relative_file_name(path: &RelativePathBuf, fallback: &str) -> String { - Path::new(path.as_str()) - .file_name() - .and_then(|name| name.to_str()) - .filter(|name| !name.is_empty()) - .unwrap_or(fallback) - .to_string() -} - -fn unique_file_name(dir: &Path, preferred: &str) -> String { - let sanitized = sanitize_filename(preferred); - let sanitized = if sanitized.is_empty() { - "file".to_string() - } else { - sanitized - }; - - let path = Path::new(&sanitized); - let stem = path - .file_stem() - .and_then(|value| value.to_str()) - .filter(|value| !value.is_empty()) - .unwrap_or("file") - .to_string(); - let extension = path - .extension() - .and_then(|value| value.to_str()) - .filter(|value| !value.is_empty()) - .map(|value| value.to_string()); - - let mut counter = 0; - loop { - let candidate = if counter == 0 { - sanitized.clone() - } else if let Some(extension) = &extension { - format!("{stem}-{counter}.{extension}") - } else { - format!("{stem}-{counter}") - }; - - if !dir.join(&candidate).exists() { - return candidate; - } - - counter += 1; - } -} - -fn copy_file_to_relative_path( - source_path: &Path, - target_project_path: &Path, - target_relative_path: &RelativePathBuf, -) -> Result<(), String> { - let target_path = target_relative_path.to_path(target_project_path); - - if let Some(parent) = target_path.parent() { - std::fs::create_dir_all(parent) - .map_err(|e| format!("Failed to create import directory: {e}"))?; - } - - std::fs::copy(source_path, &target_path) - .map(|_| ()) - .map_err(|e| format!("Failed to copy {}: {e}", source_path.display())) -} - -fn copy_video_meta( - source_project_path: &Path, - target_project_path: &Path, - source: &VideoMeta, - target_relative_dir: &str, - name: &str, - required: bool, -) -> Result, String> { - let Some(source_path) = source_asset_path( - source_project_path, - &source.path, - "video", - VIDEO_IMPORT_EXTENSIONS, - )? - else { - if required { - return Err(format!( - "Missing video file: {}", - source.path.to_path(source_project_path).display() - )); - } - return Ok(None); - }; - - let can_decode = probe_video_can_decode(&source_path) - .map_err(|e| format!("Cannot decode video {}: {e}", source_path.display()))?; - if !can_decode { - if required { - return Err(format!("Unsupported video file: {}", source_path.display())); - } - return Ok(None); - } - - let extension = relative_file_extension(&source.path, "mp4"); - let target_relative_path = - RelativePathBuf::from(format!("{target_relative_dir}/{name}.{extension}")); - copy_file_to_relative_path(&source_path, target_project_path, &target_relative_path)?; - - let mut copied = source.clone(); - copied.path = target_relative_path; - Ok(Some(copied)) -} - -fn copy_audio_meta( - source_project_path: &Path, - target_project_path: &Path, - source: &AudioMeta, - target_relative_dir: &str, - name: &str, -) -> Result, String> { - let Some(source_path) = source_asset_path( - source_project_path, - &source.path, - "audio", - AUDIO_IMPORT_EXTENSIONS, - )? - else { - return Ok(None); - }; - - let extension = relative_file_extension(&source.path, "ogg"); - let target_relative_path = - RelativePathBuf::from(format!("{target_relative_dir}/{name}.{extension}")); - copy_file_to_relative_path(&source_path, target_project_path, &target_relative_path)?; - - let mut copied = source.clone(); - copied.path = target_relative_path; - Ok(Some(copied)) -} - -fn copy_keyboard_path( - source_meta: &RecordingMeta, - source_segment: &MultipleSegment, - target_project_path: &Path, - target_relative_dir: &str, -) -> Result, String> { - if let Some(source_relative_path) = &source_segment.keyboard { - let file_name = - relative_file_name(source_relative_path, cap_project::KEYBOARD_EVENTS_FILE_NAME); - let Some(source_path) = source_asset_path( - &source_meta.project_path, - source_relative_path, - "keyboard events", - KEYBOARD_IMPORT_EXTENSIONS, - )? - else { - return Ok(None); - }; - - let target_relative_path = RelativePathBuf::from(format!( - "{target_relative_dir}/{}", - sanitize_filename(&file_name) - )); - copy_file_to_relative_path(&source_path, target_project_path, &target_relative_path)?; - - return Ok(Some(target_relative_path)); - }; - - let Some(display_dir) = source_segment.display.path.parent() else { - return Ok(None); - }; - - for file_name in [ - cap_project::KEYBOARD_EVENTS_FILE_NAME, - cap_project::LEGACY_KEYBOARD_EVENTS_FILE_NAME, - ] { - let source_relative_path = display_dir.join(file_name); - let Some(source_path) = source_asset_path( - &source_meta.project_path, - &source_relative_path, - "keyboard events", - KEYBOARD_IMPORT_EXTENSIONS, - )? - else { - continue; - }; - - let target_relative_path = RelativePathBuf::from(format!( - "{target_relative_dir}/{}", - sanitize_filename(file_name) - )); - copy_file_to_relative_path(&source_path, target_project_path, &target_relative_path)?; - - return Ok(Some(target_relative_path)); - } - - Ok(None) -} - -fn normalize_cursors_to_correct(cursors: &mut Cursors) -> &mut HashMap { - if let Cursors::Old(old) = cursors { - let converted = old - .iter() - .map(|(id, path)| { - ( - id.clone(), - CursorMeta { - image_path: RelativePathBuf::from(path.as_str()), - hotspot: XY::new(0.0, 0.0), - shape: None, - }, - ) - }) - .collect(); - *cursors = Cursors::Correct(converted); - } - - match cursors { - Cursors::Correct(map) => map, - Cursors::Old(_) => unreachable!(), - } -} - -fn unique_cursor_id( - cursors: &HashMap, - import_token: &str, - source_id: &str, -) -> String { - let source_id = if source_id.is_empty() { - "cursor" - } else { - source_id - }; - let base = format!("{import_token}-{source_id}"); - if !cursors.contains_key(&base) { - return base; - } - - let mut counter = 1; - loop { - let candidate = format!("{base}-{counter}"); - if !cursors.contains_key(&candidate) { - return candidate; - } - counter += 1; - } -} - -fn copy_source_cursor_images( - source_meta: &RecordingMeta, - source_cursors: &Cursors, - target_project_path: &Path, - target_cursors: &mut Cursors, - import_token: &str, -) -> Result, String> { - let target_cursor_dir = target_project_path.join("content").join("cursors"); - std::fs::create_dir_all(&target_cursor_dir) - .map_err(|e| format!("Failed to create cursor directory: {e}"))?; - - let target_cursors = normalize_cursors_to_correct(target_cursors); - let mut id_map = HashMap::new(); - - match source_cursors { - Cursors::Correct(source_map) => { - for (source_id, cursor) in source_map { - let Some(source_path) = source_asset_path( - &source_meta.project_path, - &cursor.image_path, - "cursor image", - IMAGE_IMPORT_EXTENSIONS, - )? - else { - continue; - }; - - let new_id = unique_cursor_id(target_cursors, import_token, source_id); - let source_file_name = relative_file_name(&cursor.image_path, "cursor.png"); - let target_file_name = - unique_file_name(&target_cursor_dir, &format!("{new_id}-{source_file_name}")); - let target_relative_path = - RelativePathBuf::from(format!("content/cursors/{target_file_name}")); - - copy_file_to_relative_path( - &source_path, - target_project_path, - &target_relative_path, - )?; - - target_cursors.insert( - new_id.clone(), - CursorMeta { - image_path: target_relative_path, - hotspot: cursor.hotspot, - shape: cursor.shape, - }, - ); - id_map.insert(source_id.clone(), new_id); - } - } - Cursors::Old(source_map) => { - for (source_id, source_path) in source_map { - let source_relative_path = legacy_cursor_relative_path(source_path)?; - let Some(source_path) = source_asset_path( - &source_meta.project_path, - &source_relative_path, - "cursor image", - IMAGE_IMPORT_EXTENSIONS, - )? - else { - continue; - }; - - let new_id = unique_cursor_id(target_cursors, import_token, source_id); - let source_file_name = relative_file_name(&source_relative_path, "cursor.png"); - let target_file_name = - unique_file_name(&target_cursor_dir, &format!("{new_id}-{source_file_name}")); - let target_relative_path = - RelativePathBuf::from(format!("content/cursors/{target_file_name}")); - - copy_file_to_relative_path( - &source_path, - target_project_path, - &target_relative_path, - )?; - - target_cursors.insert( - new_id.clone(), - CursorMeta { - image_path: target_relative_path, - hotspot: XY::new(0.0, 0.0), - shape: None, - }, - ); - id_map.insert(source_id.clone(), new_id); - } - } - } - - Ok(id_map) -} - -fn copy_cursor_events_path( - source_meta: &RecordingMeta, - source_relative_path: &RelativePathBuf, - target_project_path: &Path, - target_relative_dir: &str, - cursor_id_map: &HashMap, -) -> Result, String> { - let Some(source_path) = source_asset_path( - &source_meta.project_path, - source_relative_path, - "cursor events", - CURSOR_EVENTS_IMPORT_EXTENSIONS, - )? - else { - return Ok(None); - }; - - let target_relative_path = RelativePathBuf::from(format!("{target_relative_dir}/cursor.json")); - let target_path = target_relative_path.to_path(target_project_path); - if let Some(parent) = target_path.parent() { - std::fs::create_dir_all(parent) - .map_err(|e| format!("Failed to create cursor event directory: {e}"))?; - } - - if cursor_id_map.is_empty() { - std::fs::copy(&source_path, &target_path) - .map(|_| ()) - .map_err(|e| format!("Failed to copy cursor events: {e}"))?; - return Ok(Some(target_relative_path)); - } - - match CursorEvents::load_from_file(&source_path) { - Ok(mut events) => { - for event in &mut events.moves { - if let Some(new_id) = cursor_id_map.get(&event.cursor_id) { - event.cursor_id = new_id.clone(); - } - } - for event in &mut events.clicks { - if let Some(new_id) = cursor_id_map.get(&event.cursor_id) { - event.cursor_id = new_id.clone(); - } - } - - let file = std::fs::File::create(&target_path) - .map_err(|e| format!("Failed to create cursor event file: {e}"))?; - serde_json::to_writer_pretty(file, &events) - .map_err(|e| format!("Failed to write cursor event file: {e}"))?; - } - Err(_) => { - std::fs::copy(&source_path, &target_path) - .map(|_| ()) - .map_err(|e| format!("Failed to copy cursor events: {e}"))?; - } - } - - Ok(Some(target_relative_path)) -} - -fn single_segment_to_multiple(segment: &SingleSegment) -> MultipleSegment { - MultipleSegment { - display: segment.display.clone(), - camera: segment.camera.clone(), - mic: segment.audio.clone(), - system_audio: None, - cursor: segment.cursor.clone(), - keyboard: None, - display_notch: None, - } -} - -fn studio_segments_for_import(studio_meta: &StudioRecordingMeta) -> Vec { - match studio_meta { - StudioRecordingMeta::SingleSegment { segment } => { - vec![single_segment_to_multiple(segment)] - } - StudioRecordingMeta::MultipleSegments { inner } => inner.segments.clone(), - } -} - -fn source_timeline_segments_for_import( - source_meta: &RecordingMeta, - source_segments: &[MultipleSegment], -) -> Result, String> { - let source_config = ProjectConfiguration::load(&source_meta.project_path).unwrap_or_default(); - let Some(timeline) = source_config.timeline else { - return full_timeline_for_source_segments(source_meta, source_segments); - }; - - if timeline.segments.is_empty() { - return full_timeline_for_source_segments(source_meta, source_segments); - } - - let mut duration_cache = HashMap::new(); - let mut imported_segments = Vec::new(); - - for segment in timeline.segments { - let source_index = segment.recording_clip; - let Some(source_segment) = source_segments.get(source_index as usize) else { - continue; - }; - - let max_duration = if let Some(duration) = duration_cache.get(&source_index) { - *duration - } else { - let duration = get_source_video_duration_secs(source_meta, &source_segment.display)?; - duration_cache.insert(source_index, duration); - duration - }; - - if max_duration <= 0.0 { - continue; - } - - let raw_start = if segment.start.is_finite() { - segment.start - } else { - 0.0 - }; - let raw_end = if segment.end.is_finite() { - segment.end - } else { - max_duration - }; - let start = raw_start.clamp(0.0, max_duration); - let end = raw_end.clamp(start, max_duration); - if end <= start { - continue; - } - - imported_segments.push(TimelineSegment { - recording_clip: source_index, - timescale: if segment.timescale.is_finite() && segment.timescale > 0.0 { - segment.timescale - } else { - 1.0 - }, - start, - end, - name: None, - speed_audio_mode: segment.speed_audio_mode, - hide_cursor: segment.hide_cursor, - volume: segment.volume, - }); - } - - if imported_segments.is_empty() { - full_timeline_for_source_segments(source_meta, source_segments) - } else { - Ok(imported_segments) - } -} - -fn copy_source_segment( - source_meta: &RecordingMeta, - source_segment: &MultipleSegment, - target_project_path: &Path, - target_relative_dir: &str, - cursor_id_map: &HashMap, -) -> Result { - let display = copy_video_meta( - &source_meta.project_path, - target_project_path, - &source_segment.display, - target_relative_dir, - "display", - true, - )? - .ok_or_else(|| "Missing display video".to_string())?; - - let camera = source_segment - .camera - .as_ref() - .map(|camera| { - copy_video_meta( - &source_meta.project_path, - target_project_path, - camera, - target_relative_dir, - "camera", - false, - ) - }) - .transpose()? - .flatten(); - - let mic = source_segment - .mic - .as_ref() - .map(|mic| { - copy_audio_meta( - &source_meta.project_path, - target_project_path, - mic, - target_relative_dir, - "mic", - ) - }) - .transpose()? - .flatten(); - - let system_audio = source_segment - .system_audio - .as_ref() - .map(|system_audio| { - copy_audio_meta( - &source_meta.project_path, - target_project_path, - system_audio, - target_relative_dir, - "system-audio", - ) - }) - .transpose()? - .flatten(); - - let cursor = source_segment - .cursor - .as_ref() - .map(|cursor| { - copy_cursor_events_path( - source_meta, - cursor, - target_project_path, - target_relative_dir, - cursor_id_map, - ) - }) - .transpose()? - .flatten(); - - let keyboard = copy_keyboard_path( - source_meta, - source_segment, - target_project_path, - target_relative_dir, - )?; - - Ok(MultipleSegment { - display, - camera, - mic, - system_audio, - cursor, - keyboard, - display_notch: source_segment.display_notch, - }) -} - fn get_video_stream_info( input: &avformat::context::Input, ) -> Result<(usize, VideoInfo), ImportError> { @@ -1771,121 +1054,21 @@ async fn append_cap_project_to_editor_project( ) -> Result { let source_meta = RecordingMeta::load_for_project(&source_project_path) .map_err(|e| format!("Failed to load source project metadata: {e}"))?; - - let RecordingMetaInner::Studio(source_studio_meta) = &source_meta.inner else { - return match &source_meta.inner { - RecordingMetaInner::Instant(InstantRecordingMeta::Complete { .. }) => { - append_mp4_to_editor_project(app, target_project_path, source_meta.output_path()) - .await - } - RecordingMetaInner::Instant(InstantRecordingMeta::InProgress { .. }) => { - Err("Source Cap project is still recording".to_string()) - } - RecordingMetaInner::Instant(InstantRecordingMeta::Failed { error }) => { - Err(format!("Source Cap project failed: {error}")) - } - RecordingMetaInner::Studio(_) => unreachable!(), - }; - }; - - let source_segments = studio_segments_for_import(source_studio_meta); - if source_segments.is_empty() { - return Err("Source Cap project has no recording segments".to_string()); - } - - let source_timeline = source_timeline_segments_for_import(&source_meta, &source_segments)?; - let source_cursors = match source_studio_meta.as_ref() { - StudioRecordingMeta::MultipleSegments { inner } => Some(&inner.cursors), - StudioRecordingMeta::SingleSegment { .. } => None, - }; - - let mut target_meta = RecordingMeta::load_for_project(&target_project_path) - .map_err(|e| format!("Failed to load target project metadata: {e}"))?; - let mut config = ProjectConfiguration::load(&target_project_path).unwrap_or_default(); - let existing_segments = { - let inner = ensure_multiple_segments(&mut target_meta)?; - inner.status = Some(StudioRecordingStatus::Complete); - inner.segments.clone() - }; - ensure_project_timeline(&mut config, &target_project_path, &existing_segments)?; - - let (base_index, copied_segments, source_to_target_index) = { - let inner = ensure_multiple_segments(&mut target_meta)?; - inner.status = Some(StudioRecordingStatus::Complete); - let base_index = inner.segments.len() as u32; - let import_token = format!("import-{}", uuid::Uuid::new_v4().simple()); - let cursor_id_map = if let Some(source_cursors) = source_cursors { - copy_source_cursor_images( - &source_meta, - source_cursors, - &target_project_path, - &mut inner.cursors, - &import_token, - )? - } else { - HashMap::new() - }; - - let mut copied_segments = Vec::new(); - let mut source_to_target_index = HashMap::new(); - - for (source_index, source_segment) in source_segments.iter().enumerate() { - let target_index = base_index + copied_segments.len() as u32; - let (_, target_relative_dir) = - unique_segment_dir(&target_project_path, target_index) - .map_err(|e| format!("Failed to create imported segment directory: {e}"))?; - let copied_segment = copy_source_segment( - &source_meta, - source_segment, - &target_project_path, - &target_relative_dir, - &cursor_id_map, - )?; - - inner.segments.push(copied_segment.clone()); - copied_segments.push(copied_segment); - source_to_target_index.insert(source_index as u32, target_index); + match &source_meta.inner { + RecordingMetaInner::Studio(_) => cap_editor::append_studio_cap_project_to_editor_project( + target_project_path, + source_project_path, + ), + RecordingMetaInner::Instant(InstantRecordingMeta::Complete { .. }) => { + append_mp4_to_editor_project(app, target_project_path, source_meta.output_path()).await } - - (base_index, copied_segments, source_to_target_index) - }; - - if copied_segments.is_empty() { - return Err("Source Cap project has no importable recording segments".to_string()); - } - - { - let timeline = - ensure_project_timeline(&mut config, &target_project_path, &existing_segments)?; - for source_segment in source_timeline { - let Some(target_index) = source_to_target_index.get(&source_segment.recording_clip) - else { - continue; - }; - - timeline.segments.push(TimelineSegment { - recording_clip: *target_index, - timescale: source_segment.timescale, - start: source_segment.start, - end: source_segment.end, - name: None, - speed_audio_mode: source_segment.speed_audio_mode, - hide_cursor: source_segment.hide_cursor, - volume: source_segment.volume, - }); + RecordingMetaInner::Instant(InstantRecordingMeta::InProgress { .. }) => { + Err("Source Cap project is still recording".to_string()) + } + RecordingMetaInner::Instant(InstantRecordingMeta::Failed { error }) => { + Err(format!("Source Cap project failed: {error}")) } } - - add_clip_configs(&mut config, base_index, &copied_segments); - - target_meta - .save_for_project() - .map_err(|e| format!("Failed to save project metadata: {e:?}"))?; - config - .write(&target_project_path) - .map_err(|e| format!("Failed to save project config: {e}"))?; - - Ok(copied_segments.len()) } #[tauri::command] @@ -2142,125 +1325,4 @@ mod tests { assert_eq!((referenced.width(), referenced.height()), (12, 8)); assert_eq!(referenced.pts(), Some(93)); } - - #[test] - fn source_asset_path_allows_file_inside_source_project() { - let source_project = tempfile::tempdir().unwrap(); - let source_relative_path = RelativePathBuf::from("content/segments/segment-0/display.mp4"); - let source_path = source_relative_path.to_path(source_project.path()); - std::fs::create_dir_all(source_path.parent().unwrap()).unwrap(); - std::fs::write(&source_path, b"video").unwrap(); - - let resolved = source_asset_path( - source_project.path(), - &source_relative_path, - "video", - VIDEO_IMPORT_EXTENSIONS, - ) - .unwrap() - .unwrap(); - - assert_eq!(resolved, source_path.canonicalize().unwrap()); - } - - #[test] - fn source_asset_path_allows_backslash_separators() { - let source_project = tempfile::tempdir().unwrap(); - let source_relative_path = - RelativePathBuf::from("content\\segments\\segment-0\\display.mp4"); - let source_path = RelativePathBuf::from("content/segments/segment-0/display.mp4") - .to_path(source_project.path()); - std::fs::create_dir_all(source_path.parent().unwrap()).unwrap(); - std::fs::write(&source_path, b"video").unwrap(); - - let resolved = source_asset_path( - source_project.path(), - &source_relative_path, - "video", - VIDEO_IMPORT_EXTENSIONS, - ) - .unwrap() - .unwrap(); - - assert_eq!(resolved, source_path.canonicalize().unwrap()); - } - - #[test] - fn source_asset_path_rejects_parent_traversal() { - let source_project = tempfile::tempdir().unwrap(); - let source_relative_path = RelativePathBuf::from("../secret.mp4"); - - let error = source_asset_path( - source_project.path(), - &source_relative_path, - "video", - VIDEO_IMPORT_EXTENSIONS, - ) - .unwrap_err(); - - assert!(error.contains("Invalid video path")); - } - - #[test] - fn source_asset_path_rejects_absolute_path() { - let source_project = tempfile::tempdir().unwrap(); - let source_relative_path = RelativePathBuf::from("/tmp/secret.mp4"); - - let error = source_asset_path( - source_project.path(), - &source_relative_path, - "video", - VIDEO_IMPORT_EXTENSIONS, - ) - .unwrap_err(); - - assert!(error.contains("Invalid video path")); - } - - #[test] - fn legacy_cursor_relative_path_rejects_windows_absolute_path() { - let error = legacy_cursor_relative_path("C:\\Users\\me\\cursor.png").unwrap_err(); - - assert!(error.contains("Invalid cursor image path")); - } - - #[test] - fn source_asset_path_rejects_unsupported_extension() { - let source_project = tempfile::tempdir().unwrap(); - let source_relative_path = RelativePathBuf::from("content/segments/segment-0/display.txt"); - - let error = source_asset_path( - source_project.path(), - &source_relative_path, - "video", - VIDEO_IMPORT_EXTENSIONS, - ) - .unwrap_err(); - - assert!(error.contains("Unsupported video file type")); - } - - #[cfg(unix)] - #[test] - fn source_asset_path_rejects_symlink_escape() { - let source_project = tempfile::tempdir().unwrap(); - let external_dir = tempfile::tempdir().unwrap(); - let external_file = external_dir.path().join("cursor.png"); - std::fs::write(&external_file, b"cursor").unwrap(); - - let source_relative_path = RelativePathBuf::from("content/cursors/cursor.png"); - let source_path = source_relative_path.to_path(source_project.path()); - std::fs::create_dir_all(source_path.parent().unwrap()).unwrap(); - std::os::unix::fs::symlink(&external_file, &source_path).unwrap(); - - let error = source_asset_path( - source_project.path(), - &source_relative_path, - "cursor image", - IMAGE_IMPORT_EXTENSIONS, - ) - .unwrap_err(); - - assert!(error.contains("cursor image path escapes source project")); - } } diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index 87cae9cedb8..71def2eb94a 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -6634,6 +6634,7 @@ pub(crate) fn recording_timeline( keyboard_segments: Vec::new(), audio_segments: Vec::new(), camera3d_segments: Vec::new(), + waveform_segments: Vec::new(), } } @@ -6819,15 +6820,7 @@ fn apply_screen_recording_presentation_defaults( } pub fn default_project_config() -> ProjectConfiguration { - let mut config = ProjectConfiguration::default(); - - apply_screen_recording_presentation_defaults(&mut config, None, false, None); - - if config.background.rounding <= f64::EPSILON { - config.background.rounding = DEFAULT_SCREEN_RECORDING_BACKGROUND_ROUNDING_PERCENT; - } - - config + cap_editor::default_screen_recording_project_config() } #[tauri::command] diff --git a/apps/desktop/src/routes/editor/CanvasElementsOverlay.tsx b/apps/desktop/src/routes/editor/CanvasElementsOverlay.tsx index 3b017e92ce4..a01dd93bef3 100644 --- a/apps/desktop/src/routes/editor/CanvasElementsOverlay.tsx +++ b/apps/desktop/src/routes/editor/CanvasElementsOverlay.tsx @@ -62,6 +62,7 @@ type SnapExclude = | { text: number } | { mask: number } | { image: number } + | { waveform: number } | null; /** @@ -78,7 +79,8 @@ export function useCanvasSnapTargets() { const layout = latestFrameLayout(); if (layout) { - if (exclude !== "display") rects.push(normRect(layout.display, layout)); + if (exclude !== "display" && !project.hideDisplay) + rects.push(normRect(layout.display, layout)); if (layout.camera && exclude !== "camera") rects.push(normRect(layout.camera, layout)); } @@ -147,6 +149,22 @@ export function useCanvasSnapTargets() { h, }); }); + project.timeline?.waveformSegments?.forEach((segment, index) => { + if ( + typeof exclude === "object" && + exclude !== null && + "waveform" in exclude && + exclude.waveform === index + ) + return; + if (!segment.enabled || t < segment.start || t >= segment.end) return; + rects.push({ + x: segment.center.x - segment.size.x / 2, + y: segment.center.y - segment.size.y / 2, + w: segment.size.x, + h: segment.size.y, + }); + }); // The classic camera-inset margin lines only make sense for the camera // and display; for text/mask boxes they just cause spurious re-snaps // right next to the frame-edge lines. @@ -275,7 +293,10 @@ export function CanvasElementsOverlay(props: { size: Size }) { return mode === "splitScreen" || mode === "floating"; }; const showDisplay = () => - overlayVisible() && !paneScene() && sceneModeAt(time()) !== "cameraOnly"; + overlayVisible() && + !project.hideDisplay && + !paneScene() && + sceneModeAt(time()) !== "cameraOnly"; const showCamera = () => overlayVisible() && !paneScene(); // The rendered display rect is zoom-transformed while a zoom segment is // active, but drags write base-layout config — lock it to avoid a diff --git a/apps/desktop/src/routes/editor/CaptionOverlay.tsx b/apps/desktop/src/routes/editor/CaptionOverlay.tsx index 306ae44f1df..0b054512bbe 100644 --- a/apps/desktop/src/routes/editor/CaptionOverlay.tsx +++ b/apps/desktop/src/routes/editor/CaptionOverlay.tsx @@ -11,6 +11,7 @@ import { import { produce } from "solid-js/store"; import { defaultCaptionSettings } from "~/store/captions"; import type { CaptionTrackSegment } from "~/utils/tauri"; +import { spanIndexAt, spansInOrder } from "./active-caption"; import { useCanvasSnapTargets } from "./CanvasElementsOverlay"; import { FPS, useEditorContext } from "./context"; import { SNAP_PX, type SnapTargets, snapMovingRect } from "./snapping"; @@ -63,18 +64,29 @@ export function CaptionOverlay(props: CaptionOverlayProps) { ...project.captions?.settings, })); - const activeCaption = createMemo(() => { - if (!settings().enabled) return null; - const time = currentAbsoluteTime(); - const segments = project.timeline?.captionSegments ?? []; - const index = segments.findIndex( - (segment) => time >= segment.start && time < segment.end, - ); - if (index < 0) return null; - const segment = segments[index]; - if (!segment) return null; - return { index, segment }; - }); + const captionSegments = () => project.timeline?.captionSegments ?? []; + + const captionsInOrder = createMemo(() => spansInOrder(captionSegments())); + + const activeCaption = createMemo( + () => { + if (!settings().enabled) return null; + const segments = captionSegments(); + const index = spanIndexAt( + segments, + currentAbsoluteTime(), + captionsInOrder(), + ); + if (index < 0) return null; + const segment = segments[index]; + if (!segment) return null; + return { index, segment }; + }, + undefined, + { + equals: (a, b) => a?.index === b?.index && a?.segment === b?.segment, + }, + ); const text = createMemo(() => activeCaption()?.segment.text ?? ""); diff --git a/apps/desktop/src/routes/editor/CaptionsTab.tsx b/apps/desktop/src/routes/editor/CaptionsTab.tsx index ea3d58c0e02..0ba28f0be30 100644 --- a/apps/desktop/src/routes/editor/CaptionsTab.tsx +++ b/apps/desktop/src/routes/editor/CaptionsTab.tsx @@ -39,11 +39,10 @@ import { DEFAULT_WHISPER_CAPTION_MODEL, getCaptionGenerationErrorMessage, getModelPath, - mapEditedTimeToSource, PARAKEET_DIR_MODELS, resolveCaptionModel, - sourceCaptionId, supportsParakeetTranscription, + syncCaptionSourceFromTrack, syncCaptionWordsWithText, transcribeEditorCaptions, } from "./captions"; @@ -89,6 +88,10 @@ interface LanguageOption { } const MODEL_DOWNLOAD_STATUS_POLL_MS = 1000; +const isWebEditor = import.meta.env.VITE_CAP_WEB_EDITOR === "true"; +const webCaptionsEnabled = () => + (window as Window & { capWebEditorCaptionsEnabled?: boolean }) + .capWebEditorCaptionsEnabled === true; const MODEL_OPTIONS: ModelOption[] = [ { @@ -154,6 +157,10 @@ const LANGUAGE_OPTIONS: LanguageOption[] = [ { code: "ta", label: "Tamil" }, ]; +const WEB_LANGUAGE_OPTIONS = LANGUAGE_OPTIONS.filter( + (option) => option.code !== "pa", +); + const STYLE_PRESET_KEYS = new Set([ "font", "fontWeight", @@ -250,6 +257,11 @@ export function CaptionsTab(props: { }) { const { project, setProject, editorInstance, editorState, setEditorState } = useEditorContext(); + const [webCaptionsProEnabled, setWebCaptionsProEnabled] = createSignal( + isWebEditor && webCaptionsEnabled(), + ); + const syncWebCaptionPlan = () => + setWebCaptionsProEnabled(webCaptionsEnabled()); const selectedCaptionIndex = () => editorState.timeline.selection?.type === "caption" && @@ -281,40 +293,10 @@ export function CaptionsTab(props: { // Route content/timing onto the source-time caption master so the // edit persists across future clip changes. Style overrides stay on // the track and are carried across by source id when re-derived. - const sourceId = sourceCaptionId(timelineSegment.id); - const source = currentProject.captions?.segments?.find( - (segment) => segment.id === sourceId, - ); - if (!source) return; - - const recordingSegments = editorInstance.recordings.segments; - const sourceRange = { start: source.start, end: source.end }; - const start = mapEditedTimeToSource( - timelineSegment.start, - timeline.segments, - recordingSegments, - timeline.transitions ?? [], - sourceRange, - "outgoing", - timeline.textSegments, - ); - const end = mapEditedTimeToSource( - timelineSegment.end, - timeline.segments, - recordingSegments, - timeline.transitions ?? [], - sourceRange, - "outgoing", - timeline.textSegments, - ); - if (start !== null) source.start = start; - if (end !== null) source.end = end; - source.text = timelineSegment.text; - source.words = syncCaptionWordsWithText( - source.text, - source.words, - source.start, - source.end, + syncCaptionSourceFromTrack( + currentProject, + index, + editorInstance.recordings.segments, ); }), ); @@ -419,6 +401,9 @@ export function CaptionsTab(props: { const setIsGenerating = (value: boolean) => setEditorState("captions", "isGenerating", value); const [hasAudio, setHasAudio] = createSignal(false); + const [generationError, setGenerationError] = createSignal( + null, + ); const availableModelOptions = createMemo(() => supportsParakeetTranscription() ? MODEL_OPTIONS @@ -561,6 +546,30 @@ export function CaptionsTab(props: { }; onMount(async () => { + if (isWebEditor) { + window.addEventListener( + "cap-web-editor-captions-plan", + syncWebCaptionPlan, + ); + syncWebCaptionPlan(); + const savedLanguage = localStorage.getItem( + "selectedTranscriptionLanguage", + ); + if ( + savedLanguage && + WEB_LANGUAGE_OPTIONS.some((option) => option.code === savedLanguage) + ) { + setSelectedLanguage(savedLanguage); + } + if (editorInstance?.recordings) { + setHasAudio( + editorInstance.recordings.segments.some( + (segment) => segment.mic !== null || segment.system_audio !== null, + ), + ); + } + return; + } try { unlistenDownloadProgress = await events.downloadProgress.listen( (event) => { @@ -621,6 +630,12 @@ export function CaptionsTab(props: { }); onCleanup(() => { + if (isWebEditor) { + window.removeEventListener( + "cap-web-editor-captions-plan", + syncWebCaptionPlan, + ); + } if (unlistenDownloadProgress) unlistenDownloadProgress(); stopDownloadStatusPolling(); }); @@ -725,6 +740,7 @@ export function CaptionsTab(props: { } setIsGenerating(true); + setGenerationError(null); try { const result = await transcribeEditorCaptions( @@ -749,13 +765,15 @@ export function CaptionsTab(props: { toast.success("Captions generated successfully!"); } else { - toast.error( - "No captions were generated. The audio might be too quiet or unclear.", - ); + const message = + "No captions were generated. The audio might be too quiet or unclear."; + setGenerationError(message); + toast.error(message); } } catch (error) { console.error("Error generating captions:", error); const errorMessage = getCaptionGenerationErrorMessage(error); + setGenerationError(errorMessage); toast.error(`Failed to generate captions: ${errorMessage}`); } finally { setIsGenerating(false); @@ -777,264 +795,377 @@ export function CaptionsTab(props: { -

- - - options={availableModelOptions().map((model) => model.name)} - value={selectedModel()} - onChange={(value: string | null) => { - if (value) setSelectedModel(value); - }} - itemComponent={(props) => { - const model = availableModelOptions().find( - (option) => option.name === props.item.rawValue, - ); - - return ( - - as={KSelect.Item} - item={props.item} - > -
-
-
- - {model?.label ?? props.item.rawValue} - + +
+

+ Cap Pro captions use the same AssemblyAI transcription as your + shareable link. +

+ + + Upgrade to Cap Pro + + + + + + options={WEB_LANGUAGE_OPTIONS.map((option) => option.code)} + value={selectedLanguage()} + onChange={(value: string | null) => { + if (value) setSelectedLanguage(value); + }} + itemComponent={(props) => ( + + as={KSelect.Item} + item={props.item} + > + + { + WEB_LANGUAGE_OPTIONS.find( + (option) => option.code === props.item.rawValue, + )?.label + } + + + )} + > + + class="truncate"> + {(state) => { + const language = WEB_LANGUAGE_OPTIONS.find( + (option) => option.code === state.selectedOption(), + ); + return ( + {language?.label || "Select a language"} + ); + }} + + + + + + + + as={KSelect.Content} + class={topLeftAnimateClasses} + > + + class="overflow-y-auto max-h-48" + as={KSelect.Listbox} + /> + + + + + + + {(message) => ( + + )} + + +
+ + +
+ + + options={availableModelOptions().map((model) => model.name)} + value={selectedModel()} + onChange={(value: string | null) => { + if (value) setSelectedModel(value); + }} + itemComponent={(props) => { + const model = availableModelOptions().find( + (option) => option.name === props.item.rawValue, + ); + + return ( + + as={KSelect.Item} + item={props.item} + > +
+
+
+ + {model?.label ?? props.item.rawValue} + + + + + + +
- - - +
+ {model?.description} +
-
- {model?.description} -
+ + {model?.size} +
- - - {model?.size} - + + ); + }} + > + +
+
+ + {selectedModelOption()?.label || "Select a model"} + + + + +
- - ); - }} - > - -
-
- - {selectedModelOption()?.label || "Select a model"} - - - - +
+ {selectedModelOption()?.description} +
-
- {selectedModelOption()?.description} -
+ + {selectedModelOption()?.size} + -
- - - {selectedModelOption()?.size} - - - - - - - - - as={KSelect.Content} - class={topLeftAnimateClasses} - > - as={KSelect.Listbox} /> - - - - + + + + + + + as={KSelect.Content} + class={topLeftAnimateClasses} + > + as={KSelect.Listbox} /> + + + + + + +

+ Parakeet caption models are unavailable on Intel Macs. Whisper + models remain available. +

+ -

- Parakeet caption models are unavailable on Intel Macs. Whisper - models remain available. + One time download to your system. All captions are stored locally.

- - -

- One time download to your system. All captions are stored locally. -

- - - - options={LANGUAGE_OPTIONS.map((l) => l.code)} - value={selectedLanguage()} - onChange={(value: string | null) => { - if (value) setSelectedLanguage(value); - }} - itemComponent={(props) => ( - - as={KSelect.Item} - item={props.item} - > - - { - LANGUAGE_OPTIONS.find((l) => l.code === props.item.rawValue) - ?.label - } - - - )} - > - - class="truncate"> - {(state) => { - const language = LANGUAGE_OPTIONS.find( - (l) => l.code === state.selectedOption(), - ); - return {language?.label || "Select a language"}; - }} - - - - - - - - as={KSelect.Content} - class={topLeftAnimateClasses} - > - - class="overflow-y-auto max-h-48" - as={KSelect.Listbox} - /> - - - - - - - - -
-
+ { + LANGUAGE_OPTIONS.find( + (l) => l.code === props.item.rawValue, + )?.label + } + + + )} + > + + class="truncate"> + {(state) => { + const language = LANGUAGE_OPTIONS.find( + (l) => l.code === state.selectedOption(), + ); + return ( + {language?.label || "Select a language"} + ); + }} + + + + + + + + as={KSelect.Content} + class={topLeftAnimateClasses} + > + + class="overflow-y-auto max-h-48" + as={KSelect.Listbox} + /> + + + + + + + + +
+ class="overflow-hidden w-full h-1.5 rounded-full bg-ed-ctl" + role="progressbar" + aria-valuemin="0" + aria-valuemax="100" + aria-valuenow={downloadPercent()} + > +
+
+

+ {downloadMessage() || + "Keep Cap open while the model downloads. Editor reloads will reconnect automatically."} +

-

- {downloadMessage() || - "Keep Cap open while the model downloads. Editor reloads will reconnect automatically."} + +

+ } + > +
+ + + + + {(message) => ( + -
+ )} -
- } - > -
- - - -
- - - - {selectedModelOption()?.label ?? "Caption"} model downloaded +
+ + + + {selectedModelOption()?.label ?? "Caption"} model downloaded + - - } - onClick={deleteModel} - disabled={ - isGenerating() || - isDownloading() || - deletingModel() === selectedModel() - } - > - {deletingModel() === selectedModel() ? "Deleting..." : "Delete"} - + } + onClick={deleteModel} + disabled={ + isGenerating() || + isDownloading() || + deletingModel() === selectedModel() + } + > + {deletingModel() === selectedModel() + ? "Deleting..." + : "Delete"} + +
-
- -
+ +
+ -
+
@@ -1466,6 +1597,7 @@ export function CaptionsTab(props: { { if (!Number.isFinite(seconds) || seconds <= 0) return "0:00"; @@ -41,177 +31,6 @@ const formatClipDuration = (seconds: number) => { return `${mins}:${secs.toString().padStart(2, "0")}`; }; -const thumbnailCache = new Map(); -const thumbnailInflight = new Map>(); -const thumbnailQueue: (() => void)[] = []; -let activeThumbnailLoads = 0; -let thumbnailPumpScheduled = false; - -const MAX_THUMBNAIL_LOADS = 2; - -const clipThumbnailKey = ( - projectPath: string, - recordingSegment: number, - start: number, -) => `${projectPath}::${recordingSegment}-${Math.round(start * 1000)}`; - -const scheduleThumbnailWork = (callback: () => void) => { - const requestIdle = ( - window as Window & { - requestIdleCallback?: ( - callback: () => void, - options?: { timeout: number }, - ) => number; - } - ).requestIdleCallback; - - if (requestIdle) { - requestIdle(callback, { timeout: 150 }); - } else { - requestAnimationFrame(callback); - } -}; - -const pumpThumbnailQueue = () => { - if (thumbnailPumpScheduled) return; - thumbnailPumpScheduled = true; - - scheduleThumbnailWork(() => { - thumbnailPumpScheduled = false; - - while ( - activeThumbnailLoads < MAX_THUMBNAIL_LOADS && - thumbnailQueue.length > 0 - ) { - const run = thumbnailQueue.shift(); - if (!run) return; - activeThumbnailLoads += 1; - run(); - } - }); -}; - -const loadClipThumbnail = ( - recordingSegment: number, - start: number, - key: string, -) => { - let promise = thumbnailInflight.get(key); - if (promise) return promise; - - promise = new Promise((resolve) => { - thumbnailQueue.push(() => { - commands - .getClipThumbnail(recordingSegment, start) - .then((path) => { - const url = convertFileSrc(path); - thumbnailCache.set(key, url); - resolve(url); - }) - .catch((error) => { - console.error("Failed to load clip thumbnail", error); - resolve(null); - }) - .finally(() => { - activeThumbnailLoads -= 1; - thumbnailInflight.delete(key); - pumpThumbnailQueue(); - }); - }); - pumpThumbnailQueue(); - }); - thumbnailInflight.set(key, promise); - - return promise; -}; - -function ClipThumbnail(props: { - projectPath: string; - recordingSegment: number; - start: number; - index: number; -}) { - const [src, setSrc] = createSignal(null); - const [loaded, setLoaded] = createSignal(false); - const [visible, setVisible] = createSignal(false); - let container: HTMLDivElement | undefined; - let disposed = false; - - const cacheKey = createMemo(() => - clipThumbnailKey(props.projectPath, props.recordingSegment, props.start), - ); - - const applySrc = (key: string, url: string | null) => { - if (disposed || key !== cacheKey() || !url) return; - setSrc(url); - }; - - const load = (key: string, recordingSegment: number, start: number) => { - const cached = thumbnailCache.get(key); - if (cached) { - applySrc(key, cached); - return; - } - - const promise = loadClipThumbnail(recordingSegment, start, key); - void promise.then((url) => applySrc(key, url)); - }; - - onMount(() => { - if (!container) return; - const observer = new IntersectionObserver( - (entries) => { - if (entries.some((entry) => entry.isIntersecting)) setVisible(true); - }, - { rootMargin: "300px" }, - ); - observer.observe(container); - onCleanup(() => observer.disconnect()); - }); - - createEffect(() => { - const key = cacheKey(); - const cached = thumbnailCache.get(key); - if (cached) { - setSrc(cached); - return; - } - setSrc(null); - setLoaded(false); - if (visible()) load(key, props.recordingSegment, props.start); - }); - - onCleanup(() => { - disposed = true; - }); - - return ( -
-
- - {props.index + 1} - -
- - {(url) => ( - setLoaded(true)} - class={cx( - "absolute inset-0 object-cover pointer-events-none size-full transition-opacity duration-200", - loaded() ? "opacity-100" : "opacity-0", - )} - /> - )} - -
- ); -} - export function ClipsSidebar(props: { class?: string }) { const { project, @@ -395,75 +214,8 @@ export function ClipsSidebar(props: { class?: string }) { let listRef: HTMLDivElement | undefined; const moveClip = (from: number, insertionIndex: number) => { - let to = insertionIndex; - if (from < insertionIndex) to -= 1; - if (from === to) return; setProject( - produce((project) => { - const timeline = project.timeline; - if (!timeline) return; - const proposedSegments = [...timeline.segments]; - const [proposedMoved] = proposedSegments.splice(from, 1); - proposedSegments.splice(to, 0, proposedMoved); - const { kept, dropped } = transitionsAfterClipMove( - timeline.segments.length, - timeline.transitions ?? [], - from, - to, - ); - dropped.sort((a, b) => b.segmentIndex - a.segmentIndex); - - for (const transition of dropped) { - const effective = getClipTransition( - timeline.segments, - timeline.transitions, - transition.segmentIndex, - ); - if (!effective) continue; - const boundary = effectiveToOutput( - holdWindows(timeline.textSegments), - clipTimelineOffsets(timeline.segments, timeline.transitions)[ - transition.segmentIndex - ] + effective.duration, - ); - timeline.transitions = timeline.transitions.filter( - (candidate) => candidate.segmentIndex !== transition.segmentIndex, - ); - const camera3dSegments = timeline.camera3dSegments ?? []; - const previousCamera3dDurations = camera3dSegments.map( - (segment) => segment.end - segment.start, - ); - for (const track of [ - timeline.styleSegments, - timeline.imageSegments, - timeline.zoomSegments, - timeline.sceneSegments ?? [], - timeline.maskSegments, - timeline.textSegments, - timeline.captionSegments ?? [], - timeline.audioSegments ?? [], - camera3dSegments, - ]) { - rippleTimelineTrack(track, boundary, effective.duration); - } - rippleKeyboardTrack( - timeline.keyboardSegments ?? [], - boundary, - effective.duration, - ); - for (let index = 0; index < camera3dSegments.length; index++) { - const segment = camera3dSegments[index]; - const previousDuration = previousCamera3dDurations[index]; - const nextDuration = segment.end - segment.start; - if (previousDuration <= 0 || previousDuration === nextDuration) - continue; - scaleKeyframeTimes(segment.tracks, nextDuration / previousDuration); - } - } - - timeline.segments = proposedSegments; - timeline.transitions = kept; - }), + produce((project) => moveTimelineClip(project, from, insertionIndex)), ); setEditorState("timeline", "selection", null); }; diff --git a/apps/desktop/src/routes/editor/ConfigSidebar.tsx b/apps/desktop/src/routes/editor/ConfigSidebar.tsx index 8d1e8615992..234fa5e6927 100644 --- a/apps/desktop/src/routes/editor/ConfigSidebar.tsx +++ b/apps/desktop/src/routes/editor/ConfigSidebar.tsx @@ -12,8 +12,9 @@ import { Tabs as KTabs } from "@kobalte/core/tabs"; import { createElementBounds } from "@solid-primitives/bounds"; import { createEventListenerMap } from "@solid-primitives/event-listener"; import { createQuery } from "@tanstack/solid-query"; -import { convertFileSrc } from "@tauri-apps/api/core"; +import { convertFileSrc, invoke } from "@tauri-apps/api/core"; import { appDataDir, resolveResource } from "@tauri-apps/api/path"; +import { open } from "@tauri-apps/plugin-dialog"; import { BaseDirectory, exists, @@ -87,6 +88,7 @@ import { AnimatedGradientEditor, copyAnimatedGradientConfig, } from "./AnimatedGradientEditor"; + import { AudioLibraryPanel } from "./AudioLibrary"; import { AUDIO_TRACK_BG_CLASS, @@ -101,7 +103,10 @@ import { CursorStylePicker, isExplicitCursorFamily, } from "./CursorStylePicker"; -import { syncCaptionWordsWithText } from "./captions"; +import { + syncCaptionSourceFromTrack, + syncCaptionWordsWithText, +} from "./captions"; import { type ClipTransition, clipSourceTimeAt } from "./clip-transitions"; import { hexToRgb, RgbInput } from "./color-utils"; import { @@ -151,8 +156,13 @@ import { topSlideAnimateClasses, } from "./ui"; import { formatTime } from "./utils"; +import type { WaveformSegment } from "./waveform"; +import { WaveformSegmentConfig } from "./waveform-segment-config"; import { ZoomModeHelper } from "./ZoomModeHelper"; +const isWebEditor = import.meta.env.VITE_CAP_WEB_EDITOR === "true"; +const ZOOM_PREVIEW_MAX_RETRIES = 4; + // Split out of the sidebar chunk: the captions tab is not visible at first // paint (Kobalte only mounts the selected tab), and its code is heavy. The // render site wraps it in a local so the chunk load stays inside @@ -302,6 +312,13 @@ type WallpaperOption = { const isCurrentDesktopBackgroundPath = (path: string | null | undefined) => { if (!path) return false; + if ( + isWebEditor && + /^content\/images\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.(png|jpg|webp|gif|bmp|tiff)$/.test( + path, + ) + ) + return true; const filename = path.split(/[\\/]/).pop(); if (!filename) return false; return ( @@ -703,10 +720,9 @@ function ConfigSidebarContent() { icon: IconCapCursor, disabled: !meta().hasRecordedCursorData, }, - { - id: TAB_IDS.keyboard, - icon: IconLucideKeyboard, - }, + ...(isWebEditor + ? [] + : [{ id: TAB_IDS.keyboard, icon: IconLucideKeyboard }]), { id: TAB_IDS.captions, icon: IconCapMessageBubble, @@ -714,10 +730,11 @@ function ConfigSidebarContent() { // { id: "hotkeys" as const, icon: IconCapHotkeys }, ].filter( (item) => - !selectedStyle() || - item.id === "background" || - item.id === "camera" || - item.id === "cursor", + !(meta().audioOnly && item.id === "cursor") && + (!selectedStyle() || + item.id === "background" || + item.id === "camera" || + item.id === "cursor"), )} > {(item) => ( @@ -1553,6 +1570,75 @@ function ConfigSidebarContent() {
)} + { + const waveformSelection = selection(); + if (waveformSelection.type !== "waveform") return; + + const segments = waveformSelection.indices + .map((index) => ({ + index, + segment: project.timeline?.waveformSegments?.[index], + })) + .filter( + ( + item, + ): item is { index: number; segment: WaveformSegment } => + item.segment !== undefined, + ); + + if (segments.length === 0) { + setEditorState("timeline", "selection", null); + return; + } + return { selection: waveformSelection, segments }; + })()} + > + {(value) => ( +
+
+
+ + setEditorState("timeline", "selection", null) + } + leftIcon={} + > + Done + + + {value().segments.length === 1 + ? "Audio waveform" + : `${value().segments.length} waveforms selected`} + +
+ + projectActions.deleteOverlaySegments( + "waveform", + value().segments.map((s) => s.index), + ) + } + leftIcon={} + > + Delete + +
+ + {(item) => ( +
+ +
+ )} + +
+ )} + { const zoomSelection = selection(); @@ -1837,8 +1923,15 @@ function BackgroundConfig(props: { scrollRef: HTMLDivElement; brandColorSwatches: OrganizationBrandColorSwatch[]; }) { - const { project, setProject, editorInstance, projectHistory, selectedStyle } = - useEditorContext(); + const { + project, + setProject, + editorInstance, + projectHistory, + selectedStyle, + meta, + } = useEditorContext(); + const screenHidden = () => meta().audioOnly || !!project.hideDisplay; const notchXMax = () => { const width = project.background.notch?.width ?? editorInstance.notchBase.width; @@ -1998,6 +2091,8 @@ function BackgroundConfig(props: { const assetsDir = `${editorInstance.path}/assets`; try { + if (isWebEditor) + return await invoke("webEditorStoredDesktopBackground"); const importedPrefix = `${CURRENT_DESKTOP_BACKGROUND_BASENAME}-`; let newest: { path: string; timestamp: number } | null = null; for (const entry of await readDir(assetsDir)) { @@ -2180,9 +2275,20 @@ function BackgroundConfig(props: { if (importingDesktopBackground()) return; setImportingDesktopBackground(true); try { - const path = await commands.importCurrentDesktopBackground( - editorInstance.path, - ); + let source = editorInstance.path; + if (isWebEditor) { + const selected = await open({ + filters: [ + { + name: "Images", + extensions: [...BACKGROUND_IMAGE_EXTENSIONS, "jpeg"], + }, + ], + }); + if (typeof selected !== "string") return; + source = selected; + } + const path = await commands.importCurrentDesktopBackground(source); const addingFromBlankBackground = isNoneBackground(); batch(() => { setCurrentDesktopBackgroundPath(path); @@ -2435,7 +2541,9 @@ function BackgroundConfig(props: {
- Use the wallpaper from your desktop + {isWebEditor + ? "Choose your desktop wallpaper" + : "Use the wallpaper from your desktop"} {importingDesktopBackground() ? "Importing..." - : "Import desktop background"} + : isWebEditor + ? "Choose desktop wallpaper" + : "Import desktop background"}
} @@ -2748,7 +2858,7 @@ function BackgroundConfig(props: { }} />
+ + + setProject("hideDisplay", !show)} + /> + + - - setBackgroundDimension("padding", v[0])} - minValue={0} - maxValue={40} - step={0.1} - formatTooltip="%" - /> - - -
- - Custom screen position (dragged on canvas) - - setProject("background", "displayPosition", null)} - > - Reset - -
- - - setBackgroundDimension("rounding", v[0])} - minValue={0} - maxValue={100} - step={0.1} - formatTooltip="%" - /> - - - - setProject("background", "roundingType", value) - } - /> - - + { - const value = v[0] ?? 0; - batch(() => { - setProject("cursor", "motionBlur", value); - setProject("screenMotionBlur", value); - }); - }} + value={[project.background.padding]} + onChange={(v) => setBackgroundDimension("padding", v[0])} minValue={0} - maxValue={1} - step={0.01} - formatTooltip={(value) => `${Math.round(value * 100)}%`} + maxValue={40} + step={0.1} + formatTooltip="%" /> - - - { - const prev = project.background.border ?? { - enabled: false, - width: 5.0, - color: [0, 0, 0], - opacity: 50.0, - }; - - if (props.scrollRef && enabled) { - setTimeout( - () => - props.scrollRef.scrollTo({ - top: props.scrollRef.scrollHeight, - behavior: "smooth", - }), - 100, - ); + +
+ + Custom screen position (dragged on canvas) + + + setProject("background", "displayPosition", null) + } + > + Reset + +
+ + + setBackgroundDimension("rounding", v[0])} + minValue={0} + maxValue={100} + step={0.1} + formatTooltip="%" + /> + + + + setProject("background", "roundingType", value) } + /> + + + + { + const value = v[0] ?? 0; + batch(() => { + setProject("cursor", "motionBlur", value); + setProject("screenMotionBlur", value); + }); + }} + minValue={0} + maxValue={1} + step={0.01} + formatTooltip={(value) => `${Math.round(value * 100)}%`} + /> + + + + { + const prev = project.background.border ?? { + enabled: false, + width: 5.0, + color: [0, 0, 0], + opacity: 50.0, + }; - setProject("background", "border", { - ...prev, - enabled, - }); - }} - /> - - - -
- - - setProject("background", "border", { - ...(project.background.border ?? { - enabled: true, - width: 5.0, - color: [0, 0, 0], - opacity: 50.0, + if (props.scrollRef && enabled) { + setTimeout( + () => + props.scrollRef.scrollTo({ + top: props.scrollRef.scrollHeight, + behavior: "smooth", }), - width: v[0], - }) - } - minValue={1} - maxValue={20} - step={0.1} - formatTooltip="px" - /> - - -
- + 100, + ); + } + + setProject("background", "border", { + ...prev, + enabled, + }); + }} + /> + + + +
+ + setProject("background", "border", { ...(project.background.border ?? { enabled: true, @@ -2952,204 +3052,229 @@ function BackgroundConfig(props: { color: [0, 0, 0], opacity: 50.0, }), - color, + width: v[0], }) } + minValue={1} + maxValue={20} + step={0.1} + formatTooltip="px" /> - + +
+ + setProject("background", "border", { + ...(project.background.border ?? { + enabled: true, + width: 5.0, + color: [0, 0, 0], + opacity: 50.0, + }), + color, + }) + } + /> + +
+ + + + setProject("background", "border", { + ...(project.background.border ?? { + enabled: true, + width: 5.0, + color: [0, 0, 0], + opacity: 50.0, + }), + opacity: v[0], + }) + } + minValue={0} + maxValue={100} + step={0.1} + formatTooltip="%" /> -
- - - - setProject("background", "border", { - ...(project.background.border ?? { - enabled: true, - width: 5.0, - color: [0, 0, 0], - opacity: 50.0, - }), - opacity: v[0], - }) + +
+ + + + + setProject("background", "notch", { + ...(project.background.notch ?? UNPLACED_NOTCH), + enabled, + }) + } + /> + + + +
+

+ Draws a MacBook notch over the recording. Recordings made on a + Mac with a notch use their own measurements; otherwise start + from the size below and adjust to match. +

+ - -
- - - - - setProject("background", "notch", { - ...(project.background.notch ?? UNPLACED_NOTCH), - enabled, - }) - } - /> - - - -
-

- Draws a MacBook notch over the recording. Recordings made on a - Mac with a notch use their own measurements; otherwise start - from the size below and adjust to match. -

- - {(field) => { - const notchValue = () => - field.key === "x" - ? Math.min( - project.background.notch?.x ?? - editorInstance.notchBase.x, - notchXMax(), - ) - : (project.background.notch?.[field.key] ?? - editorInstance.notchBase[field.key]); + > + {(field) => { + const notchValue = () => + field.key === "x" + ? Math.min( + project.background.notch?.x ?? + editorInstance.notchBase.x, + notchXMax(), + ) + : (project.background.notch?.[field.key] ?? + editorInstance.notchBase[field.key]); - return ( - - { - const base = editorInstance.notchBase; - const prev = - project.background.notch ?? UNPLACED_NOTCH; - const next: NotchConfiguration = { - ...prev, - enabled: true, - }; - if (field.key === "x") { - next.x = Math.min(v[0], notchXMax()); - } else { - next[field.key] = v[0]; - } + return ( + + { + const base = editorInstance.notchBase; + const prev = + project.background.notch ?? UNPLACED_NOTCH; + const next: NotchConfiguration = { + ...prev, + enabled: true, + }; + if (field.key === "x") { + next.x = Math.min(v[0], notchXMax()); + } else { + next[field.key] = v[0]; + } - if (field.key === "width") { - // Resize about the centre rather than dragging the - // left edge along with the width. - const centre = - (prev.x ?? base.x) + - (prev.width ?? base.width) / 2; - next.x = Math.min( - Math.max(centre - v[0] / 2, 0), - 1 - v[0], - ); - } + if (field.key === "width") { + // Resize about the centre rather than dragging the + // left edge along with the width. + const centre = + (prev.x ?? base.x) + + (prev.width ?? base.width) / 2; + next.x = Math.min( + Math.max(centre - v[0] / 2, 0), + 1 - v[0], + ); + } - setProject("background", "notch", next); - }} - minValue={0} - maxValue={field.key === "x" ? notchXMax() : field.max} - step={0.001} - formatTooltip={(value) => - `${(value * 100).toFixed(1)}%` - } - /> - - ); - }} - -
- - - - { - batch(() => { - setProject("background", "shadow", v[0]); - // Initialize advanced shadow settings if they don't exist and shadow is enabled - if (v[0] > 0 && !project.background.advancedShadow) { - setProject("background", "advancedShadow", { + setProject("background", "notch", next); + }} + minValue={0} + maxValue={field.key === "x" ? notchXMax() : field.max} + step={0.001} + formatTooltip={(value) => + `${(value * 100).toFixed(1)}%` + } + /> + + ); + }} + +
+ + + + { + batch(() => { + setProject("background", "shadow", v[0]); + // Initialize advanced shadow settings if they don't exist and shadow is enabled + if (v[0] > 0 && !project.background.advancedShadow) { + setProject("background", "advancedShadow", { + size: 50, + opacity: 18, + blur: 50, + }); + } + }); + }} + minValue={0} + maxValue={100} + step={0.1} + formatTooltip="%" + /> + + { + setProject("background", "advancedShadow", { + ...(project.background.advancedShadow ?? { size: 50, opacity: 18, blur: 50, - }); - } - }); + }), + size: v[0], + }); + }, + }} + opacity={{ + value: [project.background.advancedShadow?.opacity ?? 18], + onChange: (v) => { + setProject("background", "advancedShadow", { + ...(project.background.advancedShadow ?? { + size: 50, + opacity: 18, + blur: 50, + }), + opacity: v[0], + }); + }, + }} + blur={{ + value: [project.background.advancedShadow?.blur ?? 50], + onChange: (v) => { + setProject("background", "advancedShadow", { + ...(project.background.advancedShadow ?? { + size: 50, + opacity: 18, + blur: 50, + }), + blur: v[0], + }); + }, }} - minValue={0} - maxValue={100} - step={0.1} - formatTooltip="%" /> - - { - setProject("background", "advancedShadow", { - ...(project.background.advancedShadow ?? { - size: 50, - opacity: 18, - blur: 50, - }), - size: v[0], - }); - }, - }} - opacity={{ - value: [project.background.advancedShadow?.opacity ?? 18], - onChange: (v) => { - setProject("background", "advancedShadow", { - ...(project.background.advancedShadow ?? { - size: 50, - opacity: 18, - blur: 50, - }), - opacity: v[0], - }); - }, - }} - blur={{ - value: [project.background.advancedShadow?.blur ?? 50], - onChange: (v) => { - setProject("background", "advancedShadow", { - ...(project.background.advancedShadow ?? { - size: 50, - opacity: 18, - blur: 50, - }), - blur: v[0], - }); - }, - }} - /> - - + + + {/* }> @@ -3273,11 +3398,15 @@ function CameraConfig(props: { scrollRef: HTMLDivElement }) { - options={cameraBackgroundOptions(ostype() === "macos")} + options={cameraBackgroundOptions( + isWebEditor || ostype() === "macos", + )} optionValue="value" optionTextValue="name" value={ - cameraBackgroundOptions(ostype() === "macos").find( + cameraBackgroundOptions( + isWebEditor || ostype() === "macos", + ).find( (option) => option.value === (project.camera.backgroundBlur?.mode ?? "off"), @@ -3834,7 +3963,7 @@ function CaptionSegmentConfig(props: { segmentIndex: number; segment: CaptionTrackSegment; }) { - const { setProject } = useEditorContext(); + const { setProject, editorInstance } = useEditorContext(); const updateSegment = (fn: (segment: CaptionTrackSegment) => void) => { setProject( @@ -3844,16 +3973,11 @@ function CaptionSegmentConfig(props: { if (!timelineSegment) return; fn(timelineSegment); - - const captionSegment = project.captions?.segments?.[props.segmentIndex]; - if (!captionSegment) return; - - captionSegment.start = timelineSegment.start; - captionSegment.end = timelineSegment.end; - captionSegment.text = timelineSegment.text; - captionSegment.words = timelineSegment.words?.map((word) => ({ - ...word, - })); + syncCaptionSourceFromTrack( + project, + props.segmentIndex, + editorInstance.recordings.segments, + ); }), ); }; @@ -4269,11 +4393,23 @@ function ZoomSegmentPreview(props: { ); } +function createAutoZoomBlocker() { + const generalSettings = generalSettingsStore.createQuery(); + const { meta } = useEditorContext(); + return () => { + if (!meta().hasRecordedCursorData) + return "Auto mode follows the cursor, and this recording has no cursor data."; + if (!generalSettings.data?.custom_cursor_capture2) + return 'Auto mode needs cursor capture. Enable "Custom cursor capture (Studio)" in Settings → General.'; + return null; + }; +} + function ZoomSegmentConfig(props: { segmentIndex: number; segment: ZoomSegment; }) { - const generalSettings = generalSettingsStore.createQuery(); + const autoZoomBlocker = createAutoZoomBlocker(); const { project, setProject, editorInstance, projectHistory } = useEditorContext(); @@ -4326,7 +4462,7 @@ function ZoomSegmentConfig(props: { Auto @@ -4341,11 +4477,8 @@ function ZoomSegmentConfig(props: {
- -

- Auto mode needs cursor capture. Enable "Custom cursor capture - (Studio)" in Settings → General. -

+ + {(reason) =>

{reason()}

} { - const path = convertFileSrc( - `${editorInstance.path}/content/segments/segment-${ - source().recordingSegment - }/display.mp4`, - ); - video.src = path; - video.preload = "auto"; - // Force reload if video fails to load - video.load(); - }); + // A
- {ostype() === "windows" && ( + {!isWebEditor && ostype() === "windows" && ( )}
@@ -363,7 +398,7 @@ function NameEditor(props: { /> {prettyName()} diff --git a/apps/desktop/src/routes/editor/Player.tsx b/apps/desktop/src/routes/editor/Player.tsx index 3ef4105cf0f..d3502106023 100644 --- a/apps/desktop/src/routes/editor/Player.tsx +++ b/apps/desktop/src/routes/editor/Player.tsx @@ -8,6 +8,7 @@ import { type as ostype } from "@tauri-apps/plugin-os"; import { cx } from "cva"; import { createEffect, + createMemo, createSignal, For, on, @@ -27,8 +28,14 @@ import { import { CaptionOverlay } from "./CaptionOverlay"; import { CaptionsRegenerateBadge } from "./CaptionsRegenerateBadge"; import { createCaptionTrackSegments } from "./captions"; -import { type EditorPreviewQuality, FPS, useEditorContext } from "./context"; +import { + type EditorPreviewQuality, + FPS, + MAX_ZOOM_IN, + useEditorContext, +} from "./context"; import { FrameButton } from "./FrameButton"; +import type { FocusMode } from "./focus-mode"; import { ImageOverlay } from "./image-overlay"; import { MaskOverlay } from "./MaskOverlay"; import { PerformanceOverlay } from "./PerformanceOverlay"; @@ -40,11 +47,18 @@ import { } from "./preview-bounds"; import { SplitScreenOverlay } from "./SplitScreenOverlay"; import { TextOverlay } from "./TextOverlay"; +import { sliderToZoom, ZOOM_STEP, zoomToSlider } from "./Timeline/zoom"; import { EditorButton, Slider } from "./ui"; import { useEditorShortcuts } from "./useEditorShortcuts"; import { formatTime } from "./utils"; +import { WaveformOverlay } from "./waveform-overlay"; + +const READOUT_INTERVAL_MS = 100; -export function PlayerContent(props: { compactness?: number }) { +export function PlayerContent(props: { + compactness?: number; + focusMode?: FocusMode; +}) { const { previewStyle, selectedStyle, @@ -65,6 +79,7 @@ export function PlayerContent(props: { compactness?: number }) { playbackIntent, requestHandoffPlayback, handoffPlaybackPending, + meta, } = useEditorContext(); let panelRef: HTMLDivElement | undefined; @@ -186,10 +201,25 @@ export function PlayerContent(props: { compactness?: number }) { } }); - const isAtEnd = () => { + const isAtEnd = createMemo(() => { const total = totalDuration(); return total > 0 && total - editorState.playbackTime <= 0.1; - }; + }); + + // While playing, the readout's frame digits change faster than they can be + // read, and each change costs the page a layout; it keeps up at 10 Hz. + let readoutAt = 0; + const readoutSeconds = createMemo((shown: number) => { + const seconds = Math.max( + editorState.previewTime ?? editorState.playbackTime, + 0, + ); + if (!editorState.playing) return seconds; + const now = performance.now(); + if (now - readoutAt < READOUT_INTERVAL_MS) return shown; + readoutAt = now; + return seconds; + }, 0); const cropDialogHandler = async () => { const background = selectedStyle() @@ -334,7 +364,7 @@ export function PlayerContent(props: { compactness?: number }) { combo: "Mod+=", handler: () => editorState.timeline.transform.updateZoom( - editorState.timeline.transform.zoom / 1.1, + editorState.timeline.transform.zoom / ZOOM_STEP, editorState.playbackTime, ), }, @@ -342,10 +372,17 @@ export function PlayerContent(props: { compactness?: number }) { combo: "Mod+-", handler: () => editorState.timeline.transform.updateZoom( - editorState.timeline.transform.zoom * 1.1, + editorState.timeline.transform.zoom * ZOOM_STEP, editorState.playbackTime, ), }, + { + combo: "Mod+Digit0", + handler: () => { + editorState.timeline.transform.updateZoom(zoomOutLimit(), 0); + editorState.timeline.transform.setPosition(0); + }, + }, { combo: "Space", handler: async () => { @@ -382,18 +419,24 @@ export function PlayerContent(props: { compactness?: number }) { - } - > - Crop - - + + } + > + Crop + + +
- Preview + + + Preview quality + +
{ if (event.button === 0) setPreviewPointerDown(true); }} @@ -434,19 +478,14 @@ export function PlayerContent(props: { compactness?: number }) { style={{ height: `${48 - 4 * (props.compactness ?? 0)}px` }} >
-
+ +
+ ); + }} +
diff --git a/apps/desktop/src/routes/editor/PresetsDropdown.tsx b/apps/desktop/src/routes/editor/PresetsDropdown.tsx index bd0f7d05394..21cb1e0f5c1 100644 --- a/apps/desktop/src/routes/editor/PresetsDropdown.tsx +++ b/apps/desktop/src/routes/editor/PresetsDropdown.tsx @@ -1,10 +1,15 @@ import { DropdownMenu as KDropdownMenu } from "@kobalte/core/dropdown-menu"; +import { invoke } from "@tauri-apps/api/core"; import { cx } from "cva"; import { For, Show, Suspense } from "solid-js"; import { reconcile } from "solid-js/store"; import toast from "solid-toast"; import { commands } from "~/utils/tauri"; -import { normalizeProject, useEditorContext } from "./context"; +import { + normalizeProject, + serializeProjectConfiguration, + useEditorContext, +} from "./context"; import { DropdownItem, dropdownContainerClasses, @@ -15,6 +20,8 @@ import { topCenterAnimateClasses, } from "./ui"; +const isWebEditor = import.meta.env.VITE_CAP_WEB_EDITOR === "true"; + export function PresetsDropdown() { const { setDialog, presets, setProject, project } = useEditorContext(); @@ -32,6 +39,21 @@ export function PresetsDropdown() { ); } + async function saveDefaultStyle() { + try { + await invoke("webEditorSaveDefaultStyle", { + config: JSON.parse( + JSON.stringify(serializeProjectConfiguration(project)), + ), + }); + toast.success("New recordings will use this style"); + } catch (error) { + toast.error( + error instanceof Error ? error.message : "Default style was not saved", + ); + } + } + return ( @@ -99,7 +121,28 @@ export function PresetsDropdown() { } > {(preset, i) => { - function applyPreset() { + async function applyPreset() { + if (import.meta.env.VITE_CAP_WEB_EDITOR === "true") { + try { + const preparePresetBackground = ( + window as Window & { + capWebEditorPreparePresetBackground?: ( + config: unknown, + ) => Promise; + } + ).capWebEditorPreparePresetBackground; + if (!preparePresetBackground) + throw new Error("Preset background is unavailable"); + await preparePresetBackground(preset.config); + } catch (error) { + toast.error( + error instanceof Error + ? error.message + : "Preset background could not be loaded", + ); + return; + } + } const normalizedConfig = normalizeProject({ ...preset.config, timeline: project.timeline ?? null, @@ -114,9 +157,7 @@ export function PresetsDropdown() { as={KDropdownMenu.SubTrigger} class="h-10" - onClick={() => { - applyPreset(); - }} + onClick={() => void applyPreset()} > {preset.name} @@ -134,11 +175,7 @@ export function PresetsDropdown() { dropdownContainerClasses, )} > - { - applyPreset(); - }} - > + void applyPreset()}> Apply Create new preset + + void saveDefaultStyle()}> + Use as my default style + + diff --git a/apps/desktop/src/routes/editor/ShareButton.tsx b/apps/desktop/src/routes/editor/ShareButton.tsx index 2e1424d721b..e2a612dda0e 100644 --- a/apps/desktop/src/routes/editor/ShareButton.tsx +++ b/apps/desktop/src/routes/editor/ShareButton.tsx @@ -1,4 +1,5 @@ import { Select as KSelect } from "@kobalte/core/select"; +import { cx } from "cva"; import { createSignal, Show } from "solid-js"; import Tooltip from "~/components/Tooltip"; import IconLucideExternalLink from "~icons/lucide/external-link"; @@ -89,12 +90,25 @@ function ShareButton() { rel="noreferrer" title={linkToDisplay() ?? "Open link"} aria-label="Open recording link" - class="w-full truncate max-w-[200px] max-[1400px]:w-4 max-[1400px]:shrink-0" + class={cx( + "w-full truncate max-w-[200px]", + "max-[1400px]:w-4 max-[1400px]:shrink-0", + )} > - + {linkToDisplay()} -
); } +/// Compact segments closer than this merge into one run. +const RUN_GAP_PX = 1; +/// Pressing a run zooms in this far around it, enough for its segments to +/// render and be edited. +const RUN_ZOOM_STEP = 8; +const RUN_HIT_SLOP_PX = 2; + +/// Draws the track's compact segments, merged into runs, on one canvas: a +/// zoomed out timeline shows where thousands of short segments are without an +/// element for each. Pressing a run zooms in on it. +function CompactSegmentRuns() { + const { editorState } = useEditorContext(); + const { secsPerPixel, trackBounds, compactSegments } = useTrackContext(); + const { renderPosition, followExtent } = useTimelineContext(); + let canvas: HTMLCanvasElement | undefined; + let runs: [number, number][] = []; + + const trackRoot = () => canvas?.closest("[data-track-root]"); + const runAt = (event: MouseEvent) => { + const track = trackRoot(); + if (!canvas || event.target !== track) return null; + const x = event.clientX - canvas.getBoundingClientRect().left; + return runs.some( + ([start, end]) => + x >= start - RUN_HIT_SLOP_PX && x <= end + RUN_HIT_SLOP_PX, + ) + ? x + : null; + }; + + onMount(() => { + const track = trackRoot(); + if (!track) return; + let hovering = false; + const onMouseDown = (event: MouseEvent) => { + if (event.button !== 0) return; + const x = runAt(event); + if (x === null) return; + event.stopPropagation(); + event.preventDefault(); + const { transform } = editorState.timeline; + transform.updateZoom( + transform.zoom / RUN_ZOOM_STEP, + renderPosition() + x * secsPerPixel(), + ); + }; + const onMouseMove = (event: MouseEvent) => { + const over = runAt(event) !== null; + if (over === hovering) return; + hovering = over; + track.style.cursor = over ? "zoom-in" : ""; + }; + track.addEventListener("mousedown", onMouseDown, { capture: true }); + track.addEventListener("mousemove", onMouseMove); + onCleanup(() => { + track.removeEventListener("mousedown", onMouseDown, { capture: true }); + track.removeEventListener("mousemove", onMouseMove); + }); + }); + + createEffect(() => { + const segments = compactSegments(); + const ctx = canvas?.getContext("2d"); + runs = []; + if (!canvas || !ctx) return; + const width = + (trackBounds.width ?? 0) + + (trackBounds.width ? followExtent() / secsPerPixel() : 0); + const height = trackBounds.height ?? 0; + if (segments.size === 0 || width <= 0 || height <= 0) { + ctx.clearRect(0, 0, canvas.width, canvas.height); + return; + } + const dpr = window.devicePixelRatio || 1; + const pixelWidth = Math.round(width * dpr); + const pixelHeight = Math.round(height * dpr); + if (canvas.width !== pixelWidth) canvas.width = pixelWidth; + if (canvas.height !== pixelHeight) canvas.height = pixelHeight; + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + ctx.clearRect(0, 0, width, height); + + const position = renderPosition(); + const perPixel = secsPerPixel(); + const byColor = new Map(); + for (const segment of segments) { + const x0 = (segment.start - position) / perPixel; + const x1 = (segment.end - position) / perPixel; + if (x1 < 0 || x0 > width) continue; + let spans = byColor.get(segment.color); + if (!spans) { + spans = []; + byColor.set(segment.color, spans); + } + spans.push([x0, x1]); + } + + const style = getComputedStyle(canvas); + const fallback = style.getPropertyValue("--ed-text-3").trim(); + ctx.lineWidth = 1; + for (const [color, spans] of byColor) { + const variable = color?.match(/var\((--[^)]+)\)/)?.[1]; + const resolved = + (variable ? style.getPropertyValue(variable).trim() : color) || + fallback; + ctx.fillStyle = resolved; + ctx.strokeStyle = resolved; + spans.sort((a, b) => a[0] - b[0]); + let [runStart, runEnd] = spans[0] as [number, number]; + // Tinted with an outline like a segment, so a run reads as the + // segments it stands for. + const fill = () => { + const x = Math.max(runStart, 0); + const w = Math.max(Math.min(runEnd, width) - x, 1); + runs.push([x, x + w]); + if (w < 4) { + ctx.globalAlpha = 0.5; + ctx.fillRect(x, 2, w, height - 4); + return; + } + ctx.beginPath(); + ctx.roundRect(x + 0.5, 2.5, w - 1, height - 5, 6); + ctx.globalAlpha = 0.18; + ctx.fill(); + ctx.globalAlpha = 0.6; + ctx.stroke(); + }; + for (const [x0, x1] of spans.slice(1)) { + if (x0 <= runEnd + RUN_GAP_PX) runEnd = Math.max(runEnd, x1); + else { + fill(); + runStart = x0; + runEnd = x1; + } + } + fill(); + } + ctx.globalAlpha = 1; + }); + + return ( +
+ +
+
+
+
Promise | void; +}; + +type FullscreenTarget = HTMLElement & { + webkitRequestFullscreen?: () => Promise | void; +}; + +const fullscreenDocument = () => document as FullscreenDocument; + +const fullscreenElement = () => + fullscreenDocument().fullscreenElement ?? + fullscreenDocument().webkitFullscreenElement ?? + null; + +const fullscreenAvailable = () => { + const doc = fullscreenDocument(); + const root = document.documentElement as FullscreenTarget; + return ( + (doc.fullscreenEnabled === true && !!root.requestFullscreen) || + (doc.webkitFullscreenEnabled === true && !!root.webkitRequestFullscreen) + ); +}; + +export type FocusMode = { + active: Accessor; + /** True while focus mode fills the screen; false for the in-page fallback. */ + fullscreen: Accessor; + toggle: () => void; + exit: () => void; + /** Runs just before the layout changes, so the preview can note where it was. */ + onBeforeChange: (callback: () => void) => void; +}; + +export function createFocusMode(): FocusMode { + const [active, setActive] = createSignal(false); + const [fullscreen, setFullscreen] = createSignal(false); + const beforeChange = new Set<() => void>(); + let changing = false; + // A request to leave that arrives mid-change runs once the change settles. + let exitQueued = false; + + const notifyBeforeChange = () => { + for (const callback of beforeChange) callback(); + }; + + // The embedding page hears about a change once it has settled, so its bar + // doesn't step aside for a moment while the browser goes fullscreen. + const announce = () => { + if (window.parent === window) return; + window.parent.postMessage( + { + kind: "cap-editor-focus", + version: 1, + active: active(), + fullscreen: fullscreen(), + }, + window.location.origin, + ); + }; + + const settle = (pending: Promise | void | undefined) => { + void Promise.resolve(pending) + .catch(() => undefined) + .then(() => { + changing = false; + setFullscreen(fullscreenElement() !== null); + announce(); + if (exitQueued) { + exitQueued = false; + exit(); + } + }); + }; + + // The layout changes first, inside the click or key press, and the + // fullscreen request follows in the same turn so the browser still counts + // it as the user's gesture. A refused request (no gesture, or a page that + // forbids it) leaves the in-page layout, which works on its own. + const enter = () => { + if (active() || changing) return; + changing = true; + notifyBeforeChange(); + setActive(true); + let pending: Promise | void | undefined; + if (fullscreenAvailable()) { + const root = document.documentElement as FullscreenTarget; + try { + pending = root.requestFullscreen + ? root.requestFullscreen({ navigationUI: "hide" }) + : root.webkitRequestFullscreen?.(); + } catch {} + } + settle(pending); + }; + + const exit = () => { + if (!active()) return; + if (changing) { + exitQueued = true; + return; + } + changing = true; + notifyBeforeChange(); + setActive(false); + let pending: Promise | void | undefined; + if (fullscreenElement()) { + const doc = fullscreenDocument(); + try { + pending = doc.exitFullscreen + ? doc.exitFullscreen() + : doc.webkitExitFullscreen?.(); + } catch {} + } + settle(pending); + }; + + // The browser leaves fullscreen on its own when Esc is pressed (the page + // never sees that key) or the tab is switched; focus mode follows it out. + const onFullscreenChange = () => { + const inFullscreen = fullscreenElement() !== null; + if (changing) { + if (!inFullscreen && active()) exitQueued = true; + return; + } + if (inFullscreen) { + setFullscreen(true); + announce(); + return; + } + if (!active() || !fullscreen()) return; + notifyBeforeChange(); + setFullscreen(false); + setActive(false); + announce(); + }; + createEventListener(document, "fullscreenchange", onFullscreenChange); + createEventListener(document, "webkitfullscreenchange", onFullscreenChange); + + onCleanup(() => { + if (!active()) return; + const doc = fullscreenDocument(); + if (fullscreenElement()) + void Promise.resolve( + doc.exitFullscreen + ? doc.exitFullscreen() + : doc.webkitExitFullscreen?.(), + ).catch(() => undefined); + setActive(false); + setFullscreen(false); + announce(); + }); + + return { + active, + fullscreen, + toggle: () => (active() ? exit() : enter()), + exit, + onBeforeChange: (callback) => { + beforeChange.add(callback); + onCleanup(() => beforeChange.delete(callback)); + }, + }; +} diff --git a/apps/desktop/src/routes/editor/images.ts b/apps/desktop/src/routes/editor/images.ts index 1757311d8f6..294bf388b56 100644 --- a/apps/desktop/src/routes/editor/images.ts +++ b/apps/desktop/src/routes/editor/images.ts @@ -308,6 +308,10 @@ export async function pickImage( ], }); if (typeof source !== "string") return null; + if (import.meta.env.VITE_CAP_WEB_EDITOR === "true") { + const { invoke } = await import("@tauri-apps/api/core"); + return invoke("webEditorImportImage", { source }); + } return importImagePath(projectPath, source, { read: async (path) => readBoundedImage(await fs.open(path, { read: true })), mkdir: (path) => fs.mkdir(path, { recursive: true }), diff --git a/apps/desktop/src/routes/editor/preview-bounds.test.ts b/apps/desktop/src/routes/editor/preview-bounds.test.ts index d08bc2b6270..01eb38a3726 100644 --- a/apps/desktop/src/routes/editor/preview-bounds.test.ts +++ b/apps/desktop/src/routes/editor/preview-bounds.test.ts @@ -72,17 +72,19 @@ afterEach(() => { }); describe("preview first-frame bounds", () => { - it("flushes the actual first-frame bounds and cancels a stale trailing write", () => { + it("follows the layout without delay until the first frame, then flushes its actual bounds", () => { const state = harness(); state.update({ width: 835, height: 542 }, false); state.update({ width: 820, height: 520 }, false); - expect(vi.getTimerCount()).toBe(1); + expect(state.current()).toEqual({ width: 820, height: 520 }); + expect(vi.getTimerCount()).toBe(0); state.update({ width: 820, height: 520 }, true); expect(state.current()).toEqual({ width: 835, height: 494 }); expect(vi.getTimerCount()).toBe(0); vi.advanceTimersByTime(1000); expect(state.commit.mock.calls).toEqual([ [{ width: 835, height: 542 }], + [{ width: 820, height: 520 }], [{ width: 835, height: 494 }], ]); }); @@ -137,7 +139,7 @@ describe("preview first-frame bounds", () => { state.dispose(); state.update({ width: 900, height: 400 }, true); vi.advanceTimersByTime(1000); - expect(state.current()).toEqual({ width: 835, height: 542 }); + expect(state.current()).toEqual({ width: 300, height: 600 }); expect(state.measure).not.toHaveBeenCalled(); expect(vi.getTimerCount()).toBe(0); }); diff --git a/apps/desktop/src/routes/editor/preview-bounds.ts b/apps/desktop/src/routes/editor/preview-bounds.ts index 35de0a1094d..27ab97a1a5b 100644 --- a/apps/desktop/src/routes/editor/preview-bounds.ts +++ b/apps/desktop/src/routes/editor/preview-bounds.ts @@ -54,6 +54,13 @@ export function createPreviewBoundsUpdater(options: { } return; } + // Until the first frame the preview is hidden and sizes the canvas + // it will first draw into, so it follows the layout without delay. + if (!hasFrame) { + options.cancel(); + options.commit(bounds); + return; + } const current = options.current(); if (current.width === 0 && current.height === 0) { options.commit(bounds); diff --git a/apps/desktop/src/routes/editor/project-history.test.ts b/apps/desktop/src/routes/editor/project-history.test.ts new file mode 100644 index 00000000000..48458bec889 --- /dev/null +++ b/apps/desktop/src/routes/editor/project-history.test.ts @@ -0,0 +1,123 @@ +// @vitest-environment jsdom + +import { createRoot } from "solid-js"; +import { createStore } from "solid-js/store"; +import { describe, expect, it, vi } from "vitest"; +import type { EditorProjectConfiguration } from "./context"; +import { createStoreHistory, withEditorTimeline } from "./project-history"; + +vi.mock("solid-js", () => vi.importActual("solid-js/dist/solid.js")); +vi.mock("solid-js/store", () => + vi.importActual("solid-js/store/dist/store.js"), +); +vi.mock("solid-js/web", () => vi.importActual("solid-js/web/dist/web.js")); + +type Project = { title: string; clips: { start: number; end: number }[] }; + +function setup() { + return createRoot((dispose) => { + const [project, setProject] = createStore({ + title: "first", + clips: [{ start: 0, end: 10 }], + }); + const history = createStoreHistory(project, setProject); + return { project, setProject, history, dispose }; + }); +} + +describe("editor project history", () => { + it("starts with nothing to undo", () => { + const { history, dispose } = setup(); + expect(history.canUndo()).toBe(false); + expect(history.canRedo()).toBe(false); + dispose(); + }); + + it("records nothing for a pause that changed nothing", () => { + const { history, dispose } = setup(); + const resume = history.pause(); + resume(); + expect(history.canUndo()).toBe(false); + dispose(); + }); + + it("records one step for everything written while paused", () => { + const { project, setProject, history, dispose } = setup(); + const resume = history.pause(); + setProject("clips", 0, "end", 8); + setProject("clips", 0, "end", 6); + resume(); + expect(history.canUndo()).toBe(true); + history.undo(); + expect(project.clips[0]?.end).toBe(10); + expect(history.canUndo()).toBe(false); + expect(history.canRedo()).toBe(true); + history.redo(); + expect(project.clips[0]?.end).toBe(6); + dispose(); + }); + + it("records a step per change and ignores writes of the same value", () => { + const { project, setProject, history, dispose } = setup(); + setProject("title", "second"); + setProject("clips", [{ start: 0, end: 10 }]); + history.undo(); + expect(project.title).toBe("first"); + expect(history.canUndo()).toBe(false); + dispose(); + }); + + it("keeps undoing correctly after an undo followed by a new edit", () => { + const { project, setProject, history, dispose } = setup(); + setProject("title", "second"); + history.undo(); + setProject("title", "third"); + expect(history.canRedo()).toBe(false); + history.undo(); + expect(project.title).toBe("first"); + expect(history.canUndo()).toBe(false); + dispose(); + }); + + it("records the same edit again after undoing it", () => { + const { project, setProject, history, dispose } = setup(); + setProject("title", "second"); + history.undo(); + setProject("title", "second"); + expect(history.canRedo()).toBe(false); + history.undo(); + expect(project.title).toBe("first"); + expect(history.canUndo()).toBe(false); + dispose(); + }); +}); + +describe("opening a never-edited project", () => { + const base = {} as EditorProjectConfiguration; + + it("gets the whole recording as one clip before any history exists", () => { + const project = withEditorTimeline({ ...base, timeline: null }, 12.5); + expect(project.timeline?.segments).toEqual([ + { timescale: 1, start: 0, end: 12.5 }, + ]); + expect(project.timeline?.zoomSegments).toEqual([]); + expect(project.timeline?.transitions).toEqual([]); + }); + + it("keeps an existing timeline and fills only missing tracks", () => { + const segments = [{ timescale: 1, start: 2, end: 4 }]; + const project = withEditorTimeline( + { + ...base, + timeline: { + segments, + zoomSegments: [], + } as unknown as EditorProjectConfiguration["timeline"], + }, + 30, + ); + expect(project.timeline?.segments).toBe(segments); + expect(project.timeline?.textSegments).toEqual([]); + expect(project.timeline?.camera3dSegments).toEqual([]); + }); +}); diff --git a/apps/desktop/src/routes/editor/project-history.ts b/apps/desktop/src/routes/editor/project-history.ts new file mode 100644 index 00000000000..ab2011d84d2 --- /dev/null +++ b/apps/desktop/src/routes/editor/project-history.ts @@ -0,0 +1,138 @@ +import { trackStore } from "@solid-primitives/deep"; +import { createEventListener } from "@solid-primitives/event-listener"; +import { createUndoHistory } from "@solid-primitives/history"; +import { createSignal } from "solid-js"; +import { type createStore, reconcile, unwrap } from "solid-js/store"; +import type { EditorProjectConfiguration } from "./context"; + +export function createStoreHistory( + state: T, + setState: ReturnType>[1], + onRestore?: () => void, +) { + // not working properly yet + // const getDelta = captureStoreUpdates(state); + + const [pauseCount, setPauseCount] = createSignal(0); + let recorded: unknown; + + const history = createUndoHistory(() => { + if (pauseCount() > 0) return; + + trackStore(state); + + const copy = structuredClone(unwrap(state)); + // Resuming after a pause re-runs this even when nothing changed, which + // would leave an undo step that restores the same state. + if (recorded !== undefined && sameValue(recorded, copy)) return; + recorded = copy; + + return () => { + onRestore?.(); + setState(reconcile(copy)); + }; + }); + + createEventListener(window, "keydown", (e) => { + switch (e.code) { + case "KeyZ": { + if (!(e.ctrlKey || e.metaKey)) return; + if (e.shiftKey) history.redo(); + else history.undo(); + break; + } + case "KeyY": { + if (!(e.ctrlKey || e.metaKey)) return; + history.redo(); + break; + } + default: { + return; + } + } + + e.preventDefault(); + e.stopPropagation(); + }); + + return Object.assign(history, { + pause() { + setPauseCount(pauseCount() + 1); + + return () => { + setPauseCount(pauseCount() - 1); + }; + }, + isPaused: () => pauseCount() > 0, + }); +} + +function sameValue(a: unknown, b: unknown): boolean { + if (Object.is(a, b)) return true; + if ( + typeof a !== "object" || + typeof b !== "object" || + a === null || + b === null || + Array.isArray(a) !== Array.isArray(b) + ) { + return false; + } + if (Array.isArray(a) && Array.isArray(b)) { + return ( + a.length === b.length && + a.every((value, index) => sameValue(value, b[index])) + ); + } + const left = a as Record; + const right = b as Record; + const keys = Object.keys(left); + return ( + keys.length === Object.keys(right).length && + keys.every( + (key) => Object.hasOwn(right, key) && sameValue(left[key], right[key]), + ) + ); +} + +type Static = + | { + [K in number | string]: T; + } + | T[]; + +/// Fills in the timeline the editor needs before the store (and its undo +/// history) exists, so opening a never-edited project records no edit. +export function withEditorTimeline( + project: EditorProjectConfiguration, + recordingDuration: number, +): EditorProjectConfiguration { + const timeline = project.timeline ?? { + segments: [{ timescale: 1, start: 0, end: recordingDuration }], + zoomSegments: [], + sceneSegments: [], + maskSegments: [], + textSegments: [], + styleSegments: [], + imageSegments: [], + captionSegments: [], + keyboardSegments: [], + camera3dSegments: [], + transitions: [], + }; + return { + ...project, + timeline: { + ...timeline, + zoomSegments: timeline.zoomSegments ?? [], + sceneSegments: timeline.sceneSegments ?? [], + maskSegments: timeline.maskSegments ?? [], + textSegments: timeline.textSegments ?? [], + styleSegments: timeline.styleSegments ?? [], + imageSegments: timeline.imageSegments ?? [], + captionSegments: timeline.captionSegments ?? [], + keyboardSegments: timeline.keyboardSegments ?? [], + camera3dSegments: timeline.camera3dSegments ?? [], + }, + }; +} diff --git a/apps/desktop/src/routes/editor/style.ts b/apps/desktop/src/routes/editor/style.ts index ce69db1f757..97f13f7d308 100644 --- a/apps/desktop/src/routes/editor/style.ts +++ b/apps/desktop/src/routes/editor/style.ts @@ -53,8 +53,14 @@ export function resolveStyle>( segments: readonly StyleSegment[], time: number, ): T { + return applyStyleSegments(base, activeStyleSegments(segments, time)); +} + +export function applyStyleSegments< + T extends Pick, +>(base: T, active: ReturnType): T { const result = { ...base }; - for (const { segment } of activeStyleSegments(segments, time)) { + for (const { segment } of active) { for (const group of ["background", "camera", "cursor"] as const) { const value = segment.overrides[group]; if (value != null) Object.assign(result, { [group]: value }); diff --git a/apps/desktop/src/routes/editor/templates-gallery.tsx b/apps/desktop/src/routes/editor/templates-gallery.tsx new file mode 100644 index 00000000000..aae601ad13d --- /dev/null +++ b/apps/desktop/src/routes/editor/templates-gallery.tsx @@ -0,0 +1,793 @@ +import { DropdownMenu as KDropdownMenu } from "@kobalte/core/dropdown-menu"; +import { Popover as KPopover } from "@kobalte/core/popover"; +import { convertFileSrc, invoke } from "@tauri-apps/api/core"; +import { resolveResource } from "@tauri-apps/api/path"; +import { cx } from "cva"; +import { + createMemo, + createResource, + createSignal, + For, + type JSX, + Show, +} from "solid-js"; +import { produce, reconcile } from "solid-js/store"; +import toast from "solid-toast"; +import type { + AspectRatio, + BackgroundSource, + Camera, + ProjectConfiguration, + SceneMode, +} from "~/utils/tauri"; +import { commands } from "~/utils/tauri"; +import IconLucideCheck from "~icons/lucide/check"; +import IconLucideMoreHorizontal from "~icons/lucide/more-horizontal"; +import IconLucidePlus from "~icons/lucide/plus"; +import IconLucideSwatchBook from "~icons/lucide/swatch-book"; +import { clipDuration } from "./clip-transitions"; +import { normalizeProject, useEditorContext } from "./context"; +import { + applyTemplate, + type DefaultLook, + defaultLookKey, + EDITOR_TEMPLATES, + type EditorTemplate, + fullSpanScene, + lookKey, + templateBackgroundSource, + templateDefaultConfig, + templateDefaultKey, + templateLookKey, + withoutTemplateScene, +} from "./templates"; +import { + DropdownItem, + EditorButton, + PopperContent, + topCenterAnimateClasses, +} from "./ui"; + +const isWebEditor = import.meta.env.VITE_CAP_WEB_EDITOR === "true"; + +const CAP_DEFAULT_KEY = "cap-default"; + +const GROUPS: Array<{ id: EditorTemplate["group"]; title: string }> = [ + { id: "layout", title: "Layouts" }, + { id: "look", title: "Looks" }, + { id: "social", title: "Social" }, +]; + +type Filter = "all" | EditorTemplate["group"] | "presets"; + +const FILTERS: Array<{ id: Filter; title: string }> = [ + { id: "all", title: "All" }, + ...GROUPS, + { id: "presets", title: "Your presets" }, +]; + +type PreviewLook = { + aspectRatio: AspectRatio | null; + background: string; + padding: number; + rounding: number; + shadow: number; + camera: Pick; + scene?: SceneMode; +}; + +const wallpaperPath = (id: string) => + resolveResource(`assets/backgrounds/${id}.jpg`); + +const rgb = (value: [number, number, number]) => + `rgb(${value[0]} ${value[1]} ${value[2]})`; + +function backgroundCss(source: BackgroundSource) { + switch (source.type) { + case "color": + return rgb(source.value); + case "gradient": + return `linear-gradient(${source.angle ?? 90}deg, ${rgb(source.from)}, ${rgb(source.to)})`; + case "wallpaper": + case "image": + return source.path + ? `center / cover url("${convertFileSrc( + source.type === "wallpaper" + ? source.path.replace(/\.jpg$/, "-thumbnail.jpg") + : source.path, + )}")` + : "#1c1c1f"; + default: + return "linear-gradient(135deg, #3b2f6b, #1d4f91)"; + } +} + +/** + * Ready-made looks and layouts, previewed with the person's own recording, + * next to the presets they've saved. Any of them can become the look new + * recordings start with. + */ +export function TemplatesGallery() { + const { project, setProject, presets, setDialog, totalDuration } = + useEditorContext(); + const [open, setOpen] = createSignal(false); + const [filter, setFilter] = createSignal("all"); + const [defaultConfig] = createResource(open, () => + commands.getDefaultProjectConfig().catch(() => null), + ); + // null is Cap's own look; undefined is not known yet. + const [savedStyle, { mutate: setSavedStyle }] = createResource(open, () => + invoke("webEditorDefaultStyle").catch(() => undefined), + ); + const [pendingDefault, setPendingDefault] = createSignal(null); + + const [frames] = createResource(open, async () => { + const first = project.timeline?.segments[0]; + const middle = first ? first.start + clipDuration(first) / 2 : 1; + const [screen, camera] = await Promise.all([ + commands + .getClipThumbnail(first?.recordingSegment ?? 0, middle) + .catch(() => null), + isWebEditor + ? invoke("webEditorCameraThumbnail").catch(() => null) + : null, + ]); + return { screen: screen ? convertFileSrc(screen) : null, camera }; + }); + + const [wallpapers] = createResource(open, async () => { + const ids = EDITOR_TEMPLATES.flatMap((template) => + template.background.type === "wallpaper" ? [template.background.id] : [], + ); + const entries = await Promise.all( + ids.map(async (id) => [id, await wallpaperPath(id)] as const), + ); + return new Map(entries); + }); + + const templateLook = (template: EditorTemplate): PreviewLook => { + const background = template.background; + const path = + background.type === "wallpaper" + ? (wallpapers()?.get(background.id) ?? null) + : null; + return { + ...template, + background: + background.type === "wallpaper" + ? backgroundCss({ type: "wallpaper", path }) + : backgroundCss(background), + }; + }; + + let templateScene: SceneMode | null = null; + + const apply = async (template: EditorTemplate) => { + try { + const source = await templateBackgroundSource( + template.background, + wallpaperPath, + ); + setProject( + produce((draft) => + applyTemplate(draft, template, source, totalDuration()), + ), + ); + templateScene = template.scene ?? null; + } catch (error) { + toast.error( + error instanceof Error + ? error.message + : "Template could not be applied", + ); + } + }; + + const applyConfig = async (config: ProjectConfiguration) => { + if (isWebEditor) { + const prepare = ( + window as Window & { + capWebEditorPreparePresetBackground?: ( + config: unknown, + ) => Promise; + } + ).capWebEditorPreparePresetBackground; + try { + await prepare?.(config); + } catch (error) { + toast.error( + error instanceof Error + ? error.message + : "Preset background could not be loaded", + ); + return; + } + } + const timeline = project.timeline; + const scene = templateScene; + templateScene = null; + setProject( + reconcile( + normalizeProject({ + ...config, + timeline: + timeline && scene + ? { + ...timeline, + sceneSegments: withoutTemplateScene( + timeline.sceneSegments, + scene, + totalDuration(), + ), + } + : (timeline ?? null), + overlayOrder: project.overlayOrder ?? [], + clips: project.clips, + }), + ), + ); + }; + + const applyPreset = (index: number) => { + const preset = presets.query.data?.presets[index]; + if (preset) return applyConfig(preset.config); + }; + + // A card shows as chosen while the project still looks the way it made it. + const currentLook = createMemo(() => + lookKey( + project, + fullSpanScene(project.timeline?.sceneSegments, totalDuration()), + ), + ); + const showsConfig = (config: ProjectConfiguration) => + currentLook() === lookKey(config, null); + const showsTemplate = (template: EditorTemplate) => + currentLook() === templateLookKey(template, wallpaperFor(template)); + + const savedPresets = () => presets.query.data?.presets ?? []; + + const defaultKey = () => { + const pending = pendingDefault(); + if (pending) return pending; + const style = savedStyle(); + if (style === undefined) return undefined; + return style === null ? CAP_DEFAULT_KEY : defaultLookKey(style); + }; + + const wallpaperFor = (template: EditorTemplate) => + template.background.type === "wallpaper" + ? (wallpapers()?.get(template.background.id) ?? null) + : null; + + const defaultName = createMemo(() => { + const key = defaultKey(); + if (key === undefined) return undefined; + if (key === CAP_DEFAULT_KEY) return "Cap default"; + const preset = savedPresets().find( + (preset) => defaultLookKey(preset.config) === key, + ); + if (preset) return preset.name; + const template = EDITOR_TEMPLATES.find( + (template) => + !template.scene && + templateDefaultKey(template, wallpaperFor(template)) === key, + ); + return template?.name ?? null; + }); + + const makeDefault = async ( + key: string, + name: string, + config: ProjectConfiguration | null, + ) => { + setPendingDefault(key); + try { + setSavedStyle( + await invoke("webEditorSaveDefaultStyle", { + config: config && JSON.parse(JSON.stringify(config)), + }), + ); + toast.success(`New recordings will start with ${name}`); + } catch (error) { + toast.error( + error instanceof Error ? error.message : "Default was not saved", + ); + } finally { + setPendingDefault(null); + } + }; + + const makeTemplateDefault = (template: EditorTemplate) => { + const base = defaultConfig(); + const config = + base && templateDefaultConfig(base, template, wallpaperFor(template)); + if (!config) return undefined; + return () => + void makeDefault( + templateDefaultKey(template, wallpaperFor(template)), + template.name, + config, + ); + }; + + const presetLook = (config: ProjectConfiguration): PreviewLook => ({ + aspectRatio: config.aspectRatio, + background: backgroundCss(config.background.source), + padding: config.background.padding, + rounding: config.background.rounding, + shadow: config.background.shadow, + camera: config.camera, + }); + + const savePreset = () => { + setOpen(false); + setDialog({ type: "createPreset", open: true }); + }; + + const visibleGroups = () => + filter() === "all" + ? GROUPS + : GROUPS.filter((group) => group.id === filter()); + const showPresets = () => filter() === "all" || filter() === "presets"; + + return ( + + + as={KPopover.Trigger} + leftIcon={} + tooltipText="Presets" + > + Presets + + + +
+
+ + Presets + + + Start from a look, or make one the default for new recordings. + Everything stays editable. + +
+ +
+
+ + {(item) => ( + + )} + +
+
+ +
+

+ Your presets +

+
+ + {(config) => ( + void applyConfig(config())} + onMakeDefault={() => + void makeDefault(CAP_DEFAULT_KEY, "Cap default", null) + } + preview={ + + } + /> + )} + + + {(preset, index) => ( + void applyPreset(index())} + onMakeDefault={ + // Backgrounds from this browser's uploads stay with + // the project, so they can't start other recordings. + preset.config.background.source.type === "image" + ? undefined + : () => + void makeDefault( + defaultLookKey(preset.config), + preset.name, + preset.config, + ) + } + menu={ + { + await presets.saveToPreset(index(), project); + toast.success(`Saved to "${preset.name}"`); + }} + onRename={() => { + setOpen(false); + setDialog({ + type: "renamePreset", + presetIndex: index(), + open: true, + }); + }} + onDelete={() => { + setOpen(false); + setDialog({ + type: "deletePreset", + presetIndex: index(), + open: true, + }); + }} + /> + } + preview={ + + } + /> + )} + + +
+
+ + + {(group) => ( +
+

+ {group.title} +

+
+ template.group === group.id, + )} + > + {(template) => ( + void apply(template)} + onMakeDefault={makeTemplateDefault(template)} + preview={ + + } + /> + )} + +
+
+ )} + +
+ +
+ New recordings start with{" "} + + {(name) => ( + {name()} + )} + +
+ + + + + ); +} + +function TemplateCard(props: { + name: string; + description: string; + active: boolean; + isDefault: boolean; + preview: JSX.Element; + menu?: JSX.Element; + onSelect: () => void; + onMakeDefault?: () => void; +}) { + return ( +
+ +
+ + {(makeDefault) => ( + + )} + + } + > + + + Default + + +
+ {props.menu} +
+ ); +} + +function PresetMenu(props: { + onSaveHere: () => void; + onRename: () => void; + onDelete: () => void; +}) { + return ( + + + + + + + as={KDropdownMenu.Content} + class={cx("z-70 w-48 p-1", topCenterAnimateClasses)} + > + + Save current style here + + Rename + Delete + + + + ); +} + +function TemplatePreview(props: { + look: PreviewLook; + screen: string | null; + camera: string | null; +}) { + const frameAspect = createMemo(() => + props.look.aspectRatio === "vertical" + ? 9 / 16 + : props.look.aspectRatio === "square" + ? 1 + : props.look.aspectRatio === "classic" + ? 4 / 3 + : 16 / 9, + ); + const inset = () => `${props.look.padding * 0.9}%`; + const radius = () => `${Math.min(props.look.rounding, 40) * 0.18}%`; + const shadow = () => + props.look.shadow > 0 + ? `0 ${props.look.shadow / 20}px ${props.look.shadow / 6}px rgba(0,0,0,${Math.min(0.5, props.look.shadow / 150)})` + : "none"; + const screen = (style: JSX.CSSProperties) => ( +
+ + {(src) => ( + + )} + +
+ ); + const camera = (style: JSX.CSSProperties) => ( +
+ + {(src) => ( + + )} + +
+ ); + const cameraSide = () => `${props.look.camera.size * 0.62}%`; + const cameraPlacement = (): JSX.CSSProperties => { + const { x, y } = props.look.camera.position; + return { + width: cameraSide(), + "aspect-ratio": "1", + "border-radius": `${props.look.camera.rounding * 0.3}%`, + ...(x === "left" + ? { left: "5%" } + : x === "center" + ? { left: "50%", transform: "translateX(-50%)" } + : { right: "5%" }), + ...(y === "top" ? { top: "7%" } : { bottom: "7%" }), + }; + }; + + return ( + + ); +} diff --git a/apps/desktop/src/routes/editor/templates.test.ts b/apps/desktop/src/routes/editor/templates.test.ts new file mode 100644 index 00000000000..e1ba2ab2aa2 --- /dev/null +++ b/apps/desktop/src/routes/editor/templates.test.ts @@ -0,0 +1,154 @@ +import { describe, expect, it } from "vitest"; +import type { EditorProjectConfiguration } from "./context"; +import { + applyTemplate, + defaultLookKey, + EDITOR_TEMPLATES, + fullSpanScene, + lookKey, + templateDefaultConfig, + templateDefaultKey, + templateLookKey, + templateSource, + withoutTemplateScene, +} from "./templates"; + +const project = () => + ({ + aspectRatio: null, + background: { + source: { type: "color", value: [255, 255, 255] }, + padding: 0, + rounding: 0, + shadow: 0, + }, + camera: { + hide: false, + position: { x: "left", y: "top" }, + size: 30, + rounding: 100, + shape: "square", + }, + timeline: { sceneSegments: [{ start: 2, end: 4, mode: "cameraOnly" }] }, + }) as unknown as EditorProjectConfiguration; + +const currentLook = (config: EditorProjectConfiguration) => + lookKey(config, fullSpanScene(config.timeline?.sceneSegments, 10)); + +describe("template looks", () => { + it("recognise a template until the project changes", () => { + for (const template of EDITOR_TEMPLATES) { + const config = project(); + const wallpaper = + template.background.type === "wallpaper" ? "/wallpaper.jpg" : null; + applyTemplate( + config, + template, + templateSource(template.background, wallpaper), + 10, + ); + expect(currentLook(config)).toBe(templateLookKey(template, wallpaper)); + config.background.padding += 1; + expect(currentLook(config)).not.toBe( + templateLookKey(template, wallpaper), + ); + } + }); + + it("tell presets apart from a template's whole-video scene", () => { + const config = project(); + expect(currentLook(config)).toBe(lookKey(config, null)); + const sideBySide = EDITOR_TEMPLATES.find((t) => t.scene === "floating"); + if (!sideBySide) throw new Error("Side by side template is missing"); + applyTemplate( + config, + sideBySide, + templateSource(sideBySide.background, "/wallpaper.jpg"), + 10, + ); + expect(currentLook(config)).not.toBe(lookKey(config, null)); + expect(config.timeline?.sceneSegments).toContainEqual({ + start: 2, + end: 4, + mode: "cameraOnly", + }); + }); + + it("drop only the whole-video scene a template added", () => { + const scenes = [ + { start: 0, end: 10, mode: "floating" as const }, + { start: 0, end: 10, mode: "cameraOnly" as const }, + { start: 2, end: 4, mode: "floating" as const }, + ]; + expect(withoutTemplateScene(scenes, "floating", 10)).toEqual([ + { start: 0, end: 10, mode: "cameraOnly" }, + { start: 2, end: 4, mode: "floating" }, + ]); + }); +}); + +// What the server stores: no camera visibility, extra fields, and object keys +// in whatever order MySQL's JSON column returns them. +const savedStyle = (config: EditorProjectConfiguration) => { + const reversed = (value: unknown): unknown => + typeof value === "object" && value !== null && !Array.isArray(value) + ? Object.fromEntries( + Object.entries(value) + .reverse() + .map(([key, field]) => [key, reversed(field)]), + ) + : value; + const { hide: _hide, ...camera } = config.camera; + return reversed({ + version: 1, + aspectRatio: config.aspectRatio, + background: { ...config.background, blur: 0 }, + camera, + cursor: { size: 100 }, + }) as Parameters[0]; +}; + +describe("default looks", () => { + it("recognise the template a saved default came from", () => { + for (const template of EDITOR_TEMPLATES.filter((t) => !t.scene)) { + const wallpaper = + template.background.type === "wallpaper" ? "/wallpaper.jpg" : null; + const config = templateDefaultConfig(project(), template, wallpaper); + if (!config) throw new Error(`${template.id} has no default config`); + expect(defaultLookKey(savedStyle(config))).toBe( + templateDefaultKey(template, wallpaper), + ); + for (const other of EDITOR_TEMPLATES.filter( + (t) => !t.scene && t !== template, + )) { + expect(defaultLookKey(savedStyle(config))).not.toBe( + templateDefaultKey(other, wallpaper), + ); + } + } + }); + + it("leave out templates whose scene can't reach a new recording", () => { + for (const template of EDITOR_TEMPLATES.filter((t) => t.scene)) { + expect(templateDefaultConfig(project(), template, null)).toBeNull(); + } + }); + + it("ignore camera visibility but not the look", () => { + const config = project(); + const hidden = project(); + hidden.camera.hide = true; + expect(defaultLookKey(hidden)).toBe(defaultLookKey(config)); + hidden.background.padding = 4; + expect(defaultLookKey(hidden)).not.toBe(defaultLookKey(config)); + }); + + it("match animated gradients whatever order their fields come back in", () => { + const config = project(); + config.background.source = { + type: "animatedGradient", + config: { speed: 1, colors: ["#fff", "#000"] }, + } as unknown as EditorProjectConfiguration["background"]["source"]; + expect(defaultLookKey(savedStyle(config))).toBe(defaultLookKey(config)); + }); +}); diff --git a/apps/desktop/src/routes/editor/templates.ts b/apps/desktop/src/routes/editor/templates.ts new file mode 100644 index 00000000000..5b6194d7c0a --- /dev/null +++ b/apps/desktop/src/routes/editor/templates.ts @@ -0,0 +1,450 @@ +import type { + AspectRatio, + BackgroundConfiguration, + BackgroundSource, + Camera, + ProjectConfiguration, + SceneMode, + SceneSegment, +} from "~/utils/tauri"; +import type { EditorProjectConfiguration } from "./context"; + +type RGB = [number, number, number]; + +export type TemplateBackground = + | { type: "wallpaper"; id: string } + | { type: "color"; value: RGB } + | { type: "gradient"; from: RGB; to: RGB; angle: number }; + +export type EditorTemplate = { + id: string; + name: string; + description: string; + group: "layout" | "look" | "social"; + aspectRatio: AspectRatio | null; + background: TemplateBackground; + padding: number; + rounding: number; + shadow: number; + camera: Pick; + scene?: SceneMode; +}; + +const bubble = ( + x: Camera["position"]["x"], + y: Camera["position"]["y"], + size = 30, +): EditorTemplate["camera"] => ({ + hide: false, + position: { x, y }, + size, + rounding: 100, + shape: "square", +}); + +export const EDITOR_TEMPLATES: EditorTemplate[] = [ + { + id: "classic", + name: "Classic", + description: "Soft wallpaper, camera in the corner", + group: "layout", + aspectRatio: "wide", + background: { type: "wallpaper", id: "macOS/tahoe-dusk-min" }, + padding: 10, + rounding: 12, + shadow: 60, + camera: bubble("right", "bottom"), + }, + { + id: "side-by-side", + name: "Side by side", + description: "Screen and camera as two cards", + group: "layout", + aspectRatio: "wide", + background: { type: "wallpaper", id: "dark/3" }, + padding: 6, + rounding: 12, + shadow: 50, + camera: bubble("right", "bottom"), + scene: "floating", + }, + { + id: "split", + name: "Split screen", + description: "Full-bleed halves, edge to edge", + group: "layout", + aspectRatio: "wide", + background: { type: "color", value: [12, 12, 14] }, + padding: 0, + rounding: 0, + shadow: 0, + camera: bubble("right", "bottom"), + scene: "splitScreen", + }, + { + id: "presenter", + name: "Presenter", + description: "A big camera beside your screen", + group: "layout", + aspectRatio: "wide", + background: { type: "wallpaper", id: "blue/3" }, + padding: 12, + rounding: 14, + shadow: 60, + camera: bubble("left", "bottom", 46), + }, + { + id: "camera-only", + name: "Just you", + description: "Camera full frame, no screen", + group: "layout", + aspectRatio: "wide", + background: { type: "color", value: [18, 18, 20] }, + padding: 0, + rounding: 0, + shadow: 0, + camera: bubble("right", "bottom"), + scene: "cameraOnly", + }, + { + id: "screen-only", + name: "Screen only", + description: "No camera, framed screen", + group: "layout", + aspectRatio: "wide", + background: { type: "wallpaper", id: "macOS/sequoia-light" }, + padding: 9, + rounding: 10, + shadow: 55, + camera: { ...bubble("right", "bottom"), hide: true }, + }, + { + id: "midnight", + name: "Midnight", + description: "Deep blue gradient, camera top right", + group: "look", + aspectRatio: "wide", + background: { + type: "gradient", + from: [20, 24, 64], + to: [74, 36, 120], + angle: 135, + }, + padding: 11, + rounding: 14, + shadow: 70, + camera: bubble("right", "top", 26), + }, + { + id: "paper", + name: "Paper", + description: "Warm off-white, quiet and clean", + group: "look", + aspectRatio: "wide", + background: { type: "color", value: [244, 241, 234] }, + padding: 8, + rounding: 10, + shadow: 35, + camera: { ...bubble("right", "bottom", 28), rounding: 40 }, + }, + { + id: "sunset", + name: "Sunset", + description: "Warm orange light", + group: "look", + aspectRatio: "wide", + background: { type: "wallpaper", id: "orange/2" }, + padding: 10, + rounding: 14, + shadow: 60, + camera: bubble("left", "bottom"), + }, + { + id: "studio", + name: "Studio", + description: "Dark and focused", + group: "look", + aspectRatio: "wide", + background: { type: "wallpaper", id: "macOS/tahoe-night-min" }, + padding: 10, + rounding: 12, + shadow: 70, + camera: bubble("right", "bottom"), + }, + { + id: "full-bleed", + name: "Full bleed", + description: "Your screen, edge to edge", + group: "look", + aspectRatio: null, + background: { type: "color", value: [0, 0, 0] }, + padding: 0, + rounding: 0, + shadow: 0, + camera: bubble("right", "bottom", 24), + }, + { + id: "city", + name: "City lights", + description: "Photo backdrop, camera bottom left", + group: "look", + aspectRatio: "wide", + background: { type: "wallpaper", id: "cities/nyc" }, + padding: 12, + rounding: 14, + shadow: 70, + camera: bubble("left", "bottom", 26), + }, + { + id: "shorts", + name: "Shorts", + description: "Vertical, screen over camera", + group: "social", + aspectRatio: "vertical", + background: { type: "wallpaper", id: "purple/2" }, + padding: 5, + rounding: 10, + shadow: 50, + camera: bubble("center", "bottom", 40), + scene: "floating", + }, + { + id: "square", + name: "Square post", + description: "1:1 for feeds", + group: "social", + aspectRatio: "square", + background: { + type: "gradient", + from: [255, 122, 89], + to: [255, 196, 102], + angle: 160, + }, + padding: 9, + rounding: 14, + shadow: 55, + camera: bubble("right", "bottom", 32), + }, + { + id: "story", + name: "Story", + description: "Vertical, big camera", + group: "social", + aspectRatio: "vertical", + background: { type: "wallpaper", id: "blue/5" }, + padding: 6, + rounding: 12, + shadow: 50, + camera: bubble("center", "bottom", 60), + }, +]; + +export async function templateBackgroundSource( + background: TemplateBackground, + resolveWallpaper: (id: string) => Promise, +): Promise { + return templateSource( + background, + background.type === "wallpaper" + ? await resolveWallpaper(background.id) + : null, + ); +} + +export function templateSource( + background: TemplateBackground, + wallpaperPath: string | null, +): BackgroundSource { + if (background.type === "wallpaper") { + return { type: "wallpaper", path: wallpaperPath }; + } + if (background.type === "color") { + return { type: "color", value: background.value }; + } + return { + type: "gradient", + from: background.from, + to: background.to, + angle: background.angle, + }; +} + +/** + * Applies a template's look to the project, keeping the edit itself. Scenes a + * template added span the whole video, so those are swapped out; scenes the + * person placed themselves stay. + */ +export function applyTemplate( + project: EditorProjectConfiguration, + template: EditorTemplate, + source: BackgroundSource, + duration: number, +) { + project.aspectRatio = template.aspectRatio; + project.background.source = source; + project.background.padding = template.padding; + project.background.rounding = template.rounding; + project.background.shadow = template.shadow; + project.background.displayPosition = null; + Object.assign(project.camera, template.camera, { manualPosition: null }); + + const timeline = project.timeline; + if (!timeline) return; + const scenes = (timeline.sceneSegments ?? []).filter( + (scene) => !spansVideo(scene, duration), + ); + if (template.scene && duration > 0) { + scenes.push({ start: 0, end: duration, mode: template.scene }); + } + timeline.sceneSegments = scenes; +} + +const spansVideo = (scene: SceneSegment, duration: number) => + scene.start <= 0.001 && scene.end >= duration - 0.001; + +/** The timeline's scenes without the whole-video scene a template added. */ +export function withoutTemplateScene( + scenes: SceneSegment[] | undefined, + mode: SceneMode, + duration: number, +) { + return (scenes ?? []).filter( + (scene) => !(spansVideo(scene, duration) && scene.mode === mode), + ); +} + +/** The scene a template laid over the whole video, if there is one. */ +export function fullSpanScene( + scenes: SceneSegment[] | undefined, + duration: number, +): SceneMode | null { + return scenes?.find((scene) => spansVideo(scene, duration))?.mode ?? null; +} + +type Look = { + aspectRatio: AspectRatio | null; + background: Pick< + BackgroundConfiguration, + "source" | "padding" | "rounding" | "shadow" + >; + camera: EditorTemplate["camera"]; +}; + +// Saved styles come back from MySQL, whose JSON columns reorder object keys. +const sortedJson = (value: unknown) => + JSON.stringify(value, (_key, field: unknown) => + typeof field === "object" && field !== null && !Array.isArray(field) + ? Object.fromEntries( + Object.entries(field).sort(([a], [b]) => (a < b ? -1 : 1)), + ) + : field, + ); + +function sourceKey(source: BackgroundSource) { + switch (source.type) { + case "color": + return [source.type, ...source.value, source.alpha ?? 255]; + case "gradient": + return [source.type, ...source.from, ...source.to, source.angle ?? 90]; + case "animatedGradient": + return [source.type, sortedJson(source.config)]; + default: + return [source.type, source.path]; + } +} + +/** + * What a template or preset sets, as one comparable value, so the gallery + * can tell which one the project still shows. + */ +export function lookKey(look: Look, scene: SceneMode | null) { + const { background, camera } = look; + return JSON.stringify([ + look.aspectRatio ?? null, + sourceKey(background.source), + background.padding, + background.rounding, + background.shadow, + camera.hide, + camera.position.x, + camera.position.y, + camera.size, + camera.rounding, + camera.shape, + scene, + ]); +} + +function templateLook(template: EditorTemplate, wallpaperPath: string | null) { + return { + aspectRatio: template.aspectRatio, + background: { + source: templateSource(template.background, wallpaperPath), + padding: template.padding, + rounding: template.rounding, + shadow: template.shadow, + }, + camera: template.camera, + } satisfies Look; +} + +export function templateLookKey( + template: EditorTemplate, + wallpaperPath: string | null, +) { + return lookKey(templateLook(template, wallpaperPath), template.scene ?? null); +} + +/** A saved default style, or anything shaped like a project's look. */ +export type DefaultLook = { + aspectRatio?: AspectRatio | null; + background?: Partial; + camera?: Partial> & { + position?: Partial; + }; +}; + +/** + * What new recordings start with, as one comparable value. A default style + * leaves out whether the camera shows and whole-video scenes, since both + * belong to one recording. + */ +export function defaultLookKey(look: DefaultLook) { + const { background, camera } = look; + return JSON.stringify([ + look.aspectRatio ?? null, + background?.source ? sourceKey(background.source) : null, + background?.padding ?? null, + background?.rounding ?? null, + background?.shadow ?? null, + camera?.position?.x ?? null, + camera?.position?.y ?? null, + camera?.size ?? null, + camera?.rounding ?? null, + camera?.shape ?? null, + ]); +} + +/** + * The project config that makes a template the default. Scenes can't carry + * over to a new recording, so a template with one can't be a default. + */ +export function templateDefaultConfig< + T extends Pick, +>(base: T, template: EditorTemplate, wallpaperPath: string | null): T | null { + if (template.scene) return null; + const look = templateLook(template, wallpaperPath); + return { + ...base, + aspectRatio: look.aspectRatio, + background: { ...base.background, ...look.background }, + camera: { ...base.camera, ...look.camera, manualPosition: null }, + }; +} + +export function templateDefaultKey( + template: EditorTemplate, + wallpaperPath: string | null, +) { + return defaultLookKey(templateLook(template, wallpaperPath)); +} diff --git a/apps/desktop/src/routes/editor/timeline-tracks.test.ts b/apps/desktop/src/routes/editor/timeline-tracks.test.ts index 2910126a1e7..60c51c28116 100644 --- a/apps/desktop/src/routes/editor/timeline-tracks.test.ts +++ b/apps/desktop/src/routes/editor/timeline-tracks.test.ts @@ -111,6 +111,25 @@ describe("mixed visual layers", () => { ); }); + it("stacks waveform lanes between text and images like the renderer", () => { + const lane = [{ track: 0, start: 0, end: 4 }]; + expect( + getOverlayTrackRows({ + timeline: { + textSegments: lane, + waveformSegments: lane, + imageSegments: lane, + maskSegments: lane, + }, + }), + ).toEqual([ + { kind: "text", track: 0 }, + { kind: "waveform", track: 0 }, + { kind: "image", track: 0 }, + { kind: "mask", track: 0 }, + ]); + }); + it("remaps references when an entire lane is deleted", () => { const order: OverlayTrack[] = [{ kind: "image", track: 2 }, ...visualStack]; expect(removeOverlayTrack(order, "image", 0)).toEqual([ diff --git a/apps/desktop/src/routes/editor/timeline-utils.ts b/apps/desktop/src/routes/editor/timeline-utils.ts index fdd616d13db..5d8e714b1e3 100644 --- a/apps/desktop/src/routes/editor/timeline-utils.ts +++ b/apps/desktop/src/routes/editor/timeline-utils.ts @@ -330,6 +330,7 @@ export function rippleDeleteAllTracks( transitions?: ClipTransition[] | null; styleSegments?: Array<{ start: number; end: number }> | null; imageSegments?: Array<{ start: number; end: number }> | null; + waveformSegments?: Array<{ start: number; end: number }> | null; zoomSegments?: Array<{ start: number; end: number }> | null; sceneSegments?: Array<{ start: number; end: number }> | null; maskSegments?: RippleMaskSegment[] | null; @@ -408,7 +409,11 @@ export function rippleDeleteAllTracks( const overlayShift = shiftDuration + (overlayCutEnd - overlayCutStart - (trackCutEnd - trackCutStart)); - for (const track of [timeline.styleSegments, timeline.imageSegments]) { + for (const track of [ + timeline.styleSegments, + timeline.imageSegments, + timeline.waveformSegments, + ]) { if (track) rippleDeleteFromTrack( track, diff --git a/apps/desktop/src/routes/editor/timelineTracks.ts b/apps/desktop/src/routes/editor/timelineTracks.ts index b98eb401ad2..9432fde113c 100644 --- a/apps/desktop/src/routes/editor/timelineTracks.ts +++ b/apps/desktop/src/routes/editor/timelineTracks.ts @@ -124,13 +124,19 @@ type OverlayProject = { overlayOrder?: OverlayTrack[]; timeline?: { textSegments?: TrackSegment[]; + waveformSegments?: TrackSegment[]; imageSegments?: TrackSegment[]; maskSegments?: TrackSegment[]; } | null; }; export function isOverlayTrackKind(kind: string): kind is OverlayTrackKind { - return kind === "text" || kind === "image" || kind === "mask"; + return ( + kind === "text" || + kind === "waveform" || + kind === "image" || + kind === "mask" + ); } export function sameOverlayTrack(a: OverlayTrack, b: OverlayTrack) { @@ -165,7 +171,7 @@ export function getOverlayTrackRows( ) { const timeline = project.timeline; const available: OverlayTrack[] = []; - for (const kind of ["text", "image", "mask"] as const) { + for (const kind of ["text", "waveform", "image", "mask"] as const) { const segments = timeline?.[`${kind}Segments`] ?? []; for (const track of getTrackRowsWithCount( segments, diff --git a/apps/desktop/src/routes/editor/ui.tsx b/apps/desktop/src/routes/editor/ui.tsx index ab46db4b68a..4e831450957 100644 --- a/apps/desktop/src/routes/editor/ui.tsx +++ b/apps/desktop/src/routes/editor/ui.tsx @@ -4,7 +4,6 @@ import { DropdownMenu } from "@kobalte/core/dropdown-menu"; import { Polymorphic, type PolymorphicProps } from "@kobalte/core/polymorphic"; import { Slider as KSlider } from "@kobalte/core/slider"; import { Tooltip as KTooltip } from "@kobalte/core/tooltip"; -import { createElementBounds } from "@solid-primitives/bounds"; import { createEventListener } from "@solid-primitives/event-listener"; import { cva, cx, type VariantProps } from "cva"; @@ -168,27 +167,18 @@ export function Slider( const [thumbRef, setThumbRef] = createSignal(); - let boundsFrame: number | undefined; - const deferBoundsUpdate = (update: () => void) => () => { - if (boundsFrame !== undefined) return; - boundsFrame = requestAnimationFrame(() => { - boundsFrame = undefined; - update(); - }); - }; - onCleanup(() => { - if (boundsFrame !== undefined) cancelAnimationFrame(boundsFrame); - }); - const thumbBounds = createElementBounds(thumbRef, { - trackResize: deferBoundsUpdate, - trackMutation: deferBoundsUpdate, - }); - const [dragging, setDragging] = createSignal(false); + // A value set elsewhere (e.g. resizing on the canvas) can sit outside the + // slider's range; draw it at the nearest end instead of past the track. + const value = () => + props.value?.map((v) => + Math.min(Math.max(v, props.minValue ?? 0), props.maxValue ?? 100), + ); return ( { - return { - x: thumbBounds.left ?? undefined, - y: thumbBounds.top ?? undefined, - width: thumbBounds.width ?? undefined, - height: thumbBounds.height ?? undefined, - }; + const rect = thumbRef()?.getBoundingClientRect(); + return rect + ? { + x: rect.left, + y: rect.top, + width: rect.width, + height: rect.height, + } + : undefined; }} content={ props.value?.[0] !== undefined diff --git a/apps/desktop/src/routes/editor/utils.ts b/apps/desktop/src/routes/editor/utils.ts index 17b265a9cc2..4d565311687 100644 --- a/apps/desktop/src/routes/editor/utils.ts +++ b/apps/desktop/src/routes/editor/utils.ts @@ -1,9 +1,13 @@ export function formatTime(secs: number, fps?: number) { - const minutes = Math.floor(secs / 60); + const hours = Math.floor(secs / 3600); + const minutes = Math.floor(secs / 60) - hours * 60; const seconds = Math.floor(secs % 60); const frames = fps === undefined ? undefined : Math.floor((secs % 1) * fps); - let str = `${minutes}:${seconds.toString().padStart(2, "0")}`; + let str = + hours > 0 + ? `${hours}:${minutes.toString().padStart(2, "0")}:${seconds.toString().padStart(2, "0")}` + : `${minutes}:${seconds.toString().padStart(2, "0")}`; if (frames !== undefined) { str += `.${frames.toString().padStart(2, "0 ")}`; diff --git a/apps/desktop/src/routes/editor/waveform-overlay.tsx b/apps/desktop/src/routes/editor/waveform-overlay.tsx new file mode 100644 index 00000000000..e3b53da0e60 --- /dev/null +++ b/apps/desktop/src/routes/editor/waveform-overlay.tsx @@ -0,0 +1,246 @@ +import { createEventListener } from "@solid-primitives/event-listener"; +import { For, onCleanup, Show } from "solid-js"; +import { unwrap } from "solid-js/store"; +import { useCanvasSnapTargets } from "./CanvasElementsOverlay"; +import { useEditorContext } from "./context"; +import { resizeImage } from "./images"; +import { createOverlaySegments } from "./overlay-segments"; +import { SNAP_PX, snapMovingRect } from "./snapping"; +import { getOverlayZIndex } from "./timelineTracks"; + +const CORNERS = [ + { x: -1, y: -1 }, + { x: 1, y: -1 }, + { x: -1, y: 1 }, + { x: 1, y: 1 }, +]; + +export function WaveformOverlay(props: { + size: { width: number; height: number }; +}) { + const { + project, + setProject, + editorState, + setEditorState, + projectHistory, + setSnapGuides, + } = useEditorContext(); + const snapTargets = useCanvasSnapTargets(); + const time = () => editorState.previewTime ?? editorState.playbackTime; + const { visible } = createOverlaySegments( + () => project.timeline?.waveformSegments ?? [], + time, + ); + const selected = (index: number) => + editorState.timeline.selection?.type === "waveform" && + editorState.timeline.selection.indices.includes(index); + const clampUnit = (value: number) => Math.max(0, Math.min(1, value)); + let endDrag: (() => void) | undefined; + onCleanup(() => endDrag?.()); + + function drag( + event: MouseEvent, + index: number, + mode: "move" | { x: number; y: number }, + ) { + if (event.button !== 0 || editorState.playing) return; + event.preventDefault(); + event.stopPropagation(); + endDrag?.(); + const source = project.timeline?.waveformSegments?.[index]; + if (!source) return; + setEditorState("timeline", "selection", { + type: "waveform", + indices: [index], + }); + const initial = structuredClone(unwrap(source)); + const canvas = { ...props.size }; + const targets = snapTargets({ waveform: index }); + const resume = projectHistory.pause(); + let moved = false; + let frame: number | undefined; + let pending: MouseEvent | undefined; + const move = (next: MouseEvent) => { + if (project.timeline?.waveformSegments?.[index] !== source) return; + const delta = { + x: next.clientX - event.clientX, + y: next.clientY - event.clientY, + }; + if (!moved && Math.hypot(delta.x, delta.y) < 2) return; + moved = true; + if (mode === "move") { + const raw = { + x: initial.center.x + delta.x / canvas.width - initial.size.x / 2, + y: initial.center.y + delta.y / canvas.height - initial.size.y / 2, + w: initial.size.x, + h: initial.size.y, + }; + const snap = next.shiftKey + ? { dx: 0, dy: 0, guides: [] } + : snapMovingRect( + raw, + targets, + SNAP_PX / canvas.width, + SNAP_PX / canvas.height, + ); + setSnapGuides(snap.guides); + setProject("timeline", "waveformSegments", index, "center", { + x: clampUnit(raw.x + raw.w / 2 + snap.dx), + y: clampUnit(raw.y + raw.h / 2 + snap.dy), + }); + } else { + const resized = resizeImage( + { ...initial, rotation: 0, lockAspect: false }, + delta, + mode, + canvas, + ); + setProject("timeline", "waveformSegments", index, { + center: resized.center, + size: { + x: Math.min(1, resized.size.x), + y: Math.min(1, resized.size.y), + }, + }); + } + }; + const scheduleMove = (next: MouseEvent) => { + pending = next; + if (frame !== undefined) return; + frame = requestAnimationFrame(() => { + frame = undefined; + const latest = pending; + pending = undefined; + if (latest) move(latest); + }); + }; + const finish = (next?: MouseEvent, cancelled = false) => { + if (!endDrag) return; + if (frame !== undefined) cancelAnimationFrame(frame); + frame = undefined; + pending = undefined; + if (next) move(next); + window.removeEventListener("mousemove", scheduleMove); + window.removeEventListener("mouseup", finish); + window.removeEventListener("blur", cancel); + window.removeEventListener("keydown", keydown, true); + endDrag = undefined; + if ( + cancelled && + moved && + project.timeline?.waveformSegments?.[index] === source + ) + setProject("timeline", "waveformSegments", index, { + center: initial.center, + size: initial.size, + }); + setSnapGuides([]); + resume(); + }; + const cancel = () => finish(undefined, true); + const keydown = (next: KeyboardEvent) => { + if (next.key !== "Escape") return; + next.preventDefault(); + next.stopImmediatePropagation(); + cancel(); + }; + endDrag = cancel; + window.addEventListener("mousemove", scheduleMove); + window.addEventListener("mouseup", finish); + window.addEventListener("blur", cancel); + window.addEventListener("keydown", keydown, true); + } + + createEventListener(window, "keydown", (event) => { + if ( + editorState.playing || + (event.target instanceof HTMLElement && + (event.target.isContentEditable || + ["INPUT", "TEXTAREA", "SELECT"].includes(event.target.tagName))) + ) + return; + if ( + event.key === "Escape" && + editorState.timeline.selection?.type === "waveform" + ) { + event.preventDefault(); + setEditorState("timeline", "selection", null); + return; + } + const direction = { + ArrowLeft: [-1, 0], + ArrowRight: [1, 0], + ArrowUp: [0, -1], + ArrowDown: [0, 1], + }[event.key]; + if (!direction) return; + const items = visible().filter(({ index }) => selected(index)); + if (!items.length) return; + event.preventDefault(); + event.stopPropagation(); + const step = event.shiftKey ? 10 : 1; + const resume = projectHistory.pause(); + for (const { segment, index } of items) + setProject("timeline", "waveformSegments", index, "center", { + x: clampUnit( + segment.center.x + (direction[0] * step) / props.size.width, + ), + y: clampUnit( + segment.center.y + (direction[1] * step) / props.size.height, + ), + }); + resume(); + }); + + return ( +
+ + + {({ segment, index }) => ( +
drag(event, index, "move")} + > +
+ + + Waveform + + + {(corner) => ( +
+ )} + + +
+ ); +} diff --git a/apps/desktop/src/routes/editor/waveform-segment-config.tsx b/apps/desktop/src/routes/editor/waveform-segment-config.tsx new file mode 100644 index 00000000000..ae04ac9ea6b --- /dev/null +++ b/apps/desktop/src/routes/editor/waveform-segment-config.tsx @@ -0,0 +1,314 @@ +import { cx } from "cva"; +import { For, Index, type JSX, Show } from "solid-js"; +import { Toggle } from "~/components/Toggle"; +import type { OrganizationBrandColorSwatch } from "~/utils/organization-branding"; +import { useEditorContext } from "./context"; +import { HexColorInput } from "./text-style"; +import { Field, Section, Slider } from "./ui"; +import { + placeWaveform, + WAVEFORM_MAX_BAR_COUNT, + WAVEFORM_MIN_BAR_COUNT, + WAVEFORM_PLACEMENTS, + WAVEFORM_SOURCES, + WAVEFORM_STYLES, + type WaveformSegment, + type WaveformStyle, + waveformPlacementActive, +} from "./waveform"; + +const DEFAULT_SECONDARY_COLOR = "#7C9CFF"; +const PREVIEW_LEVELS = [0.35, 0.6, 0.95, 0.55, 0.8, 0.4, 0.7, 0.3]; + +function StylePreview(props: { style: WaveformStyle }) { + const step = 44 / PREVIEW_LEVELS.length; + const x = (index: number) => 2 + step * index + step / 2; + const shapes = (): JSX.Element => { + switch (props.style) { + case "bars": + return ( + + {(level, index) => ( + + )} + + ); + case "mirrored": + return ( + + {(level, index) => ( + + )} + + ); + case "line": + return ( + `${x(index)},${20 - level * 18}`, + ).join(" ")} + /> + ); + case "dots": + return ( + + {(level, index) => ( + + {(_, dot) => ( + + )} + + )} + + ); + } + }; + return ( + + ); +} + +function Choice(props: { + active: boolean; + onClick: () => void; + children: JSX.Element; + class?: string; +}) { + return ( + + ); +} + +export function WaveformSegmentConfig(props: { + segment: WaveformSegment; + segmentIndex: number; + brandColorSwatches: OrganizationBrandColorSwatch[]; +}) { + const { setProject } = useEditorContext(); + const update = (value: Partial) => + setProject("timeline", "waveformSegments", props.segmentIndex, value); + + return ( +
+ + update({ enabled })} + /> + + +
+
+ + {(option) => ( + update({ style: option.value })} + > + + {option.label} + + )} + +
+
+ +
+ update({ color })} + /> + + + update({ + secondaryColor: enabled ? DEFAULT_SECONDARY_COLOR : null, + }) + } + /> + + + {(secondaryColor) => ( + update({ secondaryColor: color })} + /> + )} + +
+ +
+
+ + `${value}`} + onChange={(value) => update({ barCount: value[0] })} + /> + + + update({ barWidth: value[0] / 100 })} + /> + + + update({ rounding: value[0] / 100 })} + /> + + + update({ opacity: value[0] / 100 })} + /> + +
+
+ +
+
+ + `${value.toFixed(2)}x`} + onChange={(value) => update({ sensitivity: value[0] })} + /> + + + update({ smoothing: value[0] / 100 })} + /> + +
+ +
+ + {(option) => ( + update({ source: option.value })} + > + {option.label} + + )} + +
+ +
+ +
+
+ + {(option) => ( + + update(placeWaveform(props.segment, option.value)) + } + > + {option.label} + + )} + +
+

+ Drag the waveform on the canvas to move it, or pull a corner to + resize. +

+
+
+ ); +} diff --git a/apps/desktop/src/routes/editor/waveform.test.ts b/apps/desktop/src/routes/editor/waveform.test.ts new file mode 100644 index 00000000000..0403f89af84 --- /dev/null +++ b/apps/desktop/src/routes/editor/waveform.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import type { AspectRatio, BackgroundSource } from "~/utils/tauri"; +import { + applyAudioOnlySetup, + defaultWaveformSegment, + needsAudioOnlySetup, + placeWaveform, + waveformGapAt, + waveformPlacementActive, +} from "./waveform"; + +const plainProject = () => ({ + hideDisplay: false, + aspectRatio: null as AspectRatio | null, + background: { + source: { type: "color", value: [255, 255, 255] } as BackgroundSource, + }, + timeline: { waveformSegments: [] }, +}); + +describe("waveform placement", () => { + it("fills the free stretch of a lane around the requested time", () => { + expect(waveformGapAt([], 3, 20)).toEqual({ start: 0, end: 20 }); + expect( + waveformGapAt( + [ + { start: 0, end: 4 }, + { start: 12, end: 20 }, + ], + 6, + 20, + ), + ).toEqual({ start: 4, end: 12 }); + expect(waveformGapAt([{ start: 0, end: 20 }], 6, 20)).toBeNull(); + }); + + it("keeps the size when snapping to a preset and reports the active one", () => { + const segment = defaultWaveformSegment(0, 10, 0); + const bottom = placeWaveform(segment, "bottom"); + expect(bottom.size).toEqual(segment.size); + expect(bottom.center.y + bottom.size.y / 2).toBeLessThan(1); + expect(waveformPlacementActive({ ...segment, ...bottom }, "bottom")).toBe( + true, + ); + expect(waveformPlacementActive({ ...segment, ...bottom }, "top")).toBe( + false, + ); + const wide = placeWaveform(segment, "fullWidth"); + expect(wide.size.x).toBe(1); + expect(wide.center.y).toBe(segment.center.y); + }); +}); + +describe("audio-only setup", () => { + it("hides the screen and adds one centred waveform over the whole recording", () => { + const project = plainProject(); + expect(needsAudioOnlySetup(project)).toBe(true); + applyAudioOnlySetup(project, 42); + expect(project.hideDisplay).toBe(true); + expect(project.aspectRatio).toBe("wide"); + expect(project.background.source.type).toBe("gradient"); + expect(project.timeline.waveformSegments).toMatchObject([ + { start: 0, end: 42, style: "mirrored", center: { x: 0.5, y: 0.5 } }, + ]); + expect(needsAudioOnlySetup(project)).toBe(false); + }); + + it("leaves an edited project and a chosen background alone", () => { + const project = plainProject(); + project.background.source = { type: "wallpaper", path: "dark/1" }; + applyAudioOnlySetup(project, 10); + expect(project.background.source.type).toBe("wallpaper"); + expect( + needsAudioOnlySetup({ + ...plainProject(), + timeline: { waveformSegments: [defaultWaveformSegment(0, 1, 0)] }, + }), + ).toBe(false); + }); +}); diff --git a/apps/desktop/src/routes/editor/waveform.ts b/apps/desktop/src/routes/editor/waveform.ts new file mode 100644 index 00000000000..d668d2f678b --- /dev/null +++ b/apps/desktop/src/routes/editor/waveform.ts @@ -0,0 +1,172 @@ +import type { + BackgroundSource, + ProjectConfiguration, + WaveformSegment, + WaveformSource, + WaveformStyle, + XY, +} from "~/utils/tauri"; + +export type { + WaveformSegment, + WaveformSource, + WaveformStyle, +} from "~/utils/tauri"; + +export const WAVEFORM_MIN_BAR_COUNT = 8; +export const WAVEFORM_MAX_BAR_COUNT = 256; + +export const WAVEFORM_STYLES: { value: WaveformStyle; label: string }[] = [ + { value: "bars", label: "Bars" }, + { value: "mirrored", label: "Mirrored" }, + { value: "line", label: "Line" }, + { value: "dots", label: "Dots" }, +]; + +export const WAVEFORM_SOURCES: { value: WaveformSource; label: string }[] = [ + { value: "mix", label: "Mix" }, + { value: "mic", label: "Microphone" }, + { value: "system", label: "Screen audio" }, +]; + +export const waveformStyleLabel = (style: WaveformStyle) => + WAVEFORM_STYLES.find((option) => option.value === style)?.label ?? "Waveform"; + +export function defaultWaveformSegment( + start: number, + end: number, + track: number, + overrides: Partial = {}, +): WaveformSegment { + return { + start, + end, + track, + enabled: true, + center: { x: 0.5, y: 0.86 }, + size: { x: 0.6, y: 0.16 }, + opacity: 1, + style: "bars", + color: "#FFFFFF", + secondaryColor: null, + barCount: 64, + barWidth: 0.6, + rounding: 1, + sensitivity: 1, + smoothing: 0.5, + source: "mix", + ...overrides, + }; +} + +const PLACEMENT_MARGIN = 0.06; + +export type WaveformPlacement = "bottom" | "center" | "top" | "fullWidth"; + +export const WAVEFORM_PLACEMENTS: { + value: WaveformPlacement; + label: string; +}[] = [ + { value: "bottom", label: "Bottom" }, + { value: "center", label: "Center" }, + { value: "top", label: "Top" }, + { value: "fullWidth", label: "Full width" }, +]; + +export function placeWaveform( + segment: Pick, + placement: WaveformPlacement, +): { center: XY; size: XY } { + const size = { ...segment.size }; + if (placement === "fullWidth") { + size.x = 1; + return { center: { x: 0.5, y: segment.center.y }, size }; + } + const y = + placement === "bottom" + ? 1 - size.y / 2 - PLACEMENT_MARGIN + : placement === "top" + ? size.y / 2 + PLACEMENT_MARGIN + : 0.5; + return { center: { x: 0.5, y: Math.max(0, Math.min(1, y)) }, size }; +} + +export function waveformPlacementActive( + segment: Pick, + placement: WaveformPlacement, +) { + const target = placeWaveform(segment, placement); + const close = (a: number, b: number) => Math.abs(a - b) < 0.005; + if (placement === "fullWidth") return close(segment.size.x, 1); + return ( + close(segment.center.x, target.center.x) && + close(segment.center.y, target.center.y) + ); +} + +// The free stretch of a lane around `time`, so a new waveform fills the gap it +// was added into (the whole video on an empty lane). +export function waveformGapAt( + segments: { start: number; end: number }[], + time: number, + totalDuration: number, +): { start: number; end: number } | null { + if (!(totalDuration > 0)) return null; + let start = 0; + let end = totalDuration; + for (const segment of segments) { + if (segment.start <= time && time < segment.end) return null; + if (segment.end <= time) start = Math.max(start, segment.end); + else end = Math.min(end, segment.start); + } + return end - start >= Math.min(0.5, totalDuration) ? { start, end } : null; +} + +const AUDIO_ONLY_BACKGROUND: BackgroundSource = { + type: "gradient", + from: [15, 18, 38], + to: [58, 38, 96], + angle: 135, +}; + +function isPlainBackground(source: BackgroundSource) { + if (source.type !== "color") return false; + const [r, g, b] = source.value; + return (r === g && g === b && (r === 0 || r === 255)) || source.alpha === 0; +} + +type AudioOnlyProject = Pick< + ProjectConfiguration, + "hideDisplay" | "aspectRatio" +> & { + background: { source: BackgroundSource }; + timeline?: { waveformSegments?: WaveformSegment[] } | null; +}; + +export function needsAudioOnlySetup(project: AudioOnlyProject) { + return ( + !project.hideDisplay && + (project.timeline?.waveformSegments?.length ?? 0) === 0 + ); +} + +// First open of an audio-only recording: hide the (empty) screen and give the +// audio something to look at. +export function applyAudioOnlySetup( + project: AudioOnlyProject, + duration: number, +) { + project.hideDisplay = true; + project.aspectRatio = "wide"; + if (isPlainBackground(project.background.source)) + project.background.source = { ...AUDIO_ONLY_BACKGROUND }; + if (project.timeline) + project.timeline.waveformSegments = [ + defaultWaveformSegment(0, duration, 0, { + style: "mirrored", + center: { x: 0.5, y: 0.5 }, + size: { x: 0.7, y: 0.36 }, + barCount: 72, + }), + ]; +} diff --git a/apps/desktop/src/routes/editor/web-drop-import.tsx b/apps/desktop/src/routes/editor/web-drop-import.tsx new file mode 100644 index 00000000000..b3fa802892b --- /dev/null +++ b/apps/desktop/src/routes/editor/web-drop-import.tsx @@ -0,0 +1,94 @@ +import { createSignal, onCleanup, onMount, Show } from "solid-js"; +import toast from "solid-toast"; +import { pathForDroppedFile } from "~/utils/dropped-files"; +import { createWebMediaImport } from "./web-media-import"; + +const AUDIO_FILE = /\.(mp3|wav|m4a|aac|ogg|oga|opus|flac|weba)$/i; +const VIDEO_FILE = /\.(mp4|mov|m4v|webm|mkv)$/i; +const CAP_FILE = /\.capbundle$/i; + +const hasFiles = (event: DragEvent) => + Array.from(event.dataTransfer?.types ?? []).includes("Files"); + +const kindOf = (file: File) => + CAP_FILE.test(file.name) + ? "clip" + : file.type.startsWith("audio/") || AUDIO_FILE.test(file.name) + ? "audio" + : file.type.startsWith("video/") || VIDEO_FILE.test(file.name) + ? "clip" + : null; + +/** + * Drop a file anywhere on the web editor: videos and Cap recordings become + * clips at the end of the timeline, audio lands on a lane of its own. + */ +export function WebDropImport() { + const { busy, addAudio, addClip } = createWebMediaImport(); + const [dragging, setDragging] = createSignal(false); + let depth = 0; + + const importFile = (file: File) => { + const kind = kindOf(file); + const path = kind ? pathForDroppedFile(file) : null; + if (!kind || !path) { + toast.error("Drop a video, an audio file or a Cap recording"); + return; + } + if (kind === "audio") void addAudio(path, file.name); + else void addClip(path, file.name); + }; + + onMount(() => { + const enter = (event: DragEvent) => { + if (!hasFiles(event) || busy()) return; + event.preventDefault(); + depth += 1; + setDragging(true); + }; + const over = (event: DragEvent) => { + if (!hasFiles(event) || busy()) return; + event.preventDefault(); + if (event.dataTransfer) event.dataTransfer.dropEffect = "copy"; + }; + const leave = () => { + depth = Math.max(0, depth - 1); + if (depth === 0) setDragging(false); + }; + const drop = (event: DragEvent) => { + if (!hasFiles(event)) return; + event.preventDefault(); + depth = 0; + setDragging(false); + const file = event.dataTransfer?.files[0]; + if (file && !busy()) importFile(file); + }; + window.addEventListener("dragenter", enter); + window.addEventListener("dragover", over); + window.addEventListener("dragleave", leave); + window.addEventListener("drop", drop); + onCleanup(() => { + window.removeEventListener("dragenter", enter); + window.removeEventListener("dragover", over); + window.removeEventListener("dragleave", leave); + window.removeEventListener("drop", drop); + }); + }); + + return ( + +
+
+
+ + Drop to add to your project + + + Videos and Cap recordings go on the end of your timeline. Audio gets + a track of its own. + +
+
+ + ); +} diff --git a/apps/desktop/src/routes/editor/web-media-import.ts b/apps/desktop/src/routes/editor/web-media-import.ts new file mode 100644 index 00000000000..2b1241ab195 --- /dev/null +++ b/apps/desktop/src/routes/editor/web-media-import.ts @@ -0,0 +1,73 @@ +import { createSignal } from "solid-js"; +import toast from "solid-toast"; +import { commands } from "~/utils/tauri"; +import { rememberClipInsert } from "./clip-insert"; +import { useEditorContext } from "./context"; + +/** + * Adds files to the open web project: videos and Cap recordings become clips + * (the editor reloads to pick them up), audio lands on a lane of its own. + */ +export function createWebMediaImport() { + const { + project, + projectActions, + flushProjectConfig, + editorInstance, + editorState, + setEditorState, + } = useEditorContext(); + const [busy, setBusy] = createSignal(false); + + const pause = async () => { + if (!editorState.playing) return; + await commands.stopPlayback(); + setEditorState("playing", false); + }; + + const addAudio = async (path: string, name: string) => { + if (busy()) return; + setBusy(true); + const toastId = toast.loading("Adding audio…"); + try { + await pause(); + const imported = await commands.importAudioTrackFile(path); + const lanes = (project.timeline?.audioSegments ?? []).map( + (segment) => segment.track ?? 0, + ); + projectActions.addAudioSegment( + lanes.length > 0 ? Math.max(...lanes) + 1 : 0, + imported, + ); + toast.success(`Added ${name}`, { id: toastId }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + toast.error(`Couldn't add ${name}: ${message}`, { id: toastId }); + } finally { + setBusy(false); + } + }; + + const addClip = async (path: string, name: string, insertAt?: number) => { + if (busy()) return; + setBusy(true); + const toastId = toast.loading("Adding clip…"); + try { + await pause(); + await flushProjectConfig(); + const clipCount = project.timeline?.segments.length ?? 0; + rememberClipInsert(editorInstance.path, insertAt ?? clipCount, clipCount); + const count = await commands.addExistingRecordingToEditor(path); + toast.success(count === 1 ? "Clip added" : `${count} clips added`, { + id: toastId, + }); + window.location.reload(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + toast.error(`Couldn't add ${name}: ${message}`, { id: toastId }); + setBusy(false); + } + }; + + return { busy, addAudio, addClip }; +} diff --git a/apps/desktop/src/routes/editor/web-publish-controls.tsx b/apps/desktop/src/routes/editor/web-publish-controls.tsx new file mode 100644 index 00000000000..0567f280226 --- /dev/null +++ b/apps/desktop/src/routes/editor/web-publish-controls.tsx @@ -0,0 +1,308 @@ +import { Channel, invoke } from "@tauri-apps/api/core"; +import { cx } from "cva"; +import { + createSignal, + Match, + onCleanup, + onMount, + Show, + Switch, +} from "solid-js"; +import toast from "solid-toast"; +import Tooltip from "~/components/Tooltip"; +import { trackEvent } from "~/utils/analytics"; +import IconLucideCheck from "~icons/lucide/check"; +import IconLucideDownload from "~icons/lucide/download"; +import { useEditorContext } from "./context"; +import { EditorButton } from "./ui"; + +type SaveStatus = { + state: "idle" | "rendering" | "ready" | "error"; + exportId: string | null; + progress: number; + playable: boolean; + hlsUrl: string | null; + error: string | null; + /** The share link shows, or is rendering, the project as saved now. */ + current?: boolean; +}; + +type SaveStart = + | { renderer: "farm"; shareUrl: string } + | { renderer: "browser"; reason: string }; + +type BrowserSaveProgress = { + stage: "rendering" | "uploading"; + progress: number; +}; + +const POLL_MS = 2000; +const FIRST_POLL_MS = 1000; +// A farm render that shows no progress for this long is treated as stuck and +// the save renders in the browser instead. +const FARM_STALL_MS = 90_000; +// Opened straight from the recorder: the recording publishes in its default +// style without waiting for a Save when nothing is rendering it yet. +const publishOnOpen = + new URLSearchParams(window.location.search).get("publish") === "recording"; + +export function WebPublishControls() { + const { + flushProjectConfig, + projectRevision, + exportState, + setExportState, + setDialog, + setEditorState, + } = useEditorContext(); + const [starting, setStarting] = createSignal(false); + const [status, setStatus] = createSignal(null); + const [browserSave, setBrowserSave] = + createSignal(null); + // The revision this session last saved. Nothing counts as saved until a + // Save, unless the share link already shows the project as it was opened. + const [savedRevision, setSavedRevision] = createSignal(null); + let timer: ReturnType | undefined; + let disposed = false; + // The farm save this editor started, watched so a failed or stuck render + // falls back to rendering in the browser. + let farmSave: { progress: number; progressAt: number } | null = null; + + const poll = async ( + resuming = false, + ): Promise => { + clearTimeout(timer); + try { + const next = await invoke("webEditorSaveStatus"); + if (disposed) return next.state; + if (resuming && next.current) { + setSavedRevision(0); + setStatus(next); + } + if (resuming && next.state !== "rendering") return next.state; + if (farmSave && !resuming) { + const now = Date.now(); + if (next.progress > farmSave.progress || next.playable) { + farmSave = { progress: next.progress, progressAt: now }; + } + if ( + next.state === "error" || + (next.state === "rendering" && + !next.playable && + now - farmSave.progressAt > FARM_STALL_MS) + ) { + farmSave = null; + console.warn( + "Cap save renders in this browser:", + next.error ?? "the render farm stopped making progress", + ); + await saveInBrowser(); + return; + } + } + setStatus(next); + if (next.state === "rendering") timer = setTimeout(poll, POLL_MS); + else { + farmSave = null; + if (next.state === "error" && !resuming) + toast.error(next.error ?? "Save failed"); + } + return next.state; + } catch { + if (!disposed && !resuming) timer = setTimeout(poll, POLL_MS * 2); + } + }; + + const saveInBrowser = async () => { + setStatus(null); + setBrowserSave({ stage: "rendering", progress: 0 }); + try { + await invoke("webEditorSaveInBrowser", { + channel: new Channel((progress) => { + if (!disposed) setBrowserSave(progress); + }), + }); + if (disposed) return; + setStatus({ + state: "ready", + exportId: null, + progress: 1, + playable: false, + hlsUrl: null, + error: null, + }); + } catch (cause) { + if (disposed) return; + const error = cause instanceof Error ? cause.message : "Save failed"; + setStatus({ + state: "error", + exportId: null, + progress: 0, + playable: false, + hlsUrl: null, + error, + }); + toast.error(error); + } finally { + setBrowserSave(null); + } + }; + + onMount(() => { + void poll(true).then((state) => { + if (publishOnOpen && state === "idle") void save(true); + }); + }); + onCleanup(() => { + disposed = true; + clearTimeout(timer); + }); + + const rendering = () => + status()?.state === "rendering" || browserSave() !== null; + const hasUnsavedEdits = () => projectRevision() !== savedRevision(); + const upToDate = () => + !hasUnsavedEdits() && !rendering() && status()?.state === "ready"; + + // The page asks before closing while this session has edits its share link + // doesn't show yet: not saved, still rendering in this tab, or failed. + const unpublishedEdits = () => + projectRevision() !== (savedRevision() ?? 0) || + browserSave() !== null || + status()?.state === "error"; + const savingHere = () => browserSave() !== null; + const editorWindow = window as Window & { + capWebEditorUnpublishedEdits?: () => boolean; + capWebEditorSavingHere?: () => boolean; + }; + editorWindow.capWebEditorUnpublishedEdits = unpublishedEdits; + editorWindow.capWebEditorSavingHere = savingHere; + onCleanup(() => { + if (editorWindow.capWebEditorUnpublishedEdits === unpublishedEdits) + delete editorWindow.capWebEditorUnpublishedEdits; + if (editorWindow.capWebEditorSavingHere === savingHere) + delete editorWindow.capWebEditorSavingHere; + }); + + const save = async (automatic = false) => { + if (starting() || rendering()) return; + setStarting(true); + try { + // Save renders the stored project, so edits still debouncing land first. + await flushProjectConfig(); + const revision = projectRevision(); + const started = await invoke("webEditorSave"); + setSavedRevision(revision); + if (started.renderer === "browser") { + console.info("Cap save renders in this browser:", started.reason); + setStarting(false); + await saveInBrowser(); + return; + } + farmSave = { progress: 0, progressAt: Date.now() }; + setStatus({ + state: "rendering", + exportId: null, + progress: 0, + playable: false, + hlsUrl: null, + error: null, + }); + timer = setTimeout(poll, FIRST_POLL_MS); + } catch (cause) { + const message = cause instanceof Error ? cause.message : "Save failed"; + if (automatic) + console.warn("Cap could not publish the recording:", message); + else toast.error(message); + } finally { + setStarting(false); + } + }; + + // Both can render on this device, so the renderer starts loading as soon + // as the person reaches for either. + const prewarmExport = () => { + void invoke("webEditorPrewarmExport").catch(() => undefined); + }; + + const openDownload = () => { + setEditorState("timeline", "selection", null); + trackEvent("export_button_clicked"); + if (exportState.type === "done") setExportState({ type: "idle" }); + setDialog({ type: "export", open: true }); + }; + + return ( +
+ } + onClick={openDownload} + onPointerEnter={prewarmExport} + onFocus={prewarmExport} + > + Download + + + + +
+ ); +} + +function browserSaveLabel({ stage, progress }: BrowserSaveProgress) { + const percent = Math.floor(progress * 100); + if (stage === "uploading") return `Uploading ${percent}%`; + return percent > 0 ? `Rendering ${percent}%` : "Rendering"; +} diff --git a/apps/desktop/src/styles/theme.css b/apps/desktop/src/styles/theme.css index 166b2ebf933..9a604dc0dcd 100644 --- a/apps/desktop/src/styles/theme.css +++ b/apps/desktop/src/styles/theme.css @@ -35,6 +35,7 @@ --track-scene: #8b5cf6; --track-audio: #22c55e; --track-3d: #6366f1; + --track-waveform: #84cc16; /* Editor surfaces: a layered, low-contrast system (window < card < control) * with one accent. Mirrored 1:1 by EditorPalette in diff --git a/apps/desktop/src/utils/dropped-files.ts b/apps/desktop/src/utils/dropped-files.ts new file mode 100644 index 00000000000..30169f72b51 --- /dev/null +++ b/apps/desktop/src/utils/dropped-files.ts @@ -0,0 +1,9 @@ +/** + * A path the editor's import commands accept for a file dropped on the + * window. The desktop app receives native drops as real paths through Tauri, + * so there is nothing to register here; the web editor swaps this module for + * one that hands the File to its file-picker bridge. + */ +export function pathForDroppedFile(_file: File): string | null { + return null; +} diff --git a/apps/desktop/src/utils/organization-branding.ts b/apps/desktop/src/utils/organization-branding.ts index 9eb38b9986b..d461fe3a568 100644 --- a/apps/desktop/src/utils/organization-branding.ts +++ b/apps/desktop/src/utils/organization-branding.ts @@ -180,6 +180,16 @@ function normalizeDesktopOrganizations(values: unknown) { }); } +const [webOrganizations, setWebOrganizations] = createSignal< + DesktopOrganization[] +>([]); +const [webAvailability, setWebAvailability] = + createSignal("loading"); +const [webRefreshing, setWebRefreshing] = createSignal(false); +let webOrganizationRefreshPromise: Promise | null = null; +let webOrganizationUserId = ""; +let webOrganizationGeneration = 0; + function hasLocalOrganizationAuth( auth: CachedAuthStore | null | undefined, ): auth is CachedAuthWithLocalSession { @@ -261,7 +271,73 @@ export function hasAvailableOrganizationCache( return hasCompleteOrganizationCache(auth, organizations, now); } +function createWebOrganizationsQuery() { + const userId = + (window as Window & { capWebEditorUserId?: string }).capWebEditorUserId ?? + ""; + if (userId !== webOrganizationUserId) { + webOrganizationUserId = userId; + webOrganizationGeneration++; + webOrganizationRefreshPromise = null; + setWebOrganizations([]); + setWebAvailability("loading"); + } + const refresh = async () => { + if (webOrganizationRefreshPromise) return webOrganizationRefreshPromise; + const generation = webOrganizationGeneration; + setWebRefreshing(true); + const promise = (async () => { + const response = await fetch("/api/desktop/organizations", { + credentials: "same-origin", + cache: "no-store", + }); + if (response.status === 401) { + if (generation !== webOrganizationGeneration) return; + setWebOrganizations([]); + setWebAvailability("signed-out"); + return; + } + if (!response.ok) throw new Error("Unable to load organizations"); + const result: unknown = await response.json(); + if (!Array.isArray(result)) + throw new Error("Organization list was invalid"); + if (generation !== webOrganizationGeneration) return; + setWebOrganizations(normalizeDesktopOrganizations(result)); + setWebAvailability("available"); + })() + .catch((error: unknown) => { + if (generation === webOrganizationGeneration) + setWebAvailability("unavailable"); + throw error; + }) + .finally(() => { + if (generation === webOrganizationGeneration) setWebRefreshing(false); + if (webOrganizationRefreshPromise === promise) + webOrganizationRefreshPromise = null; + }); + webOrganizationRefreshPromise = promise; + setWebAvailability("loading"); + return promise; + }; + + createEffect(() => { + if (webAvailability() !== "loading" || webOrganizationRefreshPromise) + return; + void refresh().catch(console.error); + }); + + return { + availability: webAvailability, + organizations: webOrganizations, + refresh, + refreshing: webRefreshing, + signedIn: () => webAvailability() === "available", + }; +} + export function createDesktopOrganizationsQuery() { + if (import.meta.env.VITE_CAP_WEB_EDITOR === "true") + return createWebOrganizationsQuery(); const auth = authStore.createQuery(); const [refreshing, setRefreshing] = createSignal(false); @@ -438,6 +514,40 @@ export async function updateOrganizationBranding( organizationId: string, body: OrganizationBrandingPatchBody, ) { + if (import.meta.env.VITE_CAP_WEB_EDITOR === "true") { + const generation = webOrganizationGeneration; + const response = await fetch( + `/api/desktop/organizations/${encodeURIComponent(organizationId)}/branding`, + { + method: "PATCH", + headers: { "Content-Type": "application/json" }, + credentials: "same-origin", + body: JSON.stringify(body), + }, + ); + const result: unknown = response.headers + .get("Content-Type") + ?.startsWith("application/json") + ? await response.json() + : { + error: + response.status === 401 + ? "Please sign in to continue" + : "Failed to update organization branding", + }; + if (!response.ok) throw new Error(getResponseError(result)); + const organization = normalizeDesktopOrganization(result); + if (!organization || organization.id !== organizationId) + throw new Error("Organization branding response was invalid"); + if (generation === webOrganizationGeneration) { + setWebOrganizations((current) => + current.map((existing) => + existing.id === organization.id ? organization : existing, + ), + ); + } + return organization; + } const response = await apiClient.desktop.updateOrganizationBranding({ params: { organizationId }, headers: await protectedHeaders(), diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index 6f27d9856cb..8d8efe88e88 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -1179,7 +1179,7 @@ export type OnEscapePress = null export type Organization = { id: string; name: string; ownerId: string; role?: string; canEditBrand?: boolean; iconUrl?: string | null; brandColors?: OrganizationBrandColors } export type OrganizationBrandColors = { primary: string | null; secondary: string | null; accent: string | null; background: string | null } export type OverlayTrack = { kind: OverlayTrackKind; track: number } -export type OverlayTrackKind = "mask" | "image" | "text" +export type OverlayTrackKind = "mask" | "image" | "text" | "waveform" export type Phase = "awaitingShortcut" | "starting" | "recording" | "pausing" | "paused" | "resuming" | "resumeFailed" | "restarting" | "stopping" | "restoring" export type PhysicalSize = { width: number; height: number } export type Plan = { upgraded: boolean; manual: boolean; last_checked: number } @@ -1193,7 +1193,7 @@ export type PreparingEditorSeed = { title: string; tracks: string[] } export type PreparingPlaybackState = { playheadSeconds: number; playing: boolean; buffering: boolean } export type Preset = { name: string; config: ProjectConfiguration } export type PresetsStore = { presets: Preset[]; default: number | null } -export type ProjectConfiguration = { aspectRatio: AspectRatio | null; background: BackgroundConfiguration; camera: Camera; audio: AudioConfiguration; cursor: CursorConfiguration; hotkeys: HotkeysConfiguration; timeline: TimelineConfiguration | null; overlayOrder: OverlayTrack[]; captions: CaptionsData | null; keyboard: KeyboardData | null; clips: ClipConfiguration[]; annotations: Annotation[]; screenMotionBlur?: number; screenMovementSpring?: ScreenMovementSpring; +export type ProjectConfiguration = { aspectRatio: AspectRatio | null; background: BackgroundConfiguration; camera: Camera; audio: AudioConfiguration; cursor: CursorConfiguration; hotkeys: HotkeysConfiguration; timeline: TimelineConfiguration | null; overlayOrder: OverlayTrack[]; hideDisplay?: boolean; captions: CaptionsData | null; keyboard: KeyboardData | null; clips: ClipConfiguration[]; annotations: Annotation[]; screenMotionBlur?: number; screenMovementSpring?: ScreenMovementSpring; /** * Per-layer cinematic color grades. Field-level default keeps old * project files (and old saved presets) deserializing to the identity @@ -1350,7 +1350,7 @@ letterSpacing?: number; lineHeight?: number; opacity?: number; shadow?: number; * segment edges when `layout` is not `Overlay`. */ layoutTransition?: number } -export type TimelineConfiguration = { segments: TimelineSegment[]; transitions: ClipTransition[]; zoomSegments: ZoomSegment[]; sceneSegments?: SceneSegment[]; maskSegments?: MaskSegment[]; textSegments?: TextSegment[]; captionSegments?: CaptionTrackSegment[]; keyboardSegments?: KeyboardTrackSegment[]; audioSegments?: AudioTrackSegment[]; styleSegments: StyleSegment[]; imageSegments: ImageSegment[]; camera3dSegments?: Camera3DSegment[] } +export type TimelineConfiguration = { segments: TimelineSegment[]; transitions: ClipTransition[]; zoomSegments: ZoomSegment[]; sceneSegments?: SceneSegment[]; maskSegments?: MaskSegment[]; textSegments?: TextSegment[]; captionSegments?: CaptionTrackSegment[]; keyboardSegments?: KeyboardTrackSegment[]; audioSegments?: AudioTrackSegment[]; styleSegments: StyleSegment[]; imageSegments: ImageSegment[]; camera3dSegments?: Camera3DSegment[]; waveformSegments?: WaveformSegment[] } export type TimelineSegment = { recordingSegment?: number; timescale: number; start: number; end: number; name?: string | null; speedAudioMode?: ClipSpeedAudioMode | null; volume?: number | null; hideCursor?: boolean | null } export type TranscriptionEngine = "Whisper" | "Parakeet" export type Trigger = "screenshotTaken" | "studioRecordingFinished" | "instantRecordingFinished" | "recordingStarted" | "uploadCompleted" | "videoImported" | "recordingDeleted" @@ -1369,6 +1369,9 @@ export type VideoMeta = { path: string; fps?: number; start_time?: number | null export type VideoRecordingMetadata = { duration: number; size: number } export type VideoUploadInfo = { id: string; link: string; config: S3UploadMeta } export type VoiceIsolation = "light" | "balanced" | "strong" +export type WaveformSegment = { start: number; end: number; track: number; enabled: boolean; center: XY; size: XY; opacity: number; style: WaveformStyle; color: string; secondaryColor: string | null; barCount: number; barWidth: number; rounding: number; sensitivity: number; smoothing: number; source: WaveformSource } +export type WaveformSource = "mix" | "mic" | "system" +export type WaveformStyle = "bars" | "mirrored" | "line" | "dots" export type WindowExclusion = { bundleIdentifier?: string | null; ownerName?: string | null; windowTitle?: string | null } export type WindowId = string export type WindowPosition = { x: number; y: number; displayId?: DisplayId | null } diff --git a/apps/desktop/vitest.config.ts b/apps/desktop/vitest.config.ts index 71faffc677d..f4daaa82f1f 100644 --- a/apps/desktop/vitest.config.ts +++ b/apps/desktop/vitest.config.ts @@ -4,6 +4,14 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { includeSource: ["src/**/*.{js,ts,jsx,tsx}"], + server: { + deps: { + inline: [ + /@solid-primitives\/(deep|event-listener|history|utils)/, + /solid-js\/store/, + ], + }, + }, }, resolve: { alias: { diff --git a/apps/media-server/Dockerfile b/apps/media-server/Dockerfile index 5763347374a..772659984a1 100644 --- a/apps/media-server/Dockerfile +++ b/apps/media-server/Dockerfile @@ -6,10 +6,15 @@ RUN apt-get update \ WORKDIR /app -COPY apps/media-server/package.json apps/media-server/bun.lock ./ -RUN bun install --frozen-lockfile --production +COPY package.json bun.lock bunfig.toml ./ +COPY apps/media-server/package.json ./apps/media-server/package.json +COPY packages/editor-cap-bundle/package.json ./packages/editor-cap-bundle/package.json +RUN bun install --frozen-lockfile --production --filter=@cap/media-server -COPY apps/media-server/src ./src +COPY packages/editor-cap-bundle/src ./packages/editor-cap-bundle/src +COPY apps/media-server/src ./apps/media-server/src + +WORKDIR /app/apps/media-server RUN bun test src/__tests__/lib/audio-quality-policy.test.ts src/__tests__/lib/audio-quality.integration.test.ts \ && bun test src/__tests__/lib/audio-quality-formats.integration.test.ts src/__tests__/lib/audio-quality-benchmark.test.ts \ diff --git a/apps/media-server/Dockerfile.editor b/apps/media-server/Dockerfile.editor new file mode 100644 index 00000000000..2843f79bfa8 --- /dev/null +++ b/apps/media-server/Dockerfile.editor @@ -0,0 +1,87 @@ +FROM rust:1.90-trixie@sha256:e227f20ec42af3ea9a3c9c1dd1b2012aa15f12279b5e9d5fb890ca1c2bb5726c AS native-builder + +ARG TARGETARCH + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + clang libclang-dev libssl-dev libasound2-dev libx11-dev libvulkan-dev \ + ca-certificates curl \ + pkg-config ffmpeg libavcodec-dev libavformat-dev libavfilter-dev \ + libavdevice-dev libavutil-dev libswscale-dev libswresample-dev libpostproc-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /source +COPY Cargo.toml Cargo.lock ./ +COPY crates ./crates +COPY vendor ./vendor +COPY apps/cli/Cargo.toml ./apps/cli/Cargo.toml +COPY apps/cli/src ./apps/cli/src +COPY apps/desktop/src-tauri/Cargo.toml ./apps/desktop/src-tauri/Cargo.toml +COPY apps/desktop/src-tauri/src ./apps/desktop/src-tauri/src + +ENV CARGO_PROFILE_RELEASE_OPT_LEVEL=3 \ + CARGO_PROFILE_RELEASE_DEBUG=0 \ + CARGO_PROFILE_RELEASE_CODEGEN_UNITS=4 + +RUN --mount=type=cache,id=s/c1ccb15c-25d3-437c-9e17-d8ca9c6b74d7-/usr/local/cargo/registry,target=/usr/local/cargo/registry \ + --mount=type=cache,id=s/c1ccb15c-25d3-437c-9e17-d8ca9c6b74d7-/source/target,target=/source/target \ + cargo build --release --locked -p cap-export --bin web-editor -p cap-editor --bin web-editor-service --features cap-export/web-editor-camera-removal,cap-editor/web-editor-camera-removal \ + && install -Dm755 target/release/web-editor /out/web-editor \ + && install -Dm755 target/release/web-editor-service /out/web-editor-service + +RUN set -eu; \ + case "${TARGETARCH}" in \ + amd64) ort_arch=x64; ort_digest=4c436a280d650f8bf32c921a2bf4de7c42cc32884c51c90e47de991708bbb5a4 ;; \ + arm64) ort_arch=aarch64; ort_digest=15100fb88b4c692cdd6bf2cca5f4a26a3806cebca8136de6681e2aba4b2ea033 ;; \ + *) exit 1 ;; \ + esac; \ + ort_asset="onnxruntime-linux-${ort_arch}-1.24.3"; \ + ort_archive=/tmp/onnxruntime.tgz; \ + curl --fail --location --retry 3 "https://github.com/microsoft/onnxruntime/releases/download/v1.24.3/${ort_asset}.tgz" --output "${ort_archive}"; \ + printf '%s %s\n' "${ort_digest}" "${ort_archive}" | sha256sum --check; \ + tar -xzf "${ort_archive}" -C /tmp; \ + install -Dm755 "/tmp/${ort_asset}/lib/libonnxruntime.so" /out/libonnxruntime.so; \ + install -Dm755 "/tmp/${ort_asset}/lib/libonnxruntime_providers_shared.so" /out/libonnxruntime_providers_shared.so; \ + install -Dm644 "/tmp/${ort_asset}/LICENSE" /out/onnxruntime/LICENSE; \ + install -Dm644 "/tmp/${ort_asset}/ThirdPartyNotices.txt" /out/onnxruntime/ThirdPartyNotices.txt + +FROM oven/bun:1.4.0@sha256:5ff609364c049b54eb0ff560ec96319729a972078ef2c755d758f0c6ef89c2d6 AS worker-runtime + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ffmpeg libasound2t64 libvulkan1 mesa-vulkan-drivers fontconfig fonts-dejavu-core \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY package.json bun.lock bunfig.toml ./ +COPY apps/media-server/package.json ./apps/media-server/package.json +COPY packages/editor-cap-bundle/package.json ./packages/editor-cap-bundle/package.json +RUN bun install --frozen-lockfile --production --filter=@cap/media-server +COPY packages/editor-cap-bundle/src ./packages/editor-cap-bundle/src +COPY apps/media-server/src ./apps/media-server/src +COPY apps/web/lib/editor-caption-access.ts apps/web/lib/editor-caption-transport.ts apps/web/lib/editor-recording-title.ts ./apps/web/lib/ +COPY apps/desktop/src/assets/music /opt/cap/music +COPY apps/desktop/src-tauri/assets/backgrounds /opt/cap/backgrounds + +WORKDIR /app/apps/media-server + +ENV CAP_WEB_EDITOR_PREPARE_BIN=/opt/cap/bin/web-editor \ + CAP_WEB_EDITOR_SERVICE_BIN=/opt/cap/bin/web-editor-service \ + CAP_WEB_EDITOR_ENABLE_CAMERA_REMOVAL=1 \ + CAP_WEB_EDITOR_MUSIC_DIR=/opt/cap/music \ + CAP_WEB_EDITOR_WALLPAPER_DIR=/opt/cap/backgrounds \ + ORT_DYLIB_PATH=/opt/cap/bin/libonnxruntime.so \ + PORT=3457 +EXPOSE 3457 + +FROM worker-runtime +COPY --from=native-builder /out/web-editor /opt/cap/bin/web-editor +COPY --from=native-builder /out/web-editor-service /opt/cap/bin/web-editor-service +COPY --from=native-builder /out/libonnxruntime.so /opt/cap/bin/libonnxruntime.so +COPY --from=native-builder /out/libonnxruntime_providers_shared.so /opt/cap/bin/libonnxruntime_providers_shared.so +COPY --from=native-builder /out/onnxruntime /opt/cap/licenses/onnxruntime + +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \ + CMD ["bun", "-e", "fetch(`http://127.0.0.1:${process.env.PORT || 3457}/health`).then((response) => process.exit(response.ok ? 0 : 1)).catch(() => process.exit(1))"] + +CMD ["bun", "run", "src/editor-worker.ts"] diff --git a/apps/media-server/Dockerfile.editor.dockerignore b/apps/media-server/Dockerfile.editor.dockerignore new file mode 100644 index 00000000000..8bbf1a5534f --- /dev/null +++ b/apps/media-server/Dockerfile.editor.dockerignore @@ -0,0 +1,41 @@ +** +!Cargo.toml +!Cargo.lock +!package.json +!bun.lock +!bunfig.toml +!crates +!crates/** +!vendor +!vendor/** +!packages +!packages/editor-cap-bundle +!packages/editor-cap-bundle/package.json +!packages/editor-cap-bundle/src +!packages/editor-cap-bundle/src/** +!apps +!apps/cli +!apps/cli/Cargo.toml +!apps/cli/src +!apps/cli/src/** +!apps/desktop +!apps/desktop/src +!apps/desktop/src/assets +!apps/desktop/src/assets/music +!apps/desktop/src/assets/music/** +!apps/desktop/src-tauri +!apps/desktop/src-tauri/Cargo.toml +!apps/desktop/src-tauri/assets +!apps/desktop/src-tauri/assets/backgrounds +!apps/desktop/src-tauri/assets/backgrounds/** +!apps/desktop/src-tauri/src +!apps/desktop/src-tauri/src/** +!apps/media-server +!apps/media-server/package.json +!apps/media-server/src +!apps/media-server/src/** +!apps/web +!apps/web/lib +!apps/web/lib/editor-caption-access.ts +!apps/web/lib/editor-caption-transport.ts +!apps/web/lib/editor-recording-title.ts diff --git a/apps/media-server/Dockerfile.standalone b/apps/media-server/Dockerfile.standalone index 00de144faab..2a60a230551 100644 --- a/apps/media-server/Dockerfile.standalone +++ b/apps/media-server/Dockerfile.standalone @@ -6,7 +6,8 @@ RUN apt-get update \ WORKDIR /app -COPY package.json bun.lock ./ +COPY package.standalone.json ./package.json +COPY bun.lock ./bun.lock RUN bun install --frozen-lockfile --production COPY src ./src diff --git a/apps/media-server/editor-worker.md b/apps/media-server/editor-worker.md new file mode 100644 index 00000000000..96bd19ad53b --- /dev/null +++ b/apps/media-server/editor-worker.md @@ -0,0 +1,53 @@ +# Web editor worker + +The Solid editor uses a dedicated Bun service that runs the desktop Rust renderer and exporter. The regular media server handles recording processing; the editor worker holds live projects, socket tickets, previews, and exports in one process. + +Create a separate service from the repository root with `apps/media-server/Dockerfile.editor` and set its Railway config file path to `/apps/media-server/railway.editor.toml`. The image builds `web-editor` and `web-editor-service` for Linux and includes ffmpeg, ffprobe, fonts, and a software Vulkan renderer. The worker starts at `src/editor-worker.ts` and answers `/health` only when both native binaries, ffprobe, and its public origin are available. +The image also includes the desktop wallpaper originals and thumbnails. `/health` requires a wallpaper asset so a broken image cannot accept editor sessions. +The image includes ONNX Runtime 1.24.3 for both Linux architectures. Its download is pinned to the SHA-256 digest published with the release asset. The web binaries enable camera background removal on Linux; desktop binaries retain their existing platform behavior. `/health` also requires the ONNX Runtime library when this effect is enabled. The paired-source worker test changes a camera effect and re-renders the same frame, then exercises the export path. +Local amd64 and arm64 images both built, passed `/health`, and completed the paired-source route test with camera removal enabled. The current amd64 image passed all three editor route tests and the camera-effect preview/export benchmark under emulation. Its 42 fps removal result is not a native-host performance measurement; production performance still needs measurement. + +The 1080p screen and 720p camera benchmark runs a 30-second paired recording through the Linux worker with a live H.264 viewer. The web-only renderer feature reuses the processed camera texture when a 25 fps decoded camera frame repeats at the 60 fps output clock. On a local arm64 software Vulkan container capped at eight CPUs and 6 GB, 12-second replays rendered 59.8 fps with camera removal off and 52.0 fps with it on; removal rendered 41.3 fps before this optimization. Four CPUs and 4 GB rendered 40.1 fps without removal and 31.1 fps with it, saturating the four CPU cores. No H.264 fallback occurred. The benchmark caught a stop-to-preview race: an immediate same-frame preview after playback initially timed out even though the next attempt took about 200 ms. The web service now waits for the playback engine to become inactive before acknowledging stop; both immediate previews returned within about 90 ms when warm, and switching camera removal returned in about 120 ms on the eight-CPU replay. Preview hashes were stable before and after playback. A saved two-second removal configuration exported 60 MP4 frames; its frame at one second measured 23.9 dB RGB PSNR against the native RGBA preview, versus 16.0 dB against an effect-off preview. The saturated moving-color fixture magnifies H.264 chroma differences. Removal remains below 60 fps on this CPU-only worker. Run `bun run bench:native-editor-camera-effects` inside a built Linux worker image to repeat the fixture, then test representative footage and the production host before sizing or enabling broad traffic. + +The same benchmark can bind-mount the repo-owned `apps/web/public/videos/home-two/webcam.mp4` and set `CAP_EDITOR_CAMERA_REAL_VIDEO` to its container path. It loops and scales that 19-second marketing webcam clip to 720×720 at 25 fps while retaining the 1080p screen source. On the eight-CPU arm64 software-rendered replay, the real-person camera reached 54.0 fps with removal, paused preview hashes remained stable, and the two-second export retained the subject matte. In the webcam region, the exported frame measured 24.1 dB PSNR against the removal preview and 10.0 dB against the effect-off preview. Visual inspection confirmed that preview and export cut out the person from the purple background. Set `CAP_EDITOR_CAMERA_BENCH_ARTIFACT_DIR` to a mounted directory to retain the RGBA previews and MP4 frame for inspection. This public marketing clip checks one person and lighting setup; representative production footage and the chosen host remain unverified. + +Configure the worker with the same `MEDIA_SERVER_WEBHOOK_SECRET` used by Next.js and set `CAP_WEB_EDITOR_PUBLIC_ORIGIN` to its public HTTPS origin. The browser connects to that origin for authenticated WebSockets and one-use export downloads. Set `CAP_WEB_EDITOR_WORKER_URL` on the Next.js service to an origin its server can reach. A Railway private hostname works only when Next.js runs in the same Railway project and environment; otherwise use the worker's public HTTPS origin. The regular media server does not host editor sessions; configure a dedicated editor worker for local development too. + +For more simultaneous editors, deploy individually addressed worker services and set a stable `CAP_WEB_EDITOR_WORKER_ID` on each. Configure Next.js with `CAP_WEB_EDITOR_WORKER_POOL`, a JSON array such as `[ { "id": "east", "url": "https://editor-east.example.com" }, { "id": "west", "url": "https://editor-west.example.com" } ]`. Each entry must use a distinct origin reachable from Next.js, and each worker needs its own public origin. Next.js picks a preferred worker from the video ID and tries another when the first is busy or unavailable. Preparation and session IDs carry the worker ID, so subsequent API calls, sockets, and downloads return to the process that created them. + +Keep one replica per worker service. Preparations, sessions, tickets, and export jobs live in process memory; Railway does not provide sticky routing between replicas of one service. Each worker admits one active editor session by default; set `CAP_WEB_EDITOR_MAX_SESSIONS` (1 to 16) on an instance sized for more. Idle sessions close after ten minutes. Saved project configuration survives reopening; live playback and running exports do not survive a worker restart. Keep worker IDs and private origins stable across deploys while sessions are active. + +Start capacity testing at four CPUs and 4 GB RAM. In a local Linux arm64 container limited to four CPUs, the 120-second paired fixture held about 60 preview frames per second at 960×540 and exported 1280×720 H.264/AAC in about 79 seconds, with roughly 691 MB of container memory during export. At two CPUs, preview produced about 27 frames per second. These are local fixture measurements, so production sizing requires the same tests on the chosen host and representative recordings. + +The 15-minute, 1080p paired-source browser replay on a local Apple M4 Max exposed the PNG bandwidth limit: at 30 frames per second, PNG packing averaged 24.6 ms and produced about 174 Mbps for this fixture. Chrome's WebSocket download throttled to 20 Mbps and 50 ms latency painted only 3.2 frames per second on that transport. The browser benchmark counts decoded pixels separately from wire packets. + +The desktop editor requests 60 preview frames per second. At that rate, the renderer produced enough frames, but PNG packing averaged 24.8 ms and delivered 41.0 frames per second to the native socket and 39.7 frames per second to Chromium. The web service now uses H.264 Annex B packets through the authenticated frame socket during playback and retains PNG for paused frames and previews. A ticketed browser socket starts at the lower bitrate, measures its own worker WebSocket with a one-use 512 KiB random probe, then selects the higher bitrate only when delivery exceeds 12 Mbps. The native service skips an uncompressed stale motion frame when a new socket opens, and browser decoding returns the same RGBA frame shape as desktop. The browser resets its decoder on a codec change and switches to PNG if the codec is unsupported or the packet stream loses continuity. + +With a 1920×1080 screen, a separate 1280×720 webcam, and 15-minute looped sources, repeated Chromium replays at 60 requested frames per second painted about 60 frames per second through H.264 on the local Apple M4 Max. At a CDP-emulated 20 Mbps download and 50 ms latency, the higher bitrate used about 18 Mbps. At 10 Mbps with the same latency, the lower bitrate used about 6 Mbps during browser playback. Both profiles passed a live seek, a paused preview whose browser pixels matched the native renderer's SHA-256 hash exactly, a 20-second memory check, and a no-long-task check. Moving H.264 frames are lossy; paused edits and previews remain lossless. These fixture measurements do not establish production host capacity, arbitrary network quality, full UI parity, or visual equivalence during motion. Run `CAP_EDITOR_BENCHMARK_FPS=60 CAP_EDITOR_BENCHMARK_NATIVE_H264=1 CAP_EDITOR_BENCHMARK_SEEK_DURING_H264=1 CAP_EDITOR_BENCHMARK_DOWNLOAD_MBPS=10 bun src/__tests__/routes/editor-1080p-benchmark.ts` from `apps/media-server` with `CAP_WEB_EDITOR_PREPARE_BIN` and `CAP_WEB_EDITOR_SERVICE_BIN` pointing to the built native binaries. Change the download value to `20` to exercise the higher bitrate. + +Three high-motion frame samples at 10 Mbps measured 28.3–28.5 dB RGB PSNR and 4.2 mean absolute RGB error against exact native previews. At 20 Mbps, the same comparison measured 28.8–28.9 dB and 3.6 mean absolute RGB error. Use `CAP_EDITOR_BENCHMARK_MOVING_QUALITY=1` to repeat this check. The moving preview is responsive but does not meet byte-for-byte visual equality; the final export and paused preview use the native renderer. + +Playwright's WebKit engine also painted 60.2 frames per second through H.264 with a live seek, no long tasks or warnings, stable native worker memory during the 20-second sample, and paused pixels matching the native SHA-256 hash. Its decoded `VideoFrame.copyTo` returned planar bytes when the bridge requested packed RGBA, which visibly corrupted moving frames despite the frame-rate result. The bridge now checks the RGBA allocation size and returned plane layout, converting through an offscreen canvas when packed copy is unavailable. Three WebKit moving samples after the fix measured 27.8–27.9 dB RGB PSNR and 2.3 mean absolute RGB error against native previews. The Chromium regression replay still painted 60.2 frames per second and measured 28.8–28.9 dB PSNR with 3.6 mean absolute RGB error. These are Playwright browser-engine measurements on macOS; they do not establish identical results in every installed Safari or Chrome version. Set `CAP_EDITOR_BENCHMARK_BROWSER=webkit CAP_EDITOR_BENCHMARK_MOVING_QUALITY=1` to repeat the WebKit comparison. + +Isolated Chromium and WebKit UI replays mount the production Solid editor build with the production Next.js host bridge against a paired native session. They check a separate camera layer, the camera background-removal control, and the rendered export preview. A free-account replay shows the Cap Pro upgrade and no caption-generation action. A Pro replay clicks Generate Captions through the host bridge and confirms the returned caption words appear on the timeline; that replay simulates the caption route, while the web caption workflow test exercises the shared AssemblyAI transcription options against a real audio fixture. Both account flows passed in both browser engines without failed requests or page errors. Headless Chromium used the Canvas 2D fallback; WebKit used its normal renderer path. Each replay also blocks the shared Editor JavaScript chunk in a fresh page and verifies that the host receives a visible load failure. The Solid entrypoint retries failed preloads, reports mount success or failure over the existing message port, and cancels pending mounts when disposed. These replays do not exercise a signed-in Next.js page or all desktop editor interactions. Set both native binary paths and run `bun run test:native-editor-ui` from `apps/media-server` after `bun run --cwd packages/editor-solid-web build`. Set `CAP_EDITOR_SOLID_PUBLIC_DIR` to test an isolated build output and `CAP_EDITOR_UI_BROWSER=webkit` to run the WebKit flow. +Set `CAP_EDITOR_UI_PRO_CAPTIONS=1` to repeat it with a display recording that contains audio. That replay clicks Generate Captions, returns a fixture transcript through the production Next.js host bridge, and checks that the generated words appear on the editor timeline. The Free replay checks that the generation action is absent and no caption request is sent. Neither replay sends audio to AssemblyAI; the existing caption workflow tests and SDK stream replay cover the AssemblyAI job separately. Both replays check that local model download controls are absent from the web editor. + +Set `CAP_EDITOR_UI_SHARE_REPLAY=1` to run Reupload from the Solid UI through native MP4 rendering and the browser multipart client. The replay's local upload endpoint compares every uploaded byte, part size and ETag, and completion metadata against the native MP4, then requires the Reupload completion screen. Run it with the Free and Pro fixtures in Chromium and WebKit. This mode skips the separate Crop and camera background-removal checks, which remain in the normal UI replay and Linux worker tests. Multipart publication is mocked; this replay does not exercise authenticated Next.js routing or customer storage. + +The Pro UI replay also clicks Play after caption generation, waits for the visible clock to advance, and pauses playback. That path initializes the browser audio worklet and native playback in both Chromium and WebKit without page errors. Headless automation does not establish audible sound quality on a physical output device. + +An offline probe of 31 consecutive native frames found that fast lossless XOR-delta compression still needed about 139 Mbps with one full frame per second. The same benchmark encoded 60 rendered frames as 1080p H.264 with four threads and a zero-latency preset: CRF 18 averaged 14.6 Mbps and encoded at about 432 frames per second on the local Apple M4 Max. That two-second result is separate from the live encoder and Chromium measurements above. `CAP_EDITOR_BENCHMARK_DELTA_PROBE=1` and `CAP_EDITOR_BENCHMARK_H264_PROBE=1` run the optional probes. + +Imported audio and images upload directly to the owner's storage bucket, then the worker downloads the signed object into the editor project. The worker validates size, object identity, and decoded media before accepting the import. Image inspection uses the same Rust decoder and EXIF orientation handling as desktop, including TIFF files. + +The clip sidebar requests thumbnails only for visible clips. Both desktop and web use the same bounded FFmpeg decoder and JPEG settings; the worker caches each requested split time in the live project and returns a small JPEG data URL over the authenticated command socket. + +The shared Solid build copies all 106 desktop wallpaper files byte for byte. Saved configurations keep portable `cap-web-wallpaper://` paths. The worker maps those paths to `CAP_WEB_EDITOR_WALLPAPER_DIR` before rendering or exporting and maps them back when the browser reads configuration. The Docker image sets that directory to `/opt/cap/backgrounds`; local development can use the checked-in desktop assets. + +Configuration updates and saved projects accept media paths only under the session project. Wallpaper paths resolve from the bundled desktop assets, and imported image, audio, and video references must use their staged project paths. The same mapping covers style-track background overrides and rejects worker-local paths before forwarding a save or live update to Rust. + +The pull-request workflow builds amd64 and arm64 images and exercises paired-source preparation, imports, clip thumbnails, wallpapers, preview, WebSockets, export, and download inside each image. The web build produces the shared Solid editor assets before running Next.js. + +The saved-clip replay verifies two native timeline segments, a separate 25 fps camera source with its capture offset, the imported clip's audio, preview pixels before and after the clip boundary, and a 150-frame MP4 export. It runs in the Linux worker workflow. For a local replay after building `web-editor` and `web-editor-service`, set `CAP_WEB_EDITOR_PREPARE_BIN` and `CAP_WEB_EDITOR_SERVICE_BIN` to the built binaries and run `bun run test:native-editor-clips` from `apps/media-server`. + +The separate `bun run bench:native-editor-clips` command builds a 15-minute source from repeated 640×360 test footage and fills the project with 49 screen clips and 49 camera clips. On a macOS arm64 debug build, that synthetic project prepared in 2.13 seconds; ten sampled previews across the timeline took 6.4–24 ms, and the native editor service reached 214.5 MB resident memory after the seeks. These measurements exercise clip count and long-timeline navigation, but repeated low-resolution footage cannot stand in for representative 1080p recordings or a signed-in browser session. diff --git a/apps/media-server/package.json b/apps/media-server/package.json index a3b6619804c..271fd5f7ca7 100644 --- a/apps/media-server/package.json +++ b/apps/media-server/package.json @@ -7,9 +7,15 @@ "start": "bun src/index.ts", "build": "echo 'No build step needed for Bun'", "test": "bun test", + "test:browser-cap-import": "bun src/__tests__/routes/editor-browser-cap-directory-replay.ts", + "test:native-editor-clips": "bun src/__tests__/routes/editor-clips-replay.ts", + "test:native-editor-ui": "bun src/__tests__/routes/editor-solid-ui-replay.ts", + "bench:native-editor-clips": "bun src/__tests__/routes/editor-clips-benchmark.ts", + "bench:native-editor-camera-effects": "bun src/__tests__/routes/editor-camera-effects-benchmark.ts", "test:watch": "bun test --watch" }, "dependencies": { + "@cap/editor-cap-bundle": "workspace:*", "@mediabunny/server": "^1.45.2", "hono": "^4.12.12", "mediabunny": "^1.45.2", @@ -17,6 +23,7 @@ "zod": "^3.24.2" }, "devDependencies": { + "@playwright/test": "^1.51.1", "@types/bun": "latest" }, "trustedDependencies": [ diff --git a/apps/media-server/package.standalone.json b/apps/media-server/package.standalone.json new file mode 100644 index 00000000000..1bfa99680b4 --- /dev/null +++ b/apps/media-server/package.standalone.json @@ -0,0 +1,19 @@ +{ + "name": "@cap/media-server", + "private": true, + "type": "module", + "scripts": { + "start": "bun src/index.ts" + }, + "dependencies": { + "@mediabunny/server": "^1.45.2", + "hono": "^4.12.12", + "mediabunny": "^1.45.2", + "node-av": "^5.2.4", + "zod": "^3.24.2" + }, + "devDependencies": { + "@types/bun": "latest" + }, + "trustedDependencies": ["node-av"] +} diff --git a/apps/media-server/railway.editor.toml b/apps/media-server/railway.editor.toml new file mode 100644 index 00000000000..ed14eed77a9 --- /dev/null +++ b/apps/media-server/railway.editor.toml @@ -0,0 +1,9 @@ +[build] +dockerfilePath = "apps/media-server/Dockerfile.editor" + +[deploy] +startCommand = "bun run src/editor-worker.ts" +healthcheckPath = "/health" +healthcheckTimeout = 60 +restartPolicyType = "ALWAYS" +restartPolicyMaxRetries = 50 diff --git a/apps/media-server/src/__tests__/editor-worker-app.test.ts b/apps/media-server/src/__tests__/editor-worker-app.test.ts new file mode 100644 index 00000000000..2bfcb5cabe5 --- /dev/null +++ b/apps/media-server/src/__tests__/editor-worker-app.test.ts @@ -0,0 +1,77 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { randomUUID } from "node:crypto"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import app from "../editor-worker-app"; +import { editorWallpaperDirectory } from "../lib/editor-wallpapers"; + +const originalPrepare = process.env.CAP_WEB_EDITOR_PREPARE_BIN; +const originalService = process.env.CAP_WEB_EDITOR_SERVICE_BIN; +const originalPublicOrigin = process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN; +const originalWallpaperDirectory = process.env.CAP_WEB_EDITOR_WALLPAPER_DIR; +const originalCameraRemoval = process.env.CAP_WEB_EDITOR_ENABLE_CAMERA_REMOVAL; +const originalOrtPath = process.env.ORT_DYLIB_PATH; + +afterEach(() => { + if (originalPrepare === undefined) + delete process.env.CAP_WEB_EDITOR_PREPARE_BIN; + else process.env.CAP_WEB_EDITOR_PREPARE_BIN = originalPrepare; + if (originalService === undefined) + delete process.env.CAP_WEB_EDITOR_SERVICE_BIN; + else process.env.CAP_WEB_EDITOR_SERVICE_BIN = originalService; + if (originalPublicOrigin === undefined) + delete process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN; + else process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN = originalPublicOrigin; + if (originalWallpaperDirectory === undefined) + delete process.env.CAP_WEB_EDITOR_WALLPAPER_DIR; + else process.env.CAP_WEB_EDITOR_WALLPAPER_DIR = originalWallpaperDirectory; + if (originalCameraRemoval === undefined) + delete process.env.CAP_WEB_EDITOR_ENABLE_CAMERA_REMOVAL; + else process.env.CAP_WEB_EDITOR_ENABLE_CAMERA_REMOVAL = originalCameraRemoval; + if (originalOrtPath === undefined) delete process.env.ORT_DYLIB_PATH; + else process.env.ORT_DYLIB_PATH = originalOrtPath; +}); + +describe("dedicated editor worker", () => { + test("reports readiness only with both native binaries and a public origin", async () => { + delete process.env.CAP_WEB_EDITOR_ENABLE_CAMERA_REMOVAL; + delete process.env.CAP_WEB_EDITOR_PREPARE_BIN; + delete process.env.CAP_WEB_EDITOR_SERVICE_BIN; + delete process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN; + const unavailable = await app.request("/health"); + expect(unavailable.status).toBe(503); + + process.env.CAP_WEB_EDITOR_PREPARE_BIN = process.execPath; + process.env.CAP_WEB_EDITOR_SERVICE_BIN = process.execPath; + process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN = "http://127.0.0.1:3457"; + const wallpaperDirectory = editorWallpaperDirectory(); + process.env.CAP_WEB_EDITOR_WALLPAPER_DIR = join( + tmpdir(), + `cap-editor-missing-wallpapers-${randomUUID()}`, + ); + expect((await app.request("/health")).status).toBe(503); + process.env.CAP_WEB_EDITOR_WALLPAPER_DIR = wallpaperDirectory; + const ready = await app.request("/health"); + expect(ready.status).toBe(200); + expect(await ready.json()).toEqual({ status: "ok" }); + process.env.CAP_WEB_EDITOR_ENABLE_CAMERA_REMOVAL = "1"; + delete process.env.ORT_DYLIB_PATH; + expect((await app.request("/health")).status).toBe(503); + process.env.ORT_DYLIB_PATH = join( + tmpdir(), + `cap-editor-missing-ort-${randomUUID()}`, + ); + expect((await app.request("/health")).status).toBe(503); + process.env.ORT_DYLIB_PATH = process.execPath; + expect((await app.request("/health")).status).toBe(200); + }); + + test("serves editor routes without the media processing routes", async () => { + const editor = await app.request("/editor/preparations", { + method: "POST", + }); + expect(editor.status).toBe(401); + const media = await app.request("/video/process", { method: "POST" }); + expect(media.status).toBe(404); + }); +}); diff --git a/apps/media-server/src/__tests__/fixtures/editor-clips/camera-green.webm b/apps/media-server/src/__tests__/fixtures/editor-clips/camera-green.webm new file mode 100644 index 00000000000..ad8f7e6df52 Binary files /dev/null and b/apps/media-server/src/__tests__/fixtures/editor-clips/camera-green.webm differ diff --git a/apps/media-server/src/__tests__/fixtures/editor-clips/clip-blue-audio.mp4 b/apps/media-server/src/__tests__/fixtures/editor-clips/clip-blue-audio.mp4 new file mode 100644 index 00000000000..19cbff93774 Binary files /dev/null and b/apps/media-server/src/__tests__/fixtures/editor-clips/clip-blue-audio.mp4 differ diff --git a/apps/media-server/src/__tests__/fixtures/editor-clips/display-red.webm b/apps/media-server/src/__tests__/fixtures/editor-clips/display-red.webm new file mode 100644 index 00000000000..3cab79fad50 Binary files /dev/null and b/apps/media-server/src/__tests__/fixtures/editor-clips/display-red.webm differ diff --git a/apps/media-server/src/__tests__/fixtures/exif-orientation-6.jpg b/apps/media-server/src/__tests__/fixtures/exif-orientation-6.jpg new file mode 100644 index 00000000000..17e8fb8c2a4 Binary files /dev/null and b/apps/media-server/src/__tests__/fixtures/exif-orientation-6.jpg differ diff --git a/apps/media-server/src/__tests__/fixtures/rgb-tiff.tiff b/apps/media-server/src/__tests__/fixtures/rgb-tiff.tiff new file mode 100644 index 00000000000..1c625b6b723 Binary files /dev/null and b/apps/media-server/src/__tests__/fixtures/rgb-tiff.tiff differ diff --git a/apps/media-server/src/__tests__/fixtures/test-fragmented.mp4 b/apps/media-server/src/__tests__/fixtures/test-fragmented.mp4 new file mode 100644 index 00000000000..8cf97de96ec Binary files /dev/null and b/apps/media-server/src/__tests__/fixtures/test-fragmented.mp4 differ diff --git a/apps/media-server/src/__tests__/lib/editor-assets.test.ts b/apps/media-server/src/__tests__/lib/editor-assets.test.ts new file mode 100644 index 00000000000..e27da38df41 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/editor-assets.test.ts @@ -0,0 +1,106 @@ +import { expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdir, mkdtemp, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + stageSignedEditorAudioAsset, + validateEditorAudioAsset, +} from "../../lib/editor-assets"; + +test("uploaded audio is verified, staged, and reusable after reopening", async () => { + const previousHttp = process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = "1"; + const root = await mkdtemp(join(tmpdir(), "cap-editor-audio-asset-test-")); + const project = join(root, "recording.cap"); + const source = join(root, "track.mp3"); + let server: ReturnType | null = null; + try { + await mkdir(project); + const generated = spawnSync("ffmpeg", [ + "-v", + "error", + "-f", + "lavfi", + "-i", + "sine=frequency=880:sample_rate=48000:duration=2", + "-c:a", + "libmp3lame", + "-b:a", + "128k", + source, + ]); + expect(generated.status).toBe(0); + const bytes = (await stat(source)).size; + server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + const path = new URL(request.url).pathname; + if (path === "/track.mp3") { + return new Response(Bun.file(source), { + headers: { + "Content-Length": String(bytes), + ETag: JSON.stringify("asset-1"), + "Content-Type": "audio/mpeg", + }, + }); + } + if (path === "/invalid.mp3") { + return new Response("not audio", { + headers: { ETag: JSON.stringify("asset-1") }, + }); + } + return new Response("Not found", { status: 404 }); + }, + }); + const path = `assets/audio/import-${randomUUID()}.mp3`; + const asset = { + path, + name: "Imported tone", + url: `http://127.0.0.1:${server.port}/track.mp3`, + size: bytes, + contentType: "audio/mpeg", + objectIdentity: JSON.stringify("asset-1"), + }; + const first = await stageSignedEditorAudioAsset(project, asset); + expect(first).toMatchObject({ path, name: "Imported tone" }); + expect(first.duration).toBeGreaterThan(1.9); + expect(await Bun.file(join(project, path)).arrayBuffer()).toEqual( + await Bun.file(source).arrayBuffer(), + ); + expect(await stageSignedEditorAudioAsset(project, asset)).toEqual(first); + expect(() => + validateEditorAudioAsset({ ...asset, path: "../private.mp3" }), + ).toThrow("Invalid editor audio asset"); + expect(() => + validateEditorAudioAsset({ ...asset, contentType: "video/mp4" }), + ).toThrow("Invalid editor audio asset"); + const changed = { + ...asset, + path: `assets/audio/import-${randomUUID()}.mp3`, + objectIdentity: JSON.stringify("other"), + }; + await expect(stageSignedEditorAudioAsset(project, changed)).rejects.toThrow( + "identity changed", + ); + expect(await Bun.file(join(project, changed.path)).exists()).toBe(false); + const invalid = { + ...asset, + path: `assets/audio/import-${randomUUID()}.mp3`, + url: `http://127.0.0.1:${server.port}/invalid.mp3`, + size: 9, + }; + await expect(stageSignedEditorAudioAsset(project, invalid)).rejects.toThrow( + "no audio stream", + ); + expect(await Bun.file(join(project, invalid.path)).exists()).toBe(false); + } finally { + server?.stop(true); + await rm(root, { recursive: true, force: true }); + if (previousHttp === undefined) + delete process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + else process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = previousHttp; + } +}); diff --git a/apps/media-server/src/__tests__/lib/editor-audio-library.test.ts b/apps/media-server/src/__tests__/lib/editor-audio-library.test.ts new file mode 100644 index 00000000000..9e08ca8d490 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/editor-audio-library.test.ts @@ -0,0 +1,49 @@ +import { expect, test } from "bun:test"; +import { lstat, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + addEditorAudioLibraryTrack, + listEditorAudioLibrary, + stageSavedEditorAudioLibrary, +} from "../../lib/editor-audio-library"; + +test("the bundled desktop music can be added and restored on reopening", async () => { + const catalog = listEditorAudioLibrary(); + expect(catalog).toHaveLength(10); + expect(catalog[0]).toEqual({ + id: "lofi-beats-mirostar", + name: "Lofi Beats", + category: "Lo-Fi", + }); + const projectPath = await mkdtemp(join(tmpdir(), "cap-editor-music-test-")); + try { + const added = await addEditorAudioLibraryTrack( + projectPath, + "lofi-beats-mirostar", + ); + expect(added.path).toBe("assets/audio/library-lofi-beats-mirostar.mp3"); + expect(added.name).toBe("Lofi Beats"); + expect(added.duration).toBeGreaterThan(30); + const stagedPath = join(projectPath, added.path); + const first = await lstat(stagedPath); + expect(first.size).toBeGreaterThan(100_000); + await rm(stagedPath); + await stageSavedEditorAudioLibrary(projectPath, { + timeline: { + audioSegments: [ + { path: added.path }, + { path: added.path }, + { path: "assets/audio/library-foreign.mp3" }, + ], + }, + }); + const restored = await lstat(stagedPath); + expect(restored.size).toBe(first.size); + expect( + addEditorAudioLibraryTrack(projectPath, "../../recording"), + ).rejects.toThrow("Unknown library track"); + } finally { + await rm(projectPath, { recursive: true, force: true }); + } +}); diff --git a/apps/media-server/src/__tests__/lib/editor-cap-assets.test.ts b/apps/media-server/src/__tests__/lib/editor-cap-assets.test.ts new file mode 100644 index 00000000000..290e7243288 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/editor-cap-assets.test.ts @@ -0,0 +1,38 @@ +import { expect, test } from "bun:test"; +import { CAP_BUNDLE_CONTENT_TYPE } from "@cap/editor-cap-bundle"; +import { + type EditorCapAsset, + validateEditorCapAsset, +} from "../../lib/editor-cap-assets"; + +const valid: EditorCapAsset = { + path: "content/imports/00000000-0000-4000-8000-000000000000.capbundle", + name: "Studio recording", + url: "https://storage.cap.so/recording.capbundle", + size: 1024, + contentType: CAP_BUNDLE_CONTENT_TYPE, + objectIdentity: JSON.stringify("recording-etag"), +}; + +test("accepts a bounded, signed Cap project source", () => { + expect(() => validateEditorCapAsset(valid)).not.toThrow(); +}); + +test("rejects project path, type, URL, size, and identity mismatches", () => { + for (const invalid of [ + { ...valid, path: "content/imports/../../source.capbundle" }, + { + ...valid, + path: "content/videos/00000000-0000-4000-8000-000000000000.capbundle", + }, + { ...valid, contentType: "video/mp4" }, + { ...valid, url: "http://localhost/private" }, + { ...valid, url: "https://user:password@storage.cap.so/source" }, + { ...valid, size: 0 }, + { ...valid, size: 13 * 1024 * 1024 * 1024 }, + { ...valid, objectIdentity: "" }, + { ...valid, name: "../source" }, + ]) { + expect(() => validateEditorCapAsset(invalid)).toThrow(); + } +}); diff --git a/apps/media-server/src/__tests__/lib/editor-cap-bundle.test.ts b/apps/media-server/src/__tests__/lib/editor-cap-bundle.test.ts new file mode 100644 index 00000000000..29b6cf3ba1f --- /dev/null +++ b/apps/media-server/src/__tests__/lib/editor-cap-bundle.test.ts @@ -0,0 +1,73 @@ +import { expect, test } from "bun:test"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createCapBundle } from "@cap/editor-cap-bundle"; +import { extractEditorCapBundle } from "../../lib/editor-cap-bundle"; + +test("extracts separate screen, camera, audio, cursor, and keyboard files", async () => { + const content = [ + ["recording-meta.json", JSON.stringify({ version: 1 })], + ["content/display.mp4", "screen"], + ["content/camera.mp4", "camera"], + ["content/mic.wav", "microphone"], + ["content/system.wav", "system"], + ["content/cursor.json", "cursor"], + ["content/keyboard.json", "keyboard"], + ] as const; + const root = await mkdtemp(join(tmpdir(), "cap-bundle-test-")); + try { + const bundle = createCapBundle( + content.map(([path, value]) => ({ path, file: new Blob([value]) })), + ); + const path = join(root, "recording.capbundle"); + await writeFile(path, new Uint8Array(await bundle.arrayBuffer())); + const extracted = await extractEditorCapBundle(path); + try { + for (const [file, value] of content) { + expect(await readFile(join(extracted.path, file), "utf8")).toBe(value); + } + } finally { + await extracted.cleanup(); + } + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("rejects truncated or tampered bundles before creating project files", async () => { + const root = await mkdtemp(join(tmpdir(), "cap-bundle-test-")); + try { + const bundle = createCapBundle([ + { path: "recording-meta.json", file: new Blob(["{}"]) }, + { path: "content/display.mp4", file: new Blob(["screen"]) }, + ]); + const bytes = new Uint8Array(await bundle.arrayBuffer()); + const path = join(root, "recording.capbundle"); + await writeFile(path, bytes.slice(0, -1)); + await expect(extractEditorCapBundle(path)).rejects.toThrow("manifest"); + bytes[0] = 0; + await writeFile(path, bytes); + await expect(extractEditorCapBundle(path)).rejects.toThrow("header"); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("stops an import when its session is canceled", async () => { + const root = await mkdtemp(join(tmpdir(), "cap-bundle-test-")); + try { + const bundle = createCapBundle([ + { path: "recording-meta.json", file: new Blob(["{}"]) }, + ]); + const path = join(root, "recording.capbundle"); + await writeFile(path, new Uint8Array(await bundle.arrayBuffer())); + const controller = new AbortController(); + controller.abort(); + await expect( + extractEditorCapBundle(path, controller.signal), + ).rejects.toThrow("canceled"); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/apps/media-server/src/__tests__/lib/editor-clip-thumbnails.test.ts b/apps/media-server/src/__tests__/lib/editor-clip-thumbnails.test.ts new file mode 100644 index 00000000000..6cdb3b03b7f --- /dev/null +++ b/apps/media-server/src/__tests__/lib/editor-clip-thumbnails.test.ts @@ -0,0 +1,106 @@ +import { expect, test } from "bun:test"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { renderEditorClipThumbnail } from "../../lib/editor-clip-thumbnails"; + +type NativeSession = Parameters[0]; + +function fakeNative(response: Response, paths: string[]) { + return { + request: async (path: string) => { + paths.push(path); + return response; + }, + } as unknown as NativeSession; +} + +test("clip thumbnails preserve native JPEG bytes at the requested split time", async () => { + const jpeg = await readFile( + join(import.meta.dir, "../fixtures/exif-orientation-6.jpg"), + ); + const paths: string[] = []; + const result = await renderEditorClipThumbnail( + fakeNative( + new Response(jpeg, { + headers: { + "Content-Type": "image/jpeg", + "Content-Length": String(jpeg.length), + }, + }), + paths, + ), + 2, + 1.5, + ); + expect(paths).toEqual(["/clip-thumbnail/2/1500"]); + expect( + Buffer.from(result.slice("data:image/jpeg;base64,".length), "base64"), + ).toEqual(jpeg); +}); + +test("clip thumbnail requests reject invalid times and segments before native access", async () => { + const paths: string[] = []; + const native = fakeNative(new Response(null), paths); + for (const [segment, time] of [ + [-1, 0], + [0, -1], + [0.5, 0], + [0, Number.NaN], + [0, 43_201], + ]) { + await expect( + renderEditorClipThumbnail(native, segment, time), + ).rejects.toThrow("Invalid clip thumbnail request"); + } + expect(paths).toHaveLength(0); +}); + +test("clip thumbnails reject failed, oversized, and damaged native responses", async () => { + const paths: string[] = []; + await expect( + renderEditorClipThumbnail( + fakeNative(new Response(null, { status: 404 }), paths), + 0, + 0, + ), + ).rejects.toThrow("404"); + await expect( + renderEditorClipThumbnail( + fakeNative( + new Response("bad", { + headers: { "Content-Type": "text/plain", "Content-Length": "3" }, + }), + paths, + ), + 0, + 0, + ), + ).rejects.toThrow("invalid format"); + await expect( + renderEditorClipThumbnail( + fakeNative( + new Response("bad", { + headers: { + "Content-Type": "image/jpeg", + "Content-Length": String(256 * 1024 + 1), + }, + }), + paths, + ), + 0, + 0, + ), + ).rejects.toThrow("invalid size"); + await expect( + renderEditorClipThumbnail( + fakeNative( + new Response("bad", { + headers: { "Content-Type": "image/jpeg", "Content-Length": "3" }, + }), + paths, + ), + 0, + 0, + ), + ).rejects.toThrow("invalid size"); +}); diff --git a/apps/media-server/src/__tests__/lib/editor-command-socket.test.ts b/apps/media-server/src/__tests__/lib/editor-command-socket.test.ts new file mode 100644 index 00000000000..1bb80e8ed80 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/editor-command-socket.test.ts @@ -0,0 +1,438 @@ +import { expect, test } from "bun:test"; +import { + dispatchEditorSocketCommand, + type EditorCommandState, + type EditorSocketRequest, + parseEditorSocketRequest, +} from "../../lib/editor-command-socket"; + +function state(): EditorCommandState { + return { + frameNumber: 0, + fps: 60, + resolutionBase: { x: 960, y: 540 }, + }; +} + +function command(name: string, args: unknown[] = []): EditorSocketRequest { + return { kind: "invoke", id: 7, name, args }; +} + +test("the command socket rejects malformed envelopes before dispatch", () => { + expect( + parseEditorSocketRequest({ + kind: "invoke", + id: 0, + name: "seekTo", + args: [1], + }), + ).toBeNull(); + expect( + parseEditorSocketRequest({ + kind: "invoke", + id: 1, + name: "seekTo", + args: "1", + }), + ).toBeNull(); + expect( + parseEditorSocketRequest({ + kind: "emit", + id: 1, + name: "renderFrameEvent", + args: [{}], + }), + ).not.toBeNull(); +}); + +test("the browser instance exposes an opaque project path and no renderer loopback socket", async () => { + const native = async (path: string) => { + if (path === "/instance") { + return { + instanceId: "native-id", + path: "/private/user-media.cap", + framesSocketUrl: "ws://127.0.0.1:5000/frames", + audioSocketUrl: "ws://127.0.0.1:5000/audio", + eventsSocketUrl: "ws://127.0.0.1:5000/events", + }; + } + return null; + }; + const reply = await dispatchEditorSocketCommand( + "session-id", + command("createEditorInstance"), + state(), + native, + ); + expect(reply.kind).toBe("result"); + if (reply.kind !== "result") return; + expect(reply.value).toMatchObject({ + instanceId: "native-id", + path: "cap-web-editor://session/session-id", + framesSocketUrl: "", + audioSocketUrl: "", + eventsSocketUrl: "", + }); + const invalidPath = await dispatchEditorSocketCommand( + "session-id", + command("getRecordingMetaByPath", ["/private/user-media.cap"]), + state(), + native, + ); + expect(invalidPath).toMatchObject({ + kind: "error", + error: "Invalid editor project path", + }); +}); + +test("shareable link checks use the authenticated web editor session", async () => { + const native = async (path: string) => + path === "/instance" ? { recordingDuration: 420 } : null; + const projectPath = "cap-web-editor://session/session-id"; + const metadata = await dispatchEditorSocketCommand( + "session-id", + command("getVideoMetadata", [projectPath]), + state(), + native, + ); + expect(metadata).toEqual({ + kind: "result", + id: 7, + value: { duration: 420, size: (8_192_000 * 420) / (8 * 1024 * 1024) }, + }); + const invalid = await dispatchEditorSocketCommand( + "session-id", + command("getVideoMetadata", ["/private/user-media.cap"]), + state(), + native, + ); + expect(invalid).toMatchObject({ + kind: "error", + error: "Invalid editor project path", + }); + const plan = await dispatchEditorSocketCommand( + "session-id", + command("checkUpgradedAndUpdate"), + state(), + native, + ); + expect(plan).toEqual({ kind: "result", id: 7, value: false }); + expect( + await dispatchEditorSocketCommand( + "session-id", + command("checkUpgradedAndUpdate"), + { ...state(), captionsEnabled: true }, + native, + ), + ).toEqual({ kind: "result", id: 7, value: true }); +}); + +test("slider edits update memory while socket saves cannot bypass persistence", async () => { + const calls: Array<{ + path: string; + method: string; + body: unknown; + eventOnly: boolean; + }> = []; + const native = async ( + path: string, + method: string, + body?: unknown, + eventOnly = false, + ) => { + calls.push({ path, method, body, eventOnly }); + return null; + }; + const current = state(); + const config = { camera: { mirror: true } }; + const updated = await dispatchEditorSocketCommand( + "session-id", + command("updateProjectConfigInMemory", [ + config, + 120, + 60, + { x: 960, y: 540 }, + ]), + current, + native, + ); + expect(updated.kind).toBe("result"); + expect(calls.map((call) => call.path)).toEqual([ + "/config/memory", + "/preview", + ]); + expect(calls[1]?.eventOnly).toBe(true); + expect(calls[1]?.body).toEqual({ + frameNumber: 120, + fps: 60, + resolutionBase: { x: 960, y: 540 }, + }); + expect(current.frameNumber).toBe(120); + const saved = await dispatchEditorSocketCommand( + "session-id", + command("setProjectConfig", [config]), + current, + native, + ); + expect(saved).toMatchObject({ + kind: "error", + error: "Editor project saves require the authenticated web API", + }); + expect(calls).toHaveLength(2); + const stock = await dispatchEditorSocketCommand( + "session-id", + command("getDefaultProjectConfig"), + current, + native, + ); + expect(stock.kind).toBe("result"); + expect(calls[2]?.path).toBe("/default-config"); +}); + +test("a Free command socket cannot load captions or preview paid caption layers", async () => { + const calls: string[] = []; + const native = async (path: string) => { + calls.push(path); + return path === "/config" + ? { captions: { segments: [{ text: "Paid" }] } } + : null; + }; + const free = { ...state(), captionsEnabled: false }; + expect( + await dispatchEditorSocketCommand( + "session-id", + command("loadCaptions"), + free, + native, + ), + ).toEqual({ kind: "result", id: 7, value: null }); + expect(calls).toEqual([]); + expect( + await dispatchEditorSocketCommand( + "session-id", + command("updateProjectConfigInMemory", [ + { + captions: { + segments: [{ text: "Paid" }], + settings: { enabled: true }, + }, + }, + null, + null, + null, + ]), + free, + native, + ), + ).toMatchObject({ kind: "error", error: "Cap Pro is required for captions" }); + expect(calls).toEqual([]); + const pro = { ...state(), captionsEnabled: true }; + expect( + await dispatchEditorSocketCommand( + "session-id", + command("loadCaptions"), + pro, + native, + ), + ).toMatchObject({ + kind: "result", + value: { segments: [{ text: "Paid" }] }, + }); +}); + +test("two-hour word-level captions fit the editor memory command while oversized projects are rejected", async () => { + const words = Array.from({ length: 18_000 }, (_, index) => ({ + text: index % 7 === 0 ? "recording" : "process", + start: index * 0.4, + end: index * 0.4 + 0.24, + })); + const segments = Array.from({ length: 3_000 }, (_, index) => ({ + id: `segment-${index}`, + start: index * 2.4, + end: index * 2.4 + 2.24, + text: "recording process process process process process", + words: words.slice(index * 6, index * 6 + 6), + })); + const config = { + captions: { sourceTimed: true, segments }, + timeline: { captionSegments: segments }, + }; + expect(Buffer.byteLength(JSON.stringify(config), "utf8")).toBeGreaterThan( + 512 * 1024, + ); + const nativeConfigs: unknown[] = []; + const native = async (_path: string, _method: string, body?: unknown) => { + nativeConfigs.push(body); + return body; + }; + const current = state(); + const reply = await dispatchEditorSocketCommand( + "session-id", + command("updateProjectConfigInMemory", [config, null, null, null]), + current, + native, + ); + expect(reply.kind).toBe("result"); + expect(current.captionCache?.ref).toHaveLength(64); + const compact = { + captions: { sourceTimed: true }, + timeline: {}, + webCaptionRef: current.captionCache?.ref, + }; + expect(JSON.stringify(compact).length).toBeLessThan( + JSON.stringify(config).length / 20, + ); + const reused = await dispatchEditorSocketCommand( + "session-id", + command("updateProjectConfigInMemory", [compact, null, null, null]), + current, + native, + ); + expect(reused.kind).toBe("result"); + expect(nativeConfigs[1]).toEqual(config); + const mismatch = await dispatchEditorSocketCommand( + "session-id", + command("updateProjectConfigInMemory", [ + { ...compact, webCaptionRef: "0".repeat(64) }, + null, + null, + null, + ]), + current, + native, + ); + expect(mismatch).toMatchObject({ + kind: "error", + error: "Caption payload cache is unavailable", + }); + const oversized = await dispatchEditorSocketCommand( + "session-id", + command("updateProjectConfigInMemory", [ + { data: "x".repeat(8 * 1024 * 1024) }, + null, + null, + null, + ]), + state(), + native, + ); + expect(oversized.kind).toBe("error"); +}); + +test("seek and playback reuse the current frame and reject invalid dimensions", async () => { + const calls: Array<{ path: string; body: unknown }> = []; + const native = async (path: string, _method: string, body?: unknown) => { + calls.push({ path, body }); + return null; + }; + const current = state(); + const seek = await dispatchEditorSocketCommand( + "session-id", + command("seekTo", [90]), + current, + native, + ); + expect(seek.kind).toBe("result"); + const start = await dispatchEditorSocketCommand( + "session-id", + command("startPlayback", [30, { x: 640, y: 360 }]), + current, + native, + ); + expect(start.kind).toBe("result"); + expect(calls[0]).toMatchObject({ path: "/seek", body: { frameNumber: 90 } }); + expect(calls[1]).toEqual({ + path: "/playback", + body: { frameNumber: 90, fps: 30, resolutionBase: { x: 640, y: 360 } }, + }); + const invalid = await dispatchEditorSocketCommand( + "session-id", + command("startPlayback", [30, { x: 0, y: 360 }]), + current, + native, + ); + expect(invalid.kind).toBe("error"); + expect(calls).toHaveLength(2); +}); + +test("gradient randomization uses the desktop gradient generator", async () => { + const calls: string[] = []; + const gradient = { colorStops: [{ color: "#fff", position: 0 }] }; + const native = async (path: string) => { + calls.push(path); + return gradient; + }; + const reply = await dispatchEditorSocketCommand( + "session-id", + command("randomAnimatedGradient"), + state(), + native, + ); + expect(reply).toEqual({ kind: "result", id: 7, value: gradient }); + expect(calls).toEqual(["/animated-gradients/random"]); +}); + +test("keyboard generation uses the native project timeline and rejects invalid settings", async () => { + const calls: Array<{ path: string; method: string; body: unknown }> = []; + const native = async (path: string, method: string, body?: unknown) => { + calls.push({ path, method, body }); + return [{ start: 0.5, end: 1.25, text: "A" }]; + }; + const reply = await dispatchEditorSocketCommand( + "session-id", + command("generateKeyboardSegments", [350, 1800, true, false]), + state(), + native, + ); + expect(reply).toEqual({ + kind: "result", + id: 7, + value: [{ start: 0.5, end: 1.25, text: "A" }], + }); + expect(calls).toEqual([ + { + path: "/keyboard-segments", + method: "POST", + body: { + groupingThresholdMs: 350, + lingerDurationMs: 1800, + showModifiers: true, + showSpecialKeys: false, + }, + }, + ]); + const rejected = await dispatchEditorSocketCommand( + "session-id", + command("generateKeyboardSegments", [-1, 1800, true, false]), + state(), + native, + ); + expect(rejected.kind).toBe("error"); + expect(calls).toHaveLength(1); +}); + +test("auto zoom generation passes the saved zoom amount to the native cursor timeline", async () => { + const calls: Array<{ path: string; body: unknown }> = []; + const native = async (path: string, _method: string, body?: unknown) => { + calls.push({ path, body }); + return [{ start: 0.9, end: 6.7, amount: 1.8 }]; + }; + const reply = await dispatchEditorSocketCommand( + "session-id", + command("generateZoomSegmentsFromClicks", [1.8]), + state(), + native, + ); + expect(reply).toMatchObject({ kind: "result", value: [{ amount: 1.8 }] }); + expect(calls).toEqual([ + { path: "/auto-zoom-segments", body: { zoomAmount: 1.8 } }, + ]); + const invalid = await dispatchEditorSocketCommand( + "session-id", + command("generateZoomSegmentsFromClicks", [Infinity]), + state(), + native, + ); + expect(invalid.kind).toBe("error"); + expect(calls).toHaveLength(1); +}); diff --git a/apps/media-server/src/__tests__/lib/editor-config-paths.test.ts b/apps/media-server/src/__tests__/lib/editor-config-paths.test.ts new file mode 100644 index 00000000000..249ac46ea39 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/editor-config-paths.test.ts @@ -0,0 +1,110 @@ +import { expect, test } from "bun:test"; +import { join } from "node:path"; +import { + mapEditorConfigPaths, + mapEditorInstanceConfigPaths, +} from "../../lib/editor-config-paths"; +import { editorWallpaperDirectory } from "../../lib/editor-wallpapers"; + +const projectPath = "/tmp/cap-editor-config-test.cap"; +const image = "content/images/22222222-2222-4222-8222-222222222222.png"; +const video = "content/videos/33333333-3333-4333-8333-333333333333.mp4"; +const audio = "assets/audio/import-44444444-4444-4444-8444-444444444444.mp3"; +const wallpaper = + "cap-web-wallpaper://assets/backgrounds/macOS/tahoe-dusk-min.jpg"; + +test("web project paths resolve only staged media and portable wallpapers", () => { + const original = { + background: { source: { type: "wallpaper", path: wallpaper } }, + timeline: { + audioSegments: [{ path: audio }], + imageSegments: [{ path: image }], + videoSegments: [{ path: video }], + styleSegments: [ + { + overrides: { + background: { source: { type: "wallpaper", path: wallpaper } }, + }, + }, + ], + }, + }; + const native = mapEditorConfigPaths( + original, + "native", + projectPath, + ) as typeof original; + const nativeWallpaper = join( + editorWallpaperDirectory(), + "macOS/tahoe-dusk-min.jpg", + ); + expect(native.background.source.path).toBe(nativeWallpaper); + expect( + native.timeline.styleSegments[0]?.overrides.background.source.path, + ).toBe(nativeWallpaper); + expect(native.timeline.audioSegments[0]?.path).toBe(audio); + expect(native.timeline.imageSegments[0]?.path).toBe(image); + expect(native.timeline.videoSegments[0]?.path).toBe(video); + expect(mapEditorConfigPaths(native, "browser", projectPath)).toEqual( + original, + ); + expect( + mapEditorInstanceConfigPaths( + { savedProjectConfig: native, recordingDuration: 4 }, + "browser", + projectPath, + ), + ).toEqual({ savedProjectConfig: original, recordingDuration: 4 }); +}); + +test("background images use a project-local native path and a relative saved path", () => { + const original = { background: { source: { type: "image", path: image } } }; + const native = mapEditorConfigPaths( + original, + "native", + projectPath, + ) as typeof original; + expect(native.background.source.path).toBe(join(projectPath, image)); + expect(mapEditorConfigPaths(native, "browser", projectPath)).toEqual( + original, + ); +}); + +test("a chosen desktop wallpaper remains portable across editor sessions", () => { + const original = { + background: { source: { type: "wallpaper", path: image } }, + }; + const native = mapEditorConfigPaths( + original, + "native", + projectPath, + ) as typeof original; + expect(native.background.source.path).toBe(join(projectPath, image)); + expect(mapEditorConfigPaths(native, "browser", projectPath)).toEqual( + original, + ); +}); + +test("native configuration rejects worker-local and traversing media paths", () => { + for (const [track, path] of [ + ["audioSegments", "/tmp/secret.mp3"], + ["imageSegments", "content/images/../../secret.png"], + ["videoSegments", "content/videos/../../secret.mp4"], + ]) { + expect(() => + mapEditorConfigPaths( + { timeline: { [track]: [{ path }] } }, + "native", + projectPath, + ), + ).toThrow(); + } + for (const source of [ + { type: "wallpaper", path: "/tmp/secret.jpg" }, + { type: "image", path: "/tmp/secret.png" }, + ]) { + expect(() => + mapEditorConfigPaths({ background: { source } }, "native", projectPath), + ).toThrow(); + } +}); diff --git a/apps/media-server/src/__tests__/lib/editor-image-assets.test.ts b/apps/media-server/src/__tests__/lib/editor-image-assets.test.ts new file mode 100644 index 00000000000..49263993249 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/editor-image-assets.test.ts @@ -0,0 +1,179 @@ +import { expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdir, mkdtemp, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + stageSignedEditorImageAsset, + validateEditorImageAsset, +} from "../../lib/editor-image-assets"; + +test.skipIf(!process.env.CAP_WEB_EDITOR_PREPARE_BIN)( + "uploaded images are decoded, oriented, staged, and reusable after reopening", + async () => { + const previousHttp = process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = "1"; + const root = await mkdtemp(join(tmpdir(), "cap-editor-image-asset-test-")); + const project = join(root, "recording.cap"); + const source = join(root, "overlay.png"); + let server: ReturnType | null = null; + try { + await mkdir(project); + const generated = spawnSync("ffmpeg", [ + "-v", + "error", + "-f", + "lavfi", + "-i", + "color=c=magenta:s=120x80:duration=1", + "-frames:v", + "1", + "-threads", + "1", + source, + ]); + expect(generated.status).toBe(0); + const bytes = (await stat(source)).size; + const rotatedSource = join( + import.meta.dirname, + "../fixtures/exif-orientation-6.jpg", + ); + const rotatedBytes = (await stat(rotatedSource)).size; + const tiffSource = join(import.meta.dirname, "../fixtures/rgb-tiff.tiff"); + const tiffBytes = (await stat(tiffSource)).size; + server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + const path = new URL(request.url).pathname; + if (path === "/overlay.png") { + return new Response(Bun.file(source), { + headers: { + "Content-Length": String(bytes), + ETag: JSON.stringify("image-1"), + "Content-Type": "image/png", + }, + }); + } + if (path === "/rotated.jpg") { + return new Response(Bun.file(rotatedSource), { + headers: { + "Content-Length": String(rotatedBytes), + ETag: JSON.stringify("image-2"), + "Content-Type": "image/jpeg", + }, + }); + } + if (path === "/rgb-tiff.tiff") { + return new Response(Bun.file(tiffSource), { + headers: { + "Content-Length": String(tiffBytes), + ETag: JSON.stringify("image-3"), + "Content-Type": "image/tiff", + }, + }); + } + if (path === "/damaged.png") { + return new Response("not an image", { + headers: { ETag: JSON.stringify("image-1") }, + }); + } + return new Response("Not found", { status: 404 }); + }, + }); + const path = `content/images/${randomUUID()}.png`; + const asset = { + path, + name: "Overlay", + url: `http://127.0.0.1:${server.port}/overlay.png`, + size: bytes, + contentType: "image/png", + objectIdentity: JSON.stringify("image-1"), + }; + const first = await stageSignedEditorImageAsset(project, asset); + expect(first).toEqual({ path, name: "Overlay", width: 120, height: 80 }); + expect(await Bun.file(join(project, path)).arrayBuffer()).toEqual( + await Bun.file(source).arrayBuffer(), + ); + expect(await stageSignedEditorImageAsset(project, asset)).toEqual(first); + const rotated = { + ...asset, + path: `content/images/${randomUUID()}.jpg`, + url: `http://127.0.0.1:${server.port}/rotated.jpg`, + size: rotatedBytes, + contentType: "image/jpeg", + objectIdentity: JSON.stringify("image-2"), + }; + expect(await stageSignedEditorImageAsset(project, rotated)).toEqual({ + path: rotated.path, + name: "Overlay", + width: 80, + height: 120, + }); + expect(await Bun.file(join(project, rotated.path)).arrayBuffer()).toEqual( + await Bun.file(rotatedSource).arrayBuffer(), + ); + const tiff = { + ...asset, + path: `content/images/${randomUUID()}.tiff`, + name: "TIFF", + url: `http://127.0.0.1:${server.port}/rgb-tiff.tiff`, + size: tiffBytes, + contentType: "image/tiff", + objectIdentity: JSON.stringify("image-3"), + }; + expect(await stageSignedEditorImageAsset(project, tiff)).toEqual({ + path: tiff.path, + name: "TIFF", + width: 120, + height: 80, + }); + expect(await Bun.file(join(project, tiff.path)).arrayBuffer()).toEqual( + await Bun.file(tiffSource).arrayBuffer(), + ); + expect(() => + validateEditorImageAsset({ ...asset, path: "../private.png" }), + ).toThrow("Invalid editor image asset"); + expect(() => + validateEditorImageAsset({ ...asset, contentType: "video/mp4" }), + ).toThrow("Invalid editor image asset"); + const changed = { + ...asset, + path: `content/images/${randomUUID()}.png`, + objectIdentity: JSON.stringify("other"), + }; + await expect( + stageSignedEditorImageAsset(project, changed), + ).rejects.toThrow("identity changed"); + expect(await Bun.file(join(project, changed.path)).exists()).toBe(false); + const damaged = { + ...asset, + path: `content/images/${randomUUID()}.png`, + url: `http://127.0.0.1:${server.port}/damaged.png`, + size: 12, + }; + await expect( + stageSignedEditorImageAsset(project, damaged), + ).rejects.toThrow("no decodable image"); + expect(await Bun.file(join(project, damaged.path)).exists()).toBe(false); + const wrongFormat = { + ...asset, + path: `content/images/${randomUUID()}.jpg`, + contentType: "image/jpeg", + }; + await expect( + stageSignedEditorImageAsset(project, wrongFormat), + ).rejects.toThrow("format does not match"); + expect(await Bun.file(join(project, wrongFormat.path)).exists()).toBe( + false, + ); + } finally { + server?.stop(true); + await rm(root, { recursive: true, force: true }); + if (previousHttp === undefined) + delete process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + else process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = previousHttp; + } + }, +); diff --git a/apps/media-server/src/__tests__/lib/editor-media-duration.test.ts b/apps/media-server/src/__tests__/lib/editor-media-duration.test.ts new file mode 100644 index 00000000000..e225f5eb50d --- /dev/null +++ b/apps/media-server/src/__tests__/lib/editor-media-duration.test.ts @@ -0,0 +1,67 @@ +import { expect, test } from "bun:test"; +import { execFile } from "node:child_process"; +import { mkdtemp, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { downloadEditorMedia } from "../../lib/editor-media"; + +const runFile = promisify(execFile); + +test("finalizes a live WebM source before native editor preparation", async () => { + const root = await mkdtemp(join(tmpdir(), "cap-editor-live-webm-")); + const sourcePath = join(root, "live.webm"); + const previousHttpMedia = process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + let server: ReturnType | null = null; + try { + await runFile("ffmpeg", [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=size=160x90:rate=15:duration=2", + "-c:v", + "libvpx", + "-f", + "webm", + "-live", + "1", + sourcePath, + ]); + server = Bun.serve({ + port: 0, + fetch: () => new Response(Bun.file(sourcePath)), + }); + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = "1"; + const downloaded = await downloadEditorMedia({ + url: `http://127.0.0.1:${server.port}/live.webm`, + contentType: "video/webm", + size: (await stat(sourcePath)).size, + }); + try { + const { stdout } = await runFile("ffprobe", [ + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + downloaded.path, + ]); + expect(Number(stdout.trim())).toBeGreaterThan(1.8); + expect(downloaded.size).toBe((await stat(downloaded.path)).size); + } finally { + await downloaded.cleanup(); + } + } finally { + server?.stop(true); + if (previousHttpMedia === undefined) { + delete process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + } else { + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = previousHttpMedia; + } + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/apps/media-server/src/__tests__/lib/editor-process-env-replay.ts b/apps/media-server/src/__tests__/lib/editor-process-env-replay.ts new file mode 100644 index 00000000000..581aa474dbf --- /dev/null +++ b/apps/media-server/src/__tests__/lib/editor-process-env-replay.ts @@ -0,0 +1,30 @@ +import assert from "node:assert/strict"; +import { editorProcessEnv, runEditorFile } from "../../lib/editor-process"; + +const previousWebhook = process.env.MEDIA_SERVER_WEBHOOK_SECRET; +const previousAws = process.env.AWS_SECRET_ACCESS_KEY; +process.env.MEDIA_SERVER_WEBHOOK_SECRET = "webhook-secret-sentinel"; +process.env.AWS_SECRET_ACCESS_KEY = "aws-secret-sentinel"; +try { + const env = editorProcessEnv("internal-token-sentinel"); + assert.equal(env.CAP_WEB_EDITOR_INTERNAL_TOKEN, "internal-token-sentinel"); + assert.equal(env.MEDIA_SERVER_WEBHOOK_SECRET, undefined); + assert.equal(env.AWS_SECRET_ACCESS_KEY, undefined); + const { stdout } = await runEditorFile("env", [], { + env: { MEDIA_SERVER_WEBHOOK_SECRET: "overridden-sentinel" }, + }); + assert.equal(stdout.includes("MEDIA_SERVER_WEBHOOK_SECRET="), false); + assert.equal(stdout.includes("AWS_SECRET_ACCESS_KEY="), false); + console.log(JSON.stringify({ subprocessCredentials: "excluded" })); +} finally { + if (previousWebhook === undefined) { + delete process.env.MEDIA_SERVER_WEBHOOK_SECRET; + } else { + process.env.MEDIA_SERVER_WEBHOOK_SECRET = previousWebhook; + } + if (previousAws === undefined) { + delete process.env.AWS_SECRET_ACCESS_KEY; + } else { + process.env.AWS_SECRET_ACCESS_KEY = previousAws; + } +} diff --git a/apps/media-server/src/__tests__/lib/editor-project-bundles.test.ts b/apps/media-server/src/__tests__/lib/editor-project-bundles.test.ts new file mode 100644 index 00000000000..93b13c6f989 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/editor-project-bundles.test.ts @@ -0,0 +1,156 @@ +import { afterAll, afterEach, describe, test } from "bun:test"; +import assert from "node:assert/strict"; +import { + mkdir, + mkdtemp, + readFile, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { + CAP_BUNDLE_HEADER_BYTES, + parseCapBundleManifest, + readCapBundleManifestLength, +} from "@cap/editor-cap-bundle"; +import { extractEditorCapBundle } from "../../lib/editor-cap-bundle"; +import { + consumeEditorProjectBundleDownloadTicket, + createEditorProjectBundleDownloadTicket, + editorProjectBundleResponse, +} from "../../lib/editor-project-bundles"; + +const previousOrigin = process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN; +process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN = "http://127.0.0.1:43017"; +const roots: string[] = []; + +afterEach(async () => { + await Promise.all( + roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), + ); +}); + +afterAll(() => { + if (previousOrigin === undefined) + delete process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN; + else process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN = previousOrigin; +}); + +async function makeProject() { + const root = await mkdtemp(join(tmpdir(), "cap-bundle-test-")); + roots.push(root); + const project = join(root, "source.cap"); + await mkdir(join(project, "content/segments/segment-0"), { + recursive: true, + }); + await mkdir(join(project, "assets/audio"), { recursive: true }); + await mkdir(join(project, "screenshots"), { recursive: true }); + const metadata = Buffer.from( + JSON.stringify({ prettyName: "Test recording" }), + ); + const config = Buffer.from( + JSON.stringify({ background: { source: "none" } }), + ); + const video = Buffer.alloc(3 * 1024 * 1024, 0x37); + const audio = Buffer.alloc(1024 * 1024, 0x52); + const screenshot = Buffer.alloc(10_000, 0x29); + await Promise.all([ + writeFile(join(project, "recording-meta.json"), metadata), + writeFile(join(project, "project-config.json"), config), + writeFile(join(project, "content/segments/segment-0/display.mp4"), video), + writeFile(join(project, "assets/audio/import-test.mp3"), audio), + writeFile(join(project, "screenshots/preview.jpg"), screenshot), + ]); + return { project, metadata, config, video, audio, screenshot }; +} + +describe("editor project bundle downloads", () => { + test("streams a portable snapshot after the editor project closes", async () => { + const { project, metadata, config, video, audio, screenshot } = + await makeProject(); + const created = await createEditorProjectBundleDownloadTicket( + "session-test", + project, + "Test recording.capbundle", + ); + const url = new URL(created.url); + assert.equal( + url.pathname, + "/editor/sessions/session-test/project-bundle/download", + ); + const token = url.searchParams.get("ticket"); + const ticket = consumeEditorProjectBundleDownloadTicket( + "session-test", + token, + ); + assert.ok(ticket); + assert.equal( + consumeEditorProjectBundleDownloadTicket("session-test", token), + null, + ); + const snapshotConfig = await readFile( + join(ticket.root, "project-config.json"), + ); + assert.deepEqual(snapshotConfig, config); + await writeFile( + join(project, "project-config.json"), + "updated after snapshot", + ); + await rm(project, { recursive: true, force: true }); + const response = editorProjectBundleResponse(ticket); + assert.equal(response.headers.get("Content-Length"), String(ticket.size)); + assert.match( + response.headers.get("Content-Disposition") ?? "", + /Test recording\.capbundle/, + ); + const bytes = Buffer.from(await response.arrayBuffer()); + assert.equal(bytes.length, ticket.size); + const length = readCapBundleManifestLength( + bytes.subarray(0, CAP_BUNDLE_HEADER_BYTES), + ); + assert.ok(length); + const manifest = parseCapBundleManifest( + bytes.subarray(CAP_BUNDLE_HEADER_BYTES, CAP_BUNDLE_HEADER_BYTES + length), + bytes.length, + ); + assert.ok(manifest); + const expected = new Map([ + ["recording-meta.json", metadata], + ["project-config.json", config], + ["content/segments/segment-0/display.mp4", video], + ["assets/audio/import-test.mp3", audio], + ["screenshots/preview.jpg", screenshot], + ]); + for (const file of manifest.files) { + const original = expected.get(file.path); + assert.ok(original); + const start: number = CAP_BUNDLE_HEADER_BYTES + length + file.offset; + assert.deepEqual(bytes.subarray(start, start + file.size), original); + } + assert.equal(manifest.files.length, expected.size); + const archivePath = join(dirname(project), "export.capbundle"); + await writeFile(archivePath, bytes); + const extracted = await extractEditorCapBundle(archivePath); + try { + for (const [path, original] of expected) + assert.deepEqual(await readFile(join(extracted.path, path)), original); + } finally { + await extracted.cleanup(); + } + await assert.rejects(readFile(join(ticket.root, "recording-meta.json"))); + }); + + test("rejects symlinks in project media", async () => { + const { project } = await makeProject(); + await symlink( + join(project, "recording-meta.json"), + join(project, "content/segments/segment-0/unsafe.json"), + ); + await assert.rejects( + createEditorProjectBundleDownloadTicket("session-test", project), + /unsupported file/, + ); + }); +}); diff --git a/apps/media-server/src/__tests__/lib/editor-render-project.test.ts b/apps/media-server/src/__tests__/lib/editor-render-project.test.ts new file mode 100644 index 00000000000..6859431a6ce --- /dev/null +++ b/apps/media-server/src/__tests__/lib/editor-render-project.test.ts @@ -0,0 +1,139 @@ +import { afterAll, afterEach, describe, expect, test } from "bun:test"; +import { link, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + listEditorRenderProject, + uploadEditorRenderProject, +} from "../../lib/editor-render-project"; + +const previousHttp = process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; +const previousWallpapers = process.env.CAP_WEB_EDITOR_WALLPAPER_DIR; +process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = "1"; +const roots: string[] = []; +const received = new Map(); +const server = Bun.serve({ + port: 0, + async fetch(request) { + if (request.method !== "PUT") return new Response(null, { status: 405 }); + const url = new URL(request.url); + if (url.pathname === "/fail") return new Response(null, { status: 403 }); + received.set(url.pathname, (await request.arrayBuffer()).byteLength); + return new Response(null, { status: 200 }); + }, +}); + +afterEach(async () => { + received.clear(); + await Promise.all( + roots.splice(0).map((root) => rm(root, { recursive: true, force: true })), + ); +}); + +afterAll(() => { + server.stop(true); + if (previousHttp === undefined) + delete process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + else process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = previousHttp; + if (previousWallpapers === undefined) + delete process.env.CAP_WEB_EDITOR_WALLPAPER_DIR; + else process.env.CAP_WEB_EDITOR_WALLPAPER_DIR = previousWallpapers; +}); + +async function makeProject() { + const root = await mkdtemp(join(tmpdir(), "cap-render-project-test-")); + roots.push(root); + const project = join(root, "source.cap"); + const segment = join(project, "content/segments/segment-1"); + await mkdir(segment, { recursive: true }); + await mkdir(join(project, "content/videos"), { recursive: true }); + await mkdir(join(project, "output"), { recursive: true }); + const wallpapers = join(root, "backgrounds"); + await mkdir(join(wallpapers, "blue"), { recursive: true }); + process.env.CAP_WEB_EDITOR_WALLPAPER_DIR = wallpapers; + await Promise.all([ + writeFile( + join(project, "recording-meta.json"), + JSON.stringify({ segments: [{ display: { path: "x" } }] }), + ), + writeFile(join(project, "project-config.json"), "{}"), + writeFile(join(project, "content/videos/clip.webm"), Buffer.alloc(900)), + writeFile(join(project, "output/result.mp4"), Buffer.alloc(10)), + writeFile(join(wallpapers, "blue/sky.jpg"), Buffer.alloc(77)), + ]); + await link( + join(project, "content/videos/clip.webm"), + join(segment, "display.webm"), + ); + return project; +} + +describe("editor render projects", () => { + test("list project files without exports and share inodes for staged links", async () => { + const project = await makeProject(); + const listing = await listEditorRenderProject(project); + expect(listing.recordingMeta).toEqual({ + segments: [{ display: { path: "x" } }], + }); + const paths = listing.files.map((file) => file.path); + expect(paths).not.toContain("output/result.mp4"); + const clip = listing.files.find( + (file) => file.path === "content/videos/clip.webm", + ); + const staged = listing.files.find( + (file) => file.path === "content/segments/segment-1/display.webm", + ); + expect(clip?.size).toBe(900); + expect(staged?.inode).toBe(clip?.inode ?? ""); + }); + + test("upload requested files and wallpapers and report stored sizes", async () => { + const project = await makeProject(); + const origin = `http://127.0.0.1:${server.port}`; + const result = await uploadEditorRenderProject(project, { + files: [ + { path: "project-config.json", url: `${origin}/config` }, + { + path: "content/segments/segment-1/display.webm", + url: `${origin}/display`, + }, + ], + wallpapers: [{ file: "blue/sky.jpg", url: `${origin}/wallpaper` }], + }); + expect(result.files).toEqual([ + { path: "project-config.json", size: 2 }, + { path: "content/segments/segment-1/display.webm", size: 900 }, + ]); + expect(result.wallpapers).toEqual([{ file: "blue/sky.jpg", size: 77 }]); + expect(received.get("/display")).toBe(900); + expect(received.get("/wallpaper")).toBe(77); + }); + + test("refuse files outside the project, escaping wallpapers and failed uploads", async () => { + const project = await makeProject(); + const origin = `http://127.0.0.1:${server.port}`; + for (const uploads of [ + { + files: [{ path: "output/result.mp4", url: `${origin}/a` }], + wallpapers: [], + }, + { files: [{ path: "../secret", url: `${origin}/a` }], wallpapers: [] }, + { + files: [], + wallpapers: [{ file: "../blue/sky.jpg", url: `${origin}/a` }], + }, + { + files: [{ path: "project-config.json", url: `${origin}/fail` }], + wallpapers: [], + }, + { + files: [{ path: "project-config.json", url: "ftp://127.0.0.1/a" }], + wallpapers: [], + }, + ]) { + await expect( + uploadEditorRenderProject(project, uploads), + ).rejects.toThrow(); + } + }); +}); diff --git a/apps/media-server/src/__tests__/lib/editor-security.test.ts b/apps/media-server/src/__tests__/lib/editor-security.test.ts new file mode 100644 index 00000000000..4bf85050ed4 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/editor-security.test.ts @@ -0,0 +1,124 @@ +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { downloadEditorMedia } from "../../lib/editor-media"; +import { editorProcessEnv } from "../../lib/editor-process"; + +const expectedIdentity = JSON.stringify("expected"); +const body = "abc"; +const server = Bun.serve({ + port: 0, + fetch(request) { + const path = new URL(request.url).pathname; + if (path === "/redirect") { + return new Response(null, { + status: 302, + headers: { location: "/matching" }, + }); + } + return new Response(body, { + headers: { + "content-length": String(body.length), + ...(path === "/matching" + ? { etag: expectedIdentity } + : path === "/changed" + ? { etag: JSON.stringify("changed") } + : {}), + }, + }); + }, +}); + +const previousHttpMedia = process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; +beforeAll(() => { + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = "1"; +}); +afterAll(() => { + if (previousHttpMedia === undefined) { + delete process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + } else { + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = previousHttpMedia; + } + server.stop(true); +}); + +function source( + path: string, + objectIdentity: string | null = expectedIdentity, +) { + return { + url: `http://127.0.0.1:${server.port}${path}`, + contentType: "video/mp4" as const, + size: body.length, + objectIdentity, + }; +} + +test("native editor process environment excludes worker credentials", () => { + const previousWebhook = process.env.MEDIA_SERVER_WEBHOOK_SECRET; + const previousAws = process.env.AWS_SECRET_ACCESS_KEY; + process.env.MEDIA_SERVER_WEBHOOK_SECRET = "webhook-secret-sentinel"; + process.env.AWS_SECRET_ACCESS_KEY = "aws-secret-sentinel"; + try { + const env = editorProcessEnv("internal-token-sentinel"); + expect(env.CAP_WEB_EDITOR_INTERNAL_TOKEN).toBe("internal-token-sentinel"); + expect(env.MEDIA_SERVER_WEBHOOK_SECRET).toBeUndefined(); + expect(env.AWS_SECRET_ACCESS_KEY).toBeUndefined(); + } finally { + if (previousWebhook === undefined) { + delete process.env.MEDIA_SERVER_WEBHOOK_SECRET; + } else { + process.env.MEDIA_SERVER_WEBHOOK_SECRET = previousWebhook; + } + if (previousAws === undefined) { + delete process.env.AWS_SECRET_ACCESS_KEY; + } else { + process.env.AWS_SECRET_ACCESS_KEY = previousAws; + } + } +}); + +test("native editor project root is passed explicitly, not inherited", () => { + const previousRoot = process.env.CAP_WEB_EDITOR_PROJECT_ROOT; + process.env.CAP_WEB_EDITOR_PROJECT_ROOT = "/untrusted-worker-root"; + try { + expect(editorProcessEnv().CAP_WEB_EDITOR_PROJECT_ROOT).toBeUndefined(); + expect( + editorProcessEnv("internal-token", "/trusted-worker-root") + .CAP_WEB_EDITOR_PROJECT_ROOT, + ).toBe("/trusted-worker-root"); + } finally { + if (previousRoot === undefined) { + delete process.env.CAP_WEB_EDITOR_PROJECT_ROOT; + } else { + process.env.CAP_WEB_EDITOR_PROJECT_ROOT = previousRoot; + } + } +}); + +test("rejects a source when its pinned object identity is missing", async () => { + await expect(downloadEditorMedia(source("/missing"))).rejects.toThrow( + "Editor source identity changed before download", + ); +}); + +test("rejects a source when its pinned object identity changes", async () => { + await expect(downloadEditorMedia(source("/changed"))).rejects.toThrow( + "Editor source identity changed before download", + ); +}); + +test("rejects source redirects before downloading another object", async () => { + await expect(downloadEditorMedia(source("/redirect"))).rejects.toThrow( + "Editor source download failed: 302", + ); +}); + +test("downloads the exact pinned object and supports legacy unpinned sources", async () => { + for (const input of [source("/matching"), source("/missing", null)]) { + const result = await downloadEditorMedia(input); + try { + expect(result.size).toBe(body.length); + } finally { + await result.cleanup(); + } + } +}); diff --git a/apps/media-server/src/__tests__/lib/editor-session-capacity.test.ts b/apps/media-server/src/__tests__/lib/editor-session-capacity.test.ts new file mode 100644 index 00000000000..9a321ee9384 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/editor-session-capacity.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "bun:test"; +import { + type EditorSessionActivity, + editorSessionIdle, + editorSessionLimit, +} from "../../lib/editor-sessions"; + +describe("editor session limit", () => { + it("admits one session unless the deployment sizes for more", () => { + expect(editorSessionLimit(undefined)).toBe(1); + expect(editorSessionLimit("")).toBe(1); + expect(editorSessionLimit("3")).toBe(3); + }); + + it("refuses limits that are not whole numbers from 1 to 16", () => { + for (const value of ["0", "-1", "1.5", "17", "two"]) { + expect(() => editorSessionLimit(value)).toThrow( + "Invalid CAP_WEB_EDITOR_MAX_SESSIONS", + ); + } + }); +}); + +describe("abandoned editor sessions", () => { + const minute = 60 * 1000; + const session = ( + overrides: Partial = {}, + ): EditorSessionActivity => ({ + lastActive: 0, + readyAt: 0, + attached: true, + sockets: 0, + detachedAt: null, + exportRunning: false, + ...overrides, + }); + + it("keeps an idle session for ten minutes", () => { + expect(editorSessionIdle(session(), 9 * minute)).toBe(false); + expect(editorSessionIdle(session(), 11 * minute)).toBe(true); + }); + + it("frees a running export's slot soon after its client stops polling", () => { + const running = session({ exportRunning: true }); + expect(editorSessionIdle(running, 2 * minute)).toBe(false); + expect(editorSessionIdle(running, 4 * minute)).toBe(true); + expect( + editorSessionIdle({ ...running, lastActive: 3 * minute }, 4 * minute), + ).toBe(false); + }); + + it("frees a session soon after its last socket closes with no request after", () => { + const detached = session({ detachedAt: minute }); + expect(editorSessionIdle(detached, 2 * minute)).toBe(false); + expect(editorSessionIdle(detached, 4 * minute)).toBe(true); + expect( + editorSessionIdle({ ...detached, lastActive: 3 * minute }, 4 * minute), + ).toBe(false); + expect(editorSessionIdle({ ...detached, sockets: 1 }, 4 * minute)).toBe( + false, + ); + }); + + it("still frees a session nobody picked up after two minutes", () => { + expect(editorSessionIdle(session({ attached: false }), 3 * minute)).toBe( + true, + ); + }); +}); diff --git a/apps/media-server/src/__tests__/lib/editor-signed-assets.test.ts b/apps/media-server/src/__tests__/lib/editor-signed-assets.test.ts new file mode 100644 index 00000000000..c65ac349d9d --- /dev/null +++ b/apps/media-server/src/__tests__/lib/editor-signed-assets.test.ts @@ -0,0 +1,147 @@ +import { expect, test } from "bun:test"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { stageSignedEditorAsset } from "../../lib/editor-signed-assets"; + +test("signed assets recover from changed identities, same-sized tampering, and missing receipts", async () => { + const previousHttp = process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = "1"; + const root = await mkdtemp(join(tmpdir(), "cap-editor-signed-asset-test-")); + const project = join(root, "recording.cap"); + let server: ReturnType | null = null; + let contents = "AAAA"; + let identity = JSON.stringify("first"); + let requests = 0; + try { + await mkdir(project); + server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + requests++; + if (request.headers.get("if-match") !== identity) { + return new Response(null, { status: 412 }); + } + return new Response(contents, { + headers: { + "Content-Length": String(contents.length), + ETag: identity, + }, + }); + }, + }); + const asset = { + path: "content/images/asset.png", + name: "Overlay", + url: `http://127.0.0.1:${server.port}/asset`, + size: 4, + contentType: "image/png", + objectIdentity: identity, + }; + const probe = async (path: string) => ({ + contents: await readFile(path, "utf8"), + }); + const first = await stageSignedEditorAsset(project, asset, probe); + expect(first.contents).toBe("AAAA"); + expect(await stageSignedEditorAsset(project, asset, probe)).toEqual(first); + expect(requests).toBe(1); + await writeFile(join(project, asset.path), "ZZZZ"); + expect((await stageSignedEditorAsset(project, asset, probe)).contents).toBe( + "AAAA", + ); + expect(requests).toBe(2); + contents = "BBBB"; + identity = JSON.stringify("second"); + const revised = { ...asset, objectIdentity: identity }; + expect( + (await stageSignedEditorAsset(project, revised, probe)).contents, + ).toBe("BBBB"); + expect(requests).toBe(3); + await rm(join(project, ".editor-asset-receipts"), { + recursive: true, + force: true, + }); + expect( + (await stageSignedEditorAsset(project, revised, probe)).contents, + ).toBe("BBBB"); + expect(requests).toBe(4); + } finally { + server?.stop(true); + await rm(root, { recursive: true, force: true }); + if (previousHttp === undefined) + delete process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + else process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = previousHttp; + } +}); + +test("concurrent staging waits for a complete signed file and never trusts an unpinned response", async () => { + const previousHttp = process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = "1"; + const root = await mkdtemp( + join(tmpdir(), "cap-editor-signed-concurrent-test-"), + ); + const project = join(root, "recording.cap"); + let server: ReturnType | null = null; + let requests = 0; + try { + await mkdir(project); + server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + requests++; + await new Promise((resolve) => setTimeout(resolve, 30)); + const noIdentity = new URL(request.url).pathname === "/no-identity"; + return new Response("DATA", { + headers: { + "Content-Length": "4", + ...(noIdentity ? {} : { ETag: JSON.stringify("same") }), + }, + }); + }, + }); + const asset = { + path: "content/images/concurrent.png", + name: "Overlay", + url: `http://127.0.0.1:${server.port}/asset`, + size: 4, + contentType: "image/png", + objectIdentity: JSON.stringify("same"), + }; + const probe = async (path: string) => ({ + contents: await readFile(path, "utf8"), + }); + const staged = await Promise.all([ + stageSignedEditorAsset(project, asset, probe), + stageSignedEditorAsset(project, asset, probe), + ]); + expect(staged.map((result) => result.contents)).toEqual(["DATA", "DATA"]); + expect(requests).toBe(1); + const withoutEtag = { + ...asset, + path: "content/images/no-identity.png", + url: `http://127.0.0.1:${server.port}/no-identity`, + }; + await expect( + stageSignedEditorAsset(project, withoutEtag, probe), + ).rejects.toThrow("identity changed"); + expect(await Bun.file(join(project, withoutEtag.path)).exists()).toBe( + false, + ); + const unpinned = { + ...asset, + path: "content/images/unpinned.png", + objectIdentity: null, + }; + await stageSignedEditorAsset(project, unpinned, probe); + await stageSignedEditorAsset(project, unpinned, probe); + expect(requests).toBe(4); + } finally { + server?.stop(true); + await rm(root, { recursive: true, force: true }); + if (previousHttp === undefined) + delete process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + else process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = previousHttp; + } +}); diff --git a/apps/media-server/src/__tests__/lib/editor-video-assets.test.ts b/apps/media-server/src/__tests__/lib/editor-video-assets.test.ts new file mode 100644 index 00000000000..24cce6ca597 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/editor-video-assets.test.ts @@ -0,0 +1,94 @@ +import { expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdir, mkdtemp, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + stageSignedEditorVideoAsset, + validateEditorVideoAsset, +} from "../../lib/editor-video-assets"; + +test("a signed imported video is probed once, staged, and reusable after reopening", async () => { + const previousHttp = process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = "1"; + const root = await mkdtemp(join(tmpdir(), "cap-editor-video-asset-test-")); + const project = join(root, "recording.cap"); + const source = join(root, "clip.mp4"); + let server: ReturnType | null = null; + try { + await mkdir(project); + const generated = spawnSync("ffmpeg", [ + "-v", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=size=160x90:rate=30:duration=2", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + source, + ]); + expect(generated.status).toBe(0); + const size = (await stat(source)).size; + server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + if (new URL(request.url).pathname === "/clip.mp4") { + return new Response(Bun.file(source), { + headers: { + "Content-Length": String(size), + "Content-Type": "video/mp4", + ETag: JSON.stringify("video-1"), + }, + }); + } + return new Response("Not found", { status: 404 }); + }, + }); + const asset = { + path: `content/videos/${randomUUID()}.mp4`, + name: "Imported clip", + url: `http://127.0.0.1:${server.port}/clip.mp4`, + size, + contentType: "video/mp4", + objectIdentity: JSON.stringify("video-1"), + }; + const imported = await stageSignedEditorVideoAsset(project, asset); + expect(imported.path).toBe(asset.path); + expect(imported.name).toBe(asset.name); + expect(imported.duration).toBeGreaterThan(0); + expect(imported.fps).toBeGreaterThan(0); + expect(imported.width).toBeGreaterThan(0); + expect(imported.height).toBeGreaterThan(0); + expect(imported.hasAudio).toBe(false); + expect(await Bun.file(join(project, asset.path)).arrayBuffer()).toEqual( + await Bun.file(source).arrayBuffer(), + ); + expect(await stageSignedEditorVideoAsset(project, asset)).toEqual(imported); + expect(() => + validateEditorVideoAsset({ ...asset, path: "../worker-local.mp4" }), + ).toThrow(); + expect(() => + validateEditorVideoAsset({ ...asset, size: 13 * 1024 * 1024 * 1024 }), + ).toThrow(); + const changed = { + ...asset, + path: `content/videos/${randomUUID()}.mp4`, + objectIdentity: JSON.stringify("other"), + }; + await expect(stageSignedEditorVideoAsset(project, changed)).rejects.toThrow( + "identity changed", + ); + expect(await Bun.file(join(project, changed.path)).exists()).toBe(false); + } finally { + server?.stop(true); + await rm(root, { recursive: true, force: true }); + if (previousHttp === undefined) + delete process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + else process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = previousHttp; + } +}); diff --git a/apps/media-server/src/__tests__/lib/editor-wallpapers.test.ts b/apps/media-server/src/__tests__/lib/editor-wallpapers.test.ts new file mode 100644 index 00000000000..7ecbc4e922b --- /dev/null +++ b/apps/media-server/src/__tests__/lib/editor-wallpapers.test.ts @@ -0,0 +1,70 @@ +import { expect, test } from "bun:test"; +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + editorWallpaperDirectory, + mapEditorInstanceWallpaper, + mapEditorWallpaperConfig, +} from "../../lib/editor-wallpapers"; + +const logical = + "cap-web-wallpaper://assets/backgrounds/macOS/tahoe-dusk-min.jpg"; + +test("saved wallpaper IDs resolve to the desktop asset and survive reopening", () => { + const original = { + background: { source: { type: "wallpaper", path: logical }, padding: 12 }, + }; + const native = mapEditorWallpaperConfig( + original, + "native", + ) as typeof original; + expect(native.background.source.path).toBe( + join(editorWallpaperDirectory(), "macOS/tahoe-dusk-min.jpg"), + ); + expect(mapEditorWallpaperConfig(native, "browser")).toEqual(original); + expect( + mapEditorInstanceWallpaper( + { savedProjectConfig: native, recordingDuration: 3 }, + "browser", + ), + ).toEqual({ savedProjectConfig: original, recordingDuration: 3 }); +}); + +test("wallpaper paths reject traversal and linked files outside the asset root", async () => { + const root = await mkdtemp(join(tmpdir(), "cap-editor-wallpaper-test-")); + try { + const outside = join(root, "outside.jpg"); + const category = join(root, "macOS"); + await writeFile(outside, "outside"); + await mkdir(category); + await symlink(outside, join(category, "linked.jpg")); + for (const path of [ + "cap-web-wallpaper://assets/backgrounds/macOS/../../outside.jpg", + "cap-web-wallpaper://assets/backgrounds/macOS/linked.jpg", + ]) { + expect(() => + mapEditorWallpaperConfig( + { background: { source: { type: "wallpaper", path } } }, + "native", + root, + ), + ).toThrow(); + } + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("other background sources keep their paths unchanged", async () => { + const root = await mkdtemp(join(tmpdir(), "cap-editor-wallpaper-test-")); + try { + const image = join(root, "personal.jpg"); + await writeFile(image, "personal"); + const original = { background: { source: { type: "image", path: image } } }; + expect(mapEditorWallpaperConfig(original, "native", root)).toBe(original); + expect(mapEditorWallpaperConfig(original, "browser", root)).toBe(original); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/apps/media-server/src/__tests__/lib/media-probe.integration.test.ts b/apps/media-server/src/__tests__/lib/media-probe.integration.test.ts index 2f41b63c301..18ca8407220 100644 --- a/apps/media-server/src/__tests__/lib/media-probe.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/media-probe.integration.test.ts @@ -7,6 +7,9 @@ const FIXTURES_DIR = join(import.meta.dir, "..", "fixtures"); const TEST_VIDEO_WITH_AUDIO_PATH = join(FIXTURES_DIR, "test-with-audio.mp4"); const TEST_VIDEO_WITH_AUDIO = `file://${join(FIXTURES_DIR, "test-with-audio.mp4")}`; const TEST_VIDEO_NO_AUDIO = `file://${join(FIXTURES_DIR, "test-no-audio.mp4")}`; +// Fragmented, with each track declaring a single frame, as the browser +// recorder's muxer writes it. The video is six seconds long. +const TEST_VIDEO_FRAGMENTED = `file://${join(FIXTURES_DIR, "test-fragmented.mp4")}`; async function expectRejected(promise: Promise): Promise { let rejected = false; @@ -20,6 +23,12 @@ async function expectRejected(promise: Promise): Promise { describe("mediaProbe integration tests", () => { describe("probeVideo", () => { + test("measures a fragmented MP4 to its end, not its declared length", async () => { + const metadata = await probeVideo(TEST_VIDEO_FRAGMENTED); + + expect(metadata.duration).toBeCloseTo(6.07, 1); + }); + test("extracts metadata from video with audio", async () => { const metadata = await probeVideo(TEST_VIDEO_WITH_AUDIO); diff --git a/apps/media-server/src/__tests__/lib/media-video.integration.test.ts b/apps/media-server/src/__tests__/lib/media-video.integration.test.ts index dd9844d100c..7dfce466aea 100644 --- a/apps/media-server/src/__tests__/lib/media-video.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/media-video.integration.test.ts @@ -1190,6 +1190,87 @@ describe("processVideo integration tests", () => { } }, 120000); + test("keeps portrait 1080p h264 input at full resolution without re-encoding", async () => { + const workDir = mkdtempSync(join(tmpdir(), "cap-portrait-h264-")); + try { + const portraitPath = join(workDir, "portrait.mp4"); + execFileSync("ffmpeg", [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + "testsrc2=size=1080x1920:rate=30", + "-t", + "1", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + portraitPath, + ]); + const originalVideoHash = readDecodedStreamHash(portraitPath, "v"); + + const metadata = await probeVideo(`file://${portraitPath}`); + expect(pickMobileSafeH264Level(metadata).value).toBe(42); + + const tempFile = await processVideo(portraitPath, metadata, {}); + tempFiles.push(tempFile.path); + + const outputMetadata = await probeVideo(`file://${tempFile.path}`); + expect(outputMetadata.width).toBe(1080); + expect(outputMetadata.height).toBe(1920); + expect(readDecodedStreamHash(tempFile.path, "v")).toBe(originalVideoHash); + + await tempFile.cleanup(); + } finally { + rmSync(workDir, { recursive: true, force: true }); + } + }, 120000); + + test("fits oversized portrait input inside a portrait 1080p box", async () => { + const workDir = mkdtempSync(join(tmpdir(), "cap-portrait-4k-")); + try { + const portraitPath = join(workDir, "portrait-4k.mp4"); + execFileSync("ffmpeg", [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + "testsrc2=size=2160x3840:rate=30", + "-t", + "1", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + portraitPath, + ]); + + const metadata = await probeVideo(`file://${portraitPath}`); + const tempFile = await processVideo(portraitPath, metadata, {}); + tempFiles.push(tempFile.path); + + const outputMetadata = await probeVideo(`file://${tempFile.path}`); + expect(outputMetadata.width).toBe(1080); + expect(outputMetadata.height).toBe(1920); + expect(readH264Level(tempFile.path)).toBeLessThanOrEqual(42); + + await tempFile.cleanup(); + } finally { + rmSync(workDir, { recursive: true, force: true }); + } + }, 120000); + test("transcodes raw webm input into a valid mp4 output", async () => { const workDir = mkdtempSync(join(tmpdir(), "cap-webm-transcode-")); try { diff --git a/apps/media-server/src/__tests__/routes/editor-1080p-benchmark.ts b/apps/media-server/src/__tests__/routes/editor-1080p-benchmark.ts new file mode 100644 index 00000000000..4ef5cebd349 --- /dev/null +++ b/apps/media-server/src/__tests__/routes/editor-1080p-benchmark.ts @@ -0,0 +1,1288 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import { mkdir, mkdtemp, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { deflateSync } from "node:zlib"; +import { type Browser, chromium, webkit } from "@playwright/test"; +import sharp from "sharp"; +import app from "../../editor-worker-app"; +import { getEditorSession } from "../../lib/editor-sessions"; +import { + editorWebSocketHandler, + handleEditorSocketUpgrade, +} from "../../lib/editor-websocket"; + +const runFile = promisify(execFile); +const secret = "editor-1080p-benchmark-secret"; +const longSeconds = 900; +const downloadMbps = process.env.CAP_EDITOR_BENCHMARK_DOWNLOAD_MBPS + ? Number(process.env.CAP_EDITOR_BENCHMARK_DOWNLOAD_MBPS) + : null; +const playbackFps = process.env.CAP_EDITOR_BENCHMARK_FPS + ? Number(process.env.CAP_EDITOR_BENCHMARK_FPS) + : 30; +const measureLosslessDelta = + process.env.CAP_EDITOR_BENCHMARK_DELTA_PROBE === "1"; +const measureH264 = process.env.CAP_EDITOR_BENCHMARK_H264_PROBE === "1"; +const nativeH264 = process.env.CAP_EDITOR_BENCHMARK_NATIVE_H264 === "1"; +const browserEngine = + process.env.CAP_EDITOR_BENCHMARK_BROWSER === "webkit" ? webkit : chromium; +const seekDuringH264 = + process.env.CAP_EDITOR_BENCHMARK_SEEK_DURING_H264 === "1"; +const measureMovingQuality = + process.env.CAP_EDITOR_BENCHMARK_MOVING_QUALITY === "1"; +const movingArtifactDirectory = + process.env.CAP_EDITOR_BENCHMARK_MOVING_ARTIFACT_DIR; +assert.ok( + downloadMbps === null || (Number.isFinite(downloadMbps) && downloadMbps > 0), + "Benchmark download bandwidth must be a positive number", +); +assert.ok( + downloadMbps === null || browserEngine === chromium, + "Bandwidth emulation requires Chromium", +); +assert.ok( + Number.isInteger(playbackFps) && playbackFps >= 1 && playbackFps <= 60, + "Benchmark frame rate must be between one and sixty", +); + +type BrowserFrameBenchmark = { + collecting: boolean; + frameTimes: number[]; + paintTimes: number[]; + frameNumbers: number[]; + lastFrameNumber: number | null; + webCodecsAvailable: boolean; + requestedModes: string[]; + socketCloseCode: number | null; + rawBytes: number; + longTasks: number; +}; + +async function makeScreen(path: string) { + await runFile( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=size=1920x1080:rate=30", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=48000", + "-t", + "24", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "28", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-b:a", + "96k", + "-movflags", + "+faststart", + path, + ], + { timeout: 180_000 }, + ); +} + +async function makeCamera(path: string) { + await runFile( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=size=1280x720:rate=25", + "-t", + "24", + "-c:v", + "libx264", + "-preset", + "veryfast", + "-crf", + "29", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + path, + ], + { timeout: 180_000 }, + ); +} + +async function loopMedia(input: string, output: string) { + await runFile( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-stream_loop", + "-1", + "-i", + input, + "-c", + "copy", + "-t", + String(longSeconds), + output, + ], + { timeout: 180_000 }, + ); +} + +async function nativeRssMb(pid: number | null) { + if (pid === null) return null; + const { stdout } = await runFile("ps", ["-o", "rss=", "-p", String(pid)]); + const kilobytes = Number(stdout.trim()); + return Number.isFinite(kilobytes) + ? Math.round((kilobytes / 1024) * 10) / 10 + : null; +} + +async function nativeCpuPercent(pid: number | null) { + if (pid === null) return null; + const { stdout } = await runFile("ps", ["-o", "%cpu=", "-p", String(pid)]); + const percentage = Number(stdout.trim()); + return Number.isFinite(percentage) ? percentage : null; +} + +async function losslessDeltaProbe(packets: Buffer[]) { + assert.ok( + packets.length >= 30, + "Lossless delta probe received too few frames", + ); + let previous: Buffer | null = null; + let changedBytes = 0; + let comparedBytes = 0; + const fullSizes: number[] = []; + const deltaSizes: number[] = []; + const frameNumbers: number[] = []; + for (const packet of packets) { + assert.equal(packet.subarray(0, 8).toString(), "CAPPNG01"); + const footerOffset = packet.length - 24; + frameNumbers.push(packet.readUInt32LE(footerOffset + 12)); + fullSizes.push(packet.length); + const decoded = await sharp(packet.subarray(8, footerOffset)) + .raw() + .toBuffer({ resolveWithObject: true }); + assert.equal(decoded.info.width, 1920); + assert.equal(decoded.info.height, 1080); + assert.equal(decoded.info.channels, 4); + if (previous) { + const difference = Buffer.allocUnsafe(decoded.data.length); + for (let index = 0; index < difference.length; index++) { + const byte = decoded.data[index] ^ previous[index]; + difference[index] = byte; + if (byte !== 0) changedBytes++; + } + comparedBytes += difference.length; + deltaSizes.push(deflateSync(difference, { level: 1 }).length); + } + previous = decoded.data; + } + const meanFullBytes = + fullSizes.reduce((sum, size) => sum + size, 0) / fullSizes.length; + const meanDeltaBytes = + deltaSizes.reduce((sum, size) => sum + size, 0) / deltaSizes.length; + return { + frames: packets.length, + firstFrameNumber: frameNumbers[0], + lastFrameNumber: frameNumbers.at(-1), + changedBytePercent: Math.round((changedBytes / comparedBytes) * 1000) / 10, + meanFullBytes: Math.round(meanFullBytes), + meanDeltaBytes: Math.round(meanDeltaBytes), + estimated30FpsMbpsWithOneKeyframePerSecond: + Math.round(((meanFullBytes + meanDeltaBytes * 29) * 8) / 100_000) / 10, + }; +} + +async function h264PreviewProbe(packets: Buffer[], temporary: string) { + assert.ok(packets.length >= 60, "H.264 probe received too few frames"); + const directory = join(temporary, "h264-probe"); + await mkdir(directory); + await Promise.all( + packets.slice(0, 60).map((packet, index) => { + assert.equal(packet.subarray(0, 8).toString(), "CAPPNG01"); + return writeFile( + join(directory, `frame-${String(index).padStart(3, "0")}.png`), + packet.subarray(8, packet.length - 24), + ); + }), + ); + const results = []; + for (const crf of [18, 20, 24]) { + const output = join(directory, `preview-crf-${crf}.h264`); + const started = performance.now(); + await runFile( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-framerate", + "30", + "-i", + join(directory, "frame-%03d.png"), + "-frames:v", + "60", + "-c:v", + "libx264", + "-threads", + "4", + "-preset", + "ultrafast", + "-tune", + "zerolatency", + "-crf", + String(crf), + "-g", + "30", + "-pix_fmt", + "yuv420p", + "-f", + "h264", + output, + ], + { timeout: 120_000 }, + ); + const size = (await stat(output)).size; + results.push({ + crf, + bytes: size, + estimatedMbps: Math.round((size * 8) / 2 / 100_000) / 10, + encodeFramesPerSecond: + Math.round((60_000 / (performance.now() - started)) * 10) / 10, + }); + } + return results; +} + +async function readySession(id: string, headers: Record) { + const deadline = Date.now() + 180_000; + while (Date.now() < deadline) { + const response = await app.request(`/editor/preparations/${id}`, { + headers, + }); + assert.equal(response.status, 200); + const status = (await response.json()) as { + status: string; + sessionId?: string; + error?: string; + }; + if (status.status === "ready") { + assert.ok(status.sessionId); + return status.sessionId; + } + if (status.status === "error") { + throw new Error(status.error ?? "Native preparation failed"); + } + await Bun.sleep(100); + } + throw new Error("Native preparation timed out"); +} + +async function seekPreview( + sessionId: string, + seconds: number, + headers: Record, +) { + const started = performance.now(); + const response = await app.request(`/editor/sessions/${sessionId}/preview`, { + method: "POST", + headers, + body: JSON.stringify({ + frameNumber: Math.floor(seconds * 30), + fps: 30, + resolutionBase: { x: 1920, y: 1080 }, + }), + }); + assert.equal(response.status, 200); + const bytes = Buffer.from(await response.arrayBuffer()); + assert.ok(bytes.length >= 1920 * 1080 * 4); + const stride = bytes.readUInt32LE(bytes.length - 24); + assert.ok(stride >= 1920 * 4); + return Math.round((performance.now() - started) * 10) / 10; +} + +assert.ok(process.env.CAP_WEB_EDITOR_PREPARE_BIN); +assert.ok(process.env.CAP_WEB_EDITOR_SERVICE_BIN); +const previousSecret = process.env.MEDIA_SERVER_WEBHOOK_SECRET; +const previousAllowHttp = process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; +const previousPublicOrigin = process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN; +process.env.MEDIA_SERVER_WEBHOOK_SECRET = secret; +process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = "1"; +const temporary = await mkdtemp(join(tmpdir(), "cap-editor-1080p-bench-")); +let server: ReturnType | null = null; +let socketServer: ReturnType | null = null; +let frameSocket: WebSocket | null = null; +let browser: Browser | null = null; +let sessionId: string | null = null; +try { + const socketModule = await Bun.build({ + entrypoints: [ + join( + import.meta.dir, + "../../../../../packages/editor-solid-web/src/websocket.ts", + ), + ], + target: "browser", + format: "esm", + splitting: false, + }); + assert.ok(socketModule.success); + const socketCode = await socketModule.outputs[0]?.text(); + assert.ok(socketCode); + const screen = join(temporary, "screen.mp4"); + const camera = join(temporary, "camera.mp4"); + const longScreen = join(temporary, "long-screen.mp4"); + const longCamera = join(temporary, "long-camera.mp4"); + await Promise.all([makeScreen(screen), makeCamera(camera)]); + await Promise.all([ + loopMedia(screen, longScreen), + loopMedia(camera, longCamera), + ]); + const paths = new Map([ + ["/screen.mp4", screen], + ["/camera.mp4", camera], + ["/long-screen.mp4", longScreen], + ["/long-camera.mp4", longCamera], + ]); + server = Bun.serve({ + port: 0, + fetch(request) { + const pathname = new URL(request.url).pathname; + if (pathname === "/bench") { + return new Response( + "", + { headers: { "Content-Type": "text/html; charset=utf-8" } }, + ); + } + if (pathname === "/bench.js") { + return new Response(socketCode, { + headers: { "Content-Type": "text/javascript; charset=utf-8" }, + }); + } + const path = paths.get(pathname); + return path + ? new Response(Bun.file(path)) + : new Response("Missing fixture", { status: 404 }); + }, + }); + const sizes = await Promise.all( + [screen, camera, longScreen, longCamera].map((path) => stat(path)), + ); + const base = `http://127.0.0.1:${server.port}`; + const screenPath = `content/videos/${randomUUID()}.mp4`; + const cameraPath = `content/videos/${randomUUID()}.mp4`; + const headers = { + "x-media-server-secret": secret, + "Content-Type": "application/json", + }; + const started = performance.now(); + const preparation = await app.request("/editor/preparations", { + method: "POST", + headers, + body: JSON.stringify({ + videoId: "editor-1080p-benchmark", + title: "1080p screen, 720p webcam, fifteen minute clip", + display: { + url: `${base}/screen.mp4`, + contentType: "video/mp4", + size: sizes[0]?.size, + fps: 30, + }, + camera: { + url: `${base}/camera.mp4`, + contentType: "video/mp4", + size: sizes[1]?.size, + fps: 25, + offsetMs: 125, + }, + mixedAudioInDisplay: true, + videoAssets: [ + { + path: screenPath, + name: "Long screen", + url: `${base}/long-screen.mp4`, + size: sizes[2]?.size, + contentType: "video/mp4", + }, + { + path: cameraPath, + name: "Long webcam", + url: `${base}/long-camera.mp4`, + size: sizes[3]?.size, + contentType: "video/mp4", + }, + ], + clips: [ + { + displayPath: screenPath, + cameraPath, + duration: longSeconds, + fps: 30, + hasAudio: true, + cameraFps: 25, + cameraOffsetMs: 125, + }, + ], + }), + }); + assert.equal(preparation.status, 202); + const created = (await preparation.json()) as { id: string }; + sessionId = await readySession(created.id, headers); + const prepareMs = Math.round(performance.now() - started); + const native = getEditorSession(sessionId); + assert.ok(native); + const instanceResponse = await app.request( + `/editor/sessions/${sessionId}/instance`, + { headers }, + ); + assert.equal(instanceResponse.status, 200); + const instance = (await instanceResponse.json()) as { + recordingDuration: number; + recordings: { segments: unknown[] }; + }; + assert.equal(instance.recordings.segments.length, 2); + const rssBeforeMb = await nativeRssMb(native.pid); + const seeks = []; + for (const seconds of [1, 10, 23, 24.5, 90, 450, 800, 900, 922]) { + seeks.push({ seconds, ms: await seekPreview(sessionId, seconds, headers) }); + } + const endpointFrame = Math.ceil(instance.recordingDuration * 30); + const endpointResponse = await app.request( + `/editor/sessions/${sessionId}/preview`, + { + method: "POST", + headers, + body: JSON.stringify({ + frameNumber: endpointFrame, + fps: 30, + resolutionBase: { x: 1920, y: 1080 }, + }), + }, + ); + assert.equal(endpointResponse.status, 200); + const placement = endpointResponse.headers.get("x-cap-frame-placement"); + assert.ok(placement); + assert.equal( + (JSON.parse(placement) as { frameNumber: number }).frameNumber, + endpointFrame - 1, + ); + for (const [path, method] of [ + ["preview", "POST"], + ["playback", "POST"], + ["seek", "PUT"], + ] as const) { + const rejectedStarted = performance.now(); + const rejected = await app.request( + `/editor/sessions/${sessionId}/${path}`, + { + method, + headers, + body: JSON.stringify({ + frameNumber: endpointFrame + 30, + fps: 30, + resolutionBase: { x: 1920, y: 1080 }, + }), + }, + ); + assert.equal(rejected.status, 400); + assert.ok(performance.now() - rejectedStarted < 1000); + } + const sorted = seeks.map((seek) => seek.ms).sort((a, b) => a - b); + const rssAfterSeeksMb = await nativeRssMb(native.pid); + const relayDiagnostics: Array<{ + kind: string; + message?: string; + mode?: string; + upstream?: string; + }> = []; + socketServer = Bun.serve({ + port: 0, + fetch(request, listener) { + const upgrade = handleEditorSocketUpgrade(request, listener); + return upgrade === null ? app.fetch(request) : upgrade; + }, + websocket: { + ...editorWebSocketHandler, + message(ws, message) { + if (ws.data.scope === "frames") { + relayDiagnostics.push({ + kind: "control", + message: + typeof message === "string" + ? message + : `binary:${message.byteLength}`, + mode: ws.data.frameMode, + upstream: ws.data.upstream?.url, + }); + } + editorWebSocketHandler.message?.(ws, message); + if (ws.data.scope === "frames") + relayDiagnostics.push({ + kind: "state", + mode: ws.data.frameMode, + upstream: ws.data.upstream?.url, + }); + }, + }, + }); + process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN = `http://127.0.0.1:${socketServer.port}`; + const ticketsResponse = await app.request( + `/editor/sessions/${sessionId}/sockets`, + { + method: "POST", + headers, + body: JSON.stringify({ origin: "http://127.0.0.1:3000" }), + }, + ); + assert.equal(ticketsResponse.status, 200); + const tickets = (await ticketsResponse.json()) as { + sockets: { frames: { url: string; ticket: string } }; + }; + const BunWebSocket = WebSocket as unknown as new ( + url: string, + options: Bun.WebSocketOptions, + ) => WebSocket; + const nativeFrameUrl = new URL("/frames-h264", native.origin); + nativeFrameUrl.protocol = "ws:"; + frameSocket = nativeH264 + ? native.connectSocket(nativeFrameUrl.toString()) + : new BunWebSocket(tickets.sockets.frames.url, { + protocols: [ + "cap-editor-v1", + `cap-editor-ticket.${tickets.sockets.frames.ticket}`, + ], + headers: { Origin: "http://127.0.0.1:3000" }, + }); + frameSocket.binaryType = "arraybuffer"; + const frameTimes: number[] = []; + let frameBytes = 0; + let collecting = false; + let h264Config: string | null = null; + let h264Keyframes = 0; + let h264SequenceGaps = 0; + let lastH264Sequence: number | null = null; + const deltaPackets: Buffer[] = []; + frameSocket.onmessage = (event: MessageEvent) => { + if (nativeH264 && typeof event.data === "string") { + const message = JSON.parse(event.data) as { + kind: string; + codec?: string; + error?: string; + }; + if (message.kind === "cap-h264-config") + h264Config = message.codec ?? null; + if (message.kind === "cap-h264-unavailable") + throw new Error(message.error ?? "Native H.264 unavailable"); + return; + } + if (!collecting || !(event.data instanceof ArrayBuffer)) return; + if (nativeH264) { + const packet = Buffer.from(event.data); + if (packet.subarray(0, 8).toString() !== "CAPH2641") return; + const sequence = Number(packet.readBigUInt64LE(8)); + if (lastH264Sequence !== null && sequence !== lastH264Sequence + 1) + h264SequenceGaps++; + lastH264Sequence = sequence; + if (packet[16] === 1) h264Keyframes++; + } + frameTimes.push(performance.now()); + frameBytes += event.data.byteLength; + if ((measureLosslessDelta || measureH264) && deltaPackets.length < 60) { + deltaPackets.push(Buffer.from(event.data)); + } + }; + await new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error("1080p frame socket timed out")), + 5000, + ); + if (!frameSocket) return reject(new Error("Missing 1080p frame socket")); + frameSocket.onopen = () => { + clearTimeout(timer); + resolve(); + }; + frameSocket.onerror = () => { + clearTimeout(timer); + reject(new Error("1080p frame socket failed")); + }; + }); + const playback = await app.request(`/editor/sessions/${sessionId}/playback`, { + method: "POST", + headers, + body: JSON.stringify({ + frameNumber: 0, + fps: playbackFps, + resolutionBase: { x: 1920, y: 1080 }, + }), + }); + assert.equal(playback.status, 204); + await Bun.sleep(1000); + collecting = true; + const playbackResourceSamples = []; + for (const second of [5, 10, 15, 20]) { + await Bun.sleep(5000); + const [rssMb, cpuPercent] = await Promise.all([ + nativeRssMb(native.pid), + nativeCpuPercent(native.pid), + ]); + playbackResourceSamples.push({ second, rssMb, cpuPercent }); + } + collecting = false; + const stopped = await app.request(`/editor/sessions/${sessionId}/playback`, { + method: "DELETE", + headers, + }); + assert.equal(stopped.status, 204); + const intervals = frameTimes + .slice(1) + .map((time, index) => time - (frameTimes[index] ?? time)) + .sort((a, b) => a - b); + const frameIntervalP95Ms = + intervals[Math.floor(intervals.length * 0.95)] ?? null; + const firstRssMb = playbackResourceSamples[0]?.rssMb; + const lastRssMb = playbackResourceSamples.at(-1)?.rssMb; + assert.ok( + frameTimes.length >= (playbackFps === 30 && !nativeH264 ? 540 : 2), + "1080p playback delivered too few frames", + ); + if (playbackFps === 30) { + assert.ok( + frameIntervalP95Ms !== null && frameIntervalP95Ms < 100, + "1080p frame delivery stalled", + ); + } + assert.ok( + firstRssMb !== null && + firstRssMb !== undefined && + lastRssMb !== null && + lastRssMb !== undefined && + lastRssMb - firstRssMb < 64, + "1080p playback memory kept growing", + ); + frameSocket.close(); + frameSocket = null; + const losslessDelta = measureLosslessDelta + ? await losslessDeltaProbe(deltaPackets) + : null; + const h264Preview = measureH264 + ? await h264PreviewProbe(deltaPackets, temporary) + : null; + const beforeBrowserMetricsResponse = await native.request("/metrics"); + assert.equal(beforeBrowserMetricsResponse.status, 200); + const beforeBrowserFrameMetrics = + (await beforeBrowserMetricsResponse.json()) as { + h264Bytes: number; + h264Frames: number; + }; + browser = await browserEngine.launch({ + headless: true, + ...(browserEngine === chromium + ? { args: ["--enable-precise-memory-info"] } + : {}), + }); + const page = await browser.newPage({ + viewport: { width: 1920, height: 1080 }, + }); + const browserWarnings: string[] = []; + page.on("console", (message) => { + if (message.type() === "warning" && message.text().includes("H.264")) + browserWarnings.push(message.text()); + }); + await page.goto(`${base}/bench`); + if (downloadMbps !== null) { + const browserNetwork = await page.context().newCDPSession(page); + await browserNetwork.send("Network.enable"); + await browserNetwork.send("Network.emulateNetworkConditions", { + offline: false, + latency: 50, + downloadThroughput: (downloadMbps * 1_000_000) / 8, + uploadThroughput: -1, + }); + } + const browserTicketsResponse = await app.request( + `/editor/sessions/${sessionId}/sockets`, + { + method: "POST", + headers, + body: JSON.stringify({ origin: base }), + }, + ); + assert.equal(browserTicketsResponse.status, 200); + const browserTickets = (await browserTicketsResponse.json()) as { + sockets: { frames: { url: string; ticket: string } }; + }; + await page.evaluate(async (credential) => { + const modulePath = "/bench.js"; + const module = (await import(modulePath)) as { + setEditorFrameSocketCredential: (value: { + url: string; + ticket: string; + }) => void; + createWS: (url: string) => WebSocket; + }; + const canvas = document.getElementById( + "bench-canvas", + ) as HTMLCanvasElement | null; + const context = canvas?.getContext("2d", { alpha: false }); + if (!canvas || !context) + throw new Error("Browser frame canvas unavailable"); + const state: BrowserFrameBenchmark = { + collecting: false, + frameTimes: [], + paintTimes: [], + frameNumbers: [], + lastFrameNumber: null, + webCodecsAvailable: + isSecureContext && + typeof VideoDecoder !== "undefined" && + typeof EncodedVideoChunk !== "undefined", + requestedModes: [], + socketCloseCode: null, + rawBytes: 0, + longTasks: 0, + }; + const browserWindow = window as typeof window & { + capFrameBenchmark?: BrowserFrameBenchmark; + capFrameSocket?: WebSocket; + capExpectedPausedFrame?: number; + capPausedFrame?: ArrayBuffer; + }; + browserWindow.capFrameBenchmark = state; + new PerformanceObserver((list) => { + if (state.collecting) state.longTasks += list.getEntries().length; + }).observe({ entryTypes: ["longtask"] }); + module.setEditorFrameSocketCredential(credential); + const socket = module.createWS(credential.url); + const nativeSend = socket.send.bind(socket); + socket.send = (data) => { + if (typeof data === "string") state.requestedModes.push(data); + nativeSend(data); + }; + socket.addEventListener("close", (event) => { + state.socketCloseCode = event.code; + }); + browserWindow.capFrameSocket = socket; + socket.addEventListener("message", (event: MessageEvent) => { + if (!(event.data instanceof ArrayBuffer)) return; + const paintedAt = performance.now(); + const footerOffset = event.data.byteLength - 24; + const meta = new DataView(event.data, footerOffset, 24); + const width = meta.getUint32(8, true); + const height = meta.getUint32(4, true); + state.lastFrameNumber = meta.getUint32(12, true); + if (state.lastFrameNumber === browserWindow.capExpectedPausedFrame) + browserWindow.capPausedFrame = event.data; + if (canvas.width !== width || canvas.height !== height) { + canvas.width = width; + canvas.height = height; + } + context.putImageData( + new ImageData( + new Uint8ClampedArray(event.data, 0, width * height * 4), + width, + height, + ), + 0, + 0, + ); + if (state.collecting) { + state.frameTimes.push(performance.now()); + state.paintTimes.push(performance.now() - paintedAt); + state.frameNumbers.push(meta.getUint32(12, true)); + state.rawBytes += event.data.byteLength; + } + }); + await new Promise((resolve, reject) => { + const timer = window.setTimeout( + () => reject(new Error("Browser frame socket timed out")), + 5000, + ); + socket.addEventListener( + "open", + () => { + window.clearTimeout(timer); + resolve(); + }, + { once: true }, + ); + socket.addEventListener( + "error", + () => { + window.clearTimeout(timer); + reject(new Error("Browser frame socket failed")); + }, + { once: true }, + ); + }); + }, browserTickets.sockets.frames); + const browserPlayback = await app.request( + `/editor/sessions/${sessionId}/playback`, + { + method: "POST", + headers, + body: JSON.stringify({ + frameNumber: 0, + fps: playbackFps, + resolutionBase: { x: 1920, y: 1080 }, + }), + }, + ); + assert.equal(browserPlayback.status, 204); + await Bun.sleep(1000); + await page.evaluate(() => { + const state = ( + window as typeof window & { capFrameBenchmark?: BrowserFrameBenchmark } + ).capFrameBenchmark; + if (!state) throw new Error("Browser frame benchmark unavailable"); + state.collecting = true; + }); + const browserResourceSamples = []; + for (const second of [5, 10, 15, 20]) { + await Bun.sleep(5000); + if (seekDuringH264 && second === 10) { + const frameNumber = 450 * playbackFps; + const position = { + frameNumber, + fps: playbackFps, + resolutionBase: { x: 1920, y: 1080 }, + }; + const moved = await app.request(`/editor/sessions/${sessionId}/seek`, { + method: "PUT", + headers, + body: JSON.stringify(position), + }); + assert.equal(moved.status, 204); + try { + await page.waitForFunction( + (expected) => + ( + window as typeof window & { + capFrameBenchmark?: BrowserFrameBenchmark; + } + ).capFrameBenchmark?.frameNumbers.some( + (frameNumber) => frameNumber >= expected, + ), + frameNumber, + { timeout: 5000 }, + ); + } catch (error) { + const browserState = await page.evaluate( + () => + ( + window as typeof window & { + capFrameBenchmark?: BrowserFrameBenchmark; + } + ).capFrameBenchmark, + ); + const metrics = await native.request("/metrics"); + throw new Error( + `Live seek did not reach the browser: ${JSON.stringify({ + browserFrames: browserState?.frameTimes.length, + lastFrameNumber: browserState?.lastFrameNumber, + requestedModes: browserState?.requestedModes, + relayDiagnostics, + browserWarnings, + nativeMetrics: await metrics.json(), + })}`, + { cause: error }, + ); + } + } + browserResourceSamples.push( + await page.evaluate((sampleSecond) => { + const memory = ( + performance as Performance & { + memory?: { usedJSHeapSize: number }; + } + ).memory; + return { + second: sampleSecond, + jsHeapMb: memory + ? Math.round((memory.usedJSHeapSize / 1024 / 1024) * 10) / 10 + : null, + }; + }, second), + ); + } + const browserResult = await page.evaluate(() => { + const browserWindow = window as typeof window & { + capFrameBenchmark?: BrowserFrameBenchmark; + capFrameSocket?: WebSocket; + }; + const state = browserWindow.capFrameBenchmark; + if (!state) throw new Error("Browser frame benchmark unavailable"); + state.collecting = false; + return state; + }); + const movingSamples: Array<{ frameNumber: number; png: string }> = []; + if (measureMovingQuality) { + assert.ok(nativeH264, "Moving quality requires native H.264 preview"); + for (let sample = 0; sample < 3; sample++) { + await Bun.sleep(250); + movingSamples.push( + await page.evaluate(() => { + const state = ( + window as typeof window & { + capFrameBenchmark?: BrowserFrameBenchmark; + } + ).capFrameBenchmark; + const canvas = document.getElementById( + "bench-canvas", + ) as HTMLCanvasElement | null; + if (!canvas || state?.lastFrameNumber === null || !state) + throw new Error("Moving frame sample was unavailable"); + return { + frameNumber: state.lastFrameNumber, + png: canvas.toDataURL("image/png"), + }; + }), + ); + } + } + const browserStopped = await app.request( + `/editor/sessions/${sessionId}/playback`, + { method: "DELETE", headers }, + ); + assert.equal(browserStopped.status, 204); + if (movingSamples.length > 0) await Bun.sleep(250); + const movingQuality: Array<{ + frameNumber: number; + psnrDb: number; + meanAbsoluteError: number; + }> = []; + for (const sample of movingSamples) { + const preview = await native.request("/preview", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + frameNumber: sample.frameNumber, + fps: playbackFps, + resolutionBase: { x: 1920, y: 1080 }, + }), + }); + assert.equal(preview.status, 200); + const exact = Buffer.from(await preview.arrayBuffer()); + const pixelBytes = 1920 * 1080 * 4; + assert.equal(exact.length, pixelBytes + 24); + const browserPng = Buffer.from( + sample.png.slice("data:image/png;base64,".length), + "base64", + ); + if (movingArtifactDirectory) { + await mkdir(movingArtifactDirectory, { recursive: true }); + await Promise.all([ + writeFile( + join(movingArtifactDirectory, `browser-${sample.frameNumber}.png`), + browserPng, + ), + writeFile( + join(movingArtifactDirectory, `native-${sample.frameNumber}.png`), + await sharp(exact.subarray(0, pixelBytes), { + raw: { width: 1920, height: 1080, channels: 4 }, + }) + .png() + .toBuffer(), + ), + ]); + } + const browserPixels = await sharp(browserPng) + .ensureAlpha() + .raw() + .toBuffer(); + assert.equal(browserPixels.length, pixelBytes); + let squaredError = 0; + let absoluteError = 0; + for (let offset = 0; offset < pixelBytes; offset += 4) { + for (let channel = 0; channel < 3; channel++) { + const difference = + (browserPixels[offset + channel] ?? 0) - + (exact[offset + channel] ?? 0); + absoluteError += Math.abs(difference); + squaredError += difference * difference; + } + } + const channels = 1920 * 1080 * 3; + const mse = squaredError / channels; + movingQuality.push({ + frameNumber: sample.frameNumber, + psnrDb: + Math.round((mse === 0 ? 100 : 10 * Math.log10(255 ** 2 / mse)) * 10) / + 10, + meanAbsoluteError: Math.round((absoluteError / channels) * 10) / 10, + }); + } + let pausedPreviewSha256: string | null = null; + if (seekDuringH264) { + await Bun.sleep(250); + const pausedFrameNumber = 700 * playbackFps; + await page.evaluate((frameNumber) => { + ( + window as typeof window & { capExpectedPausedFrame?: number } + ).capExpectedPausedFrame = frameNumber; + }, pausedFrameNumber); + const preview = await native.request("/preview", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + frameNumber: pausedFrameNumber, + fps: playbackFps, + resolutionBase: { x: 1920, y: 1080 }, + }), + }); + assert.equal(preview.status, 200); + const nativeFrame = Buffer.from(await preview.arrayBuffer()); + const nativePixelBytes = 1920 * 1080 * 4; + assert.equal(nativeFrame.length, nativePixelBytes + 24); + assert.equal(nativeFrame.readUInt32LE(nativePixelBytes), 1920 * 4); + pausedPreviewSha256 = createHash("sha256") + .update(nativeFrame.subarray(0, nativePixelBytes)) + .digest("hex"); + try { + await page.waitForFunction( + (expected) => + ( + window as typeof window & { + capFrameBenchmark?: BrowserFrameBenchmark; + } + ).capFrameBenchmark?.lastFrameNumber === expected, + pausedFrameNumber, + { timeout: 5000 }, + ); + } catch (error) { + const browserState = await page.evaluate( + () => + ( + window as typeof window & { + capFrameBenchmark?: BrowserFrameBenchmark; + } + ).capFrameBenchmark, + ); + const metrics = await native.request("/metrics"); + throw new Error( + `Paused preview did not reach the browser: ${JSON.stringify({ + lastFrameNumber: browserState?.lastFrameNumber, + requestedModes: browserState?.requestedModes, + relayDiagnostics, + browserWarnings, + nativeMetrics: await metrics.json(), + })}`, + { cause: error }, + ); + } + const browserPreviewSha256 = await page.evaluate(async () => { + const browserWindow = window as typeof window & { + capPausedFrame?: ArrayBuffer; + }; + const frame = browserWindow.capPausedFrame; + if (!frame) throw new Error("Paused browser preview was missing"); + const digest = await crypto.subtle.digest( + "SHA-256", + frame.slice(0, frame.byteLength - 24), + ); + return Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, "0"), + ).join(""); + }); + assert.equal(browserPreviewSha256, pausedPreviewSha256); + } + await page.evaluate(() => { + ( + window as typeof window & { capFrameSocket?: WebSocket } + ).capFrameSocket?.close(); + }); + const browserFrameIntervals = browserResult.frameTimes + .slice(1) + .map((time, index) => time - (browserResult.frameTimes[index] ?? time)) + .sort((a, b) => a - b); + const browserPaintTimes = browserResult.paintTimes.toSorted((a, b) => a - b); + const browserIntervalP95Ms = + browserFrameIntervals[Math.floor(browserFrameIntervals.length * 0.95)] ?? + null; + const browserPaintP95Ms = + browserPaintTimes[Math.floor(browserPaintTimes.length * 0.95)] ?? null; + assert.ok( + browserResult.frameTimes.length >= + (nativeH264 + ? playbackFps * 18 + : downloadMbps === null && playbackFps === 30 + ? 540 + : 2), + `1080p browser replay painted too few frames: ${JSON.stringify({ + frames: browserResult.frameTimes.length, + modes: browserResult.requestedModes, + relayDiagnostics, + warnings: browserWarnings, + })}`, + ); + if (downloadMbps === null && playbackFps === 30) { + assert.ok( + browserIntervalP95Ms !== null && browserIntervalP95Ms < 100, + "1080p browser replay stalled", + ); + } + assert.ok( + browserPaintP95Ms !== null && browserPaintP95Ms < 16, + "1080p browser painting blocked the main thread", + ); + const browserHeapHalfwayMb = browserResourceSamples[1]?.jsHeapMb; + const browserHeapFinalMb = browserResourceSamples.at(-1)?.jsHeapMb; + assert.ok( + browserHeapHalfwayMb === null || + browserHeapHalfwayMb === undefined || + browserHeapFinalMb === null || + browserHeapFinalMb === undefined || + browserHeapFinalMb - browserHeapHalfwayMb < 64, + "1080p browser heap kept growing", + ); + const rssAfterPlaybackMb = await nativeRssMb(native.pid); + const metricsResponse = await native.request("/metrics"); + assert.equal(metricsResponse.status, 200); + const frameMetrics = (await metricsResponse.json()) as { + packedFrames: number; + avgPackMs: number; + rawBytes: number; + sentBytes: number; + h264Frames: number; + h264Bytes: number; + }; + assert.ok(frameMetrics.packedFrames >= (nativeH264 ? 540 : 1_080)); + assert.ok(frameMetrics.avgPackMs >= 0); + assert.ok(frameMetrics.sentBytes > 0); + assert.ok(frameMetrics.sentBytes < frameMetrics.rawBytes); + if (nativeH264) + assert.ok( + frameMetrics.h264Frames >= playbackFps * 35, + "1080p browser replay failed to use H.264 preview", + ); + const estimatedRequestedFpsCompressedMbps = nativeH264 + ? null + : Math.round( + ((frameMetrics.sentBytes / frameMetrics.packedFrames) * + playbackFps * + 8) / + 100_000, + ) / 10; + const estimatedH264Mbps = + frameMetrics.h264Frames > 0 + ? Math.round( + ((frameMetrics.h264Bytes / frameMetrics.h264Frames) * + playbackFps * + 8) / + 100_000, + ) / 10 + : null; + const browserH264Bytes = + frameMetrics.h264Bytes - beforeBrowserFrameMetrics.h264Bytes; + const browserH264Frames = + frameMetrics.h264Frames - beforeBrowserFrameMetrics.h264Frames; + assert.ok(browserH264Bytes >= 0 && browserH264Frames >= 0); + const estimatedBrowserH264Mbps = + browserH264Frames > 0 + ? Math.round( + ((browserH264Bytes / browserH264Frames) * playbackFps * 8) / 100_000, + ) / 10 + : null; + process.stdout.write( + `${JSON.stringify({ + resolution: "1920x1080", + browserEngine: browserEngine.name(), + downloadMbps, + playbackFps, + nativeH264, + seekDuringH264, + movingQuality, + pausedPreviewSha256, + h264Config, + h264Keyframes, + h264SequenceGaps, + cameraResolution: "1280x720", + clipSeconds: longSeconds, + assetBytes: sizes.map((size) => size.size), + prepareMs, + seeks, + seekMedianMs: sorted[Math.floor(sorted.length / 2)], + seekP95Ms: sorted[Math.floor(sorted.length * 0.95)], + rssBeforeMb, + rssAfterSeeksMb, + playbackFramesInTwentySeconds: frameTimes.length, + playbackDeliveredFps: Math.round((frameTimes.length / 20) * 10) / 10, + playbackFrameIntervalP95Ms: frameIntervalP95Ms, + playbackBytes: frameBytes, + playbackResourceSamples, + browserFramesInTwentySeconds: browserResult.frameTimes.length, + browserPaintedFps: + Math.round((browserResult.frameTimes.length / 20) * 10) / 10, + browserFrameIntervalP95Ms: browserIntervalP95Ms, + browserPaintP95Ms, + browserDecodedBytes: browserResult.rawBytes, + frameMetrics, + estimatedRequestedFpsCompressedMbps, + estimatedH264Mbps, + estimatedBrowserH264Mbps, + browserH264Bytes, + browserH264Frames, + losslessDelta, + h264Preview, + browserLongTasks: browserResult.longTasks, + browserWarnings, + browserWebCodecsAvailable: browserResult.webCodecsAvailable, + browserRequestedModes: browserResult.requestedModes, + relayDiagnostics, + browserSocketCloseCode: browserResult.socketCloseCode, + browserResourceSamples, + rssAfterPlaybackMb, + workerRssMb: Math.round(process.memoryUsage().rss / 1024 / 1024), + })}\n`, + ); +} finally { + if (sessionId) { + await app.request(`/editor/sessions/${sessionId}`, { + method: "DELETE", + headers: { "x-media-server-secret": secret }, + }); + } + frameSocket?.close(); + await browser?.close(); + socketServer?.stop(true); + server?.stop(true); + await rm(temporary, { recursive: true, force: true }); + if (previousSecret === undefined) { + delete process.env.MEDIA_SERVER_WEBHOOK_SECRET; + } else { + process.env.MEDIA_SERVER_WEBHOOK_SECRET = previousSecret; + } + if (previousAllowHttp === undefined) { + delete process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + } else { + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = previousAllowHttp; + } + if (previousPublicOrigin === undefined) { + delete process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN; + } else { + process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN = previousPublicOrigin; + } +} diff --git a/apps/media-server/src/__tests__/routes/editor-browser-cap-directory-replay.ts b/apps/media-server/src/__tests__/routes/editor-browser-cap-directory-replay.ts new file mode 100644 index 00000000000..e32ee1441a3 --- /dev/null +++ b/apps/media-server/src/__tests__/routes/editor-browser-cap-directory-replay.ts @@ -0,0 +1,221 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { type Browser, chromium, webkit } from "@playwright/test"; + +type DialogModule = + typeof import("../../../../../packages/editor-solid-web/src/tauri-dialog"); +type DialogWindow = Window & { + editorDialog?: DialogModule; + editorOpenPromise?: Promise; + editorAlerts?: string[]; +}; + +const entrypoint = resolve( + import.meta.dir, + "../../../../../packages/editor-solid-web/src/tauri-dialog.ts", +); +const bundle = await Bun.build({ + entrypoints: [entrypoint], + target: "browser", + format: "esm", +}); +assert.ok(bundle.success, bundle.logs.map((log) => log.message).join("\n")); +assert.equal(bundle.outputs.length, 1); +const source = await bundle.outputs[0]?.text(); +assert.ok(source); + +const temporary = await mkdtemp( + join(tmpdir(), "cap-editor-browser-directory-"), +); +const good = join(temporary, "sample.cap"); +const missingMedia = join(temporary, "missing-media.cap"); +const wrongRoot = join(temporary, "ordinary-folder"); +const expected = new Map([ + [ + "recording-meta.json", + JSON.stringify({ + segments: [ + { display: { path: "content/segments/segment-0/display.mp4" } }, + ], + }), + ], + ["content/segments/segment-0/display.mp4", "screen-bytes-01"], + ["content/segments/segment-0/camera.mp4", "camera-bytes-02"], + ["content/segments/segment-0/mic.wav", "microphone-bytes-03"], +]); + +async function writeFixture(folder: string, files: Map) { + for (const [relative, content] of files) { + const path = join(folder, relative); + await mkdir(resolve(path, ".."), { recursive: true }); + await writeFile(path, content); + } +} + +async function createPage(browser: Browser) { + const page = await browser.newPage(); + const url = "https://editor.cap.so/browser-directory-replay"; + await page.route(url, (route) => + route.fulfill({ + status: 200, + contentType: "text/html", + body: "", + }), + ); + await page.goto(url); + // A native alert opening while WebKit is still reporting the folder upload + // can stall Playwright's upload on Linux, so the page records its alerts. + await page.evaluate(() => { + const browserWindow = window as DialogWindow; + browserWindow.editorAlerts = []; + window.alert = (message?: unknown) => { + browserWindow.editorAlerts?.push(String(message)); + }; + }); + await page.evaluate(async (code) => { + const moduleUrl = URL.createObjectURL( + new Blob([code], { type: "text/javascript" }), + ); + try { + (window as DialogWindow).editorDialog = (await import( + moduleUrl + )) as DialogModule; + } finally { + URL.revokeObjectURL(moduleUrl); + } + }, source); + return page; +} + +async function selectFolder( + page: Awaited>, + folder: string, +) { + const chooserPromise = page.waitForEvent("filechooser", { timeout: 10_000 }); + await page.evaluate(() => { + const dialog = (window as DialogWindow).editorDialog; + if (!dialog) throw new Error("Editor browser dialog did not load"); + (window as DialogWindow).editorOpenPromise = dialog.open({ + directory: true, + }); + }); + const chooser = await chooserPromise; + await chooser.setFiles(folder); + return page.evaluate(async () => { + const pending = (window as DialogWindow).editorOpenPromise; + if (!pending) throw new Error("Editor browser picker did not open"); + return pending; + }); +} + +async function replay(browser: Browser, name: string) { + const page = await createPage(browser); + const nativeDialogs: string[] = []; + const dialogAccepts: Promise[] = []; + page.on("dialog", (dialog) => { + nativeDialogs.push(dialog.message()); + dialogAccepts.push(dialog.accept()); + }); + const alerts = () => + page.evaluate(() => [...((window as DialogWindow).editorAlerts ?? [])]); + const nextAlert = async (count: number) => { + await page.waitForFunction( + (expected) => + ((window as DialogWindow).editorAlerts?.length ?? 0) >= expected, + count, + { timeout: 10_000 }, + ); + return (await alerts()).at(-1) ?? ""; + }; + try { + const token = await selectFolder(page, good); + assert.ok(token); + assert.match(token, /^cap-web-editor:\/\/import\/[0-9a-f-]{36}$/); + const result = await page.evaluate(async (selection) => { + const dialog = (window as DialogWindow).editorDialog; + if (!dialog) throw new Error("Editor browser dialog is unavailable"); + const file = dialog.takeEditorSelectedFile(selection); + if (!file) throw new Error("Selected recording bundle is unavailable"); + const bytes = new Uint8Array(await file.arrayBuffer()); + const magic = new TextDecoder().decode(bytes.subarray(0, 8)); + const length = new DataView(bytes.buffer).getUint32(8, true); + const manifest = JSON.parse( + new TextDecoder().decode(bytes.subarray(12, 12 + length)), + ) as { + version: number; + files: Array<{ path: string; size: number; offset: number }>; + }; + const entries = manifest.files.map((entry) => ({ + path: entry.path, + content: new TextDecoder().decode( + bytes.subarray( + 12 + length + entry.offset, + 12 + length + entry.offset + entry.size, + ), + ), + })); + return { + name: file.name, + type: file.type, + magic, + version: manifest.version, + entries, + secondTakeMissing: dialog.takeEditorSelectedFile(selection) === null, + }; + }, token); + assert.equal(result.name, "sample.capbundle"); + assert.equal(result.type, "application/vnd.cap.project-bundle"); + assert.equal(result.magic, "CAPBND01"); + assert.equal(result.version, 1); + assert.equal(result.secondTakeMissing, true); + assert.deepEqual( + new Map(result.entries.map((entry) => [entry.path, entry.content])), + expected, + ); + assert.deepEqual(await alerts(), []); + const missingToken = await selectFolder(page, missingMedia); + assert.equal(missingToken, null); + assert.match(await nextAlert(1), /missing Cap recording media/); + const wrongRootToken = await selectFolder(page, wrongRoot); + assert.equal(wrongRootToken, null); + assert.match(await nextAlert(2), /Select a \.cap recording folder/); + assert.deepEqual(nativeDialogs, []); + console.log( + JSON.stringify({ + name, + entries: result.entries.length, + alerts: await alerts(), + }), + ); + } finally { + await Promise.all(dialogAccepts); + await page.close(); + } +} + +try { + await writeFixture(good, new Map([...expected, ["notes.txt", "ignored"]])); + await writeFixture( + missingMedia, + new Map([ + ["recording-meta.json", "{}"], + ["notes.txt", "ignored"], + ]), + ); + await writeFixture(wrongRoot, expected); + for (const [name, engine] of [ + ["Chromium", chromium], + ["WebKit", webkit], + ] as const) { + const browser = await engine.launch({ headless: true }); + try { + await replay(browser, name); + } finally { + await browser.close(); + } + } +} finally { + await rm(temporary, { recursive: true, force: true }); +} diff --git a/apps/media-server/src/__tests__/routes/editor-camera-effects-benchmark.ts b/apps/media-server/src/__tests__/routes/editor-camera-effects-benchmark.ts new file mode 100644 index 00000000000..b876f7498ab --- /dev/null +++ b/apps/media-server/src/__tests__/routes/editor-camera-effects-benchmark.ts @@ -0,0 +1,577 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + copyFile, + mkdir, + mkdtemp, + readFile, + rm, + stat, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import { promisify } from "node:util"; +import app from "../../editor-worker-app"; +import { getEditorSession } from "../../lib/editor-sessions"; + +const runFile = promisify(execFile); +const secret = "editor-camera-effects-benchmark-secret"; +const benchmarkSeconds = 12; +const resolutionBase = { x: 1920, y: 1080 }; + +type NativeMetrics = { + producedFrames: number; + packedFrames: number; + rawBytes: number; + sentBytes: number; + h264Frames: number; + h264Bytes: number; + h264Viewers: number; + softwareAdapter: boolean; + gpuAdapter: string; +}; + +type ExportState = { + status: string; + error: string | null; + progress: { rendered_count: number; total_frames: number } | null; + mediaMetadata: { duration: number; width: number; height: number } | null; +}; + +async function ffmpeg(args: string[]) { + await runFile("ffmpeg", ["-hide_banner", "-loglevel", "error", ...args], { + timeout: 120_000, + }); +} + +async function makeSource(path: string, size: string, fps: number) { + await ffmpeg([ + "-f", + "lavfi", + "-i", + `testsrc2=size=${size}:rate=${fps}:duration=8`, + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-crf", + "34", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + path, + ]); +} + +async function loopSource(source: string, output: string) { + await ffmpeg([ + "-stream_loop", + "3", + "-i", + source, + "-c", + "copy", + "-t", + "30", + "-movflags", + "+faststart", + output, + ]); +} + +async function readySession(id: string, headers: Record) { + const deadline = Date.now() + 120_000; + while (Date.now() < deadline) { + const response = await app.request(`/editor/preparations/${id}`, { + headers, + }); + assert.equal(response.status, 200); + const result = (await response.json()) as { + status: string; + sessionId?: string; + error?: string; + }; + if (result.status === "ready") { + assert.ok(result.sessionId); + return result.sessionId; + } + if (result.status === "error") + throw new Error(result.error ?? "Editor preparation failed"); + await Bun.sleep(100); + } + throw new Error("Editor preparation timed out"); +} + +async function processSample(pid: number) { + const [statText, statusText] = await Promise.all([ + readFile(`/proc/${pid}/stat`, "utf8"), + readFile(`/proc/${pid}/status`, "utf8"), + ]); + const fields = statText.slice(statText.lastIndexOf(") ") + 2).split(" "); + const ticks = Number(fields[11]) + Number(fields[12]); + const rssKilobytes = Number(statusText.match(/^VmRSS:\s+(\d+)\s+kB/m)?.[1]); + assert.ok(Number.isFinite(ticks)); + assert.ok(Number.isFinite(rssKilobytes)); + return { ticks, rssMb: Math.round((rssKilobytes / 1024) * 10) / 10 }; +} + +assert.ok(process.platform === "linux"); +assert.ok(process.env.CAP_WEB_EDITOR_PREPARE_BIN); +assert.ok(process.env.CAP_WEB_EDITOR_SERVICE_BIN); +assert.equal(process.env.CAP_WEB_EDITOR_ENABLE_CAMERA_REMOVAL, "1"); +const { stdout: tickOutput } = await runFile("getconf", ["CLK_TCK"]); +const ticksPerSecond = Number(tickOutput.trim()); +assert.ok(ticksPerSecond > 0); + +const previousSecret = process.env.MEDIA_SERVER_WEBHOOK_SECRET; +const previousAllowHttp = process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; +process.env.MEDIA_SERVER_WEBHOOK_SECRET = secret; +process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = "1"; +const root = await mkdtemp(join(tmpdir(), "cap-editor-camera-bench-")); +const headers = { + "x-media-server-secret": secret, + "Content-Type": "application/json", +}; +let server: ReturnType | null = null; +let sessionId: string | null = null; +let viewer: WebSocket | null = null; +const viewerFallbacks: string[] = []; +let viewerPackets = 0; +try { + const displaySource = join(root, "display-source.mp4"); + const cameraSource = join(root, "camera-source.mp4"); + const display = join(root, "display.mp4"); + const camera = join(root, "camera.mp4"); + const realCameraVideo = process.env.CAP_EDITOR_CAMERA_REAL_VIDEO; + const displayFixture = async () => { + await makeSource(displaySource, "1920x1080", 30); + await loopSource(displaySource, display); + }; + const cameraFixture = async () => { + if (realCameraVideo) { + const source = await stat(realCameraVideo); + assert.ok(source.isFile() && source.size > 0); + await ffmpeg([ + "-stream_loop", + "1", + "-i", + realCameraVideo, + "-vf", + "fps=25,scale=720:720", + "-t", + "30", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-crf", + "23", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + camera, + ]); + } else { + await makeSource(cameraSource, "1280x720", 25); + await loopSource(cameraSource, camera); + } + }; + await Promise.all([displayFixture(), cameraFixture()]); + server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + const path = new URL(request.url).pathname; + if (path === "/display.mp4") return new Response(Bun.file(display)); + if (path === "/camera.mp4") return new Response(Bun.file(camera)); + return new Response("Missing fixture", { status: 404 }); + }, + }); + const base = `http://127.0.0.1:${server.port}`; + const preparation = await app.request("/editor/preparations", { + method: "POST", + headers, + body: JSON.stringify({ + videoId: "editor-camera-effects-benchmark", + title: "1080p screen and 720p camera effect benchmark", + captionsEnabled: false, + display: { + url: `${base}/display.mp4`, + contentType: "video/mp4", + size: (await stat(display)).size, + fps: 30, + }, + camera: { + url: `${base}/camera.mp4`, + contentType: "video/mp4", + size: (await stat(camera)).size, + fps: 25, + offsetMs: 125, + }, + }), + }); + assert.equal(preparation.status, 202); + const created = (await preparation.json()) as { id: string }; + sessionId = await readySession(created.id, headers); + const native = getEditorSession(sessionId); + assert.ok(native?.pid); + const pid = native.pid; + viewer = native.connectSocket( + `${native.origin.replace(/^http/, "ws")}/frames-h264`, + ); + viewer.binaryType = "arraybuffer"; + viewer.addEventListener("message", (event) => { + if (typeof event.data === "string") { + if (event.data.includes("cap-h264-unavailable")) + viewerFallbacks.push(event.data); + } else viewerPackets++; + }); + await new Promise((resolve, reject) => { + viewer?.addEventListener("open", () => resolve(), { once: true }); + viewer?.addEventListener( + "error", + () => reject(new Error("H.264 viewer failed")), + { + once: true, + }, + ); + }); + const instance = await app.request(`/editor/sessions/${sessionId}/instance`, { + headers, + }); + assert.equal(instance.status, 200); + const saved = (await instance.json()) as { + savedProjectConfig: { + camera: { backgroundBlur: { mode: string } }; + timeline?: Record | null; + }; + }; + const initialConfig = saved.savedProjectConfig; + const preview = async () => { + const started = performance.now(); + const response = await app.request( + `/editor/sessions/${sessionId}/preview`, + { + method: "POST", + headers, + body: JSON.stringify({ + frameNumber: 30, + fps: 30, + resolutionBase: { x: 640, y: 360 }, + }), + }, + ); + const bytes = Buffer.from(await response.arrayBuffer()); + return { + status: response.status, + ms: Math.round(performance.now() - started), + sha256: + response.status === 200 + ? createHash("sha256").update(bytes).digest("hex") + : null, + }; + }; + const metrics = async () => { + const response = await native.request("/metrics"); + assert.equal(response.status, 200); + return (await response.json()) as NativeMetrics; + }; + const measure = async (mode: "off" | "remove") => { + const config = structuredClone(initialConfig); + config.camera.backgroundBlur.mode = mode; + const update = await app.request( + `/editor/sessions/${sessionId}/config/memory`, + { + method: "PUT", + headers, + body: JSON.stringify(config), + }, + ); + assert.equal(update.status, 204); + const previewResult = await preview(); + const before = await metrics(); + assert.equal(before.h264Viewers, 1); + const processBefore = await processSample(pid); + const started = performance.now(); + const playback = await app.request( + `/editor/sessions/${sessionId}/playback`, + { + method: "POST", + headers, + body: JSON.stringify({ + frameNumber: 0, + fps: 60, + resolutionBase, + }), + }, + ); + assert.equal(playback.status, 204); + const frameSamples = []; + let previousFrames = before.producedFrames; + for (let second = 0; second < benchmarkSeconds; second++) { + await Bun.sleep(1000); + const sample = await metrics(); + frameSamples.push(sample.producedFrames - previousFrames); + previousFrames = sample.producedFrames; + } + const stopped = await app.request( + `/editor/sessions/${sessionId}/playback`, + { method: "DELETE", headers }, + ); + assert.equal(stopped.status, 204); + const elapsedSeconds = (performance.now() - started) / 1000; + const after = await metrics(); + const processAfter = await processSample(pid); + const producedFrames = after.producedFrames - before.producedFrames; + const warmedPreview = await preview(); + return { + mode, + preview: previewResult, + warmedPreview, + elapsedSeconds: Math.round(elapsedSeconds * 100) / 100, + producedFrames, + renderFps: Math.round((producedFrames / elapsedSeconds) * 10) / 10, + frameSamples, + packedFrames: after.packedFrames - before.packedFrames, + rawMb: Math.round((after.rawBytes - before.rawBytes) / 1048576), + sentMb: Math.round((after.sentBytes - before.sentBytes) / 1048576), + h264Frames: after.h264Frames - before.h264Frames, + h264Mb: Math.round((after.h264Bytes - before.h264Bytes) / 1048576), + cpuCoresUsed: + Math.round( + ((processAfter.ticks - processBefore.ticks) / + ticksPerSecond / + elapsedSeconds) * + 100, + ) / 100, + nativeRssBeforeMb: processBefore.rssMb, + nativeRssAfterMb: processAfter.rssMb, + }; + }; + const off = await measure("off"); + const remove = await measure("remove"); + const exportConfig = structuredClone(initialConfig); + exportConfig.camera.backgroundBlur.mode = "remove"; + exportConfig.timeline = { + ...(exportConfig.timeline ?? {}), + segments: [{ recordingSegment: 0, timescale: 1, start: 0, end: 2 }], + }; + const exportUpdate = await app.request( + `/editor/sessions/${sessionId}/config`, + { + method: "PUT", + headers, + body: JSON.stringify(exportConfig), + }, + ); + assert.equal(exportUpdate.status, 204); + const savedConfigResponse = await app.request( + `/editor/sessions/${sessionId}/config`, + { headers }, + ); + assert.equal(savedConfigResponse.status, 200); + const savedConfig = (await savedConfigResponse.json()) as typeof exportConfig; + const savedTimeline = savedConfig.timeline as + | { segments: Array<{ end: number }> } + | null + | undefined; + assert.equal(savedConfig.camera.backgroundBlur.mode, "remove"); + assert.equal(savedTimeline?.segments[0]?.end, 2); + const previewResponse = await app.request( + `/editor/sessions/${sessionId}/preview`, + { + method: "POST", + headers, + body: JSON.stringify({ + frameNumber: 30, + fps: 30, + resolutionBase: { x: 640, y: 360 }, + }), + }, + ); + assert.equal(previewResponse.status, 200); + const rgbaPixels = async (response: Response) => { + assert.equal(response.status, 200); + const bytes = Buffer.from(await response.arrayBuffer()); + const footer = bytes.length - 24; + assert.equal(bytes.length, 640 * 360 * 4 + 24); + assert.equal(bytes.readUInt32LE(footer), 640 * 4); + assert.equal(bytes.readUInt32LE(footer + 4), 360); + assert.equal(bytes.readUInt32LE(footer + 8), 640); + assert.equal(bytes.readUInt32LE(footer + 12), 30); + return bytes.subarray(0, footer); + }; + const previewPath = join(root, "camera-removal-preview.rgba"); + await writeFile(previewPath, await rgbaPixels(previewResponse)); + const offConfig = structuredClone(exportConfig); + offConfig.camera.backgroundBlur.mode = "off"; + const offUpdate = await app.request( + `/editor/sessions/${sessionId}/config/memory`, + { method: "PUT", headers, body: JSON.stringify(offConfig) }, + ); + assert.equal(offUpdate.status, 204); + const offPreviewResponse = await app.request( + `/editor/sessions/${sessionId}/preview`, + { + method: "POST", + headers, + body: JSON.stringify({ + frameNumber: 30, + fps: 30, + resolutionBase: { x: 640, y: 360 }, + }), + }, + ); + const offPreviewPath = join(root, "camera-off-preview.rgba"); + await writeFile(offPreviewPath, await rgbaPixels(offPreviewResponse)); + const removeUpdate = await app.request( + `/editor/sessions/${sessionId}/config/memory`, + { method: "PUT", headers, body: JSON.stringify(exportConfig) }, + ); + assert.equal(removeUpdate.status, 204); + const exportResponse = await app.request( + `/editor/sessions/${sessionId}/exports`, + { + method: "POST", + headers, + body: JSON.stringify({ + format: "Mp4", + fps: 30, + resolution_base: { x: 640, y: 360 }, + compression: "Social", + custom_bpp: null, + force_ffmpeg_decoder: true, + optimize_filesize: false, + }), + }, + ); + assert.equal(exportResponse.status, 202); + const exportId = ((await exportResponse.json()) as { id: string }).id; + let exportState: ExportState | null = null; + const exportDeadline = Date.now() + 120_000; + while (Date.now() < exportDeadline) { + const response = await app.request( + `/editor/sessions/${sessionId}/exports/${exportId}`, + { headers }, + ); + assert.equal(response.status, 200); + exportState = (await response.json()) as ExportState; + if (exportState?.status === "ready") break; + if (exportState?.status === "error") + throw new Error(exportState.error ?? "Camera removal export failed"); + await Bun.sleep(100); + } + if (!exportState || exportState.status !== "ready") + throw new Error("Camera removal export timed out"); + assert.equal(exportState.progress?.rendered_count, 60); + assert.equal(exportState.progress?.total_frames, 60); + assert.equal(exportState.mediaMetadata?.width, 640); + assert.equal(exportState.mediaMetadata?.height, 360); + const fileResponse = await app.request( + `/editor/sessions/${sessionId}/exports/${exportId}/file`, + { headers }, + ); + assert.equal(fileResponse.status, 200); + const exportPath = join(root, "camera-removal-export.mp4"); + await writeFile(exportPath, Buffer.from(await fileResponse.arrayBuffer())); + const exportFramePath = join(root, "camera-removal-export-frame.png"); + await ffmpeg([ + "-i", + exportPath, + "-vf", + "select=eq(n\\,30)", + "-frames:v", + "1", + exportFramePath, + ]); + const exportPsnr = async (sourcePath: string) => { + const { stderr } = await runFile("ffmpeg", [ + "-hide_banner", + "-loglevel", + "info", + "-f", + "rawvideo", + "-pixel_format", + "rgba", + "-video_size", + "640x360", + "-i", + sourcePath, + "-i", + exportFramePath, + "-lavfi", + "psnr", + "-f", + "null", + "-", + ]); + return Number(stderr.match(/average:([0-9.]+)/)?.[1]); + }; + const exportPsnrDb = await exportPsnr(previewPath); + const offExportPsnrDb = await exportPsnr(offPreviewPath); + assert.ok(Number.isFinite(exportPsnrDb)); + assert.ok(Number.isFinite(offExportPsnrDb)); + assert.ok(exportPsnrDb >= 20); + assert.ok(exportPsnrDb > offExportPsnrDb + 3); + const artifactDir = process.env.CAP_EDITOR_CAMERA_BENCH_ARTIFACT_DIR; + if (artifactDir) { + await mkdir(artifactDir, { recursive: true }); + for (const path of [ + previewPath, + offPreviewPath, + exportFramePath, + exportPath, + ]) + await copyFile(path, join(artifactDir, basename(path))); + } + const removalSha256 = remove.preview.sha256 ?? remove.warmedPreview?.sha256; + if (off.preview.sha256 && removalSha256) + assert.notEqual(off.preview.sha256, removalSha256); + const adapter = await metrics(); + process.stdout.write( + `${JSON.stringify({ + sources: { + display: "1920x1080@30", + camera: realCameraVideo ? "720x720@25 real subject" : "1280x720@25", + }, + requestedFps: 60, + softwareAdapter: adapter.softwareAdapter, + gpuAdapter: adapter.gpuAdapter, + workerRssMb: Math.round(process.memoryUsage().rss / 1048576), + viewerPackets, + viewerFallbacks, + off, + remove, + export: { + frames: exportState.progress?.rendered_count, + duration: exportState.mediaMetadata?.duration, + previewToMp4PsnrDb: exportPsnrDb, + offPreviewToMp4PsnrDb: offExportPsnrDb, + }, + })}\n`, + ); + for (const result of [off, remove]) { + assert.equal(result.preview.status, 200); + assert.equal(result.warmedPreview.status, 200); + } + assert.deepEqual(viewerFallbacks, []); +} finally { + viewer?.close(); + if (sessionId) + await app.request(`/editor/sessions/${sessionId}`, { + method: "DELETE", + headers, + }); + server?.stop(true); + await rm(root, { recursive: true, force: true }); + if (previousSecret === undefined) + delete process.env.MEDIA_SERVER_WEBHOOK_SECRET; + else process.env.MEDIA_SERVER_WEBHOOK_SECRET = previousSecret; + if (previousAllowHttp === undefined) + delete process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + else process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = previousAllowHttp; +} diff --git a/apps/media-server/src/__tests__/routes/editor-cap-caption-benchmark.ts b/apps/media-server/src/__tests__/routes/editor-cap-caption-benchmark.ts new file mode 100644 index 00000000000..6db4566cf68 --- /dev/null +++ b/apps/media-server/src/__tests__/routes/editor-cap-caption-benchmark.ts @@ -0,0 +1,292 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { lstat, mkdtemp, open, readdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, relative } from "node:path"; +import { promisify } from "node:util"; +import { + CAP_BUNDLE_CONTENT_TYPE, + CAP_BUNDLE_HEADER_BYTES, + CAP_BUNDLE_MAGIC, + parseCapBundleManifest, + readCapBundleManifestLength, + validCapBundlePath, +} from "@cap/editor-cap-bundle"; +import { prepareEditorCapCaptionAudio } from "../../lib/editor-cap-captions"; + +const runFile = promisify(execFile); +const binary = process.env.CAP_WEB_EDITOR_PREPARE_BIN; +assert.ok(binary); +const recordingSeconds = 900; +const root = await mkdtemp(join(tmpdir(), "cap-editor-caption-benchmark-")); + +async function durationMs(path: string) { + const { stdout } = await runFile("ffprobe", [ + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=nw=1:nk=1", + path, + ]); + const seconds = Number(stdout.trim()); + assert.ok(Number.isFinite(seconds) && seconds > 0); + return Math.round(seconds * 1000); +} + +async function bundleProject(sourceProject: string) { + const sources: Array<{ path: string; absolutePath: string; size: number }> = + []; + const directories = [sourceProject]; + while (directories.length > 0) { + const directory = directories.pop(); + assert.ok(directory); + for (const entry of await readdir(directory, { withFileTypes: true })) { + const absolutePath = join(directory, entry.name); + if (entry.isDirectory()) { + directories.push(absolutePath); + } else if (entry.isFile()) { + const path = relative(sourceProject, absolutePath).replaceAll( + "\\", + "/", + ); + if (validCapBundlePath(path)) { + sources.push({ + path, + absolutePath, + size: (await lstat(absolutePath)).size, + }); + } + } + } + } + sources.sort((a, b) => a.path.localeCompare(b.path)); + let offset = 0; + const files = sources.map((source) => { + const entry = { path: source.path, size: source.size, offset }; + offset += source.size; + return entry; + }); + const manifestBytes = new TextEncoder().encode( + JSON.stringify({ version: 1, files }), + ); + const header = new Uint8Array(CAP_BUNDLE_HEADER_BYTES); + header.set(new TextEncoder().encode(CAP_BUNDLE_MAGIC)); + new DataView(header.buffer).setUint32(8, manifestBytes.length, true); + const bundlePath = join(root, "source.capbundle"); + const destination = await open(bundlePath, "wx", 0o600); + try { + await destination.writeFile(header); + await destination.writeFile(manifestBytes); + for (const source of sources) { + for await (const chunk of createReadStream(source.absolutePath, { + highWaterMark: 1024 * 1024, + })) { + await destination.writeFile(chunk); + } + } + } finally { + await destination.close(); + } + const size = (await lstat(bundlePath)).size; + assert.equal(size, CAP_BUNDLE_HEADER_BYTES + manifestBytes.length + offset); + assert.equal(readCapBundleManifestLength(header), manifestBytes.length); + assert.ok(parseCapBundleManifest(manifestBytes, size)); + return bundlePath; +} + +try { + const display = join(root, "display.mp4"); + const camera = join(root, "camera.mp4"); + const mic = join(root, "mic.wav"); + const systemAudio = join(root, "system.wav"); + const generatedStart = performance.now(); + await Promise.all([ + runFile( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=size=1920x1080:rate=1", + "-t", + String(recordingSeconds), + "-threads", + "4", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-crf", + "38", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + display, + ], + { timeout: 5 * 60 * 1000, maxBuffer: 64 * 1024 }, + ), + runFile( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "color=c=green:s=640x360:r=1", + "-t", + String(recordingSeconds), + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + camera, + ], + { timeout: 5 * 60 * 1000, maxBuffer: 64 * 1024 }, + ), + ...[ + { frequency: 440, path: mic }, + { frequency: 660, path: systemAudio }, + ].map(({ frequency, path }) => + runFile( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + `sine=frequency=${frequency}:sample_rate=48000`, + "-t", + String(recordingSeconds), + "-ac", + "1", + "-c:a", + "pcm_s16le", + path, + ], + { timeout: 5 * 60 * 1000, maxBuffer: 64 * 1024 }, + ), + ), + ]); + const generatedMs = Math.round(performance.now() - generatedStart); + const sourceProject = join(root, `${randomUUID()}.cap`); + const manifestPath = join(root, "manifest.json"); + await writeFile( + manifestPath, + JSON.stringify({ + version: 1, + title: + "Fifteen-minute separate screen, webcam, microphone, and system audio", + displayPath: display, + displayFps: 30, + cameraPath: camera, + cameraFps: 25, + cameraOffsetMs: 0, + micPath: mic, + systemAudioPath: systemAudio, + mixedAudioInDisplay: false, + }), + ); + await runFile(binary, ["prepare", sourceProject, manifestPath], { + timeout: 5 * 60 * 1000, + maxBuffer: 64 * 1024, + }); + const bundlePath = await bundleProject(sourceProject); + const bundle = Bun.file(bundlePath); + const etag = JSON.stringify("cap-caption-900s"); + const server = Bun.serve({ + port: 0, + fetch(request) { + if (request.headers.get("if-match") !== etag) { + return new Response(null, { status: 412 }); + } + return new Response(bundle, { + headers: { + ETag: etag, + "Content-Length": String(bundle.size), + }, + }); + }, + }); + const previousAllowHttp = process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = "1"; + try { + const displayDurationMs = await durationMs(display); + const segmentDurationMs = Math.max( + displayDurationMs, + await durationMs(camera), + await durationMs(mic), + await durationMs(systemAudio), + ); + const expectedSegments = [ + { + mediaDurationMs: displayDurationMs, + segmentDurationMs, + hasAudio: true, + }, + ]; + let peakRss = process.memoryUsage().rss; + const sampler = setInterval(() => { + peakRss = Math.max(peakRss, process.memoryUsage().rss); + }, 100); + const started = performance.now(); + let prepared: Awaited>; + try { + prepared = await prepareEditorCapCaptionAudio( + { + path: `content/imports/${randomUUID()}.capbundle`, + name: "Fifteen-minute Studio source", + url: `http://127.0.0.1:${server.port}/source.capbundle`, + size: bundle.size, + contentType: CAP_BUNDLE_CONTENT_TYPE, + objectIdentity: etag, + }, + expectedSegments, + ); + } finally { + clearInterval(sampler); + } + const captionPrepareMs = Math.round(performance.now() - started); + try { + const actualDurationMs = await durationMs(prepared.path); + assert.ok(Math.abs(actualDurationMs - segmentDurationMs) <= 250); + const output = await lstat(prepared.path); + assert.equal(output.size, prepared.size); + process.stdout.write( + `${JSON.stringify({ + recordingSeconds, + generatedMs, + bundleMiB: Math.round(bundle.size / 1024 / 1024), + captionPrepareMs, + captionAudioMiB: Math.round(prepared.size / 1024 / 1024), + peakBunRssMiB: Math.round(peakRss / 1024 / 1024), + actualDurationMs, + })}\n`, + ); + } finally { + await prepared.cleanup(); + } + } finally { + server.stop(true); + if (previousAllowHttp === undefined) { + delete process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + } else { + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = previousAllowHttp; + } + } +} finally { + await rm(root, { recursive: true, force: true }); +} diff --git a/apps/media-server/src/__tests__/routes/editor-cap-import-replay.ts b/apps/media-server/src/__tests__/routes/editor-cap-import-replay.ts new file mode 100644 index 00000000000..431b73e40fc --- /dev/null +++ b/apps/media-server/src/__tests__/routes/editor-cap-import-replay.ts @@ -0,0 +1,888 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { + copyFile, + mkdir, + mkdtemp, + readdir, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, relative } from "node:path"; +import { promisify } from "node:util"; +import type { VideoMetadata } from "@cap/database/types"; +import { + CAP_BUNDLE_CONTENT_TYPE, + createCapBundle, + validCapBundlePath, +} from "@cap/editor-cap-bundle"; +import editorWorkerApp from "../../editor-worker-app"; +import { extractEditorCapBundle } from "../../lib/editor-cap-bundle"; +import { prepareEditorCapCaptionAudio } from "../../lib/editor-cap-captions"; +import { + prepareNativeEditorProject, + startNativeEditorSession, +} from "../../lib/editor-native"; + +const runFile = promisify(execFile); +const binary = process.env.CAP_WEB_EDITOR_PREPARE_BIN; +assert.ok(binary); +const fixtures = join(import.meta.dir, "../fixtures/editor-clips"); +const screen = join(fixtures, "display-red.webm"); +const camera = join(fixtures, "camera-green.webm"); +const imported = join(fixtures, "clip-blue-audio.mp4"); +const cursorImage = join(import.meta.dir, "../fixtures/exif-orientation-6.jpg"); + +type VideoPath = { path: string }; +type Segment = { + display: VideoPath; + camera?: VideoPath; + mic?: VideoPath; + system_audio?: VideoPath; + cursor?: string; + keyboard?: string; +}; +type Cursor = { imagePath: string; hotspot: { x: number; y: number } }; +type RecordingMeta = { + segments: Segment[]; + cursors: Record; +}; +type TimelineSegment = { + recordingSegment: number; + timescale: number; + start: number; + end: number; + speedAudioMode?: string; + hideCursor?: boolean; + volume?: number; + name?: string | null; +}; +type ProjectConfig = { + timeline?: { segments: TimelineSegment[]; zoomSegments: unknown[] }; +}; + +async function jsonFile(path: string): Promise { + return JSON.parse(await Bun.file(path).text()) as T; +} + +async function durationMs(path: string) { + const { stdout } = await runFile("ffprobe", [ + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=nw=1:nk=1", + path, + ]); + const seconds = Number(stdout.trim()); + assert.ok(Number.isFinite(seconds) && seconds > 0); + return Math.round(seconds * 1000); +} + +async function bundleProject(sourceProject: string, bundlePath: string) { + const sources: Array<{ path: string; file: Blob }> = []; + const directories = [sourceProject]; + while (directories.length > 0) { + const directory = directories.pop(); + assert.ok(directory); + for (const entry of await readdir(directory, { withFileTypes: true })) { + const absolutePath = join(directory, entry.name); + if (entry.isDirectory()) { + directories.push(absolutePath); + } else if (entry.isFile()) { + const path = relative(sourceProject, absolutePath).replaceAll( + "\\", + "/", + ); + if (validCapBundlePath(path)) { + sources.push({ + path, + file: new Blob([await readFile(absolutePath)]), + }); + } + } + } + } + const bundle = createCapBundle(sources); + await writeFile(bundlePath, new Uint8Array(await bundle.arrayBuffer())); +} + +const temporary = await mkdtemp(join(tmpdir(), "cap-editor-cap-import-")); +try { + const sourceProject = join(temporary, `${randomUUID()}.cap`); + const instantProject = join(temporary, `${randomUUID()}.cap`); + const targetProject = join(temporary, `${randomUUID()}.cap`); + const sourceManifest = join(temporary, "source-manifest.json"); + const targetManifest = join(temporary, "target-manifest.json"); + const mic = join(temporary, "mic.wav"); + const systemAudio = join(temporary, "system.wav"); + await runFile("ffmpeg", [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "sine=frequency=880:sample_rate=48000", + "-t", + "3", + "-ac", + "2", + mic, + ]); + await copyFile(mic, systemAudio); + await writeFile( + sourceManifest, + JSON.stringify({ + version: 1, + title: "Two source clips with separate camera and audio tracks", + displayPath: screen, + displayFps: 30, + cameraPath: camera, + cameraFps: 25, + cameraOffsetMs: 125, + micPath: mic, + micOffsetMs: 0, + systemAudioPath: systemAudio, + systemAudioOffsetMs: 0, + mixedAudioInDisplay: false, + }), + ); + await writeFile( + targetManifest, + JSON.stringify({ + version: 1, + title: "Target editor project", + displayPath: imported, + displayFps: 30, + mixedAudioInDisplay: true, + }), + ); + await Promise.all([ + runFile(binary, ["prepare", sourceProject, sourceManifest]), + runFile(binary, ["prepare", targetProject, targetManifest]), + ]); + const sourceVideos = join(sourceProject, "content/videos"); + await mkdir(sourceVideos, { recursive: true }); + const extraDisplay = join(sourceVideos, "extra-display.mp4"); + const extraCamera = join(sourceVideos, "extra-camera.webm"); + await Promise.all([ + copyFile(imported, extraDisplay), + copyFile(camera, extraCamera), + ]); + const clipManifest = join(temporary, "clip-manifest.json"); + await writeFile( + clipManifest, + JSON.stringify({ + version: 1, + clips: [ + { + displayPath: extraDisplay, + cameraPath: extraCamera, + duration: 2, + fps: 30, + hasAudio: true, + cameraFps: 25, + cameraOffsetMs: 125, + }, + ], + }), + ); + await runFile(binary, ["append-clips", sourceProject, clipManifest]); + const sourceMetaPath = join(sourceProject, "recording-meta.json"); + const sourceMeta = await jsonFile(sourceMetaPath); + assert.equal(sourceMeta.segments.length, 2); + const cursorRelative = "content/segments/segment-0/cursor.json"; + const keyboardRelative = "content/segments/segment-0/keyboard.json"; + const cursorImageRelative = "content/cursors/source-cursor.jpg"; + const cursorDirectory = join(sourceProject, "content/cursors"); + await mkdir(cursorDirectory, { recursive: true }); + await copyFile(cursorImage, join(sourceProject, cursorImageRelative)); + await writeFile( + join(sourceProject, cursorRelative), + JSON.stringify({ + clicks: [ + { + active_modifiers: [], + cursor_num: 0, + cursor_id: "source-cursor", + time_ms: 500, + down: true, + }, + ], + moves: [ + { + active_modifiers: [], + cursor_id: "source-cursor", + time_ms: 400, + x: 0.3, + y: 0.4, + }, + ], + }), + ); + await writeFile( + join(sourceProject, keyboardRelative), + JSON.stringify({ + presses: [ + { + key: "A", + keyCode: "KeyA", + timeMs: 650, + down: true, + }, + ], + }), + ); + const sourceFirstSegment = sourceMeta.segments[0]; + assert.ok(sourceFirstSegment); + sourceFirstSegment.cursor = cursorRelative; + sourceFirstSegment.keyboard = keyboardRelative; + sourceMeta.cursors = { + "source-cursor": { + imagePath: cursorImageRelative, + hotspot: { x: 4, y: 5 }, + }, + }; + await writeFile(sourceMetaPath, JSON.stringify(sourceMeta)); + const sourceConfigPath = join(sourceProject, "project-config.json"); + const sourceConfig = await jsonFile(sourceConfigPath); + sourceConfig.timeline = { + segments: [ + { + recordingSegment: 0, + timescale: 1.5, + start: 0.2, + end: 1.6, + speedAudioMode: "maintainPitch", + hideCursor: false, + volume: 0.75, + }, + { + recordingSegment: 1, + timescale: 1, + start: 0, + end: 1.2, + hideCursor: true, + }, + ], + zoomSegments: [], + }; + await writeFile(sourceConfigPath, JSON.stringify(sourceConfig)); + const sourcePreviewSettingsPath = join( + temporary, + "source-preview-settings.json", + ); + await writeFile( + sourcePreviewSettingsPath, + JSON.stringify({ + fps: 30, + resolution_base: { x: 640, y: 360 }, + compression_bpp: 0.7, + }), + ); + await runFile(binary, [ + "preview", + sourceProject, + sourceConfigPath, + "0.25", + sourcePreviewSettingsPath, + join(temporary, "source-preview.jpg"), + ]); + const bundlePath = join(temporary, "source.capbundle"); + await bundleProject(sourceProject, bundlePath); + const extracted = await extractEditorCapBundle(bundlePath); + let appendedStdout = ""; + try { + const appended = await runFile(binary, [ + "append-cap", + targetProject, + extracted.path, + ]); + appendedStdout = appended.stdout.toString(); + } finally { + await extracted.cleanup(); + } + assert.equal( + (JSON.parse(appendedStdout) as { clipCount: number }).clipCount, + 2, + ); + const targetMeta = await jsonFile( + join(targetProject, "recording-meta.json"), + ); + assert.equal(targetMeta.segments.length, 3); + const firstImported = targetMeta.segments[1]; + const secondImported = targetMeta.segments[2]; + assert.ok(firstImported?.camera?.path); + assert.ok(firstImported?.mic?.path); + assert.ok(firstImported?.system_audio?.path); + assert.ok(firstImported?.cursor); + assert.ok(firstImported?.keyboard); + assert.ok(secondImported?.camera?.path); + assert.equal( + secondImported?.system_audio?.path, + secondImported?.display.path, + ); + const sourceAudioSegments = await Promise.all( + sourceMeta.segments.map(async (segment) => { + const mediaDurationMs = await durationMs( + join(sourceProject, segment.display.path), + ); + const trackPaths = [ + segment.camera?.path, + segment.mic?.path, + segment.system_audio?.path, + ].filter((path): path is string => typeof path === "string"); + const trackDurations = await Promise.all( + trackPaths.map((path) => durationMs(join(sourceProject, path))), + ); + return { + mediaDurationMs, + segmentDurationMs: Math.max(mediaDurationMs, ...trackDurations), + hasAudio: Boolean(segment.mic || segment.system_audio), + }; + }), + ); + const importedCursorId = Object.keys(targetMeta.cursors)[0]; + assert.ok( + importedCursorId?.startsWith("import-") && + importedCursorId.endsWith("-source-cursor"), + ); + const importedCursor = targetMeta.cursors[importedCursorId]; + assert.deepEqual(importedCursor?.hotspot, { x: 4, y: 5 }); + const cursorEvents = await jsonFile<{ + moves: Array<{ cursor_id: string }>; + clicks: Array<{ cursor_id: string }>; + }>(join(targetProject, firstImported.cursor)); + assert.equal(cursorEvents.moves[0]?.cursor_id, importedCursorId); + assert.equal(cursorEvents.clicks[0]?.cursor_id, importedCursorId); + const keyboardEvents = await jsonFile<{ + presses: Array<{ keyCode: string }>; + }>(join(targetProject, firstImported.keyboard)); + assert.equal(keyboardEvents.presses[0]?.keyCode, "KeyA"); + for (const path of [ + firstImported.display.path, + firstImported.camera.path, + firstImported.mic.path, + firstImported.system_audio.path, + importedCursor.imagePath, + ]) { + assert.ok(await Bun.file(join(targetProject, path)).exists(), path); + } + const targetConfigPath = join(targetProject, "project-config.json"); + const targetConfig = await jsonFile(targetConfigPath); + assert.equal(targetConfig.timeline?.segments.length, 3); + assert.deepEqual(targetConfig.timeline?.segments.slice(1), [ + { + recordingSegment: 1, + timescale: 1.5, + start: 0.2, + end: 1.6, + speedAudioMode: "maintainPitch", + hideCursor: false, + volume: 0.75, + name: null, + }, + { + recordingSegment: 2, + timescale: 1, + start: 0, + end: 1.2, + hideCursor: true, + name: null, + }, + ]); + const { stdout: probe } = await runFile("ffprobe", [ + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=nw=1:nk=1", + imported, + ]); + const targetDuration = Number(probe.trim()); + assert.ok(Number.isFinite(targetDuration) && targetDuration > 0); + const previewSettingsPath = join(temporary, "preview-settings.json"); + const previewPath = join(temporary, "imported-preview.jpg"); + await writeFile( + previewSettingsPath, + JSON.stringify({ + fps: 30, + resolution_base: { x: 640, y: 360 }, + compression_bpp: 0.7, + }), + ); + await runFile(binary, [ + "preview", + targetProject, + targetConfigPath, + String(targetDuration + 0.25), + previewSettingsPath, + previewPath, + ]); + const preview = await readFile(previewPath); + assert.equal(preview[0], 0xff); + assert.equal(preview[1], 0xd8); + assert.ok(preview.length > 1024); + const tailPath = `content/videos/${randomUUID()}.mp4`; + await mkdir(join(targetProject, "content/videos"), { recursive: true }); + await copyFile(imported, join(targetProject, tailPath)); + const tailManifest = join(temporary, "tail-manifest.json"); + await writeFile( + tailManifest, + JSON.stringify({ + version: 1, + clips: [ + { + displayPath: join(targetProject, tailPath), + duration: targetDuration, + fps: 30, + hasAudio: true, + }, + ], + }), + ); + await runFile(binary, ["append-clip", targetProject, tailManifest]); + const configWithTail = await jsonFile(targetConfigPath); + assert.ok(configWithTail.timeline); + configWithTail.timeline.segments.push({ + recordingSegment: 3, + timescale: 1, + start: 0, + end: targetDuration, + }); + await writeFile(targetConfigPath, JSON.stringify(configWithTail)); + const tailMeta = await jsonFile( + join(targetProject, "recording-meta.json"), + ); + assert.equal(tailMeta.segments.length, 4); + await mkdir(join(instantProject, "content"), { recursive: true }); + await copyFile(imported, join(instantProject, "content/output.mp4")); + await writeFile( + join(instantProject, "recording-meta.json"), + JSON.stringify({ + pretty_name: "Completed Instant recording", + fps: 30, + sample_rate: null, + }), + ); + const instantBundlePath = join(temporary, "instant.capbundle"); + await bundleProject(instantProject, instantBundlePath); + const extractedInstant = await extractEditorCapBundle(instantBundlePath); + let instantStdout = ""; + try { + const appended = await runFile(binary, [ + "append-cap", + targetProject, + extractedInstant.path, + ]); + instantStdout = appended.stdout.toString(); + } finally { + await extractedInstant.cleanup(); + } + assert.equal( + (JSON.parse(instantStdout) as { clipCount: number }).clipCount, + 1, + ); + const configWithInstant = await jsonFile(targetConfigPath); + const instantMeta = await jsonFile( + join(targetProject, "recording-meta.json"), + ); + assert.equal(instantMeta.segments.length, 5); + assert.equal( + instantMeta.segments[4]?.system_audio?.path, + instantMeta.segments[4]?.display.path, + ); + assert.equal(configWithInstant.timeline?.segments.length, 5); + const beforeInstant = configWithInstant.timeline?.segments + .slice(0, -1) + .reduce( + (total, segment) => + total + (segment.end - segment.start) / segment.timescale, + 0, + ); + assert.ok(beforeInstant !== undefined && Number.isFinite(beforeInstant)); + await runFile(binary, [ + "preview", + targetProject, + targetConfigPath, + String(beforeInstant + 0.25), + previewSettingsPath, + join(temporary, "instant-preview.jpg"), + ]); + const sourceBlob = Bun.file(bundlePath); + const instantBlob = Bun.file(instantBundlePath); + const clipBlob = Bun.file(imported); + const etag = JSON.stringify("cap-bundle-replay"); + const server = Bun.serve({ + port: 0, + fetch(request) { + if (request.headers.get("if-match") !== etag) { + return new Response(null, { status: 412 }); + } + const pathname = new URL(request.url).pathname; + const blob = + pathname === "/source.capbundle" + ? sourceBlob + : pathname === "/instant.capbundle" + ? instantBlob + : clipBlob; + return new Response(blob, { + headers: { + ETag: etag, + "Content-Length": String(blob.size), + }, + }); + }, + }); + const previousAllowHttp = process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = "1"; + try { + const studioCapId = randomUUID(); + const instantCapId = randomUUID(); + const studioCapPath = `content/imports/${studioCapId}.capbundle`; + const instantCapPath = `content/imports/${instantCapId}.capbundle`; + let nativeStudioCaptionSegments: typeof sourceAudioSegments | null = null; + const replay = await prepareNativeEditorProject({ + title: "Ordered Cap and MP4 import replay", + display: { + path: imported, + contentType: "video/mp4", + size: clipBlob.size, + fps: 30, + }, + mixedAudioInDisplay: true, + projectConfig: configWithInstant as Record, + videoAssets: [ + { + path: tailPath, + name: "Tail MP4", + url: `http://127.0.0.1:${server.port}/tail.mp4`, + size: clipBlob.size, + contentType: "video/mp4", + objectIdentity: etag, + }, + ], + imports: [ + { + kind: "cap", + asset: { + path: studioCapPath, + name: "Studio source", + url: `http://127.0.0.1:${server.port}/source.capbundle`, + size: sourceBlob.size, + contentType: CAP_BUNDLE_CONTENT_TYPE, + objectIdentity: etag, + }, + clipCount: 2, + }, + { + kind: "clip", + clip: { + displayPath: tailPath, + duration: targetDuration, + fps: 30, + hasAudio: true, + }, + }, + { + kind: "cap", + asset: { + path: instantCapPath, + name: "Instant source", + url: `http://127.0.0.1:${server.port}/instant.capbundle`, + size: instantBlob.size, + contentType: CAP_BUNDLE_CONTENT_TYPE, + objectIdentity: etag, + }, + clipCount: 1, + }, + ], + }); + let nativeSession: Awaited< + ReturnType + > | null = null; + try { + const replayMeta = await jsonFile( + join(replay.path, "recording-meta.json"), + ); + const replayConfig = await jsonFile( + join(replay.path, "project-config.json"), + ); + assert.equal(replayMeta.segments.length, 5); + assert.deepEqual( + replayConfig.timeline?.segments, + configWithInstant.timeline?.segments, + ); + assert.ok(replayMeta.segments[1]?.camera?.path); + assert.ok(replayMeta.segments[1]?.cursor); + assert.ok(replayMeta.segments[1]?.keyboard); + assert.ok(replayMeta.segments[3]?.display.path); + assert.equal( + replayMeta.segments[4]?.system_audio?.path, + replayMeta.segments[4]?.display.path, + ); + nativeSession = await startNativeEditorSession(replay); + const nativeResponse = await nativeSession.request("/instance"); + assert.equal(nativeResponse.status, 200); + const nativeInstance: unknown = await nativeResponse.json(); + const captionMetadata: VideoMetadata = { + editorSources: { + version: 1, + display: { + key: "owner/video/result.mp4", + contentType: "video/mp4", + size: clipBlob.size, + objectIdentity: etag, + }, + }, + webEditorVideos: { + version: 1, + items: [ + { + key: `owner/video/editor-assets/recordings/${studioCapId}.capbundle`, + path: studioCapPath, + name: "Studio source", + contentType: CAP_BUNDLE_CONTENT_TYPE, + size: sourceBlob.size, + objectIdentity: etag, + }, + { + key: `owner/video/editor-assets/videos/${tailPath.split("/").at(-1)}`, + path: tailPath, + name: "Tail MP4", + contentType: "video/mp4", + size: clipBlob.size, + objectIdentity: etag, + }, + { + key: `owner/video/editor-assets/recordings/${instantCapId}.capbundle`, + path: instantCapPath, + name: "Instant source", + contentType: CAP_BUNDLE_CONTENT_TYPE, + size: instantBlob.size, + objectIdentity: etag, + }, + ], + }, + webEditorClips: { + version: 1, + items: [ + { + displayPath: tailPath, + duration: targetDuration, + fps: 30, + hasAudio: true, + }, + ], + }, + webEditorImports: { + version: 1, + items: [ + { kind: "cap", path: studioCapPath, clipCount: 2 }, + { kind: "clip", path: tailPath }, + { kind: "cap", path: instantCapPath, clipCount: 1 }, + ], + }, + }; + const nativeCaptionFixture = join( + temporary, + "native-caption-fixture.json", + ); + await writeFile( + nativeCaptionFixture, + JSON.stringify({ metadata: captionMetadata, instance: nativeInstance }), + ); + const { stdout: captionPlanOutput } = await runFile(process.execPath, [ + join( + import.meta.dir, + "../../../../web/__tests__/editor-caption-native-replay.ts", + ), + nativeCaptionFixture, + ]); + const measuredSegments = ( + JSON.parse(captionPlanOutput) as { + studioSegments: typeof sourceAudioSegments; + } + ).studioSegments; + const keyboardResponse = await nativeSession.request( + "/keyboard-segments", + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + groupingThresholdMs: 350, + lingerDurationMs: 1800, + showModifiers: true, + showSpecialKeys: true, + }), + }, + ); + assert.equal(keyboardResponse.status, 200); + const generatedKeyboardSegments: unknown = await keyboardResponse.json(); + assert.ok( + Array.isArray(generatedKeyboardSegments) && + generatedKeyboardSegments.length > 0, + ); + const autoZoomResponse = await nativeSession.request( + "/auto-zoom-segments", + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ zoomAmount: 1.8 }), + }, + ); + assert.equal(autoZoomResponse.status, 200); + const generatedZoomSegments = (await autoZoomResponse.json()) as Array<{ + start: number; + end: number; + amount: number; + }>; + assert.ok(generatedZoomSegments.length > 0); + assert.equal(generatedZoomSegments[0]?.amount, 1.8); + assert.ok( + (generatedZoomSegments[0]?.start ?? 0) > 1.5, + "imported cursor clicks must zoom after the preceding source clip", + ); + assert.equal(measuredSegments.length, sourceAudioSegments.length); + for (const [index, measured] of measuredSegments.entries()) { + const source = sourceAudioSegments[index]; + assert.ok(source); + assert.ok( + Math.abs(measured.mediaDurationMs - source.mediaDurationMs) <= 2000, + ); + assert.ok( + Math.abs(measured.segmentDurationMs - source.segmentDurationMs) <= + 2000, + ); + } + nativeStudioCaptionSegments = measuredSegments; + } finally { + if (nativeSession) await nativeSession.close(); + else await replay.cleanup(); + } + assert.ok(nativeStudioCaptionSegments); + const studioCaptionAudio = await prepareEditorCapCaptionAudio( + { + path: `content/imports/${randomUUID()}.capbundle`, + name: "Studio source", + url: `http://127.0.0.1:${server.port}/source.capbundle`, + size: sourceBlob.size, + contentType: CAP_BUNDLE_CONTENT_TYPE, + objectIdentity: etag, + }, + nativeStudioCaptionSegments, + ); + try { + assert.ok(studioCaptionAudio.size > 1024); + assert.ok( + Math.abs( + (await durationMs(studioCaptionAudio.path)) - + nativeStudioCaptionSegments.reduce( + (total, segment) => total + segment.segmentDurationMs, + 0, + ), + ) <= 250, + ); + } finally { + await studioCaptionAudio.cleanup(); + } + const previousSecret = process.env.MEDIA_SERVER_WEBHOOK_SECRET; + process.env.MEDIA_SERVER_WEBHOOK_SECRET = "cap-caption-replay-secret"; + try { + const instantDurationMs = await durationMs(imported); + const requestBody = JSON.stringify({ + asset: { + path: `content/imports/${randomUUID()}.capbundle`, + name: "Instant source", + url: `http://127.0.0.1:${server.port}/instant.capbundle`, + size: instantBlob.size, + contentType: CAP_BUNDLE_CONTENT_TYPE, + objectIdentity: etag, + }, + segments: [ + { + mediaDurationMs: instantDurationMs, + segmentDurationMs: instantDurationMs, + hasAudio: true, + }, + ], + }); + const rejected = await editorWorkerApp.request( + "/editor/caption-cap-audio", + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: requestBody, + }, + ); + assert.equal(rejected.status, 401); + const captionResponse = await editorWorkerApp.request( + "/editor/caption-cap-audio", + { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-media-server-secret": "cap-caption-replay-secret", + }, + body: requestBody, + }, + ); + assert.equal(captionResponse.status, 200); + assert.equal(captionResponse.headers.get("content-type"), "audio/mp4"); + const captionBytes = new Uint8Array(await captionResponse.arrayBuffer()); + assert.ok(captionBytes.length > 1024); + const captionAudioPath = join(temporary, "instant-caption-audio.m4a"); + await writeFile(captionAudioPath, captionBytes); + const { stdout: assemblyReplayOutput } = await runFile(process.execPath, [ + join( + import.meta.dir, + "../../../../web/__tests__/editor-caption-assemblyai-stream-replay.ts", + ), + captionAudioPath, + ]); + assert.equal( + (JSON.parse(assemblyReplayOutput) as { uploadedBytes: number }) + .uploadedBytes, + captionBytes.length, + ); + } finally { + if (previousSecret === undefined) { + delete process.env.MEDIA_SERVER_WEBHOOK_SECRET; + } else { + process.env.MEDIA_SERVER_WEBHOOK_SECRET = previousSecret; + } + } + } finally { + server.stop(true); + if (previousAllowHttp === undefined) { + delete process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + } else { + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = previousAllowHttp; + } + } + process.stdout.write( + `${JSON.stringify({ + importedClips: 4, + orderedReplay: + "Studio Cap, MP4, then Instant Cap, with saved timeline restored", + preserved: [ + "webcam", + "microphone", + "system audio", + "cursor events and image", + "keyboard events", + "source trims and speed", + "Instant embedded audio", + ], + previewBytes: preview.length, + })}\n`, + ); +} finally { + await rm(temporary, { recursive: true, force: true }); +} diff --git a/apps/media-server/src/__tests__/routes/editor-caption-cache-browser-replay.ts b/apps/media-server/src/__tests__/routes/editor-caption-cache-browser-replay.ts new file mode 100644 index 00000000000..057751b50e4 --- /dev/null +++ b/apps/media-server/src/__tests__/routes/editor-caption-cache-browser-replay.ts @@ -0,0 +1,176 @@ +import assert from "node:assert/strict"; +import { + lstat, + mkdtemp, + rmdir, + symlink, + unlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { chromium, webkit } from "@playwright/test"; + +const root = resolve(import.meta.dir, "../../../../.."); +const cachePath = resolve( + root, + "packages/editor-solid-web/src/caption-cache-memo.ts", +); +const transportPath = resolve(root, "apps/web/lib/editor-caption-transport.ts"); +const temporary = await mkdtemp(join(tmpdir(), "cap-caption-cache-browser-")); +const entrypoint = join(temporary, "entry.ts"); +const dependencies = join(temporary, "node_modules"); + +const entrySource = ` +import { createStore, produce } from "solid-js/store"; +import { EditorCaptionCacheMemo } from ${JSON.stringify(cachePath)}; +import { createEditorCaptionCache } from ${JSON.stringify(transportPath)}; + +export async function replay() { + const segments = Array.from({ length: 3_000 }, (_, index) => ({ + id: "segment-" + index, + text: "A useful recording process for everyone", + start: index * 2.4, + end: index * 2.4 + 2.2, + words: Array.from({ length: 6 }, (_, wordIndex) => ({ + text: "recording", + start: index * 2.4 + wordIndex * 0.4, + end: index * 2.4 + wordIndex * 0.4 + 0.3, + })), + })); + const [project, setProject] = createStore({ + captions: { segments }, + timeline: { captionSegments: segments }, + camera: { mirror: false }, + }); + const cache = new EditorCaptionCacheMemo(); + try { + const initialTask = cache.get(project); + const initial = await initialTask; + if (!initial) throw new Error("Long caption cache was unavailable"); + const cachedDurations = []; + for (let index = 0; index < 25; index++) { + setProject("camera", "mirror", index % 2 === 0); + const config = { + ...project, + timeline: { ...project.timeline }, + camera: { ...project.camera }, + }; + const start = performance.now(); + const task = cache.get(config); + if (task !== initialTask || (await task)?.ref !== initial.ref) + throw new Error("Camera edit rehashed unchanged captions"); + cachedDurations.push(performance.now() - start); + } + setProject("captions", "segments", 0, "words", 0, "text", "changed"); + const sourceTask = cache.get(project); + const source = await sourceTask; + if (!source || sourceTask === initialTask || source.ref === initial.ref) + throw new Error("Caption word edit reused stale payload"); + setProject("timeline", "captionSegments", 0, "words", 0, "text", "edited"); + const trackTask = cache.get(project); + const track = await trackTask; + if (!track || trackTask === sourceTask || track.ref === source.ref) + throw new Error("Caption track edit reused stale payload"); + setProject(produce((draft) => { + draft.captions.segments.push({ ...segments[0], id: "added-caption" }); + })); + const addedTask = cache.get(project); + const added = await addedTask; + if (!added || addedTask === trackTask || added.ref === track.ref) + throw new Error("Added caption reused stale payload"); + const baselineDurations = []; + for (let index = 0; index < 5; index++) { + const start = performance.now(); + const result = await createEditorCaptionCache(project); + if (result?.ref !== added.ref) + throw new Error("Fresh caption hash disagreed with the cache"); + baselineDurations.push(performance.now() - start); + } + cachedDurations.sort((left, right) => left - right); + const cachedAverageMs = + cachedDurations.reduce((sum, duration) => sum + duration, 0) / + cachedDurations.length; + const baselineAverageMs = + baselineDurations.reduce((sum, duration) => sum + duration, 0) / + baselineDurations.length; + return { + captionBytes: new TextEncoder().encode(JSON.stringify({ + sourceSegments: segments, + trackSegments: segments, + })).byteLength, + cameraEdits: cachedDurations.length, + cachedAverageMs, + cachedP95Ms: cachedDurations[23], + baselineAverageMs, + }; + } finally { + cache.dispose(); + } +} +`; + +let linked = false; +try { + await symlink(join(root, "node_modules"), dependencies, "dir"); + linked = true; + await writeFile(entrypoint, entrySource); + const bundle = await Bun.build({ + entrypoints: [entrypoint], + target: "browser", + format: "esm", + }); + assert.ok(bundle.success, bundle.logs.map((log) => log.message).join("\n")); + assert.equal(bundle.outputs.length, 1); + const source = await bundle.outputs[0]?.text(); + assert.ok(source); + for (const [name, engine] of [ + ["Chromium", chromium], + ["WebKit", webkit], + ] as const) { + const browser = await engine.launch({ headless: true }); + try { + const page = await browser.newPage(); + try { + const url = "https://editor.cap.so/caption-cache-replay"; + await page.route(url, (route) => + route.fulfill({ + status: 200, + contentType: "text/html", + body: "", + }), + ); + await page.goto(url); + const result = await page.evaluate(async (code) => { + const moduleUrl = URL.createObjectURL( + new Blob([code], { type: "text/javascript" }), + ); + try { + const module = await import(moduleUrl); + return module.replay(); + } finally { + URL.revokeObjectURL(moduleUrl); + } + }, source); + assert.ok(result.captionBytes > 128 * 1024); + assert.equal(result.cameraEdits, 25); + assert.ok( + result.cachedAverageMs < Math.max(5, result.baselineAverageMs / 10), + `${name} repeated caption cache work during camera edits`, + ); + console.log(JSON.stringify({ browser: name, ...result })); + } finally { + await page.close(); + } + } finally { + await browser.close(); + } + } +} finally { + if (await Bun.file(entrypoint).exists()) await unlink(entrypoint); + if (linked) { + assert.ok((await lstat(dependencies)).isSymbolicLink()); + await unlink(dependencies); + } + await rmdir(temporary); +} diff --git a/apps/media-server/src/__tests__/routes/editor-clips-benchmark.ts b/apps/media-server/src/__tests__/routes/editor-clips-benchmark.ts new file mode 100644 index 00000000000..9f975c73560 --- /dev/null +++ b/apps/media-server/src/__tests__/routes/editor-clips-benchmark.ts @@ -0,0 +1,264 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import app from "../../editor-worker-app"; +import { getEditorSession } from "../../lib/editor-sessions"; + +const runFile = promisify(execFile); +const secret = "editor-clip-benchmark-secret"; +const longSeconds = 900; +const clipCount = 49; + +async function makeLongMedia(input: string, output: string, loops: number) { + await runFile( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-stream_loop", + String(loops), + "-i", + input, + "-c", + "copy", + "-t", + String(longSeconds), + output, + ], + { timeout: 60_000 }, + ); + const { stdout } = await runFile( + "ffprobe", + ["-v", "error", "-show_entries", "format=duration", "-of", "json", output], + { timeout: 30_000 }, + ); + const duration = Number( + (JSON.parse(stdout) as { format: { duration: string } }).format.duration, + ); + assert.ok(Math.abs(duration - longSeconds) < 0.1); +} + +async function nativeRssMb(pid: number | null) { + if (pid === null) return null; + const { stdout } = await runFile("ps", ["-o", "rss=", "-p", String(pid)]); + const kilobytes = Number(stdout.trim()); + return Number.isFinite(kilobytes) + ? Math.round((kilobytes / 1024) * 10) / 10 + : null; +} + +async function waitForPreparation(id: string, headers: Record) { + const deadline = Date.now() + 120_000; + while (Date.now() < deadline) { + const response = await app.request(`/editor/preparations/${id}`, { + headers, + }); + assert.equal(response.status, 200); + const result = (await response.json()) as { + status: string; + sessionId?: string; + error?: string; + }; + if (result.status === "ready") { + assert.ok(result.sessionId); + return result.sessionId; + } + if (result.status === "error") + throw new Error(result.error ?? "Native preparation failed"); + await Bun.sleep(100); + } + throw new Error("Native preparation timed out"); +} + +async function seekPreview( + sessionId: string, + seconds: number, + headers: Record, +) { + const started = performance.now(); + const response = await app.request(`/editor/sessions/${sessionId}/preview`, { + method: "POST", + headers, + body: JSON.stringify({ + frameNumber: Math.floor(seconds * 30), + fps: 30, + resolutionBase: { x: 640, y: 360 }, + }), + }); + assert.equal(response.status, 200); + const bytes = Buffer.from(await response.arrayBuffer()); + const stride = bytes.readUInt32LE(bytes.length - 24); + const offset = 180 * stride + 320 * 4; + const red = bytes[offset] ?? 0; + const blue = bytes[offset + 2] ?? 0; + assert.ok( + seconds < 903 ? red - blue > 100 : blue - red > 100, + `Incorrect source at ${seconds}s: red=${red}, blue=${blue}`, + ); + return Math.round((performance.now() - started) * 10) / 10; +} + +assert.ok(process.env.CAP_WEB_EDITOR_PREPARE_BIN); +assert.ok(process.env.CAP_WEB_EDITOR_SERVICE_BIN); +const previousSecret = process.env.MEDIA_SERVER_WEBHOOK_SECRET; +const previousAllowHttp = process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; +process.env.MEDIA_SERVER_WEBHOOK_SECRET = secret; +process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = "1"; +const fixtures = join(import.meta.dir, "../fixtures/editor-clips"); +const original = join(fixtures, "display-red.webm"); +const imported = join(fixtures, "clip-blue-audio.mp4"); +const camera = join(fixtures, "camera-green.webm"); +const temporary = await mkdtemp(join(tmpdir(), "cap-editor-clip-bench-")); +let server: ReturnType | null = null; +let sessionId: string | null = null; +try { + const longDisplay = join(temporary, "long-display.webm"); + const longCamera = join(temporary, "long-camera.webm"); + await Promise.all([ + makeLongMedia(original, longDisplay, 299), + makeLongMedia(camera, longCamera, 449), + ]); + const paths = new Map([ + ["/original.webm", original], + ["/imported.mp4", imported], + ["/camera.webm", camera], + ["/long-display.webm", longDisplay], + ["/long-camera.webm", longCamera], + ]); + server = Bun.serve({ + port: 0, + fetch(request) { + const path = paths.get(new URL(request.url).pathname); + return path + ? new Response(Bun.file(path)) + : new Response("Missing fixture", { status: 404 }); + }, + }); + const headers = { + "x-media-server-secret": secret, + "Content-Type": "application/json", + }; + const base = `http://127.0.0.1:${server.port}`; + const clips = Array.from({ length: clipCount }, (_, index) => { + const duration = index === 0 ? longSeconds : 2; + return { + displayPath: `content/videos/${randomUUID()}.${index === 0 ? "webm" : "mp4"}`, + cameraPath: `content/videos/${randomUUID()}.webm`, + duration, + fps: 30, + hasAudio: index !== 0, + cameraFps: 25, + cameraOffsetMs: 125, + }; + }); + const [longDisplaySize, longCameraSize, importedSize, cameraSize] = + await Promise.all([ + stat(longDisplay), + stat(longCamera), + stat(imported), + stat(camera), + ]); + const videoAssets = clips.flatMap((clip, index) => [ + { + path: clip.displayPath, + name: `Screen clip ${index + 1}`, + url: `${base}/${index === 0 ? "long-display.webm" : "imported.mp4"}`, + size: index === 0 ? longDisplaySize.size : importedSize.size, + contentType: index === 0 ? "video/webm" : "video/mp4", + }, + { + path: clip.cameraPath, + name: `Camera clip ${index + 1}`, + url: `${base}/${index === 0 ? "long-camera.webm" : "camera.webm"}`, + size: index === 0 ? longCameraSize.size : cameraSize.size, + contentType: "video/webm", + }, + ]); + const started = performance.now(); + const preparation = await app.request("/editor/preparations", { + method: "POST", + headers, + body: JSON.stringify({ + videoId: "editor-clip-benchmark", + title: "Fifteen minute, 49 clip benchmark", + display: { + url: `${base}/original.webm`, + contentType: "video/webm", + size: (await stat(original)).size, + fps: 30, + }, + videoAssets, + clips, + projectConfig: { + camera: { hide: true }, + timeline: { + segments: [ + { recordingSegment: 0, timescale: 1, start: 0, end: 3 }, + ...clips.map((clip, index) => ({ + recordingSegment: index + 1, + timescale: 1, + start: 0, + end: clip.duration, + })), + ], + zoomSegments: [], + }, + }, + }), + }); + assert.equal(preparation.status, 202); + const created = (await preparation.json()) as { id: string }; + sessionId = await waitForPreparation(created.id, headers); + const prepareMs = Math.round(performance.now() - started); + const native = getEditorSession(sessionId); + assert.ok(native); + const instanceResponse = await app.request( + `/editor/sessions/${sessionId}/instance`, + { headers }, + ); + assert.equal(instanceResponse.status, 200); + const instance = (await instanceResponse.json()) as { + recordings: { segments: unknown[] }; + }; + assert.equal(instance.recordings.segments.length, clipCount + 1); + const nativeRssBeforeMb = await nativeRssMb(native.pid); + const seeks = []; + for (const seconds of [30, 100, 450, 800, 899, 903.5, 920, 950, 985, 998]) { + seeks.push({ seconds, ms: await seekPreview(sessionId, seconds, headers) }); + } + const sorted = seeks.map((seek) => seek.ms).sort((a, b) => a - b); + const nativeRssAfterMb = await nativeRssMb(native.pid); + process.stdout.write( + `${JSON.stringify({ + clipCount, + assetCount: videoAssets.length, + projectSeconds: 3 + longSeconds + (clipCount - 1) * 2, + prepareMs, + seeks, + seekMedianMs: sorted[Math.floor(sorted.length / 2)], + seekP95Ms: sorted[Math.floor(sorted.length * 0.95)], + nativeRssBeforeMb, + nativeRssAfterMb, + workerRssMb: Math.round(process.memoryUsage().rss / 1024 / 1024), + })}\n`, + ); +} finally { + if (sessionId) + await app.request(`/editor/sessions/${sessionId}`, { + method: "DELETE", + headers: { "x-media-server-secret": secret }, + }); + server?.stop(true); + await rm(temporary, { recursive: true, force: true }); + if (previousSecret === undefined) + delete process.env.MEDIA_SERVER_WEBHOOK_SECRET; + else process.env.MEDIA_SERVER_WEBHOOK_SECRET = previousSecret; + if (previousAllowHttp === undefined) + delete process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + else process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = previousAllowHttp; +} diff --git a/apps/media-server/src/__tests__/routes/editor-clips-replay.ts b/apps/media-server/src/__tests__/routes/editor-clips-replay.ts new file mode 100644 index 00000000000..39c1d3be7de --- /dev/null +++ b/apps/media-server/src/__tests__/routes/editor-clips-replay.ts @@ -0,0 +1,4 @@ +import { replayEditorClips } from "./editor-clips-scenario"; + +const result = await replayEditorClips(); +process.stdout.write(`${JSON.stringify(result)}\n`); diff --git a/apps/media-server/src/__tests__/routes/editor-clips-scenario.ts b/apps/media-server/src/__tests__/routes/editor-clips-scenario.ts new file mode 100644 index 00000000000..f6128f347e3 --- /dev/null +++ b/apps/media-server/src/__tests__/routes/editor-clips-scenario.ts @@ -0,0 +1,357 @@ +import assert from "node:assert/strict"; +import { randomUUID } from "node:crypto"; +import { stat } from "node:fs/promises"; +import { join } from "node:path"; +import app from "../../editor-worker-app"; +import { getEditorSession } from "../../lib/editor-sessions"; +import { + editorWebSocketHandler, + handleEditorSocketUpgrade, +} from "../../lib/editor-websocket"; + +type ExportResult = { + status: string; + error: string | null; + size: number | null; + mediaMetadata: { duration: number } | null; + progress: { rendered_count: number; total_frames: number } | null; +}; + +async function waitForPreparation(id: string, headers: Record) { + const deadline = Date.now() + 60_000; + while (Date.now() < deadline) { + const response = await app.request(`/editor/preparations/${id}`, { + headers, + }); + assert.equal(response.status, 200); + const result = (await response.json()) as { + status: string; + sessionId?: string; + error?: string; + }; + if (result.status === "ready") { + assert.ok(result.sessionId); + return result.sessionId; + } + if (result.status === "error") + throw new Error(result.error ?? "Native preparation failed"); + await Bun.sleep(100); + } + throw new Error("Native preparation timed out"); +} + +async function waitForExport( + sessionId: string, + exportId: string, + headers: Record, +) { + const deadline = Date.now() + 60_000; + while (Date.now() < deadline) { + const response = await app.request( + `/editor/sessions/${sessionId}/exports/${exportId}`, + { headers }, + ); + assert.equal(response.status, 200); + const result = (await response.json()) as ExportResult; + if (result.status === "ready") return result; + if (result.status === "error") + throw new Error(result.error ?? "Native export failed"); + await Bun.sleep(100); + } + throw new Error("Native export timed out"); +} + +async function previewColor( + sessionId: string, + frameNumber: number, + headers: Record, +) { + const response = await app.request(`/editor/sessions/${sessionId}/preview`, { + method: "POST", + headers, + body: JSON.stringify({ + frameNumber, + fps: 30, + resolutionBase: { x: 640, y: 360 }, + }), + }); + assert.equal(response.status, 200); + const bytes = Buffer.from(await response.arrayBuffer()); + const stride = bytes.readUInt32LE(bytes.length - 24); + const offset = 180 * stride + 320 * 4; + return { red: bytes[offset] ?? 0, blue: bytes[offset + 2] ?? 0 }; +} + +export async function replayEditorClips() { + assert.ok(process.env.CAP_WEB_EDITOR_PREPARE_BIN); + assert.ok(process.env.CAP_WEB_EDITOR_SERVICE_BIN); + const secret = "editor-clips-test-secret"; + const previousSecret = process.env.MEDIA_SERVER_WEBHOOK_SECRET; + const previousAllowHttp = process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + const previousPublicOrigin = process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN; + process.env.MEDIA_SERVER_WEBHOOK_SECRET = secret; + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = "1"; + const fixtures = join(import.meta.dir, "../fixtures/editor-clips"); + const original = join(fixtures, "display-red.webm"); + const imported = join(fixtures, "clip-blue-audio.mp4"); + const camera = join(fixtures, "camera-green.webm"); + const paths = new Map([ + ["/original.webm", original], + ["/imported.mp4", imported], + ["/camera.webm", camera], + ]); + let server: ReturnType | null = null; + let socketServer: ReturnType | null = null; + let frameSocket: WebSocket | null = null; + let sessionId: string | null = null; + try { + server = Bun.serve({ + port: 0, + fetch(request) { + const path = paths.get(new URL(request.url).pathname); + return path + ? new Response(Bun.file(path)) + : new Response("Missing fixture", { status: 404 }); + }, + }); + const headers = { + "x-media-server-secret": secret, + "Content-Type": "application/json", + }; + const displayPath = `content/videos/${randomUUID()}.mp4`; + const cameraPath = `content/videos/${randomUUID()}.webm`; + const base = `http://127.0.0.1:${server.port}`; + const preparation = await app.request("/editor/preparations", { + method: "POST", + headers, + body: JSON.stringify({ + videoId: "saved-clip-fixture", + title: "Saved two clip project", + display: { + url: `${base}/original.webm`, + contentType: "video/webm", + size: (await stat(original)).size, + fps: 30, + }, + videoAssets: [ + { + path: displayPath, + name: "Imported clip", + url: `${base}/imported.mp4`, + size: (await stat(imported)).size, + contentType: "video/mp4", + }, + { + path: cameraPath, + name: "Separate camera", + url: `${base}/camera.webm`, + size: (await stat(camera)).size, + contentType: "video/webm", + }, + ], + clips: [ + { + displayPath, + duration: 2, + fps: 30, + hasAudio: true, + cameraPath, + cameraFps: 25, + cameraOffsetMs: 125, + }, + ], + projectConfig: { + camera: { hide: true }, + timeline: { + segments: [ + { recordingSegment: 0, timescale: 1, start: 0, end: 3 }, + { recordingSegment: 1, timescale: 1, start: 0, end: 2 }, + ], + zoomSegments: [], + }, + }, + }), + }); + assert.equal(preparation.status, 202); + const created = (await preparation.json()) as { id: string }; + sessionId = await waitForPreparation(created.id, headers); + const instanceResponse = await app.request( + `/editor/sessions/${sessionId}/instance`, + { headers }, + ); + assert.equal(instanceResponse.status, 200); + const instance = (await instanceResponse.json()) as { + recordings: { + segments: Array<{ + camera: { + duration: number; + fps: number; + start_time: number; + } | null; + system_audio: { sample_rate: number } | null; + }>; + }; + savedProjectConfig: { timeline: { segments: unknown[] } }; + }; + assert.equal(instance.recordings.segments.length, 2); + assert.equal(instance.recordings.segments[1]?.camera?.fps, 25); + assert.ok( + Math.abs((instance.recordings.segments[1]?.camera?.duration ?? 0) - 2) < + 0.1, + ); + assert.ok( + Math.abs( + (instance.recordings.segments[1]?.camera?.start_time ?? 0) - 0.125, + ) < 0.001, + ); + assert.equal( + instance.recordings.segments[1]?.system_audio?.sample_rate, + 48_000, + ); + assert.equal(instance.savedProjectConfig.timeline.segments.length, 2); + const projectPath = getEditorSession(sessionId)?.projectPath; + assert.ok(projectPath); + assert.ok( + (await stat(join(projectPath, "content/segments/segment-1/display.mp4"))) + .size > 10_000, + ); + assert.ok( + (await stat(join(projectPath, "content/segments/segment-1/camera.webm"))) + .size > 1_000, + ); + const first = await previewColor(sessionId, 30, headers); + const second = await previewColor(sessionId, 120, headers); + assert.ok(first.red - first.blue > 100); + assert.ok(second.blue - second.red > 100); + socketServer = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request, listener) { + const upgrade = handleEditorSocketUpgrade(request, listener); + return upgrade === null ? app.fetch(request) : upgrade; + }, + websocket: editorWebSocketHandler, + }); + process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN = `http://127.0.0.1:${socketServer.port}`; + const ticketResponse = await app.request( + `/editor/sessions/${sessionId}/sockets`, + { + method: "POST", + headers, + body: JSON.stringify({ origin: "http://127.0.0.1:3000" }), + }, + ); + assert.equal(ticketResponse.status, 200); + const socketTicket = (await ticketResponse.json()) as { + sockets: { frames: { url: string; ticket: string } }; + }; + assert.equal( + new URL(socketTicket.sockets.frames.url).pathname, + `/editor/sessions/${sessionId}/frames`, + ); + const BunWebSocket = WebSocket as unknown as new ( + url: string, + options: Bun.WebSocketOptions, + ) => WebSocket; + frameSocket = new BunWebSocket(socketTicket.sockets.frames.url, { + protocols: [ + "cap-editor-v1", + `cap-editor-ticket.${socketTicket.sockets.frames.ticket}`, + ], + headers: { Origin: "http://127.0.0.1:3000" }, + }); + frameSocket.binaryType = "arraybuffer"; + const socketFrame = await new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error("Editor frame socket timed out")), + 5000, + ); + if (!frameSocket) return reject(new Error("Missing editor frame socket")); + frameSocket.onmessage = (event: MessageEvent) => { + clearTimeout(timer); + if (event.data instanceof ArrayBuffer) resolve(event.data); + else reject(new Error("Editor frame was not binary")); + }; + frameSocket.onerror = () => { + clearTimeout(timer); + reject(new Error("Editor frame socket failed")); + }; + }); + assert.ok(socketFrame.byteLength > 1000); + assert.equal( + new TextDecoder().decode(new Uint8Array(socketFrame).subarray(0, 8)), + "CAPPNG01", + ); + frameSocket.close(); + frameSocket = null; + const exportResponse = await app.request( + `/editor/sessions/${sessionId}/exports`, + { + method: "POST", + headers, + body: JSON.stringify({ + format: "Mp4", + fps: 30, + resolution_base: { x: 640, y: 360 }, + compression: "Social", + custom_bpp: null, + force_ffmpeg_decoder: true, + optimize_filesize: false, + }), + }, + ); + assert.equal(exportResponse.status, 202); + const exportId = ((await exportResponse.json()) as { id: string }).id; + const exportResult = await waitForExport(sessionId, exportId, headers); + assert.ok((exportResult.mediaMetadata?.duration ?? 0) > 4.5); + assert.ok((exportResult.size ?? 0) > 20_000); + assert.equal(exportResult.progress?.rendered_count, 150); + assert.equal(exportResult.progress?.total_frames, 150); + const downloadTicketResponse = await app.request( + `/editor/sessions/${sessionId}/exports/${exportId}/download-ticket`, + { + method: "POST", + headers, + body: JSON.stringify({ fileName: "paired-clips.mp4" }), + }, + ); + assert.equal(downloadTicketResponse.status, 200); + const downloadTicket = (await downloadTicketResponse.json()) as { + url: string; + }; + const downloadUrl = new URL(downloadTicket.url); + assert.equal( + downloadUrl.pathname, + `/editor/sessions/${sessionId}/exports/${exportId}/download`, + ); + const downloadResponse = await app.request( + `${downloadUrl.pathname}${downloadUrl.search}`, + ); + assert.equal(downloadResponse.status, 200); + assert.ok((await downloadResponse.arrayBuffer()).byteLength > 20_000); + return { + clips: instance.recordings.segments.length, + duration: exportResult.mediaMetadata?.duration, + cameraOffset: instance.recordings.segments[1]?.camera?.start_time, + frames: exportResult.progress?.rendered_count, + }; + } finally { + frameSocket?.close(); + socketServer?.stop(true); + if (sessionId) + await app.request(`/editor/sessions/${sessionId}`, { + method: "DELETE", + headers: { "x-media-server-secret": secret }, + }); + server?.stop(true); + if (previousSecret === undefined) + delete process.env.MEDIA_SERVER_WEBHOOK_SECRET; + else process.env.MEDIA_SERVER_WEBHOOK_SECRET = previousSecret; + if (previousAllowHttp === undefined) + delete process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + else process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = previousAllowHttp; + if (previousPublicOrigin === undefined) + delete process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN; + else process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN = previousPublicOrigin; + } +} diff --git a/apps/media-server/src/__tests__/routes/editor-clips.test.ts b/apps/media-server/src/__tests__/routes/editor-clips.test.ts new file mode 100644 index 00000000000..3138aa83c95 --- /dev/null +++ b/apps/media-server/src/__tests__/routes/editor-clips.test.ts @@ -0,0 +1,11 @@ +import { test } from "bun:test"; +import { replayEditorClips } from "./editor-clips-scenario"; + +const hasNativeBinaries = + !!process.env.CAP_WEB_EDITOR_PREPARE_BIN && + !!process.env.CAP_WEB_EDITOR_SERVICE_BIN; + +test.skipIf(!hasNativeBinaries)( + "a saved screen clip with a separate camera survives preparation, preview, and export", + replayEditorClips, +); diff --git a/apps/media-server/src/__tests__/routes/editor-cursor-mov-replay.ts b/apps/media-server/src/__tests__/routes/editor-cursor-mov-replay.ts new file mode 100644 index 00000000000..86416eaece6 --- /dev/null +++ b/apps/media-server/src/__tests__/routes/editor-cursor-mov-replay.ts @@ -0,0 +1,148 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +const runFile = promisify(execFile); +const binary = process.env.CAP_WEB_EDITOR_PREPARE_BIN; +assert.ok(binary, "Native web editor preparation binary is required"); +const fixtures = join(import.meta.dir, "../fixtures/editor-clips"); +const temporary = await mkdtemp(join(tmpdir(), "cap-web-cursor-mov-")); + +try { + const project = join(temporary, "cursor-project.cap"); + const eventsPath = join(temporary, "events.ndjson"); + const manifestPath = join(temporary, "source-manifest.json"); + const settingsPath = join(temporary, "mov-settings.json"); + const outputPath = join(temporary, "cursor-only.mov"); + const events = [ + { version: 1, platform: "MacOS" }, + { + kind: "move", + timeMs: 200, + x: 0.3, + y: 0.4, + cursor: "default", + button: 0, + modifiers: [], + }, + { + kind: "move", + timeMs: 1000, + x: 0.7, + y: 0.6, + cursor: "pointer", + button: 0, + modifiers: [], + }, + ]; + await writeFile( + eventsPath, + `${events.map((event) => JSON.stringify(event)).join("\n")}\n`, + ); + await writeFile( + manifestPath, + JSON.stringify({ + version: 1, + title: "Paired web cursor export", + displayPath: join(fixtures, "display-red.webm"), + displayFps: 30, + cameraPath: join(fixtures, "camera-green.webm"), + cameraFps: 25, + cameraOffsetMs: 0, + inputEventsPath: eventsPath, + mixedAudioInDisplay: false, + }), + ); + await runFile(binary, ["prepare", project, manifestPath]); + assert.ok( + (await readFile(join(project, "content/cursors/web-default.png"))).length > + 0, + ); + assert.ok( + (await readFile(join(project, "content/cursors/web-pointer.png"))).length > + 0, + ); + await writeFile( + settingsPath, + JSON.stringify({ + format: "Mov", + fps: 10, + resolution_base: { x: 160, y: 90 }, + cursor_only: true, + }), + ); + await runFile(binary, [ + "export", + project, + join(project, "project-config.json"), + settingsPath, + outputPath, + ]); + const probe = JSON.parse( + ( + await runFile("ffprobe", [ + "-v", + "error", + "-show_entries", + "stream=codec_name,pix_fmt,nb_frames:format=duration", + "-of", + "json", + outputPath, + ]) + ).stdout, + ) as { + streams: Array<{ + codec_name?: string; + pix_fmt?: string; + nb_frames?: string; + }>; + format: { duration?: string }; + }; + assert.equal(probe.streams[0]?.codec_name, "prores"); + assert.ok(probe.streams[0]?.pix_fmt?.startsWith("yuva")); + assert.ok(Number(probe.streams[0]?.nb_frames) >= 10); + assert.ok(Number(probe.format.duration) > 0); + const decoder = Bun.spawn( + [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-i", + outputPath, + "-vf", + "fps=2", + "-pix_fmt", + "rgba", + "-f", + "rawvideo", + "-", + ], + { stdout: "pipe", stderr: "pipe" }, + ); + const rgba = new Uint8Array(await new Response(decoder.stdout).arrayBuffer()); + assert.equal(await decoder.exited, 0); + const pixelsPerFrame = 160 * 90; + const frameBytes = pixelsPerFrame * 4; + assert.equal(rgba.length % frameBytes, 0); + let cursorFrames = 0; + let maxVisiblePixels = 0; + for (let frame = 0; frame < rgba.length / frameBytes; frame++) { + let visible = 0; + for (let pixel = 0; pixel < pixelsPerFrame; pixel++) { + if (rgba[frame * frameBytes + pixel * 4 + 3] > 0) visible++; + } + if (visible > 0) cursorFrames++; + maxVisiblePixels = Math.max(maxVisiblePixels, visible); + assert.ok(visible < pixelsPerFrame / 20); + } + assert.ok(cursorFrames >= 2, "Cursor-only MOV did not show its input cursor"); + process.stdout.write( + `${JSON.stringify({ format: "ProRes MOV with alpha", frames: Number(probe.streams[0]?.nb_frames), cursorFrames, maxVisiblePixels, bytes: (await readFile(outputPath)).length })}\n`, + ); +} finally { + await rm(temporary, { recursive: true, force: true }); +} diff --git a/apps/media-server/src/__tests__/routes/editor-error-ui-replay.ts b/apps/media-server/src/__tests__/routes/editor-error-ui-replay.ts new file mode 100644 index 00000000000..bbc5e677beb --- /dev/null +++ b/apps/media-server/src/__tests__/routes/editor-error-ui-replay.ts @@ -0,0 +1,200 @@ +import assert from "node:assert/strict"; +import { mkdir } from "node:fs/promises"; +import { join, resolve, sep } from "node:path"; +import { chromium, type Page, webkit } from "@playwright/test"; + +const editorPublic = resolve( + process.env.CAP_EDITOR_SOLID_PUBLIC_DIR ?? + resolve(import.meta.dir, "../../../../../apps/web/public/editor-solid"), +); +const screenshotRoot = resolve( + process.env.CAP_EDITOR_ERROR_SCREENSHOT_DIR ?? + resolve( + import.meta.dir, + "../../../../../output/playwright/cap-editor-error", + ), +); +assert.ok(await Bun.file(join(editorPublic, "index.html")).exists()); + +const hostHtml = ` + + + + + + +`; + +async function sendError( + page: Page, + state: { + message: string; + hasBrowserDraftConflict: boolean; + restoringBrowserDraft: boolean; + }, +) { + await page.evaluate((data) => { + const iframe = document.getElementById("editor") as HTMLIFrameElement; + iframe.contentWindow?.postMessage( + { kind: "cap-editor-error", version: 1, ...data }, + window.location.origin, + ); + }, state); +} + +const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + const url = new URL(request.url); + if (url.pathname === "/test-editor-error") { + return new Response(hostHtml, { + headers: { "Content-Type": "text/html; charset=utf-8" }, + }); + } + if (url.pathname === "/favicon.ico") + return new Response(null, { status: 204 }); + if (url.pathname.startsWith("/editor-solid/")) { + const staticPath = resolve( + editorPublic, + url.pathname.slice("/editor-solid/".length), + ); + if ( + staticPath.startsWith(`${editorPublic}${sep}`) && + (await Bun.file(staticPath).exists()) + ) { + const file = Bun.file(staticPath); + return new Response(file, { + headers: { "Content-Type": file.type }, + }); + } + return new Response("Missing editor asset", { status: 404 }); + } + return new Response("Not found", { status: 404 }); + }, +}); + +try { + await mkdir(screenshotRoot, { recursive: true }); + for (const [name, engine] of [ + ["chromium", chromium], + ["webkit", webkit], + ] as const) { + const browser = await engine.launch({ headless: true }); + try { + const page = await browser.newPage({ + viewport: { width: 1440, height: 900 }, + }); + const pageErrors: string[] = []; + const failedRequests: string[] = []; + page.on("pageerror", (error) => pageErrors.push(error.message)); + page.on("requestfailed", (request) => failedRequests.push(request.url())); + await page.goto(`${server.url}test-editor-error`); + const frame = page.frameLocator("#editor"); + await frame.getByRole("button", { name: "Export" }).waitFor(); + const skeletonBackground = await frame + .locator("#editor-root > div") + .evaluate((element) => getComputedStyle(element).backgroundColor); + await sendError(page, { + message: "Editor preparation could not start", + hasBrowserDraftConflict: false, + restoringBrowserDraft: false, + }); + await page.locator("body[data-ready=true]").waitFor(); + await frame + .getByRole("heading", { name: "Unable to Open Recording" }) + .waitFor(); + assert.equal( + await frame.getByRole("alert").textContent(), + "Editor preparation could not start", + ); + assert.equal( + await frame.getByRole("button", { name: "Try again" }).count(), + 1, + ); + const background = await frame + .locator("#editor-root > div") + .evaluate((element) => getComputedStyle(element).backgroundColor); + assert.equal(background, skeletonBackground); + assert.notEqual(background, "rgb(255, 255, 255)"); + await page.screenshot({ + path: join(screenshotRoot, `${name}-preparation-error.png`), + }); + await frame.getByRole("button", { name: "Try again" }).click(); + await page.locator("body[data-action=retry]").waitFor(); + await page.evaluate(() => { + document.body.dataset.action = ""; + }); + await sendError(page, { + message: + "This recording changed after the browser saved pending edits.", + hasBrowserDraftConflict: true, + restoringBrowserDraft: false, + }); + await frame + .getByRole("button", { name: "Restore browser edits" }) + .waitFor(); + assert.equal( + await frame.getByRole("button", { name: "Try again" }).count(), + 0, + ); + await page.screenshot({ + path: join(screenshotRoot, `${name}-browser-draft-conflict.png`), + }); + await frame + .getByRole("button", { name: "Restore browser edits" }) + .click(); + await page.locator("body[data-action=restore-browser]").waitFor(); + await sendError(page, { + message: + "This recording changed after the browser saved pending edits.", + hasBrowserDraftConflict: true, + restoringBrowserDraft: true, + }); + assert.equal( + await frame + .getByRole("button", { name: "Restoring browser edits…" }) + .isDisabled(), + true, + ); + assert.equal( + await frame + .getByRole("button", { name: "Open latest saved" }) + .isDisabled(), + true, + ); + await sendError(page, { + message: + "This recording changed after the browser saved pending edits.", + hasBrowserDraftConflict: true, + restoringBrowserDraft: false, + }); + await page.evaluate(() => { + document.body.dataset.action = ""; + }); + await frame.getByRole("button", { name: "Open latest saved" }).click(); + await page.locator("body[data-action=open-latest]").waitFor(); + await page.evaluate(() => { + document.body.dataset.action = ""; + }); + await frame.getByRole("button", { name: "Back to recording" }).click(); + await page.locator("body[data-action=back-to-recording]").waitFor(); + assert.deepEqual(pageErrors, []); + assert.deepEqual(failedRequests, []); + process.stdout.write( + `${JSON.stringify({ browser: name, background, skeletonBackground, pageErrors, failedRequests })}\n`, + ); + } finally { + await browser.close(); + } + } +} finally { + server.stop(true); +} diff --git a/apps/media-server/src/__tests__/routes/editor-frame-canvas-fallback-replay.ts b/apps/media-server/src/__tests__/routes/editor-frame-canvas-fallback-replay.ts new file mode 100644 index 00000000000..50d93227152 --- /dev/null +++ b/apps/media-server/src/__tests__/routes/editor-frame-canvas-fallback-replay.ts @@ -0,0 +1,460 @@ +import assert from "node:assert/strict"; +import { resolve } from "node:path"; +import { type Browser, chromium, webkit } from "@playwright/test"; + +type PreviewResult = { + rgba: number[]; + width: number; + height: number; + frameNumber: number; + closed: boolean; + decodeMs: number; + sha256?: string; + longTasks?: number; +}; + +const entrypoint = resolve( + import.meta.dir, + "../../../../../packages/editor-solid-web/src/websocket.ts", +); +const bundle = await Bun.build({ + entrypoints: [entrypoint], + target: "browser", + format: "esm", +}); +assert.ok(bundle.success, bundle.logs.map((log) => log.message).join("\n")); +assert.equal(bundle.outputs.length, 1); +const source = await bundle.outputs[0]?.text(); +assert.ok(source); + +async function createReplayPage(browser: Browser) { + const page = await browser.newPage(); + const url = "https://editor.cap.so/canvas-fallback-replay"; + await page.route(url, (route) => + route.fulfill({ + status: 200, + contentType: "text/html", + body: "", + }), + ); + await page.goto(url); + return page; +} + +async function replayPng( + browser: Browser, + options: { + noOffscreenCanvas: boolean; + noImageBitmap: boolean; + offscreenContextUnavailable?: boolean; + width?: number; + height?: number; + frames?: number; + }, +): Promise { + const page = await createReplayPage(browser); + try { + return await page.evaluate( + async ({ source, options }) => { + if (options.noOffscreenCanvas) { + Object.defineProperty(globalThis, "OffscreenCanvas", { + value: undefined, + configurable: true, + }); + } + if (options.noImageBitmap) { + Object.defineProperty(globalThis, "createImageBitmap", { + value: undefined, + configurable: true, + }); + } + if (options.offscreenContextUnavailable) { + class No2DOffscreenCanvas { + constructor( + readonly width: number, + readonly height: number, + ) {} + getContext() { + return null; + } + } + Object.defineProperty(globalThis, "OffscreenCanvas", { + value: No2DOffscreenCanvas, + configurable: true, + }); + } + const moduleUrl = URL.createObjectURL( + new Blob([source], { type: "text/javascript" }), + ); + const editor = (await import( + moduleUrl + )) as typeof import("../../../../../packages/editor-solid-web/src/websocket"); + URL.revokeObjectURL(moduleUrl); + class MockWebSocket extends EventTarget { + static readonly OPEN = 1; + readonly readyState = 1; + binaryType: BinaryType = "blob"; + closed = false; + send() {} + close() { + this.closed = true; + } + } + globalThis.WebSocket = MockWebSocket as unknown as typeof WebSocket; + const width = options.width ?? 2; + const height = options.height ?? 2; + const frameCanvas = document.createElement("canvas"); + frameCanvas.width = width; + frameCanvas.height = height; + const frameContext = frameCanvas.getContext("2d"); + if (!frameContext) throw new Error("Fixture canvas is unavailable"); + if (width === 2 && height === 2) { + const pixels = new Uint8ClampedArray([ + 255, 0, 0, 255, 0, 255, 0, 255, 0, 0, 255, 255, 255, 255, 0, 255, + ]); + frameContext.putImageData(new ImageData(pixels, 2, 2), 0, 0); + } else { + const gradient = frameContext.createLinearGradient( + 0, + 0, + width, + height, + ); + gradient.addColorStop(0, "#132a4a"); + gradient.addColorStop(1, "#ddc89a"); + frameContext.fillStyle = gradient; + frameContext.fillRect(0, 0, width, height); + frameContext.font = "32px sans-serif"; + for (let line = 0; line < 24; line++) { + frameContext.fillStyle = line % 2 === 0 ? "#ffffff" : "#141414"; + frameContext.fillText( + `Cap editor preview frame ${line}: transcript, timeline, and camera controls`, + 36, + 48 + line * 40, + ); + } + } + const pngBlob = await new Promise((resolve, reject) => { + frameCanvas.toBlob((blob) => { + if (blob) resolve(blob); + else reject(new Error("Fixture PNG could not encode")); + }, "image/png"); + }); + const png = new Uint8Array(await pngBlob.arrayBuffer()); + const packet = new ArrayBuffer(8 + png.length + 24); + const bytes = new Uint8Array(packet); + bytes.set([67, 65, 80, 80, 78, 71, 48, 49]); + bytes.set(png, 8); + const footer = new DataView(packet, packet.byteLength - 24, 24); + footer.setUint32(4, height, true); + footer.setUint32(8, width, true); + footer.setUint32(12, 7, true); + footer.setBigUint64(16, 1_234_567n, true); + const url = + "wss://editor.cap.so/editor/sessions/canvas-fallback/frames"; + editor.setEditorFrameSocketCredential({ url, ticket: "a".repeat(43) }); + const socket = editor.createWS(url) as unknown as MockWebSocket; + let longTasks = 0; + let observer: PerformanceObserver | null = null; + try { + observer = new PerformanceObserver((list) => { + longTasks += list.getEntries().length; + }); + observer.observe({ entryTypes: ["longtask"] }); + } catch { + observer = null; + } + const decodeTimes: number[] = []; + let raw = new ArrayBuffer(0); + for (let frame = 0; frame < (options.frames ?? 1); frame++) { + const startedAt = performance.now(); + raw = await new Promise((resolve, reject) => { + const timeout = window.setTimeout( + () => reject(new Error("Editor PNG frame was not delivered")), + 5000, + ); + socket.addEventListener( + "message", + (event) => { + if (!(event instanceof MessageEvent)) return; + if ( + !(event.data instanceof ArrayBuffer) || + event.data.byteLength !== width * height * 4 + 24 + ) + return; + window.clearTimeout(timeout); + resolve(event.data); + }, + { once: true }, + ); + socket.dispatchEvent(new MessageEvent("message", { data: packet })); + }); + decodeTimes.push(performance.now() - startedAt); + } + await new Promise((resolve) => window.setTimeout(resolve, 0)); + observer?.disconnect(); + const decodeMs = + [...decodeTimes].sort((a, b) => a - b)[ + Math.floor(decodeTimes.length / 2) + ] ?? 0; + const rawFooter = new DataView(raw, width * height * 4, 24); + const digest = await crypto.subtle.digest( + "SHA-256", + new Uint8Array(raw, 0, width * height * 4), + ); + const sha256 = Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, "0"), + ).join(""); + return { + rgba: Array.from(new Uint8Array(raw, 0, 16)), + width: rawFooter.getUint32(8, true), + height: rawFooter.getUint32(4, true), + frameNumber: rawFooter.getUint32(12, true), + closed: socket.closed, + decodeMs, + sha256, + longTasks, + }; + }, + { source, options }, + ); + } finally { + await page.close(); + } +} + +async function replayH264Canvas(browser: Browser): Promise { + const page = await createReplayPage(browser); + try { + return await page.evaluate(async (source) => { + if ( + typeof VideoFrame === "undefined" || + typeof EncodedVideoChunk === "undefined" + ) { + throw new Error("Chromium VideoFrame is unavailable"); + } + Object.defineProperty(globalThis, "OffscreenCanvas", { + value: undefined, + configurable: true, + }); + const moduleUrl = URL.createObjectURL( + new Blob([source], { type: "text/javascript" }), + ); + const editor = (await import( + moduleUrl + )) as typeof import("../../../../../packages/editor-solid-web/src/websocket"); + URL.revokeObjectURL(moduleUrl); + const frameCanvas = document.createElement("canvas"); + frameCanvas.width = 2; + frameCanvas.height = 2; + const frameContext = frameCanvas.getContext("2d"); + if (!frameContext) throw new Error("H.264 fixture canvas is unavailable"); + frameContext.putImageData( + new ImageData( + new Uint8ClampedArray([ + 255, 0, 0, 255, 0, 255, 0, 255, 0, 0, 255, 255, 255, 255, 0, 255, + ]), + 2, + 2, + ), + 0, + 0, + ); + class MockVideoDecoder { + static async isConfigSupported() { + return { supported: true }; + } + readonly decodeQueueSize = 0; + constructor(private readonly callbacks: VideoDecoderInit) {} + configure() {} + decode(chunk: EncodedVideoChunk) { + const frame = new VideoFrame(frameCanvas, { + timestamp: chunk.timestamp, + }); + Object.defineProperty(frame, "allocationSize", { + value: () => { + throw new Error("Use canvas frame conversion"); + }, + }); + this.callbacks.output(frame); + } + close() {} + } + globalThis.VideoDecoder = + MockVideoDecoder as unknown as typeof VideoDecoder; + class MockWebSocket extends EventTarget { + static readonly OPEN = 1; + readonly readyState = 1; + binaryType: BinaryType = "blob"; + closed = false; + send() {} + close() { + this.closed = true; + } + } + globalThis.WebSocket = MockWebSocket as unknown as typeof WebSocket; + const url = "wss://editor.cap.so/editor/sessions/h264-canvas/frames"; + editor.setEditorFrameSocketCredential({ url, ticket: "b".repeat(43) }); + const socket = editor.createWS(url) as unknown as MockWebSocket; + socket.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ + kind: "cap-h264-config", + codec: "avc1.42e01e", + width: 2, + height: 2, + }), + }), + ); + await new Promise((resolve) => window.setTimeout(resolve, 20)); + const packet = new ArrayBuffer(42); + const bytes = new Uint8Array(packet); + bytes.set([67, 65, 80, 72, 50, 54, 52, 49]); + const header = new DataView(packet, 0, 41); + header.setBigUint64(8, 1n, true); + header.setUint8(16, 1); + header.setUint32(17, 2, true); + header.setUint32(21, 2, true); + header.setUint32(25, 7, true); + header.setBigUint64(29, 1_234_567n, true); + header.setUint32(37, 1, true); + bytes[41] = 1; + const startedAt = performance.now(); + const raw = await new Promise((resolve, reject) => { + const timeout = window.setTimeout( + () => + reject(new Error("Editor H.264 canvas frame was not delivered")), + 5000, + ); + socket.addEventListener("message", (event) => { + if (!(event instanceof MessageEvent)) return; + if ( + !(event.data instanceof ArrayBuffer) || + event.data.byteLength !== 40 + ) + return; + window.clearTimeout(timeout); + resolve(event.data); + }); + socket.dispatchEvent(new MessageEvent("message", { data: packet })); + }); + const decodeMs = performance.now() - startedAt; + const footer = new DataView(raw, 16, 24); + return { + rgba: Array.from(new Uint8Array(raw, 0, 16)), + width: footer.getUint32(8, true), + height: footer.getUint32(4, true), + frameNumber: footer.getUint32(12, true), + closed: socket.closed, + decodeMs, + }; + }, source); + } finally { + await page.close(); + } +} + +for (const [name, engine] of [ + ["Chromium", chromium], + ["WebKit", webkit], +] as const) { + const browser = await engine.launch({ headless: true }); + try { + const baseline = await replayPng(browser, { + noOffscreenCanvas: false, + noImageBitmap: false, + }); + const canvasFallback = await replayPng(browser, { + noOffscreenCanvas: true, + noImageBitmap: false, + }); + const imageFallback = await replayPng(browser, { + noOffscreenCanvas: true, + noImageBitmap: true, + }); + const bitmapOnlyFallback = await replayPng(browser, { + noOffscreenCanvas: false, + noImageBitmap: true, + }); + const noOffscreen2D = await replayPng(browser, { + noOffscreenCanvas: false, + noImageBitmap: false, + offscreenContextUnavailable: true, + }); + for (const result of [ + baseline, + canvasFallback, + imageFallback, + bitmapOnlyFallback, + noOffscreen2D, + ]) { + assert.deepEqual( + result.rgba, + [255, 0, 0, 255, 0, 255, 0, 255, 0, 0, 255, 255, 255, 255, 0, 255], + ); + assert.equal(result.width, 2); + assert.equal(result.height, 2); + assert.equal(result.frameNumber, 7); + assert.equal(result.closed, false); + } + assert.deepEqual(canvasFallback.rgba, baseline.rgba); + assert.deepEqual(imageFallback.rgba, baseline.rgba); + assert.deepEqual(bitmapOnlyFallback.rgba, baseline.rgba); + assert.deepEqual(noOffscreen2D.rgba, baseline.rgba); + const output: Record = { + browser: name, + pngBaselineMs: baseline.decodeMs, + pngCanvasFallbackMs: canvasFallback.decodeMs, + pngImageFallbackMs: imageFallback.decodeMs, + pngBitmapOnlyFallbackMs: bitmapOnlyFallback.decodeMs, + pngNoOffscreen2DMs: noOffscreen2D.decodeMs, + }; + const largeOptions = { width: 1920, height: 1080, frames: 10 }; + const largeBaseline = await replayPng(browser, { + ...largeOptions, + noOffscreenCanvas: false, + noImageBitmap: false, + }); + const largeCanvasFallback = await replayPng(browser, { + ...largeOptions, + noOffscreenCanvas: true, + noImageBitmap: false, + }); + const largeImageFallback = await replayPng(browser, { + ...largeOptions, + noOffscreenCanvas: true, + noImageBitmap: true, + }); + for (const result of [ + largeBaseline, + largeCanvasFallback, + largeImageFallback, + ]) { + assert.equal(result.width, 1920); + assert.equal(result.height, 1080); + assert.equal(result.frameNumber, 7); + assert.equal(result.closed, false); + assert.ok(result.sha256); + } + assert.equal(largeCanvasFallback.sha256, largeBaseline.sha256); + assert.equal(largeImageFallback.sha256, largeBaseline.sha256); + output.large1080p = { + baselineMedianMs: largeBaseline.decodeMs, + canvasFallbackMedianMs: largeCanvasFallback.decodeMs, + imageFallbackMedianMs: largeImageFallback.decodeMs, + baselineLongTasks: largeBaseline.longTasks, + canvasFallbackLongTasks: largeCanvasFallback.longTasks, + imageFallbackLongTasks: largeImageFallback.longTasks, + sha256: largeBaseline.sha256, + }; + if (name === "Chromium") { + const h264Fallback = await replayH264Canvas(browser); + assert.deepEqual(h264Fallback.rgba, baseline.rgba); + assert.equal(h264Fallback.closed, false); + output.h264CanvasFallbackMs = h264Fallback.decodeMs; + } + console.log(JSON.stringify(output)); + } finally { + await browser.close(); + } +} diff --git a/apps/media-server/src/__tests__/routes/editor-legacy-replay.test.ts b/apps/media-server/src/__tests__/routes/editor-legacy-replay.test.ts new file mode 100644 index 00000000000..ecc919ff124 --- /dev/null +++ b/apps/media-server/src/__tests__/routes/editor-legacy-replay.test.ts @@ -0,0 +1,175 @@ +import { expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { prepareNativeEditorProject } from "../../lib/editor-native"; + +const hasNativeBinary = Boolean(process.env.CAP_WEB_EDITOR_PREPARE_BIN); + +function run(command: string, args: string[]) { + const result = spawnSync(command, args, { + encoding: "buffer", + timeout: 30_000, + maxBuffer: 1024 * 1024, + }); + if (result.status !== 0) { + throw new Error(`${command} failed: ${result.stderr.toString("utf8")}`); + } + return result.stdout; +} + +function mediaDuration(path: string) { + const result = run("ffprobe", [ + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + path, + ]); + return Number(result.toString("utf8")); +} + +function centerColor(path: string, time: number) { + return [ + ...run("ffmpeg", [ + "-v", + "error", + "-ss", + String(time), + "-i", + path, + "-frames:v", + "1", + "-vf", + "crop=1:1:iw/2:ih/2,format=rgb24", + "-f", + "rawvideo", + "pipe:1", + ]), + ]; +} + +test.skipIf(!hasNativeBinary)( + "preserved legacy MP4 keep ranges replay in a native Studio export", + async () => { + const root = await mkdtemp(join(tmpdir(), "cap-legacy-editor-replay-")); + const sourcePath = join(root, "original.mp4"); + const renderedPath = join(root, "edited.mp4"); + let cleanup: (() => Promise) | null = null; + try { + run("ffmpeg", [ + "-v", + "error", + "-f", + "lavfi", + "-i", + "color=c=red:s=320x180:r=30:d=2", + "-f", + "lavfi", + "-i", + "color=c=blue:s=320x180:r=30:d=2", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=48000:duration=4", + "-filter_complex", + "[0:v][1:v]concat=n=2:v=1:a=0[v]", + "-map", + "[v]", + "-map", + "2:a", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + sourcePath, + ]); + const originalBytes = await readFile(sourcePath); + const sourceDuration = mediaDuration(sourcePath); + expect(sourceDuration).toBeCloseTo(4, 1); + const spec = { + version: 1 as const, + sourceDuration, + keepRanges: [ + { start: 0.2, end: 0.8 }, + { start: 2.2, end: 2.8 }, + ], + }; + const input = { + title: "Older trimmed recording", + display: { + path: sourcePath, + contentType: "video/mp4" as const, + size: (await stat(sourcePath)).size, + fps: 30, + }, + mixedAudioInDisplay: true, + legacyEditSpec: spec, + }; + const project = await prepareNativeEditorProject(input); + cleanup = project.cleanup; + const configPath = join(project.path, "project-config.json"); + const config = JSON.parse(await readFile(configPath, "utf8")) as { + audio: { improve: boolean; isolation: string }; + timeline: { segments: Array<{ start: number; end: number }> }; + }; + expect( + config.timeline.segments.map(({ start, end }) => [start, end]), + ).toEqual([ + [0.2, 0.8], + [2.2, 2.8], + ]); + const settingsPath = join(root, "settings.json"); + await writeFile( + settingsPath, + JSON.stringify({ + format: "Mp4", + fps: 30, + resolution_base: { x: 640, y: 360 }, + compression: "Social", + custom_bpp: null, + force_ffmpeg_decoder: true, + optimize_filesize: false, + }), + ); + run(process.env.CAP_WEB_EDITOR_PREPARE_BIN ?? "", [ + "export", + project.path, + configPath, + settingsPath, + renderedPath, + ]); + expect(mediaDuration(renderedPath)).toBeCloseTo(1.2, 1); + const red = centerColor(renderedPath, 0.3); + const blue = centerColor(renderedPath, 0.9); + expect(red[0]).toBeGreaterThan(130); + expect(red[2]).toBeLessThan(100); + expect(blue[0]).toBeLessThan(100); + expect(blue[2]).toBeGreaterThan(130); + expect(await readFile(sourcePath)).toEqual(originalBytes); + const reopened = await prepareNativeEditorProject({ + ...input, + audioDefault: { enabledByDefault: true, isolation: "strong" }, + projectConfig: config as unknown as Record, + }); + try { + const restored = JSON.parse( + await readFile(join(reopened.path, "project-config.json"), "utf8"), + ) as typeof config; + expect(restored.timeline.segments).toEqual(config.timeline.segments); + expect(restored.audio).toEqual(config.audio); + } finally { + await reopened.cleanup(); + } + } finally { + await cleanup?.(); + await rm(root, { recursive: true, force: true }); + } + }, + 30_000, +); diff --git a/apps/media-server/src/__tests__/routes/editor-presets-browser-replay.ts b/apps/media-server/src/__tests__/routes/editor-presets-browser-replay.ts new file mode 100644 index 00000000000..c2f23fb43ff --- /dev/null +++ b/apps/media-server/src/__tests__/routes/editor-presets-browser-replay.ts @@ -0,0 +1,190 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { chromium, webkit } from "@playwright/test"; +import sharp from "sharp"; + +const sourcePath = "content/images/22222222-2222-4222-8222-222222222222.png"; +const targetPath = "content/images/33333333-3333-4333-8333-333333333333.png"; +const source = await sharp({ + create: { + width: 32, + height: 32, + channels: 4, + background: { r: 255, g: 0, b: 255, alpha: 1 }, + }, +}) + .png() + .toBuffer(); +const sourceSha256 = createHash("sha256").update(source).digest("hex"); +const root = resolve(import.meta.dir, "../../../../../"); +const temporary = await mkdtemp(join(tmpdir(), "cap-editor-presets-")); +const browserEntry = join(temporary, "browser-entry.ts"); +const browserSource = ` +import { Store, setEditorStoreNamespace } from ${JSON.stringify(join(root, "packages/editor-solid-web/src/tauri-store.ts"))}; +import { setEditorPresetAssetBase, prepareEditorPresetBackground } from ${JSON.stringify(join(root, "packages/editor-solid-web/src/preset-backgrounds.ts"))}; +import { mapEditorImportedImages, resolveEditorImportedImage } from ${JSON.stringify(join(root, "packages/editor-solid-web/src/editor-file-mapping.ts"))}; +const sourcePath = ${JSON.stringify(sourcePath)}; +const targetPath = ${JSON.stringify(targetPath)}; +const scope = "preset-browser-replay"; +setEditorStoreNamespace(scope); +const store = await Store.load("store"); +let importCount = 0; +window.capPresetReplay = { + save: async () => { + setEditorPresetAssetBase("/api/editor/sessions/session-one/file?videoId=recording-one"); + await store.set("presets", new Proxy({ + presets: [{ name: "Magenta", config: { background: { source: { type: "image", path: sourcePath } } } }], + default: null, + }, {})); + return (await store.get("presets"))?.presets?.length; + }, + apply: async () => { + setEditorPresetAssetBase("/api/editor/sessions/session-two/file?videoId=recording-two"); + const presets = await store.get("presets"); + if (!presets?.presets?.[0]) throw new Error("Saved preset was lost after reload"); + let bytes = null; + await prepareEditorPresetBackground(scope, "store", presets.presets[0].config, async (file) => { + importCount += 1; + bytes = Array.from(new Uint8Array(await file.arrayBuffer())); + return { path: targetPath }; + }); + const mapped = mapEditorImportedImages(presets.presets[0].config); + return { bytes, importCount, mappedPath: mapped.background.source.path, resolvedPath: resolveEditorImportedImage(sourcePath) }; + }, + delete: async () => { + await store.delete("presets"); + await store.reload(); + return await store.get("presets"); + }, +}; +`; + +type ReplayApi = { + save: () => Promise; + apply: () => Promise<{ + bytes: number[] | null; + importCount: number; + mappedPath: string; + resolvedPath: string; + }>; + delete: () => Promise; +}; + +declare global { + interface Window { + capPresetReplay?: ReplayApi; + } +} + +let server: ReturnType | null = null; + +try { + await writeFile(browserEntry, browserSource); + const built = await Bun.build({ + entrypoints: [browserEntry], + target: "browser", + format: "esm", + }); + assert.ok(built.success, JSON.stringify(built.logs)); + const script = await built.outputs[0]?.text(); + assert.ok(script); + let sourceAvailable = true; + let rawRequests = 0; + server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + const url = new URL(request.url); + if (url.pathname === "/") { + return new Response( + "", + { headers: { "Content-Type": "text/html; charset=utf-8" } }, + ); + } + if (url.pathname === "/browser-entry.js") { + return new Response(script, { + headers: { "Content-Type": "text/javascript; charset=utf-8" }, + }); + } + if (url.pathname === "/favicon.ico") + return new Response(null, { status: 204 }); + if ( + url.pathname === "/api/editor/sessions/session-one/file" && + url.searchParams.get("videoId") === "recording-one" && + url.searchParams.get("path") === sourcePath && + url.searchParams.get("raw") === "1" + ) { + rawRequests += 1; + return sourceAvailable + ? new Response(source, { + headers: { + "Content-Type": "image/png", + "Content-Length": String(source.length), + }, + }) + : new Response("Source recording was deleted", { status: 404 }); + } + return new Response("Missing replay resource", { status: 404 }); + }, + }); + for (const [name, engine] of [ + ["chromium", chromium], + ["webkit", webkit], + ] as const) { + sourceAvailable = true; + rawRequests = 0; + const browser = await engine.launch({ headless: true }); + try { + const page = await browser.newPage(); + const pageErrors: string[] = []; + const failedResponses: string[] = []; + page.on("pageerror", (error) => pageErrors.push(error.message)); + page.on("response", (response) => { + if (response.status() >= 400) failedResponses.push(response.url()); + }); + await page.goto(server.url.toString()); + await page.waitForFunction(() => Boolean(window.capPresetReplay)); + assert.equal( + await page.evaluate(() => window.capPresetReplay?.save()), + 1, + ); + assert.equal(rawRequests, 1); + sourceAvailable = false; + await page.reload(); + await page.waitForFunction(() => Boolean(window.capPresetReplay)); + const restored = await page.evaluate(() => + window.capPresetReplay?.apply(), + ); + assert.ok(restored); + assert.equal( + createHash("sha256") + .update(Buffer.from(restored.bytes ?? [])) + .digest("hex"), + sourceSha256, + ); + assert.equal(restored.importCount, 1); + assert.equal(restored.mappedPath, targetPath); + assert.equal(restored.resolvedPath, targetPath); + const again = await page.evaluate(() => window.capPresetReplay?.apply()); + assert.equal(again?.importCount, 1); + assert.equal(rawRequests, 1); + assert.equal( + await page.evaluate(() => window.capPresetReplay?.delete()), + undefined, + ); + assert.deepEqual(pageErrors, []); + assert.deepEqual(failedResponses, []); + process.stdout.write( + `${JSON.stringify({ browser: name, sourceSha256, rawRequests, importCount: again?.importCount, pageErrors, failedResponses })}\n`, + ); + } finally { + await browser.close(); + } + } +} finally { + server?.stop(true); + await rm(temporary, { recursive: true, force: true }); +} diff --git a/apps/media-server/src/__tests__/routes/editor-project-bundle-replay.ts b/apps/media-server/src/__tests__/routes/editor-project-bundle-replay.ts new file mode 100644 index 00000000000..fb5872cbf40 --- /dev/null +++ b/apps/media-server/src/__tests__/routes/editor-project-bundle-replay.ts @@ -0,0 +1,241 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + CAP_BUNDLE_CONTENT_TYPE, + CAP_BUNDLE_HEADER_BYTES, + parseCapBundleManifest, + readCapBundleManifestLength, +} from "@cap/editor-cap-bundle"; +import app from "../../editor-worker-app"; +import { extractEditorCapBundle } from "../../lib/editor-cap-bundle"; +import { startNativeEditorSession } from "../../lib/editor-native"; + +if ( + !process.env.CAP_WEB_EDITOR_PREPARE_BIN || + !process.env.CAP_WEB_EDITOR_SERVICE_BIN +) + throw new Error("Native editor test binaries are unavailable"); + +const previousSecret = process.env.MEDIA_SERVER_WEBHOOK_SECRET; +const previousOrigin = process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN; +const previousHttp = process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; +const secret = `bundle-replay-${randomUUID()}`; +process.env.MEDIA_SERVER_WEBHOOK_SECRET = secret; +process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = "1"; + +const root = await mkdtemp(join(tmpdir(), "cap-editor-bundle-replay-")); +const inputCap = process.env.CAP_WEB_EDITOR_BUNDLE_SOURCE; +const displayPath = inputCap + ? join(inputCap, "content/segments/segment-0/display.mp4") + : join(root, "display.mp4"); +const cameraPath = inputCap + ? join(inputCap, "content/segments/segment-0/camera.mp4") + : join(root, "camera.mp4"); +const headers = { + "x-media-server-secret": secret, + "Content-Type": "application/json", +}; +let fixture: ReturnType | null = null; +let sessionId: string | null = null; +let extracted: Awaited> | null = null; +let replay: Awaited> | null = null; + +try { + if (!inputCap) { + for (const [path, source, rate] of [ + [displayPath, "testsrc2=size=640x360:rate=30:duration=3", "30"], + [cameraPath, "testsrc2=size=320x180:rate=25:duration=3", "25"], + ]) { + const result = spawnSync( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + source, + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-r", + rate, + path, + ], + { encoding: "utf8" }, + ); + assert.equal(result.status, 0, result.stderr); + } + } + const [display, camera] = await Promise.all([ + stat(displayPath), + stat(cameraPath), + ]); + fixture = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + const path = new URL(request.url).pathname; + if (path === "/display.mp4") return new Response(Bun.file(displayPath)); + if (path === "/camera.mp4") return new Response(Bun.file(cameraPath)); + return new Response("Not found", { status: 404 }); + }, + }); + process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN = `http://127.0.0.1:${fixture.port}`; + const prepared = await app.request("/editor/preparations", { + method: "POST", + headers, + body: JSON.stringify({ + videoId: randomUUID(), + title: "Paired bundle replay", + display: { + url: `http://127.0.0.1:${fixture.port}/display.mp4`, + contentType: "video/mp4", + size: display.size, + fps: 30, + }, + camera: { + url: `http://127.0.0.1:${fixture.port}/camera.mp4`, + contentType: "video/mp4", + size: camera.size, + fps: inputCap ? 30 : 25, + offsetMs: -100, + }, + }), + }); + if (prepared.status !== 202) + throw new Error(`Editor preparation failed: ${await prepared.text()}`); + const preparationId = ((await prepared.json()) as { id: string }).id; + const deadline = Date.now() + 60_000; + while (Date.now() < deadline) { + const status = await app.request(`/editor/preparations/${preparationId}`, { + headers, + }); + assert.equal(status.status, 200); + const value = (await status.json()) as { + status: string; + sessionId?: string; + error?: string; + }; + if (value.status === "error") + throw new Error(value.error || "Native editor preparation failed"); + if (value.status === "ready") { + sessionId = value.sessionId ?? null; + break; + } + await Bun.sleep(100); + } + assert.ok(sessionId, "Native editor preparation timed out"); + const bundlePath = `/editor/sessions/${sessionId}/project-bundle`; + const forbidden = await app.request(`${bundlePath}/download-ticket`, { + method: "POST", + body: "{}", + }); + assert.equal(forbidden.status, 401); + const snapshotStart = performance.now(); + const ticketResponse = await app.request(`${bundlePath}/download-ticket`, { + method: "POST", + headers, + body: JSON.stringify({ fileName: "Paired bundle replay.capbundle" }), + }); + if (ticketResponse.status !== 200) + throw new Error( + `Editor bundle ticket failed: ${await ticketResponse.text()}`, + ); + const snapshotMs = performance.now() - snapshotStart; + const url = new URL(((await ticketResponse.json()) as { url: string }).url); + assert.equal(url.pathname, `${bundlePath}/download`); + const closed = await app.request(`/editor/sessions/${sessionId}`, { + method: "DELETE", + headers, + }); + assert.equal(closed.status, 204); + const streamStart = performance.now(); + const downloaded = await app.request(`${url.pathname}${url.search}`); + assert.equal(downloaded.status, 200); + assert.equal(downloaded.headers.get("Content-Type"), CAP_BUNDLE_CONTENT_TYPE); + const bytes = Buffer.from(await downloaded.arrayBuffer()); + const streamMs = performance.now() - streamStart; + assert.equal( + downloaded.headers.get("Content-Length"), + String(bytes.byteLength), + ); + assert.equal((await app.request(`${url.pathname}${url.search}`)).status, 404); + const manifestLength = readCapBundleManifestLength( + bytes.subarray(0, CAP_BUNDLE_HEADER_BYTES), + ); + assert.ok(manifestLength); + const manifest = parseCapBundleManifest( + bytes.subarray( + CAP_BUNDLE_HEADER_BYTES, + CAP_BUNDLE_HEADER_BYTES + manifestLength, + ), + bytes.byteLength, + ); + assert.ok(manifest); + assert.ok( + manifest.files.some( + (file) => file.path === "content/segments/segment-0/display.mp4", + ), + ); + assert.ok( + manifest.files.some( + (file) => file.path === "content/segments/segment-0/camera.mp4", + ), + ); + const archivePath = join(root, "download.capbundle"); + await writeFile(archivePath, bytes); + extracted = await extractEditorCapBundle(archivePath); + replay = await startNativeEditorSession(extracted); + const instance = await replay.request("/instance"); + assert.equal(instance.status, 200); + const instanceData = (await instance.json()) as { recordingDuration: number }; + assert.ok(instanceData.recordingDuration > 2); + const preview = await replay.request("/preview", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + frameNumber: 30, + fps: 30, + resolutionBase: { x: 640, y: 360 }, + }), + }); + if (preview.status !== 200) + throw new Error(`Editor replay preview failed: ${await preview.text()}`); + const frameBytes = (await preview.arrayBuffer()).byteLength; + assert.ok(frameBytes > 600_000); + console.log( + JSON.stringify({ + bundleBytes: bytes.byteLength, + files: manifest.files.length, + frameBytes, + snapshotMs: Math.round(snapshotMs), + streamMs: Math.round(streamMs), + }), + ); +} finally { + await replay?.close(); + if (!replay) await extracted?.cleanup(); + if (sessionId) + await app.request(`/editor/sessions/${sessionId}`, { + method: "DELETE", + headers, + }); + fixture?.stop(); + await rm(root, { recursive: true, force: true }); + if (previousSecret === undefined) + delete process.env.MEDIA_SERVER_WEBHOOK_SECRET; + else process.env.MEDIA_SERVER_WEBHOOK_SECRET = previousSecret; + if (previousOrigin === undefined) + delete process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN; + else process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN = previousOrigin; + if (previousHttp === undefined) + delete process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + else process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = previousHttp; +} diff --git a/apps/media-server/src/__tests__/routes/editor-solid-browser-only-replay.ts b/apps/media-server/src/__tests__/routes/editor-solid-browser-only-replay.ts new file mode 100644 index 00000000000..b46b4c73f07 --- /dev/null +++ b/apps/media-server/src/__tests__/routes/editor-solid-browser-only-replay.ts @@ -0,0 +1,1176 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { mkdir, stat, writeFile } from "node:fs/promises"; +import { join, resolve, sep } from "node:path"; +import { + chromium, + type FrameLocator, + firefox, + type Page, + webkit, +} from "@playwright/test"; +import sharp from "sharp"; +import { + default_project_config_json, + initSync, + random_animated_gradient_json, +} from "../../../../../packages/editor-solid-web/renderer/pkg/cap_editor_browser_renderer.js"; +import app from "../../editor-worker-app"; +import { getEditorSession } from "../../lib/editor-sessions"; +import { + editorWebSocketHandler, + handleEditorSocketUpgrade, +} from "../../lib/editor-websocket"; + +const secret = "editor-browser-only-replay-secret"; +const videoId = "editor-browser-only-replay"; +const userId = "editor-browser-only-user"; +const engine = + process.env.CAP_EDITOR_UI_BROWSER === "webkit" + ? webkit + : process.env.CAP_EDITOR_UI_BROWSER === "firefox" + ? firefox + : chromium; +const editorPublic = resolve( + process.env.CAP_EDITOR_SOLID_PUBLIC_DIR ?? + resolve(import.meta.dir, "../../../../../apps/web/public/editor-solid"), +); +const display = join( + import.meta.dir, + "../fixtures/editor-clips/display-red.webm", +); +const camera = join( + import.meta.dir, + "../fixtures/editor-clips/camera-green.webm", +); +const overlayImagePath = + "content/images/7db75f36-4b50-4ebf-9a5c-3e6be0b6e929.png"; +const overlayImage = await sharp( + Buffer.from( + "", + ), +) + .png() + .toBuffer(); + +assert.ok(process.env.CAP_WEB_EDITOR_PREPARE_BIN); +assert.ok(process.env.CAP_WEB_EDITOR_SERVICE_BIN); +assert.ok(await Bun.file(join(editorPublic, "index.html")).exists()); +initSync({ + module: readFileSync( + resolve( + import.meta.dir, + "../../../../../packages/editor-solid-web/renderer/pkg/cap_editor_browser_renderer_bg.wasm", + ), + ), +}); +let config = JSON.parse(default_project_config_json()) as { + background: { source: unknown; blur: number }; + timeline?: unknown; +}; +config.background.source = { + type: "animatedGradient", + config: JSON.parse(random_animated_gradient_json(1234)), +}; +config.background.blur = 60; +config.timeline = { + segments: [{ recordingSegment: 0, timescale: 1, start: 0, end: 3 }], + transitions: [], + zoomSegments: [], + imageSegments: [ + { + start: 0, + end: 3, + path: overlayImagePath, + center: { x: 0.5, y: 0.5 }, + size: { x: 0.42, y: 0.36 }, + opacity: 0.8, + rotation: 15, + rounding: 20, + flipX: true, + }, + ], +}; + +const hostModule = await Bun.build({ + entrypoints: [ + resolve( + import.meta.dir, + "../../../../../apps/web/app/s/[videoId]/edit/studio/editor-host.ts", + ), + ], + target: "browser", + format: "esm", + splitting: false, +}); +assert.ok(hostModule.success); +const hostCode = await hostModule.outputs[0]?.text(); +assert.ok(hostCode); + +const previousSecret = process.env.MEDIA_SERVER_WEBHOOK_SECRET; +const previousAllowHttp = process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; +const previousPublicOrigin = process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN; +process.env.MEDIA_SERVER_WEBHOOK_SECRET = secret; +process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = "1"; +const headers = { + "x-media-server-secret": secret, + "Content-Type": "application/json", +}; +let server: ReturnType | null = null; +let socketServer: ReturnType | null = null; +let browser: Awaited> | null = null; +let sessionId: string | null = null; +let savedAt: string | null = null; +let base = ""; +let bootstrapRequests = 0; +let rangeRequests = 0; +let workerRequests = 0; +let preparationRequests = 0; +let workerFixtureEnabled = false; +let currentCaptionPlan = false; +let planRequests = 0; + +async function preparationInput() { + return { + videoId, + title: "Paired browser-only editor fixture", + captionsEnabled: false, + display: { + url: `${base}/display.webm`, + contentType: "video/webm", + size: (await stat(display)).size, + fps: 30, + }, + camera: { + url: `${base}/camera.webm`, + contentType: "video/webm", + size: (await stat(camera)).size, + fps: 25, + offsetMs: 125, + }, + projectConfig: config, + imageAssets: [ + { + path: overlayImagePath, + name: "Preview parity image", + url: `${base}/overlay-image.png`, + size: overlayImage.byteLength, + contentType: "image/png", + }, + ], + }; +} + +function mediaResponse(request: Request, path: string) { + const blob = Bun.file(path); + const mediaHeaders: Record = { + "Content-Type": "video/webm", + "Accept-Ranges": "bytes", + }; + const range = /^bytes=(\d+)-(\d*)$/.exec(request.headers.get("range") ?? ""); + if (!range) return new Response(blob, { headers: mediaHeaders }); + const start = Number(range[1]); + const end = range[2] + ? Math.min(Number(range[2]), blob.size - 1) + : blob.size - 1; + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(end) || + start >= blob.size || + end < start + ) { + return new Response(null, { + status: 416, + headers: { ...mediaHeaders, "Content-Range": `bytes */${blob.size}` }, + }); + } + rangeRequests++; + return new Response(blob.slice(start, end + 1), { + status: 206, + headers: { + ...mediaHeaders, + "Content-Range": `bytes ${start}-${end}/${blob.size}`, + }, + }); +} + +async function readySession(id: string) { + const deadline = Date.now() + 60_000; + while (Date.now() < deadline) { + const response = await app.request(`/editor/preparations/${id}`, { + headers, + }); + assert.equal(response.status, 200); + const status = (await response.json()) as { + status: string; + sessionId?: string; + error?: string; + }; + if (status.status === "ready") { + assert.ok(status.sessionId); + return status.sessionId; + } + if (status.status === "error") + throw new Error(status.error ?? "Editor reference failed to prepare"); + await Bun.sleep(100); + } + throw new Error("Editor reference preparation timed out"); +} + +async function reportStageFailure( + stage: string, + page: Page, + editor: FrameLocator, + cause: unknown, + pageErrors: string[], + pageWarnings: string[], + failedResponses: string[], +) { + try { + // A frame that stopped answering must not hold back the failure itself. + const bounded = (read: Promise) => + Promise.race([ + read, + new Promise((resolve) => setTimeout(() => resolve(null), 5_000)), + ]); + const [body, canvas, videos, hostError] = await Promise.all([ + bounded( + editor + .locator("body") + .innerText() + .catch(() => null), + ), + bounded( + editor + .locator("#canvas") + .evaluate((element) => { + const canvas = element as HTMLCanvasElement; + return { + width: canvas.width, + height: canvas.height, + busy: canvas.getAttribute("aria-busy"), + opacity: getComputedStyle(canvas).opacity, + }; + }) + .catch(() => null), + ), + bounded( + editor + .locator("video") + .evaluateAll((elements) => + elements.map((element) => { + const video = element as HTMLVideoElement; + return { + readyState: video.readyState, + networkState: video.networkState, + currentTime: video.currentTime, + videoWidth: video.videoWidth, + videoHeight: video.videoHeight, + error: video.error?.message ?? null, + }; + }), + ) + .catch(() => null), + ), + bounded( + page + .evaluate( + () => + (window as typeof window & { capTestError?: string }) + .capTestError ?? null, + ) + .catch(() => null), + ), + ]); + const diagnostics = { + stage, + browserEngine: engine.name(), + error: cause instanceof Error ? cause.message : String(cause), + body, + canvas, + videos, + hostError, + pageErrors, + pageWarnings, + failedResponses, + }; + process.stderr.write(`${JSON.stringify(diagnostics)}\n`); + const artifactDir = process.env.CAP_EDITOR_UI_PARITY_ARTIFACT_DIR; + if (artifactDir) { + await mkdir(artifactDir, { recursive: true }); + await Promise.all([ + writeFile( + join(artifactDir, `${engine.name()}-${stage}.json`), + JSON.stringify(diagnostics, null, 2), + ), + page + .screenshot() + .then((image) => + writeFile( + join(artifactDir, `${engine.name()}-${stage}.png`), + image, + ), + ), + ]); + } + } catch (diagnosticError) { + process.stderr.write( + `Studio ${stage} diagnostics failed: ${String(diagnosticError)}\n`, + ); + } +} + +try { + server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + const url = new URL(request.url); + if (url.pathname === "/display.webm") + return mediaResponse(request, display); + if (url.pathname === "/camera.webm") + return mediaResponse(request, camera); + if (url.pathname === "/overlay-image.png") + return new Response(overlayImage, { + headers: { "Content-Type": "image/png" }, + }); + if (url.pathname === "/favicon.ico") + return new Response(null, { status: 204 }); + if (url.pathname === "/api/desktop/organizations") + return Response.json([]); + if (url.pathname === `/api/videos/${videoId}/render-status`) + return Response.json({ + state: "idle", + exportId: null, + progress: 0, + playable: false, + hlsUrl: null, + error: null, + current: false, + revision: null, + }); + if (url.pathname === "/test-host.js") + return new Response(hostCode, { + headers: { "Content-Type": "text/javascript; charset=utf-8" }, + }); + if (url.pathname === "/test-editor") + return new Response( + "", + { headers: { "Content-Type": "text/html; charset=utf-8" } }, + ); + if ( + url.pathname === `/api/editor/videos/${videoId}/bootstrap` && + request.method === "GET" + ) { + bootstrapRequests++; + return Response.json({ + videoId, + sources: { + videoId, + title: "Paired browser-only editor fixture", + captionsEnabled: false, + signedUrlExpiresAt: Date.now() + 20 * 60_000, + displayHasAudio: false, + projectConfig: config, + display: { + url: `${base}/display.webm`, + contentType: "video/webm", + fps: 30, + }, + camera: { + url: `${base}/camera.webm`, + contentType: "video/webm", + fps: 25, + offsetMs: 125, + }, + }, + }); + } + if (url.pathname.startsWith("/editor-solid/")) { + const staticPath = resolve( + editorPublic, + url.pathname.slice("/editor-solid/".length), + ); + if ( + staticPath.startsWith(`${editorPublic}${sep}`) && + (await Bun.file(staticPath).exists()) + ) { + const file = Bun.file(staticPath); + return new Response(file, { + headers: { "Content-Type": file.type }, + }); + } + return new Response("Missing editor asset", { status: 404 }); + } + const apiRoot = `/api/editor/videos/${videoId}`; + if (url.pathname === `${apiRoot}/file` && request.method === "GET") { + if (url.searchParams.get("path") !== overlayImagePath) + return new Response("Missing editor image", { status: 404 }); + return new Response(overlayImage, { + headers: { "Content-Type": "image/png" }, + }); + } + if (url.pathname === `${apiRoot}/plan` && request.method === "GET") { + planRequests++; + return Response.json({ pro: currentCaptionPlan }); + } + if (url.pathname === `${apiRoot}/assets` && request.method === "GET") + return Response.json({ path: null }); + if (url.pathname === `${apiRoot}/config` && request.method === "GET") + return Response.json({ savedAt }); + if (url.pathname === `${apiRoot}/config` && request.method === "PUT") { + const payload = (await request.json()) as { + config?: unknown; + expectedSavedAt?: string | null; + }; + if ( + payload.expectedSavedAt !== savedAt || + !payload.config || + Array.isArray(payload.config) || + typeof payload.config !== "object" + ) + return new Response("Editor revision changed", { status: 409 }); + config = payload.config as typeof config; + savedAt = new Date().toISOString(); + return Response.json({ saved: true, savedAt }); + } + if ( + url.pathname.startsWith("/api/editor/preparations") || + url.pathname.startsWith("/api/editor/sessions") + ) { + workerRequests++; + if (workerFixtureEnabled) { + if ( + url.pathname === "/api/editor/preparations" && + request.method === "POST" + ) { + const payload = (await request.json()) as { videoId?: string }; + if (payload.videoId !== videoId) + return new Response("Invalid recording", { status: 403 }); + preparationRequests++; + return app.request("/editor/preparations", { + method: "POST", + headers, + body: JSON.stringify(await preparationInput()), + }); + } + const preparationMatch = + /^\/api\/editor\/preparations\/([0-9a-f-]{36})$/.exec(url.pathname); + if ( + preparationMatch && + url.searchParams.get("videoId") === videoId && + (request.method === "GET" || request.method === "DELETE") + ) { + const response = await app.request( + `/editor/preparations/${preparationMatch[1]}`, + { method: request.method, headers }, + ); + if (request.method === "GET" && response.ok) { + const status = (await response.clone().json()) as { + status?: string; + sessionId?: string; + }; + if (status.status === "ready" && status.sessionId) + sessionId = status.sessionId; + } + return response; + } + const sessionMatch = + /^\/api\/editor\/sessions\/([0-9a-f-]{36})(?:\/(tickets))?$/.exec( + url.pathname, + ); + if (sessionMatch && sessionMatch[1] === sessionId) { + if (sessionMatch[2] === "tickets" && request.method === "POST") { + const payload = (await request.json()) as { videoId?: string }; + if (payload.videoId !== videoId) + return new Response("Invalid recording", { status: 403 }); + const ticketed = await app.request( + `/editor/sessions/${sessionId}/sockets`, + { + method: "POST", + headers, + body: JSON.stringify({ origin: base }), + }, + ); + if (!ticketed.ok) return ticketed; + const result = (await ticketed.json()) as { + sockets: Record; + }; + return Response.json(result.sockets); + } + if ( + !sessionMatch[2] && + request.method === "DELETE" && + url.searchParams.get("videoId") === videoId + ) { + const response = await app.request( + `/editor/sessions/${sessionId}`, + { method: "DELETE", headers }, + ); + if (response.ok) sessionId = null; + return response; + } + } + } + return new Response("Browser preview requested a native worker", { + status: 503, + }); + } + return new Response("Missing browser-only editor fixture route", { + status: 404, + }); + }, + }); + base = `http://127.0.0.1:${server.port}`; + browser = await engine.launch({ + headless: process.env.CAP_EDITOR_UI_HEADED !== "1", + ...(process.env.CAP_EDITOR_UI_HEADED === "1" && + process.env.CAP_EDITOR_UI_BROWSER === "firefox" + ? { + firefoxUserPrefs: { + "webgl.force-enabled": true, + "webgl.forbid-software": false, + }, + } + : {}), + }); + const page = await browser.newPage({ + viewport: { width: 1440, height: 900 }, + }); + const pageErrors: string[] = []; + const pageWarnings: string[] = []; + const failedResponses: string[] = []; + page.on("pageerror", (error) => pageErrors.push(error.message)); + page.on("console", (message) => { + if (message.type() === "error") pageErrors.push(message.text()); + if (message.type() === "warning") pageWarnings.push(message.text()); + }); + page.on("response", (response) => { + if (response.status() >= 400) + failedResponses.push(`${response.status()} ${response.url()}`); + }); + await page.goto(`${base}/test-editor`); + await page.locator("#editor").evaluate(async (element) => { + const iframe = element as HTMLIFrameElement; + if (iframe.contentDocument?.readyState === "complete") return; + await new Promise((resolve) => + iframe.addEventListener("load", () => resolve(), { once: true }), + ); + }); + await page.evaluate( + async ({ recordingId, ownerId }) => { + const { EditorHostBridge } = await import( + new URL("/test-host.js", location.origin).href + ); + const iframe = document.getElementById("editor") as HTMLIFrameElement; + const bridge = new EditorHostBridge( + recordingId, + `browser-${recordingId}`, + ownerId, + () => undefined, + (error: Error) => { + (window as typeof window & { capTestError?: string }).capTestError = + error.message; + }, + undefined, + undefined, + undefined, + false, + undefined, + (value: string) => { + ( + window as typeof window & { capTestSavedAt?: string | null } + ).capTestSavedAt = value; + }, + () => + (window as typeof window & { capTestSavedAt?: string | null }) + .capTestSavedAt ?? null, + true, + ); + ( + window as typeof window & { capTestBridge?: { dispose: () => void } } + ).capTestBridge = bridge; + await bridge.connect(iframe); + }, + { recordingId: videoId, ownerId: userId }, + ); + const editor = page.frameLocator("#editor"); + await editor.getByRole("button", { name: "Download", exact: true }).waitFor({ + state: "visible", + timeout: 20_000, + }); + await editor.getByRole("tab", { name: "Camera" }).waitFor({ + state: "visible", + timeout: 20_000, + }); + try { + await editor.locator("[aria-busy='false']").waitFor({ + state: "attached", + timeout: 20_000, + }); + } catch (cause) { + await reportStageFailure( + "canvas-readiness", + page, + editor, + cause, + pageErrors, + pageWarnings, + failedResponses, + ); + throw cause; + } + await editor.locator("#canvas").evaluate(async (canvas) => { + const surface = canvas.parentElement?.parentElement; + if (!surface) throw new Error("Preview surface is unavailable"); + const deadline = performance.now() + 3_000; + while (Number(getComputedStyle(surface).opacity) < 0.999) { + if (performance.now() > deadline) + throw new Error("Preview fade did not complete"); + await new Promise((resolve) => setTimeout(resolve, 25)); + } + }); + const draggableRects = await editor.locator("#canvas").evaluate((canvas) => { + const overlay = canvas.nextElementSibling; + return Array.from(overlay?.children ?? []) + .filter((element) => element.classList.contains("pointer-events-auto")) + .map((element) => { + const rect = element.getBoundingClientRect(); + return { width: rect.width, height: rect.height }; + }); + }); + assert.equal(draggableRects.length, 2); + assert.ok(draggableRects.every((rect) => rect.width > 0 && rect.height > 0)); + assert.ok(bootstrapRequests > 0); + assert.ok(rangeRequests > 0); + assert.equal(workerRequests, 0); + await editor.getByRole("tab", { name: "Captions" }).click(); + await editor.getByRole("link", { name: "Upgrade to Cap Pro" }).waitFor({ + state: "visible", + }); + const planRequestsBeforeFocus = planRequests; + currentCaptionPlan = true; + await page.evaluate(() => window.dispatchEvent(new Event("focus"))); + await editor.getByRole("link", { name: "Upgrade to Cap Pro" }).waitFor({ + state: "hidden", + }); + await editor.getByText("Font settings", { exact: true }).waitFor({ + state: "visible", + }); + assert.ok(planRequests > planRequestsBeforeFocus); + assert.equal(workerRequests, 0); + currentCaptionPlan = false; + await page.evaluate(() => + document.dispatchEvent(new Event("visibilitychange")), + ); + await editor.getByRole("link", { name: "Upgrade to Cap Pro" }).waitFor({ + state: "visible", + }); + await editor.getByText("Font settings", { exact: true }).waitFor({ + state: "hidden", + }); + assert.equal(workerRequests, 0); + await editor.getByRole("tab", { name: "Background" }).click(); + assert.ok( + ( + await editor.getByText("Blur", { exact: true }).locator("..").innerText() + ).includes("60.0%"), + ); + const canvasSize = await editor.locator("#canvas").evaluate((canvas) => { + const element = canvas as HTMLCanvasElement; + return { width: element.width, height: element.height }; + }); + const directScreenshot = await editor.locator("#canvas").screenshot(); + const webGlPreserve = await editor.locator("#canvas").evaluate((node) => { + const canvas = node as HTMLCanvasElement; + return ( + canvas.getContext("webgl2")?.getContextAttributes() + ?.preserveDrawingBuffer ?? null + ); + }); + assert.ok(webGlPreserve === null || webGlPreserve === true); + await editor.getByRole("button", { name: "Crop", exact: true }).click(); + await editor.getByText("Loading frame…").waitFor({ + state: "hidden", + timeout: 20_000, + }); + assert.ok( + (await editor + .getByRole("img", { name: "Current frame" }) + .evaluate((element) => (element as HTMLImageElement).naturalWidth)) > 0, + ); + const cropSourceSample = await editor + .getByRole("img", { name: "Current frame" }) + .evaluate((element) => { + const image = element as HTMLImageElement; + const canvas = document.createElement("canvas"); + canvas.width = 1; + canvas.height = 1; + const context = canvas.getContext("2d"); + if (!context) throw new Error("Crop source sample is unavailable"); + context.drawImage( + image, + Math.floor(image.naturalWidth / 2), + Math.floor(image.naturalHeight / 2), + 1, + 1, + 0, + 0, + 1, + 1, + ); + return [...context.getImageData(0, 0, 1, 1).data]; + }); + await editor.getByText("Rendering preview…").waitFor({ + state: "hidden", + timeout: 10_000, + }); + const cropDataUrl = await editor + .locator("canvas") + .last() + .evaluate((element) => + (element as HTMLCanvasElement).toDataURL("image/png"), + ); + const browserScreenshot = Buffer.from( + cropDataUrl.split(",")[1] ?? "", + "base64", + ); + if ((cropSourceSample[0] ?? 0) <= 200 || (cropSourceSample[1] ?? 255) >= 50) { + process.stderr.write( + `${JSON.stringify({ stage: "crop-source", browserEngine: engine.name(), cropSourceSample })}\n`, + ); + await reportStageFailure( + "crop-source", + page, + editor, + new Error("Crop did not show the red display clip"), + pageErrors, + pageWarnings, + failedResponses, + ); + } + assert.ok((cropSourceSample[0] ?? 0) > 200); + assert.ok((cropSourceSample[1] ?? 255) < 50); + await editor.getByRole("button", { name: "Cancel", exact: true }).click(); + assert.equal(workerRequests, 0); + const { data: browserPixels, info: browserInfo } = await sharp( + browserScreenshot, + ) + .ensureAlpha() + .raw() + .toBuffer({ resolveWithObject: true }); + const directScreenshotInfo = await sharp(directScreenshot).metadata(); + const { data: directPixels, info: directInfo } = await sharp(directScreenshot) + .ensureAlpha() + .resize(browserInfo.width, browserInfo.height, { kernel: "nearest" }) + .raw() + .toBuffer({ resolveWithObject: true }); + assert.equal(directInfo.channels, 4); + assert.equal(directPixels.length, browserPixels.length); + let visibleAbsolute = 0; + let visibleDifferentPixels = 0; + let visibleAlphaDifferentPixels = 0; + for (let pixel = 0; pixel < browserPixels.length; pixel += 4) { + let changed = false; + for (let channel = 0; channel < 4; channel++) { + const delta = Math.abs( + (directPixels[pixel + channel] ?? 0) - + (browserPixels[pixel + channel] ?? 0), + ); + visibleAbsolute += delta; + changed ||= delta > 20; + if (channel === 3 && delta > 20) visibleAlphaDifferentPixels++; + } + if (changed) visibleDifferentPixels++; + } + const visibleMeanAbsoluteError = + Math.round((visibleAbsolute / browserPixels.length) * 100) / 100; + const visibleMismatch = + visibleMeanAbsoluteError >= 5 || + visibleDifferentPixels >= browserInfo.width * browserInfo.height * 0.05; + if (visibleMismatch) { + process.stderr.write( + `${JSON.stringify({ stage: "visible-canvas", browserEngine: engine.name(), screenshotSize: { width: directScreenshotInfo.width, height: directScreenshotInfo.height }, retainedSize: { width: browserInfo.width, height: browserInfo.height }, visibleMeanAbsoluteError, visibleDifferentPixels, visibleAlphaDifferentPixels })}\n`, + ); + await reportStageFailure( + "visible-canvas", + page, + editor, + new Error("Visible Studio canvas differs from its retained frame"), + pageErrors, + pageWarnings, + failedResponses, + ); + const artifactDir = process.env.CAP_EDITOR_UI_PARITY_ARTIFACT_DIR; + if (artifactDir) { + await mkdir(artifactDir, { recursive: true }); + await Promise.all([ + writeFile( + join(artifactDir, `${engine.name()}-visible-canvas.png`), + directScreenshot, + ), + writeFile( + join(artifactDir, `${engine.name()}-retained-canvas.png`), + browserScreenshot, + ), + ]); + } + } + assert.ok(visibleMeanAbsoluteError < 5); + assert.ok( + visibleDifferentPixels < browserInfo.width * browserInfo.height * 0.05, + ); + await editor.getByRole("button", { name: "Play video" }).click(); + await editor + .getByRole("button", { name: "Pause video" }) + .waitFor({ state: "visible", timeout: 10_000 }) + .catch(async (error: unknown) => { + await reportStageFailure( + "first-playback", + page, + editor, + error, + pageErrors, + pageWarnings, + failedResponses, + ); + throw error; + }); + await editor + .getByText(/^0:(?:00\.(?:0[1-9]|[1-9]\d)|01\.\d\d)$/) + .first() + .waitFor({ state: "visible", timeout: 10_000 }); + await editor.getByRole("button", { name: "Pause video" }).click(); + assert.equal(workerRequests, 0); + assert.deepEqual(failedResponses, []); + assert.deepEqual(pageErrors, []); + assert.equal( + await page.evaluate( + () => + (window as typeof window & { capTestError?: string }).capTestError ?? + null, + ), + null, + ); + const preparation = await app.request("/editor/preparations", { + method: "POST", + headers, + body: JSON.stringify(await preparationInput()), + }); + assert.equal(preparation.status, 202); + const prepared = (await preparation.json()) as { id: string }; + sessionId = await readySession(prepared.id); + const native = getEditorSession(sessionId); + assert.ok(native); + const preview = await native.request("/preview", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + frameNumber: 0, + fps: 60, + resolutionBase: { x: canvasSize.width, y: canvasSize.height }, + }), + }); + assert.equal(preview.status, 200); + const exact = Buffer.from(await preview.arrayBuffer()); + const footer = exact.subarray(-24); + const stride = footer.readUInt32LE(0); + const nativeHeight = footer.readUInt32LE(4); + const nativeWidth = footer.readUInt32LE(8); + assert.equal(footer.readUInt32LE(12), 0); + assert.equal(exact.length, stride * nativeHeight + 24); + assert.ok(Math.abs(nativeWidth - browserInfo.width) <= 2); + assert.ok(Math.abs(nativeHeight - browserInfo.height) <= 2); + const packed = Buffer.alloc(nativeWidth * nativeHeight * 4); + for (let row = 0; row < nativeHeight; row++) { + exact.copy( + packed, + row * nativeWidth * 4, + row * stride, + row * stride + nativeWidth * 4, + ); + } + const nativePixels = await sharp(packed, { + raw: { width: nativeWidth, height: nativeHeight, channels: 4 }, + }) + .resize(browserInfo.width, browserInfo.height, { kernel: "nearest" }) + .raw() + .toBuffer(); + assert.equal(nativePixels.length, browserPixels.length); + let absolute = 0; + let squared = 0; + let differentPixels = 0; + for (let pixel = 0; pixel < browserPixels.length; pixel += 4) { + let changed = false; + for (let channel = 0; channel < 3; channel++) { + const delta = + (nativePixels[pixel + channel] ?? 0) - + (browserPixels[pixel + channel] ?? 0); + absolute += Math.abs(delta); + squared += delta * delta; + changed ||= Math.abs(delta) > 8; + } + if (changed) differentPixels++; + } + const samples = browserInfo.width * browserInfo.height * 3; + const mse = squared / samples; + const meanAbsoluteError = Math.round((absolute / samples) * 100) / 100; + const psnrDb = + Math.round((mse === 0 ? 100 : 10 * Math.log10(255 ** 2 / mse)) * 100) / 100; + if ( + meanAbsoluteError >= 3 || + psnrDb <= 30 || + differentPixels >= browserInfo.width * browserInfo.height * 0.05 + ) { + const samplePoints = [ + [0.05, 0.05], + [0.5, 0.05], + [0.5, 0.5], + [0.25, 0.5], + [0.95, 0.95], + ].map(([x, y]) => { + const column = Math.floor(x * (browserInfo.width - 1)); + const row = Math.floor(y * (browserInfo.height - 1)); + const offset = (row * browserInfo.width + column) * 4; + return { + x, + y, + browser: [...browserPixels.subarray(offset, offset + 3)], + native: [...nativePixels.subarray(offset, offset + 3)], + }; + }); + try { + const diagnostics = { + browserEngine: engine.name(), + meanAbsoluteError, + psnrDb, + differentPixels, + browserSize: { width: browserInfo.width, height: browserInfo.height }, + nativeSize: { width: nativeWidth, height: nativeHeight }, + samples: samplePoints, + videos: await editor.locator("video").evaluateAll((elements) => + elements.map((element) => { + const video = element as HTMLVideoElement; + return { + readyState: video.readyState, + currentTime: video.currentTime, + videoWidth: video.videoWidth, + videoHeight: video.videoHeight, + error: video.error?.message ?? null, + }; + }), + ), + pageErrors, + pageWarnings, + failedResponses, + }; + process.stderr.write(`${JSON.stringify(diagnostics)}\n`); + const artifactDir = process.env.CAP_EDITOR_UI_PARITY_ARTIFACT_DIR; + if (artifactDir) { + await mkdir(artifactDir, { recursive: true }); + await Promise.all([ + writeFile( + join(artifactDir, `${engine.name()}-direct-canvas.png`), + directScreenshot, + ), + writeFile( + join(artifactDir, `${engine.name()}-browser.png`), + browserScreenshot, + ), + writeFile( + join(artifactDir, `${engine.name()}-native.png`), + await sharp(packed, { + raw: { + width: nativeWidth, + height: nativeHeight, + channels: 4, + }, + }) + .png() + .toBuffer(), + ), + writeFile( + join(artifactDir, `${engine.name()}-diagnostics.json`), + JSON.stringify(diagnostics, null, 2), + ), + ]); + } + } catch (diagnosticError) { + process.stderr.write( + `Studio parity diagnostics failed: ${String(diagnosticError)}\n`, + ); + } + } + assert.ok(meanAbsoluteError < 3); + assert.ok(psnrDb > 30); + assert.ok(differentPixels < browserInfo.width * browserInfo.height * 0.05); + assert.equal(workerRequests, 0); + const closedReference = await app.request(`/editor/sessions/${sessionId}`, { + method: "DELETE", + headers, + }); + assert.equal(closedReference.status, 204); + sessionId = null; + socketServer = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request, listener) { + const upgrade = handleEditorSocketUpgrade(request, listener); + return upgrade === null ? app.fetch(request) : upgrade; + }, + websocket: editorWebSocketHandler, + }); + process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN = `http://127.0.0.1:${socketServer.port}`; + workerFixtureEnabled = true; + // Browsers with WebCodecs encoders export locally; this stage covers the + // worker that browsers without them prepare on demand. + await editor.locator("html").evaluate(() => { + Object.defineProperty(window, "VideoEncoder", { + configurable: true, + value: undefined, + }); + }); + const exportPreviewStartedAt = Date.now(); + await editor.getByRole("button", { name: "Download", exact: true }).click(); + await editor.getByRole("button", { name: "Back to editor" }).waitFor({ + state: "visible", + timeout: 20_000, + }); + try { + await editor.getByRole("img", { name: "Export preview" }).waitFor({ + state: "visible", + timeout: 90_000, + }); + } catch (cause) { + await reportStageFailure( + "export-preview", + page, + editor, + cause, + pageErrors, + pageWarnings, + failedResponses, + ); + throw cause; + } + const workerExportPreviewMs = Date.now() - exportPreviewStartedAt; + try { + await editor + .getByText(/(?:< 1 MB|~[0-9.]+(?:–[0-9.]+)? (?:MB|GB))/) + .first() + .waitFor({ state: "visible", timeout: 30_000 }); + } catch (cause) { + await reportStageFailure( + "export-estimate", + page, + editor, + cause, + pageErrors, + pageWarnings, + failedResponses, + ); + throw cause; + } + const workerExportEstimateMs = Date.now() - exportPreviewStartedAt; + assert.equal(preparationRequests, 1); + assert.ok(workerRequests > 0); + assert.ok(sessionId); + assert.ok(getEditorSession(sessionId)); + assert.deepEqual(failedResponses, []); + assert.deepEqual(pageErrors, []); + await editor.getByRole("button", { name: "Back to editor" }).click(); + const remountedCanvas = await editor + .locator("#canvas") + .evaluate(async (canvas) => { + const element = canvas as HTMLCanvasElement; + const deadline = performance.now() + 20_000; + while (element.width === 300 && element.height === 150) { + if (performance.now() > deadline) { + throw new Error("Editor preview did not reattach after export"); + } + await new Promise((resolve) => setTimeout(resolve, 25)); + } + return { width: element.width, height: element.height }; + }); + assert.ok(remountedCanvas.width > 0 && remountedCanvas.height > 0); + const remountedPixels = await sharp( + await editor.locator("#canvas").screenshot(), + ) + .removeAlpha() + .raw() + .toBuffer(); + let visibleRemountedPixels = 0; + for (let pixel = 0; pixel < remountedPixels.length; pixel += 3) { + if ( + (remountedPixels[pixel] ?? 0) > 20 || + (remountedPixels[pixel + 1] ?? 0) > 20 || + (remountedPixels[pixel + 2] ?? 0) > 20 + ) { + visibleRemountedPixels++; + } + } + assert.ok(visibleRemountedPixels > remountedPixels.length / 12); + await editor.getByRole("button", { name: "Full preview quality" }).click(); + await editor.getByRole("button", { name: "Skip to start" }).click(); + await editor + .getByText(/^0:00\.00$/) + .first() + .waitFor({ state: "visible", timeout: 10_000 }); + await editor.getByRole("button", { name: "Play video" }).click(); + await editor + .getByRole("button", { name: "Pause video" }) + .waitFor({ state: "visible", timeout: 10_000 }) + .catch(async (error: unknown) => { + await reportStageFailure( + "full-quality-playback", + page, + editor, + error, + pageErrors, + pageWarnings, + failedResponses, + ); + throw error; + }); + const playbackOpacities = await editor + .locator("#canvas") + .evaluate(async (canvas) => { + const surface = canvas.parentElement?.parentElement; + if (!surface) throw new Error("Preview surface is unavailable"); + const samples: number[] = []; + for (let index = 0; index < 20; index++) { + samples.push(Number(getComputedStyle(surface).opacity)); + await new Promise((resolve) => setTimeout(resolve, 50)); + } + return samples; + }); + assert.ok(playbackOpacities.every((opacity) => opacity >= 0.99)); + const pauseVideo = editor.getByRole("button", { name: "Pause video" }); + if (await pauseVideo.isVisible()) await pauseVideo.click(); + process.stdout.write( + `${JSON.stringify({ browserEngine: engine.name(), browserOnly: true, bootstrapRequests, rangeRequests, workerRequests, preparationRequests, captionPlanChangesWithoutReload: true, planRequests, playbackAdvanced: true, persistedGradient: true, webGlPreserve, workerExportPreviewVisible: true, workerExportEstimateVisible: true, workerExportPreviewMs, workerExportEstimateMs, visibleMeanAbsoluteError, visibleDifferentPixels, visibleAlphaDifferentPixels, meanAbsoluteError, psnrDb, differentPixels, totalPixels: browserInfo.width * browserInfo.height, pageErrors, failedResponses })}\n`, + ); + await page.evaluate(() => { + ( + window as typeof window & { capTestBridge?: { dispose: () => void } } + ).capTestBridge?.dispose(); + }); +} finally { + if (sessionId) + await app.request(`/editor/sessions/${sessionId}`, { + method: "DELETE", + headers, + }); + await browser?.close(); + socketServer?.stop(true); + server?.stop(true); + if (previousSecret === undefined) + delete process.env.MEDIA_SERVER_WEBHOOK_SECRET; + else process.env.MEDIA_SERVER_WEBHOOK_SECRET = previousSecret; + if (previousAllowHttp === undefined) + delete process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + else process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = previousAllowHttp; + if (previousPublicOrigin === undefined) + delete process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN; + else process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN = previousPublicOrigin; +} diff --git a/apps/media-server/src/__tests__/routes/editor-solid-ui-replay.ts b/apps/media-server/src/__tests__/routes/editor-solid-ui-replay.ts new file mode 100644 index 00000000000..59d561d9ed4 --- /dev/null +++ b/apps/media-server/src/__tests__/routes/editor-solid-ui-replay.ts @@ -0,0 +1,2323 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdir, readFile, stat, writeFile } from "node:fs/promises"; +import { join, resolve, sep } from "node:path"; +import { + type Browser, + chromium, + type Download, + firefox, + webkit, +} from "@playwright/test"; +import sharp from "sharp"; +import { hasEditorCaptionContent } from "../../../../../apps/web/lib/editor-caption-access"; +import app from "../../editor-worker-app"; +import { parseEditorSocketRequest } from "../../lib/editor-command-socket"; +import { renderEditorExportPreview } from "../../lib/editor-export-previews"; +import { getEditorSession } from "../../lib/editor-sessions"; +import { + type EditorSocketConnection, + editorWebSocketHandler, + handleEditorSocketUpgrade, +} from "../../lib/editor-websocket"; + +const secret = "editor-solid-ui-replay-secret"; +const videoId = "editor-solid-ui-replay"; +const userId = "editor-solid-ui-user"; +const proCaptions = process.env.CAP_EDITOR_UI_PRO_CAPTIONS === "1"; +const shareReplay = process.env.CAP_EDITOR_UI_SHARE_REPLAY === "1"; +const cursorMovReplay = process.env.CAP_EDITOR_UI_CURSOR_MOV === "1"; +assert.ok(!(shareReplay && cursorMovReplay)); +const canvasFallbackReplay = process.env.CAP_EDITOR_UI_CANVAS_FALLBACK === "1"; +assert.ok(!(canvasFallbackReplay && (shareReplay || cursorMovReplay))); +const coldMount = process.env.CAP_EDITOR_UI_COLD_MOUNT === "1"; +const recoveryFault = process.env.CAP_EDITOR_UI_RUNTIME_RECOVERY_FAULT === "1"; +const runtimeFault = + process.env.CAP_EDITOR_UI_RUNTIME_FAULT === "1" || recoveryFault; +const runtimeFaultMessage = recoveryFault + ? "Recording may need to be recovered" + : "Editor runtime fault for UI replay"; +const browserEngine = + process.env.CAP_EDITOR_UI_BROWSER === "webkit" + ? webkit + : process.env.CAP_EDITOR_UI_BROWSER === "firefox" + ? firefox + : chromium; +const editorPublic = resolve( + process.env.CAP_EDITOR_SOLID_PUBLIC_DIR ?? + resolve(import.meta.dir, "../../../../../apps/web/public/editor-solid"), +); +const display = join( + import.meta.dir, + proCaptions + ? "../fixtures/editor-clips/clip-blue-audio.mp4" + : "../fixtures/editor-clips/display-red.webm", +); +const camera = join( + import.meta.dir, + "../fixtures/editor-clips/camera-green.webm", +); +const image = join(import.meta.dir, "../fixtures/exif-orientation-6.jpg"); +const imagePath = "content/images/3d82ac0f-c24a-4c21-aa3c-e1749c23b24b.jpg"; +const cursorInputEvents = `${[ + { version: 1, platform: "MacOS" }, + { + kind: "move", + timeMs: 200, + x: 0.3, + y: 0.4, + cursor: "default", + button: 0, + modifiers: [], + }, + { + kind: "move", + timeMs: 1000, + x: 0.7, + y: 0.6, + cursor: "pointer", + button: 0, + modifiers: [], + }, +] + .map((event) => JSON.stringify(event)) + .join("\n")}\n`; +const imageKey = `${userId}/${videoId}/editor-assets/images/${imagePath.slice("content/images/".length)}`; + +async function readySession(id: string, headers: Record) { + const deadline = Date.now() + 60_000; + while (Date.now() < deadline) { + const response = await app.request(`/editor/preparations/${id}`, { + headers, + }); + assert.equal(response.status, 200); + const status = (await response.json()) as { + status: string; + sessionId?: string; + error?: string; + }; + if (status.status === "ready") { + assert.ok(status.sessionId); + return status.sessionId; + } + if (status.status === "error") + throw new Error(status.error ?? "Editor UI fixture could not prepare"); + await Bun.sleep(100); + } + throw new Error("Editor UI fixture preparation timed out"); +} + +async function waitForWorkerCaptionContent(expected: boolean) { + if (!sessionId) throw new Error("Editor session is unavailable"); + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + const response = await app.request(`/editor/sessions/${sessionId}/config`, { + headers, + }); + if ( + response.ok && + hasEditorCaptionContent(await response.json()) === expected + ) + return; + await Bun.sleep(100); + } + throw new Error(`Worker caption access did not become ${expected}`); +} + +assert.ok(process.env.CAP_WEB_EDITOR_PREPARE_BIN); +assert.ok(process.env.CAP_WEB_EDITOR_SERVICE_BIN); +assert.ok(await Bun.file(join(editorPublic, "index.html")).exists()); + +const hostModule = await Bun.build({ + entrypoints: [ + join( + import.meta.dir, + "../../../../../apps/web/app/s/[videoId]/edit/studio/editor-host.ts", + ), + ], + target: "browser", + format: "esm", + splitting: false, +}); +assert.ok(hostModule.success); +const hostCode = await hostModule.outputs[0]?.text(); +assert.ok(hostCode); + +const previousSecret = process.env.MEDIA_SERVER_WEBHOOK_SECRET; +const previousAllowHttp = process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; +const previousPublicOrigin = process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN; +process.env.MEDIA_SERVER_WEBHOOK_SECRET = secret; +process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = "1"; + +const headers = { + "x-media-server-secret": secret, + "Content-Type": "application/json", +}; +let server: ReturnType | null = null; +let socketServer: ReturnType | null = null; +let browser: Browser | null = null; +let sessionId: string | null = null; +let savedAt: string | null = null; +let captionRequests = 0; +let currentCaptionPlan = proCaptions; +let planRequests = 0; +let freeCaptionlessSaves = 0; +let proCaptionSaves = 0; +let savedPaidCaptions = false; +let preservingFreeSaves = 0; +let bundleTicketRequests = 0; +let browserBootstrapRequests = 0; +let browserMediaRangeRequests = 0; +let imageImports = 0; +let imagePreviewRequests = 0; +let relativeImagePreviewRequests = 0; +let uploadedImage: Uint8Array | null = null; +let renderedExportId: string | null = null; +let cursorMovTicketRequests = 0; +let cursorMovDownloadUrl: string | null = null; +let cursorMovWorkerDigest: { sha256: string; size: number } | null = null; +function requireCursorMovWorkerDigest() { + const digest = cursorMovWorkerDigest; + if (!digest) throw new Error("Cursor-only MOV worker file is unavailable"); + return digest; +} +let cursorMovVerification: { + sha256: string; + size: number; + visibleFrames: number; +} | null = null; +let cursorMovExpectedStatus404 = 0; +let cursorMovExpectedConsole404 = 0; +let multipartInitiations = 0; +let exportChunkRequests = 0; +const uploadedParts = new Map(); +let completedShare: { + sha256: string; + size: number; + parts: number; + duration: number; +} | null = null; +let base = ""; +function mediaFixtureResponse( + request: Request, + path: string, + contentType: string, +) { + const blob = Bun.file(path); + const headers: Record = { + "Content-Type": contentType, + "Accept-Ranges": "bytes", + "Access-Control-Allow-Origin": "*", + "Access-Control-Expose-Headers": "Content-Range", + }; + const range = /^bytes=(\d+)-(\d*)$/.exec(request.headers.get("range") ?? ""); + if (!range) return new Response(blob, { headers }); + const start = Number(range[1]); + const end = range[2] + ? Math.min(Number(range[2]), blob.size - 1) + : blob.size - 1; + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(end) || + start >= blob.size || + end < start + ) { + return new Response(null, { + status: 416, + headers: { ...headers, "Content-Range": `bytes */${blob.size}` }, + }); + } + browserMediaRangeRequests++; + return new Response(blob.slice(start, end + 1), { + status: 206, + headers: { + ...headers, + "Content-Range": `bytes ${start}-${end}/${blob.size}`, + }, + }); +} +try { + server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + const url = new URL(request.url); + if (url.pathname === "/display.webm" || url.pathname === "/display.mp4") + return mediaFixtureResponse( + request, + display, + proCaptions ? "video/mp4" : "video/webm", + ); + if (url.pathname === "/camera.webm") + return mediaFixtureResponse(request, camera, "video/webm"); + if (url.pathname === "/input-events.ndjson") + return new Response(cursorInputEvents, { + headers: { "Content-Type": "application/x-ndjson" }, + }); + if ( + url.pathname === "/api/upload/multipart/initiate" && + request.method === "POST" + ) { + const payload = (await request.json()) as { + videoId?: string; + contentType?: string; + subpath?: string; + replaceExisting?: boolean; + }; + if ( + payload.videoId !== videoId || + payload.contentType !== "video/mp4" || + payload.subpath !== "result.mp4" || + payload.replaceExisting !== true + ) + return new Response("Invalid replacement upload", { status: 400 }); + multipartInitiations++; + return Response.json({ uploadId: "ui-reupload", provider: "s3" }); + } + if ( + url.pathname === "/api/upload/multipart/presign-part" && + request.method === "POST" + ) { + const payload = (await request.json()) as { + videoId?: string; + uploadId?: string; + partNumber?: number; + subpath?: string; + replaceExisting?: boolean; + }; + if ( + payload.videoId !== videoId || + payload.uploadId !== "ui-reupload" || + !Number.isSafeInteger(payload.partNumber) || + !payload.partNumber || + payload.partNumber < 1 || + payload.subpath !== "result.mp4" || + payload.replaceExisting !== true + ) + return new Response("Invalid upload part", { status: 400 }); + return Response.json({ + presignedUrl: `${base}/test-multipart-part/${payload.partNumber}`, + provider: "s3", + }); + } + const uploadPart = /^\/test-multipart-part\/([1-9][0-9]*)$/.exec( + url.pathname, + ); + if (uploadPart && request.method === "PUT") { + const partNumber = Number(uploadPart[1]); + uploadedParts.set(partNumber, Buffer.from(await request.arrayBuffer())); + return new Response(null, { + status: 200, + headers: { ETag: `"ui-etag-${partNumber}"` }, + }); + } + if ( + url.pathname === "/api/upload/multipart/complete" && + request.method === "POST" + ) { + const payload = (await request.json()) as { + videoId?: string; + uploadId?: string; + subpath?: string; + replaceExisting?: boolean; + durationInSecs?: number; + width?: number; + height?: number; + fps?: number; + parts?: Array<{ partNumber: number; etag: string; size: number }>; + }; + if ( + payload.videoId !== videoId || + payload.uploadId !== "ui-reupload" || + payload.subpath !== "result.mp4" || + payload.replaceExisting !== true || + !payload.parts?.length || + !sessionId || + !renderedExportId + ) + return new Response("Invalid upload completion", { status: 400 }); + const exportRoot = `/editor/sessions/${sessionId}/exports/${renderedExportId}`; + const status = await app.request(exportRoot, { headers }); + const file = await app.request(`${exportRoot}/file`, { headers }); + if (!status.ok || !file.ok) + return new Response("Rendered MP4 is unavailable", { status: 503 }); + const source = (await status.json()) as { + mediaMetadata?: { + duration: number; + width: number; + height: number; + fps: number; + } | null; + }; + const sourceBytes = Buffer.from(await file.arrayBuffer()); + const parts = [...payload.parts].sort( + (left, right) => left.partNumber - right.partNumber, + ); + for (const part of parts) { + assert.equal(part.etag, `ui-etag-${part.partNumber}`); + assert.equal(part.size, uploadedParts.get(part.partNumber)?.length); + } + const uploaded = Buffer.concat( + parts.map( + (part) => uploadedParts.get(part.partNumber) ?? Buffer.alloc(0), + ), + ); + assert.deepEqual(uploaded, sourceBytes); + assert.ok(source.mediaMetadata); + assert.equal(payload.width, source.mediaMetadata.width); + assert.equal(payload.height, source.mediaMetadata.height); + assert.equal(payload.fps, source.mediaMetadata.fps); + assert.ok(payload.durationInSecs); + assert.ok( + Math.abs(payload.durationInSecs - source.mediaMetadata.duration) < + 0.001, + ); + completedShare = { + sha256: createHash("sha256").update(uploaded).digest("hex"), + size: uploaded.byteLength, + parts: parts.length, + duration: source.mediaMetadata.duration, + }; + return Response.json({ success: true, processingStarted: true }); + } + if ( + url.pathname === "/api/upload/multipart/abort" && + request.method === "POST" + ) + return Response.json({ success: true }); + if (url.pathname === "/test-image-upload" && request.method === "PUT") { + uploadedImage = new Uint8Array(await request.arrayBuffer()); + return new Response(null, { status: 200 }); + } + if (url.pathname === "/test-image.jpg" && uploadedImage) + return new Response(uploadedImage, { + headers: { "Content-Type": "image/jpeg" }, + }); + if (url.pathname === "/test-host.js") + return new Response(hostCode, { + headers: { "Content-Type": "text/javascript; charset=utf-8" }, + }); + if (url.pathname === "/api/desktop/organizations") + return Response.json([]); + if (url.pathname === `/api/videos/${videoId}/render-status`) + return Response.json({ + state: "idle", + exportId: null, + progress: 0, + playable: false, + hlsUrl: null, + error: null, + current: false, + revision: null, + }); + if (url.pathname === "/favicon.ico") + return new Response(null, { status: 204 }); + if (url.pathname === "/test-editor") + return new Response( + "", + { headers: { "Content-Type": "text/html; charset=utf-8" } }, + ); + if ( + url.pathname === + `/api/editor/videos/${encodeURIComponent(videoId)}/bootstrap` && + request.method === "GET" + ) { + browserBootstrapRequests++; + return Response.json({ + videoId, + sources: { + videoId, + title: "Paired editor UI fixture", + captionsEnabled: currentCaptionPlan, + signedUrlExpiresAt: Date.now() + 20 * 60_000, + displayHasAudio: proCaptions, + display: { + url: `${base}/display.${proCaptions ? "mp4" : "webm"}`, + contentType: proCaptions ? "video/mp4" : "video/webm", + fps: 30, + }, + camera: { + url: `${base}/camera.webm`, + contentType: "video/webm", + fps: 25, + offsetMs: 125, + }, + }, + }); + } + if (url.pathname.startsWith("/editor-solid/")) { + const staticPath = resolve( + editorPublic, + url.pathname.slice("/editor-solid/".length), + ); + if ( + staticPath.startsWith(`${editorPublic}${sep}`) && + (await Bun.file(staticPath).exists()) + ) { + const file = Bun.file(staticPath); + return new Response(file, { + headers: { "Content-Type": file.type }, + }); + } + return new Response("Missing editor asset", { status: 404 }); + } + if (sessionId) { + const apiRoot = `/api/editor/sessions/${encodeURIComponent(sessionId)}`; + const exportRoot = `${apiRoot}/exports`; + if (url.pathname === exportRoot && request.method === "POST") { + const payload = (await request.json()) as { + videoId?: string; + settings?: unknown; + }; + if ( + payload.videoId !== videoId || + typeof payload.settings !== "object" || + payload.settings === null + ) + return new Response("Invalid editor export", { status: 400 }); + if ( + cursorMovReplay && + (!("format" in payload.settings) || + payload.settings.format !== "Mov" || + !("cursor_only" in payload.settings) || + payload.settings.cursor_only !== true) + ) + return new Response("Invalid cursor-only MOV settings", { + status: 400, + }); + const started = await app.request( + `/editor/sessions/${sessionId}/exports`, + { + method: "POST", + headers, + body: JSON.stringify(payload.settings), + }, + ); + if (started.ok) { + const state = (await started.clone().json()) as { id: string }; + renderedExportId = state.id; + } + return started; + } + if (url.pathname.startsWith(`${exportRoot}/`)) { + const relative = url.pathname.slice(exportRoot.length + 1); + const [exportId, action, extra] = relative.split("/"); + if ( + !exportId || + extra || + (action !== "download-ticket" && + url.searchParams.get("videoId") !== videoId) || + (action && action !== "chunk" && action !== "download-ticket") + ) + return new Response("Invalid editor export request", { + status: 400, + }); + const workerPath = `/editor/sessions/${sessionId}/exports/${encodeURIComponent(exportId)}`; + if ( + !action && + (request.method === "GET" || request.method === "DELETE") + ) + return app.request(workerPath, { method: request.method, headers }); + if (action === "chunk" && request.method === "GET") { + const offset = url.searchParams.get("offset"); + const length = url.searchParams.get("length"); + if (!offset || !length) + return new Response("Invalid editor export chunk", { + status: 400, + }); + const chunk = await app.request( + `${workerPath}/chunk?offset=${offset}&length=${length}`, + { headers }, + ); + if (chunk.status === 206) exportChunkRequests++; + return chunk; + } + if (action === "download-ticket" && request.method === "POST") { + const payload = (await request.json()) as { + videoId?: string; + fileName?: string; + }; + if ( + !cursorMovReplay || + payload.videoId !== videoId || + !payload.fileName?.endsWith(".mov") + ) + return new Response("Invalid cursor-only MOV ticket", { + status: 400, + }); + const ticket = await app.request(`${workerPath}/download-ticket`, { + method: "POST", + headers, + body: JSON.stringify({ fileName: payload.fileName }), + }); + if (ticket.ok) { + const file = await app.request(`${workerPath}/file`, { + headers, + }); + assert.equal(file.status, 200); + assert.equal(file.headers.get("Content-Type"), "video/quicktime"); + const bytes = Buffer.from(await file.arrayBuffer()); + cursorMovWorkerDigest = { + sha256: createHash("sha256").update(bytes).digest("hex"), + size: bytes.length, + }; + const value = (await ticket.clone().json()) as { url: string }; + cursorMovDownloadUrl = value.url; + cursorMovTicketRequests++; + } + return ticket; + } + } + if ( + url.pathname === `${apiRoot}/project-bundle/download-ticket` && + request.method === "POST" + ) { + const payload = (await request.json()) as { videoId?: string }; + if (payload.videoId !== videoId) + return new Response("Invalid recording", { status: 403 }); + const ticketed = await app.request( + `/editor/sessions/${sessionId}/project-bundle/download-ticket`, + { + method: "POST", + headers, + body: JSON.stringify({ fileName: "Cap Recording.capbundle" }), + }, + ); + if (ticketed.ok) bundleTicketRequests++; + return ticketed; + } + if (url.pathname === `${apiRoot}/assets`) { + if (request.method === "GET") return Response.json({ path: null }); + const payload = (await request.json()) as { + kind?: string; + videoId?: string; + fileName?: string; + size?: number; + contentType?: string; + key?: string; + path?: string; + }; + if ( + payload.kind !== "image" || + payload.videoId !== videoId || + payload.fileName !== "exif-orientation-6.jpg" || + payload.contentType !== "image/jpeg" || + payload.size !== (await stat(image)).size + ) + return new Response("Invalid image import", { status: 400 }); + if (request.method === "POST") + return Response.json({ + key: imageKey, + path: imagePath, + upload: { + type: "put", + url: `${base}/test-image-upload`, + headers: {}, + }, + }); + if ( + request.method !== "PUT" || + payload.key !== imageKey || + payload.path !== imagePath || + !uploadedImage || + uploadedImage.byteLength !== payload.size + ) + return new Response("Image upload is incomplete", { status: 400 }); + const imported = await app.request( + `/editor/sessions/${sessionId}/image-assets`, + { + method: "POST", + headers, + body: JSON.stringify({ + path: imagePath, + name: "exif-orientation-6", + url: `${base}/test-image.jpg`, + size: payload.size, + contentType: "image/jpeg", + objectIdentity: null, + }), + }, + ); + if (imported.ok) imageImports++; + return imported; + } + if (url.pathname === `${apiRoot}/file`) { + const requestedPath = url.searchParams.get("path"); + if ( + request.method !== "GET" || + url.searchParams.get("videoId") !== videoId || + (requestedPath !== imagePath && + requestedPath !== + `cap-web-editor://session/${sessionId}/${imagePath}`) + ) + return new Response("Image asset is unavailable", { status: 404 }); + imagePreviewRequests++; + if (requestedPath === imagePath) relativeImagePreviewRequests++; + return Response.redirect(`${base}/test-image.jpg`); + } + if (url.pathname === `${apiRoot}/captions`) { + const requestedVideoId = + request.method === "GET" + ? url.searchParams.get("videoId") + : ((await request.json()) as { videoId?: string }).videoId; + if (!currentCaptionPlan || requestedVideoId !== videoId) + return new Response("Cap Pro is required for captions", { + status: 403, + }); + captionRequests++; + return Response.json({ + status: "ready", + captions: { + segments: [ + { + id: "assemblyai-caption", + start: 0.5, + end: 1.2, + text: "Hello Cap", + words: [ + { text: "Hello", start: 0.5, end: 0.8 }, + { text: "Cap", start: 0.8, end: 1.2 }, + ], + }, + ], + settings: null, + }, + message: null, + }); + } + if ( + url.pathname === `${apiRoot}/tickets` && + request.method === "POST" + ) { + const payload = (await request.json()) as { videoId?: string }; + if (payload.videoId !== videoId) + return new Response("Invalid recording", { status: 403 }); + const worker = await app.request( + `/editor/sessions/${sessionId}/sockets`, + { + method: "POST", + headers, + body: JSON.stringify({ origin: base }), + }, + ); + if (!worker.ok) return worker; + const ticketed = (await worker.json()) as { + sockets: Record; + }; + return Response.json(ticketed.sockets); + } + if (url.pathname === `${apiRoot}/config`) { + if (request.method === "GET") return Response.json({ savedAt }); + if (request.method === "PUT") { + const payload = (await request.json()) as { + videoId?: string; + config?: unknown; + expectedSavedAt?: string | null; + preserveExistingPaidCaptions?: boolean; + }; + if ( + payload.videoId !== videoId || + payload.expectedSavedAt !== savedAt + ) + return new Response("Editor revision changed", { status: 409 }); + if (!currentCaptionPlan && hasEditorCaptionContent(payload.config)) + return new Response("Cap Pro is required for captions", { + status: 403, + }); + if ( + payload.preserveExistingPaidCaptions === true && + hasEditorCaptionContent(payload.config) + ) + return new Response("Preserved captions must be absent", { + status: 403, + }); + const worker = await app.request( + `/editor/sessions/${sessionId}/config`, + { + method: "PUT", + headers, + body: JSON.stringify(payload.config), + }, + ); + if (!worker.ok) return worker; + if (!currentCaptionPlan) { + freeCaptionlessSaves++; + if ( + savedPaidCaptions && + payload.preserveExistingPaidCaptions === true + ) + preservingFreeSaves++; + } else if (hasEditorCaptionContent(payload.config)) { + proCaptionSaves++; + savedPaidCaptions = true; + } else if (payload.preserveExistingPaidCaptions !== true) { + savedPaidCaptions = false; + } + savedAt = new Date().toISOString(); + return Response.json({ saved: true, savedAt }); + } + } + if (url.pathname === `${apiRoot}/plan` && request.method === "GET") { + planRequests++; + return Response.json({ pro: currentCaptionPlan }); + } + if (url.pathname === `${apiRoot}/assets` && request.method === "GET") + return Response.json({ path: null }); + } + return new Response("Missing editor fixture route", { status: 404 }); + }, + }); + base = `http://127.0.0.1:${server.port}`; + const preparation = await app.request("/editor/preparations", { + method: "POST", + headers, + body: JSON.stringify({ + videoId, + title: "Paired editor UI fixture", + captionsEnabled: proCaptions, + display: { + url: `${base}/display.${proCaptions ? "mp4" : "webm"}`, + contentType: proCaptions ? "video/mp4" : "video/webm", + size: (await stat(display)).size, + fps: 30, + }, + camera: { + url: `${base}/camera.webm`, + contentType: "video/webm", + size: (await stat(camera)).size, + fps: 25, + offsetMs: 125, + }, + ...(cursorMovReplay + ? { + inputEvents: { + url: `${base}/input-events.ndjson`, + contentType: "application/x-ndjson", + size: Buffer.byteLength(cursorInputEvents), + }, + } + : {}), + }), + }); + assert.equal(preparation.status, 202); + const prepared = (await preparation.json()) as { id: string }; + sessionId = await readySession(prepared.id, headers); + const preparedNative = getEditorSession(sessionId); + assert.ok(preparedNative); + if (proCaptions && !runtimeFault) { + const endPreview = await renderEditorExportPreview( + sessionId, + preparedNative, + 2, + { + fps: 60, + resolution_base: { x: 640, y: 360 }, + compression_bpp: 0.14, + }, + new AbortController().signal, + ); + assert.equal(endPreview.total_frames, 120); + assert.ok(Buffer.from(endPreview.jpeg_base64, "base64").length > 1_000); + } + if (shareReplay) { + const gradientResponse = await preparedNative.request( + "/animated-gradients/random", + ); + assert.equal(gradientResponse.status, 200); + const gradient = await gradientResponse.json(); + const configResponse = await app.request( + `/editor/sessions/${sessionId}/config`, + { headers }, + ); + assert.equal(configResponse.status, 200); + const config = (await configResponse.json()) as { + background: { source: unknown; blur: number }; + }; + config.background.source = { type: "animatedGradient", config: gradient }; + config.background.blur = 60; + const saved = await app.request(`/editor/sessions/${sessionId}/config`, { + method: "PUT", + headers, + body: JSON.stringify(config), + }); + assert.equal(saved.status, 204); + } + const socketClosures: Array<{ + scope: string; + code: number; + reason: string; + upstreamReadyState: number | null; + atMs: number; + }> = []; + const baseSocketHandler: Bun.WebSocketHandler = + runtimeFault + ? { + ...editorWebSocketHandler, + message(ws, message) { + let command = null; + try { + command = parseEditorSocketRequest( + JSON.parse( + typeof message === "string" + ? message + : Buffer.from(message).toString("utf8"), + ), + ); + } catch {} + if ( + command?.kind === "invoke" && + command.name === "createEditorInstance" + ) { + ws.send( + JSON.stringify({ + kind: "error", + id: command.id, + error: runtimeFaultMessage, + }), + ); + return; + } + editorWebSocketHandler.message(ws, message); + }, + } + : editorWebSocketHandler; + const socketHandler: Bun.WebSocketHandler = { + ...baseSocketHandler, + close(ws, code, reason) { + socketClosures.push({ + scope: ws.data.scope, + code, + reason, + upstreamReadyState: ws.data.upstream?.readyState ?? null, + atMs: Date.now(), + }); + baseSocketHandler.close?.(ws, code, reason); + }, + }; + socketServer = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request, listener) { + const upgrade = handleEditorSocketUpgrade(request, listener); + return upgrade === null ? app.fetch(request) : upgrade; + }, + websocket: socketHandler, + }); + process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN = `http://127.0.0.1:${socketServer.port}`; + browser = await browserEngine.launch({ + headless: process.env.CAP_EDITOR_UI_HEADED !== "1", + ...(process.env.CAP_EDITOR_UI_HEADED === "1" && + process.env.CAP_EDITOR_UI_BROWSER === "firefox" + ? { + firefoxUserPrefs: { + "webgl.force-enabled": true, + "webgl.forbid-software": false, + }, + } + : {}), + }); + const page = await browser.newPage({ + viewport: { width: 1440, height: 900 }, + }); + if (canvasFallbackReplay) { + await page.addInitScript(() => { + for (const name of [ + "OffscreenCanvas", + "createImageBitmap", + "VideoDecoder", + "EncodedVideoChunk", + "VideoFrame", + ]) { + Object.defineProperty(window, name, { + value: undefined, + configurable: true, + }); + } + Object.defineProperty(navigator, "gpu", { + value: undefined, + configurable: true, + }); + }); + } + if (browserEngine.name() === "firefox" && proCaptions) { + await page.addInitScript(() => { + const browserWindow = window as typeof window & { + capTestColorPath?: Record; + }; + const record = (source: unknown) => { + if (browserWindow.capTestColorPath) return; + if (typeof VideoFrame === "function" && source instanceof VideoFrame) { + browserWindow.capTestColorPath = { + kind: "VideoFrame", + format: source.format, + matrix: source.colorSpace.matrix, + }; + } else if (source instanceof HTMLVideoElement) { + browserWindow.capTestColorPath = { kind: "HTMLVideoElement" }; + } + }; + const watch = ( + owner: T | undefined, + name: keyof T & string, + sourceOf: (args: unknown[]) => unknown, + ) => { + const original = owner?.[name]; + if (!owner || typeof original !== "function") return; + Object.defineProperty(owner, name, { + configurable: true, + writable: true, + value: new Proxy(original, { + apply(target, thisArg, args: unknown[]) { + record(sourceOf(args)); + return Reflect.apply(target, thisArg, args); + }, + }), + }); + }; + // Decoded frames reach the renderer's texture uploads directly; a + // bitmap is only made on the fallback paths. + watch(window, "createImageBitmap", (args) => args[0]); + for (const context of [ + window.WebGL2RenderingContext?.prototype, + window.WebGLRenderingContext?.prototype, + ]) { + watch(context, "texImage2D", (args) => args.at(-1)); + watch(context, "texSubImage2D", (args) => args.at(-1)); + } + const queue: unknown = Reflect.get(window, "GPUQueue"); + watch( + typeof queue === "function" + ? (queue.prototype as { copyExternalImageToTexture?: unknown }) + : undefined, + "copyExternalImageToTexture", + (args) => { + const source = args[0]; + return typeof source === "object" && source !== null + ? Reflect.get(source, "source") + : null; + }, + ); + }); + } + await page.addInitScript(() => { + const nativeClose = WebSocket.prototype.close; + WebSocket.prototype.close = function (code?: number, reason?: string) { + if (code === 4003) { + const browserWindow = window as typeof window & { + capTestInvalidFrameCloses?: number; + }; + browserWindow.capTestInvalidFrameCloses = + (browserWindow.capTestInvalidFrameCloses ?? 0) + 1; + } + if (code === undefined) return nativeClose.call(this); + if (reason === undefined) return nativeClose.call(this, code); + return nativeClose.call(this, code, reason); + }; + }); + const pageErrors: string[] = []; + const navigationRequests: string[] = []; + const frameNavigations: string[] = []; + page.on("request", (request) => { + if (request.isNavigationRequest()) navigationRequests.push(request.url()); + }); + page.on("framenavigated", (frame) => { + frameNavigations.push(frame.url()); + }); + const dialogMessages: string[] = []; + const rendererFallbacks: string[] = []; + const failedResponses: string[] = []; + const exportResponses: string[] = []; + const failedResponseReads: Promise[] = []; + let delayedSkeletonRequests = 0; + let delayedEditorRequests = 0; + if (coldMount) { + await page.addInitScript(() => { + window.addEventListener("message", (event: MessageEvent) => { + if (typeof event.data !== "object" || event.data === null) return; + if (!("kind" in event.data)) return; + if (event.data.kind === "cap-editor-connect") + document.body.dataset.editorConnectReceived = "true"; + }); + }); + await page.route( + /\/editor-solid\/assets\/editor-skeleton-[^/]+\.js(?:\?.*)?$/, + async (route) => { + delayedSkeletonRequests++; + await Bun.sleep(3_000); + await route.continue(); + }, + ); + await page.route( + /\/editor-solid\/assets\/Editor-[^/]+\.js(?:\?.*)?$/, + async (route) => { + delayedEditorRequests++; + await Bun.sleep(6_000); + await route.continue(); + }, + ); + } + page.on("pageerror", (error) => + pageErrors.push(error.stack ?? error.message), + ); + page.on("dialog", (dialog) => { + dialogMessages.push(dialog.message()); + void dialog + .dismiss() + .catch((cause: unknown) => pageErrors.push(String(cause))); + }); + page.on("console", (message) => { + if (message.type() !== "error") return; + if ( + message + .text() + .includes( + "Main thread WebGPU init failed: Error: No WebGPU adapter available", + ) + ) { + rendererFallbacks.push(message.text()); + return; + } + if ( + cursorMovReplay && + cursorMovTicketRequests === 1 && + sessionId && + renderedExportId && + message.text().includes("404") && + message.location().url === + `${base}/api/editor/sessions/${encodeURIComponent(sessionId)}/exports/${encodeURIComponent(renderedExportId)}?videoId=${videoId}` + ) { + cursorMovExpectedConsole404++; + return; + } + pageErrors.push(message.text()); + }); + page.on("response", (response) => { + if ( + response.url().includes("/api/editor/sessions/") && + response.url().includes("/exports/") + ) { + exportResponses.push(`${response.status()} ${response.url()}`); + if (exportResponses.length > 12) exportResponses.shift(); + } + if (response.status() < 400) return; + const index = + failedResponses.push(`${response.status()} ${response.url()}`) - 1; + failedResponseReads.push( + response.text().then( + (body) => { + failedResponses[index] += `: ${body.slice(0, 500)}`; + }, + (error: unknown) => { + failedResponses[index] += `: ${String(error)}`; + }, + ), + ); + }); + await page.goto(`${base}/test-editor`); + await page.locator("#editor").evaluate(async (frame) => { + const iframe = frame as HTMLIFrameElement; + if (iframe.contentDocument?.readyState === "complete") return; + await new Promise((resolve) => + iframe.addEventListener("load", () => resolve(), { once: true }), + ); + }); + const connecting = page.evaluate( + async ({ recordingId, editorSession, ownerId, captionsEnabled }) => { + const { EditorHostBridge } = await import( + new URL("/test-host.js", location.origin).href + ); + const iframe = document.getElementById("editor") as HTMLIFrameElement; + const browserWindow = window as typeof window & { + capTestEditorError?: string; + capTestEditorClosed?: boolean; + capTestBridge?: { dispose: () => void }; + capTestSavedAt?: string | null; + }; + browserWindow.capTestSavedAt = null; + const bridge = new EditorHostBridge( + recordingId, + editorSession, + ownerId, + () => { + browserWindow.capTestEditorClosed = true; + }, + (error: Error) => { + browserWindow.capTestEditorError = error.message; + }, + undefined, + undefined, + undefined, + captionsEnabled, + undefined, + (value: string) => { + browserWindow.capTestSavedAt = value; + }, + () => browserWindow.capTestSavedAt ?? null, + ); + browserWindow.capTestBridge = bridge; + await bridge.connect(iframe); + }, + { + recordingId: videoId, + editorSession: sessionId, + ownerId: userId, + captionsEnabled: proCaptions, + }, + ); + const editor = page.frameLocator("#editor"); + if (canvasFallbackReplay) { + assert.deepEqual( + await editor + .locator("body") + .evaluate(() => [ + typeof OffscreenCanvas, + typeof createImageBitmap, + typeof VideoDecoder, + ]), + ["undefined", "undefined", "undefined"], + ); + } + if (runtimeFault) { + await connecting; + await editor + .getByRole("heading", { name: "Unable to Open Recording" }) + .waitFor({ state: "visible", timeout: 20_000 }); + assert.equal(await editor.getByText(runtimeFaultMessage).count(), 1); + assert.equal( + await editor.getByRole("button", { name: "Try again" }).count(), + 1, + ); + assert.equal( + await editor.getByRole("button", { name: "Open Folder" }).count(), + 0, + ); + assert.equal( + await editor.getByRole("button", { name: "Close Window" }).count(), + 0, + ); + assert.equal( + await editor.getByRole("button", { name: "Recover Recording" }).count(), + 0, + ); + if (process.env.CAP_EDITOR_UI_RUNTIME_ERROR_SCREENSHOT_PATH) + await writeFile( + process.env.CAP_EDITOR_UI_RUNTIME_ERROR_SCREENSHOT_PATH, + await page.screenshot(), + ); + await editor.getByRole("button", { name: "Back to recording" }).click(); + await page.waitForFunction( + () => + (window as typeof window & { capTestEditorClosed?: boolean }) + .capTestEditorClosed === true, + ); + assert.deepEqual(failedResponses, []); + process.stdout.write( + `${JSON.stringify({ browserEngine: browserEngine.name(), runtimeFault: true, recoveryFault, nativeOnlyActionsHidden: true, backToRecordingClosedEditor: true, failedResponses })}\n`, + ); + } else { + if (coldMount) { + await editor + .locator("body[data-editor-connect-received=true]") + .waitFor({ state: "attached", timeout: 4_000 }); + await editor + .getByRole("button", { name: "Export", exact: true }) + .waitFor({ timeout: 4_000 }); + assert.equal( + await editor + .getByRole("button", { name: "Export", exact: true }) + .isDisabled(), + true, + ); + assert.equal(delayedSkeletonRequests, 1); + assert.equal(delayedEditorRequests, 1); + } + await connecting; + await editor + .getByRole("button", { name: "Download", exact: true }) + .waitFor({ + state: "visible", + timeout: 20_000, + }); + await editor.getByRole("tab", { name: "Camera" }).waitFor({ + state: "visible", + timeout: 20_000, + }); + try { + await editor.locator("[aria-busy='false']").waitFor({ + state: "attached", + timeout: 20_000, + }); + } catch (cause) { + throw new Error( + `Local Studio preview did not render: ${JSON.stringify({ browserBootstrapRequests, browserMediaRangeRequests, pageErrors, failedResponses, rendererFallbacks })}`, + { cause }, + ); + } + assert.ok(browserBootstrapRequests > 0); + assert.ok(browserMediaRangeRequests > 0); + assert.equal( + await editor.getByRole("tab", { name: "Camera" }).isDisabled(), + false, + ); + if (shareReplay) { + const blurControl = await editor + .getByText("Blur", { exact: true }) + .locator("..") + .innerText(); + assert.ok(blurControl.includes("60.0%")); + } + let nativePreviewParity: { + meanAbsoluteError: number; + psnrDb: number; + differentPixels: number; + totalPixels: number; + } | null = null; + let firefoxColorPath: { + webCodecsSupported: boolean; + kind: string | null; + matrix: string | null; + } | null = null; + if (!cursorMovReplay && !canvasFallbackReplay) { + const halfWidth = await editor + .locator("#canvas") + .evaluate((canvas) => (canvas as HTMLCanvasElement).width); + await editor + .getByRole("button", { name: "Full preview quality" }) + .click(); + await editor.locator("#canvas").evaluate(async (canvas, before) => { + const element = canvas as HTMLCanvasElement; + const deadline = performance.now() + 10_000; + while (element.width <= before) { + if (performance.now() > deadline) + throw new Error("Full preview quality did not resize the preview"); + await new Promise((resolve) => setTimeout(resolve, 25)); + } + }, halfWidth); + await editor.locator("[aria-busy='false']").waitFor({ + state: "attached", + timeout: 20_000, + }); + await editor.locator("#canvas").evaluate(async (canvas) => { + const previewSurface = canvas.parentElement?.parentElement; + if (!previewSurface) throw new Error("Preview surface is unavailable"); + const deadline = performance.now() + 3_000; + while (Number(getComputedStyle(previewSurface).opacity) < 0.999) { + if (performance.now() > deadline) + throw new Error("Preview fade did not complete"); + await new Promise((resolve) => setTimeout(resolve, 25)); + } + }); + if (browserEngine.name() === "firefox" && proCaptions && shareReplay) { + firefoxColorPath = await editor + .locator("#canvas") + .evaluate(async () => { + let webCodecsSupported = false; + if (typeof VideoDecoder === "function") { + try { + webCodecsSupported = + ( + await VideoDecoder.isConfigSupported({ + codec: "avc1.64001e", + codedWidth: 640, + codedHeight: 360, + }) + ).supported === true; + } catch { + webCodecsSupported = false; + } + } + const source: unknown = Reflect.get(window, "capTestColorPath"); + const path = + typeof source === "object" && source !== null ? source : null; + return { + webCodecsSupported, + kind: + path && "kind" in path && typeof path.kind === "string" + ? path.kind + : null, + matrix: + path && "matrix" in path && typeof path.matrix === "string" + ? path.matrix + : null, + }; + }); + if (firefoxColorPath.webCodecsSupported) { + assert.equal(firefoxColorPath.kind, "VideoFrame"); + } + } + const canvasSize = await editor.locator("#canvas").evaluate((canvas) => { + const element = canvas as HTMLCanvasElement; + return { width: element.width, height: element.height }; + }); + const native = getEditorSession(sessionId); + assert.ok(native); + const preview = await native.request("/preview", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + frameNumber: 0, + fps: 60, + resolutionBase: { x: canvasSize.width, y: canvasSize.height }, + }), + }); + assert.equal(preview.status, 200); + const exact = Buffer.from(await preview.arrayBuffer()); + const footer = exact.subarray(-24); + const stride = footer.readUInt32LE(0); + const nativeHeight = footer.readUInt32LE(4); + const nativeWidth = footer.readUInt32LE(8); + assert.equal(footer.readUInt32LE(12), 0); + assert.equal(exact.length, stride * nativeHeight + 24); + assert.ok(stride >= nativeWidth * 4); + const browserScreenshot = await editor.locator("#canvas").screenshot(); + const { data: browserPixels, info: browserInfo } = await sharp( + browserScreenshot, + ) + .ensureAlpha() + .raw() + .toBuffer({ resolveWithObject: true }); + assert.ok(Math.abs(nativeWidth - browserInfo.width) <= 2); + assert.ok(Math.abs(nativeHeight - browserInfo.height) <= 2); + const packed = Buffer.alloc(nativeWidth * nativeHeight * 4); + for (let row = 0; row < nativeHeight; row++) { + exact.copy( + packed, + row * nativeWidth * 4, + row * stride, + row * stride + nativeWidth * 4, + ); + } + const nativePixels = await sharp(packed, { + raw: { width: nativeWidth, height: nativeHeight, channels: 4 }, + }) + .resize(browserInfo.width, browserInfo.height, { kernel: "nearest" }) + .raw() + .toBuffer(); + assert.equal(nativePixels.length, browserPixels.length); + let absolute = 0; + let squared = 0; + let differentPixels = 0; + for (let pixel = 0; pixel < browserPixels.length; pixel += 4) { + let changed = false; + for (let channel = 0; channel < 3; channel++) { + const delta = + (nativePixels[pixel + channel] ?? 0) - + (browserPixels[pixel + channel] ?? 0); + absolute += Math.abs(delta); + squared += delta * delta; + changed ||= Math.abs(delta) > 8; + } + if (changed) differentPixels++; + } + const samples = browserInfo.width * browserInfo.height * 3; + const mse = squared / samples; + nativePreviewParity = { + meanAbsoluteError: Math.round((absolute / samples) * 100) / 100, + psnrDb: + Math.round( + (mse === 0 ? 100 : 10 * Math.log10(255 ** 2 / mse)) * 100, + ) / 100, + differentPixels, + totalPixels: browserInfo.width * browserInfo.height, + }; + if ( + nativePreviewParity.meanAbsoluteError >= 3 || + nativePreviewParity.psnrDb <= 30 || + differentPixels >= nativePreviewParity.totalPixels * 0.05 + ) { + const canvasState = await editor.locator("#canvas").evaluate((node) => { + const canvas = node as HTMLCanvasElement; + const bounds = canvas.getBoundingClientRect(); + const probe = document.createElement("canvas"); + probe.width = 1; + probe.height = 1; + const probeContext = probe.getContext("2d"); + probeContext?.drawImage( + canvas, + -Math.floor(canvas.width / 2), + -Math.floor(canvas.height / 2), + ); + const webGl = canvas.getContext("webgl2"); + let webGlCenter: number[] | null = null; + if (webGl && !webGl.isContextLost()) { + const pixel = new Uint8Array(4); + webGl.bindFramebuffer(webGl.FRAMEBUFFER, null); + webGl.readPixels( + Math.floor(canvas.width / 2), + Math.floor(canvas.height / 2), + 1, + 1, + webGl.RGBA, + webGl.UNSIGNED_BYTE, + pixel, + ); + webGlCenter = [...pixel]; + } + return { + width: canvas.width, + height: canvas.height, + bounds: { width: bounds.width, height: bounds.height }, + backgroundImage: getComputedStyle(canvas).backgroundImage, + busy: document + .querySelector("[aria-busy]") + ?.getAttribute("aria-busy"), + bitmapCenter: probeContext + ? [...probeContext.getImageData(0, 0, 1, 1).data] + : null, + webGlCenter, + webGlLost: webGl?.isContextLost() ?? null, + preserveDrawingBuffer: + webGl?.getContextAttributes()?.preserveDrawingBuffer ?? null, + colorPath: Reflect.get(window, "capTestColorPath") as unknown, + videos: [...document.querySelectorAll("video")].map((video) => ({ + readyState: video.readyState, + width: video.videoWidth, + height: video.videoHeight, + currentTime: video.currentTime, + })), + }; + }); + await page.waitForTimeout(300); + const delayedScreenshot = await editor.locator("#canvas").screenshot(); + const delayedPixels = await sharp(delayedScreenshot) + .ensureAlpha() + .raw() + .toBuffer(); + const center = + (Math.floor(browserInfo.height / 2) * browserInfo.width + + Math.floor(browserInfo.width / 2)) * + 4; + const diagnostics = { + browserEngine: browserEngine.name(), + proCaptions, + shareReplay, + firefoxColorPath, + nativePreviewParity, + centerBrowser: [...browserPixels.subarray(center, center + 3)], + centerNative: [...nativePixels.subarray(center, center + 3)], + centerDelayed: [...delayedPixels.subarray(center, center + 3)], + canvasState, + }; + process.stderr.write(`${JSON.stringify(diagnostics)}\n`); + const artifactDir = process.env.CAP_EDITOR_UI_PARITY_ARTIFACT_DIR; + if (artifactDir) { + try { + await mkdir(artifactDir, { recursive: true }); + const stem = `${browserEngine.name()}-${proCaptions ? "pro" : "free"}-native-preview-parity`; + await Promise.all([ + writeFile( + join(artifactDir, `${stem}.json`), + JSON.stringify(diagnostics, null, 2), + ), + writeFile( + join(artifactDir, `${stem}-browser.png`), + browserScreenshot, + ), + writeFile( + join(artifactDir, `${stem}-delayed.png`), + delayedScreenshot, + ), + writeFile( + join(artifactDir, `${stem}-page.png`), + await page.screenshot(), + ), + writeFile( + join(artifactDir, `${stem}-native.png`), + await sharp(packed, { + raw: { + width: nativeWidth, + height: nativeHeight, + channels: 4, + }, + }) + .png() + .toBuffer(), + ), + ]); + } catch (cause) { + process.stderr.write( + `Studio parity diagnostics failed: ${String(cause)}\n`, + ); + } + } + } + assert.ok(nativePreviewParity.meanAbsoluteError < 3); + assert.ok(nativePreviewParity.psnrDb > 30); + assert.ok(differentPixels < nativePreviewParity.totalPixels * 0.05); + } + let cropFrameLoadMs = 0; + let cropFrameSize = { width: 0, height: 0 }; + if (!shareReplay && !cursorMovReplay) { + const cropStartedAt = Date.now(); + await editor.getByRole("button", { name: "Crop", exact: true }).click(); + await editor.getByText("Loading frame…").waitFor({ + state: "hidden", + timeout: 30_000, + }); + cropFrameLoadMs = Date.now() - cropStartedAt; + cropFrameSize = await editor + .getByRole("img", { name: "Current frame" }) + .evaluate((frame) => { + const image = frame as HTMLImageElement; + return { width: image.naturalWidth, height: image.naturalHeight }; + }); + assert.ok(cropFrameSize.width > 0 && cropFrameSize.width <= 1440); + assert.ok(cropFrameSize.height > 0 && cropFrameSize.height <= 1440); + const cropSurface = editor.locator(".cropper-editor"); + await cropSurface.click({ button: "right" }); + const cropMenu = editor.getByRole("menu", { name: "Editor actions" }); + await cropMenu.waitFor({ state: "visible" }); + assert.equal( + await cropMenu.evaluate( + (menu) => getComputedStyle(menu).backgroundColor, + ), + "rgb(246, 246, 247)", + ); + const checkedCropItem = cropMenu.locator( + "[role='menuitemcheckbox'][aria-checked='true']", + ); + assert.ok((await checkedCropItem.count()) > 0); + assert.equal( + await checkedCropItem + .first() + .locator("span[aria-hidden='true']") + .innerText(), + "✓", + ); + await cropMenu.getByRole("menuitemcheckbox", { name: "16:9" }).click(); + await cropSurface.click({ button: "right" }); + const selectedRatio = cropMenu.getByRole("menuitemcheckbox", { + name: "16:9", + }); + assert.equal(await selectedRatio.getAttribute("aria-checked"), "true"); + assert.equal( + await selectedRatio.locator("span[aria-hidden='true']").innerText(), + "✓", + ); + await page.keyboard.press("Escape"); + assert.equal(await cropSurface.count(), 1); + await editor + .locator("html") + .evaluate((root) => root.classList.add("dark")); + await cropSurface.click({ button: "right" }); + assert.equal( + await cropMenu.evaluate( + (menu) => getComputedStyle(menu).backgroundColor, + ), + "rgb(32, 32, 36)", + ); + await page.keyboard.press("Escape"); + assert.equal(await cropSurface.count(), 1); + await editor + .locator("html") + .evaluate((root) => root.classList.remove("dark")); + await editor.getByRole("button", { name: "Cancel", exact: true }).click(); + } + assert.equal( + await editor.getByRole("button", { name: /recording bundle/i }).count(), + 0, + ); + await editor.getByRole("tab", { name: "Captions" }).click(); + if (proCaptions) { + await editor.getByRole("button", { name: "Generate Captions" }).waitFor({ + state: "visible", + timeout: 20_000, + }); + const languageField = editor + .getByText("Language", { exact: true }) + .last() + .locator(".."); + await languageField.getByRole("button").click(); + await editor + .getByRole("option", { name: "English" }) + .waitFor({ state: "visible" }); + assert.equal( + await editor.getByRole("option", { name: "Punjabi" }).count(), + 0, + ); + await page.keyboard.press("Escape"); + assert.equal( + await editor.getByRole("link", { name: "Upgrade to Cap Pro" }).count(), + 0, + ); + await editor.getByRole("button", { name: "Generate Captions" }).click(); + await editor + .getByRole("button", { name: "Regenerate Captions" }) + .waitFor({ state: "visible", timeout: 20_000 }); + await editor.locator("[data-caption-segment]").first().waitFor({ + state: "visible", + }); + await editor.getByText("Font settings", { exact: true }).waitFor({ + state: "visible", + }); + await waitForWorkerCaptionContent(true); + const proSaveDeadline = Date.now() + 10_000; + while (proCaptionSaves === 0 && Date.now() < proSaveDeadline) + await Bun.sleep(100); + assert.ok(proCaptionSaves > 0); + assert.equal(captionRequests, 1); + await editor.getByText("Captions", { exact: true }).last().waitFor({ + state: "visible", + }); + await editor + .getByRole("button", { name: "Captions", exact: true }) + .click(); + await editor.getByRole("button", { name: "SRT", exact: true }).waitFor({ + state: "visible", + }); + const srtDownload = page.waitForEvent("download"); + await editor.getByRole("button", { name: "SRT", exact: true }).click(); + const srt = await srtDownload; + assert.ok(srt.suggestedFilename().endsWith(".srt")); + assert.match(await Bun.file(await srt.path()).text(), /Hello Cap/); + const vttDownload = page.waitForEvent("download"); + await editor.getByRole("button", { name: "VTT", exact: true }).click(); + const vtt = await vttDownload; + assert.ok(vtt.suggestedFilename().endsWith(".vtt")); + assert.match( + await Bun.file(await vtt.path()).text(), + /WEBVTT[\s\S]*Hello Cap/, + ); + } else { + await editor.getByRole("link", { name: "Upgrade to Cap Pro" }).waitFor({ + state: "visible", + timeout: 20_000, + }); + assert.equal( + await editor.getByRole("button", { name: "Generate Captions" }).count(), + 0, + ); + assert.equal(captionRequests, 0); + await editor.getByText("Font settings", { exact: true }).waitFor({ + state: "hidden", + }); + await waitForWorkerCaptionContent(false); + } + if (proCaptions) + await editor + .getByRole("button", { name: "Back to editor" }) + .waitFor({ state: "visible" }); + const planRequestsBeforeFocus = planRequests; + currentCaptionPlan = !proCaptions; + await page.evaluate(() => { + window.dispatchEvent(new Event("focus")); + document.dispatchEvent(new Event("visibilitychange")); + }); + if (proCaptions) { + await editor + .getByRole("button", { name: "Back to editor" }) + .waitFor({ state: "hidden" }); + await editor + .getByRole("button", { name: "SRT", exact: true }) + .waitFor({ state: "hidden" }); + await editor + .getByRole("button", { name: "VTT", exact: true }) + .waitFor({ state: "hidden" }); + assert.equal( + await editor + .getByRole("button", { name: "Captions", exact: true }) + .count(), + 0, + ); + await editor.getByRole("tab", { name: "Captions" }).click(); + await editor.getByRole("link", { name: "Upgrade to Cap Pro" }).waitFor({ + state: "visible", + }); + await editor.locator("[data-caption-segment]").first().waitFor({ + state: "hidden", + }); + await editor.getByText("Font settings", { exact: true }).waitFor({ + state: "hidden", + }); + const previousFreeSaves = freeCaptionlessSaves; + const previousPreservingFreeSaves = preservingFreeSaves; + await editor.getByRole("tab", { name: "Camera" }).click(); + await editor + .getByText("Mirror Camera", { exact: true }) + .locator("..") + .locator(".cap-toggle") + .click(); + const saveDeadline = Date.now() + 10_000; + while ( + freeCaptionlessSaves === previousFreeSaves && + Date.now() < saveDeadline + ) + await Bun.sleep(100); + assert.ok(freeCaptionlessSaves > previousFreeSaves); + assert.ok(preservingFreeSaves > previousPreservingFreeSaves); + assert.equal(savedPaidCaptions, true); + let unsavedSnapshot: string | null | undefined; + const receiptDeadline = Date.now() + 10_000; + while (Date.now() < receiptDeadline) { + unsavedSnapshot = await editor.locator("body").evaluate(() => + ( + window as Window & { + capWebEditorUnsavedProjectSnapshot?: () => string | null; + } + ).capWebEditorUnsavedProjectSnapshot?.(), + ); + if (unsavedSnapshot === null) break; + await Bun.sleep(100); + } + assert.equal(unsavedSnapshot, null); + await editor.getByRole("tab", { name: "Captions" }).click(); + assert.equal( + await editor + .getByRole("button", { name: "Regenerate Captions" }) + .count(), + 0, + ); + } else { + try { + await editor + .getByRole("link", { name: "Upgrade to Cap Pro" }) + .waitFor({ state: "hidden" }); + } catch (cause) { + const iframePlan = await editor + .locator("body") + .evaluate( + () => + (window as Window & { capWebEditorCaptionsEnabled?: boolean }) + .capWebEditorCaptionsEnabled, + ); + const hostState = await page + .evaluate(async () => { + const host = window as Window & { + capTestBridge?: object; + capTestEditorError?: string; + }; + const bridge = host.capTestBridge; + let focusProbeCount = 0; + window.addEventListener( + "focus", + () => { + focusProbeCount++; + }, + { once: true }, + ); + const sequenceBeforeProbe = bridge + ? Reflect.get(bridge, "planRequestSequence") + : null; + window.dispatchEvent(new Event("focus")); + await new Promise((resolve) => setTimeout(resolve, 100)); + const controller = bridge + ? Reflect.get(bridge, "controller") + : null; + return { + hidden: document.hidden, + visibilityState: document.visibilityState, + focusProbeCount, + sequenceBeforeProbe, + sequenceAfterProbe: bridge + ? Reflect.get(bridge, "planRequestSequence") + : null, + refreshPending: bridge + ? Reflect.get(bridge, "captionPlanRefreshPending") + : null, + controllerAborted: + controller instanceof AbortController + ? controller.signal.aborted + : null, + disposed: bridge ? Reflect.get(bridge, "disposed") : null, + portAttached: bridge + ? Reflect.get(bridge, "port") !== null + : null, + editorError: host.capTestEditorError ?? null, + }; + }) + .catch(() => null); + process.stderr.write( + `${JSON.stringify({ stage: "caption-plan-focus", browserEngine: browserEngine.name(), planRequestsBeforeFocus, planRequests, currentCaptionPlan, iframePlan, hostState, socketClosures: socketClosures.slice(-12), navigationRequests: navigationRequests.slice(-12), frameNavigations: frameNavigations.slice(-12), pageErrors, failedResponses })}\n`, + ); + throw cause; + } + assert.ok(planRequests > planRequestsBeforeFocus); + await editor.getByText("Font settings", { exact: true }).waitFor({ + state: "visible", + }); + await waitForWorkerCaptionContent(false); + } + currentCaptionPlan = proCaptions; + await page.evaluate(() => + document.dispatchEvent(new Event("visibilitychange")), + ); + if (proCaptions) { + await editor + .getByRole("button", { name: "Captions", exact: true }) + .waitFor({ state: "visible" }); + await editor + .getByRole("button", { name: "Regenerate Captions" }) + .waitFor({ + state: "visible", + }); + await editor.locator("[data-caption-segment]").first().waitFor({ + state: "visible", + }); + await editor.getByText("Font settings", { exact: true }).waitFor({ + state: "visible", + }); + } else { + await editor.getByRole("link", { name: "Upgrade to Cap Pro" }).waitFor({ + state: "visible", + }); + await editor.getByText("Font settings", { exact: true }).waitFor({ + state: "hidden", + }); + } + assert.equal( + await editor + .getByText( + "Cap Pro captions use the same AssemblyAI transcription as your shareable link.", + ) + .isVisible(), + true, + ); + assert.equal(await editor.getByText("Download Whisper model").count(), 0); + await editor.getByRole("button", { name: "Add track" }).click(); + const fileChooser = page.waitForEvent("filechooser"); + await editor.getByRole("button", { name: "Image", exact: true }).click(); + await (await fileChooser).setFiles(image); + await editor.locator("[data-image-overlay]").waitFor({ + state: "visible", + timeout: 20_000, + }); + assert.equal(imageImports, 1); + assert.ok(imagePreviewRequests > 0); + let canvasFallbackPreviewPixel: number[] | null = null; + if (canvasFallbackReplay) { + const deadline = Date.now() + 15_000; + do { + const canvasImage = await editor.locator("#canvas").screenshot(); + const { data, info } = await sharp(canvasImage) + .ensureAlpha() + .raw() + .toBuffer({ resolveWithObject: true }); + const center = + (Math.floor(info.height / 2) * info.width + + Math.floor(info.width / 2)) * + info.channels; + canvasFallbackPreviewPixel = Array.from( + data.subarray(center, center + info.channels), + ); + if (Math.max(...canvasFallbackPreviewPixel.slice(0, 3)) > 20) break; + await Bun.sleep(150); + } while (Date.now() < deadline); + if (Math.max(...(canvasFallbackPreviewPixel ?? []).slice(0, 3)) <= 20) { + const browserState = await editor.locator("body").evaluate(() => { + const canvas = document.querySelector("#canvas"); + const display = document.querySelector("video"); + const sample = (source: CanvasImageSource | null) => { + if (!source) return null; + const target = document.createElement("canvas"); + target.width = 1; + target.height = 1; + const context = target.getContext("2d", { + willReadFrequently: true, + }); + if (!context) return null; + try { + context.drawImage(source, 0, 0, 1, 1); + return Array.from(context.getImageData(0, 0, 1, 1).data); + } catch { + return null; + } + }; + return { + canvasWidth: canvas?.width ?? null, + canvasHeight: canvas?.height ?? null, + canvasConnected: canvas?.isConnected ?? false, + canvasReadbackPixel: sample(canvas), + displayReadbackPixel: sample(display), + webglContextLost: + canvas?.getContext("webgl2")?.isContextLost() ?? null, + loadingFrame: document.body.textContent?.includes("Loading frame…"), + videos: Array.from(document.querySelectorAll("video")).map( + (video) => ({ + readyState: video.readyState, + videoWidth: video.videoWidth, + videoHeight: video.videoHeight, + error: video.error?.message ?? null, + }), + ), + }; + }); + console.error( + JSON.stringify({ + canvasFallbackPreviewPixel, + browserState, + pageErrors, + failedResponses, + rendererFallbacks, + }), + ); + } + assert.ok(Math.max(...canvasFallbackPreviewPixel.slice(0, 3)) > 20); + } + const screenshot = await page.screenshot(); + if (process.env.CAP_EDITOR_UI_SCREENSHOT_PATH) + await writeFile(process.env.CAP_EDITOR_UI_SCREENSHOT_PATH, screenshot); + let playbackAdvanced = false; + if (proCaptions) { + await editor.getByRole("button", { name: "Play video" }).click(); + await editor.getByRole("button", { name: "Pause video" }).waitFor({ + state: "visible", + timeout: 10_000, + }); + await editor + .getByText(/^0:(?:00\.(?:0[1-9]|[1-9]\d)|01\.\d\d)$/) + .first() + .waitFor({ + state: "visible", + timeout: 10_000, + }); + playbackAdvanced = true; + await editor.getByRole("button", { name: "Pause video" }).click(); + } + await editor.getByRole("tab", { name: "Camera" }).click(); + await editor.getByText("Hide Camera").waitFor({ state: "visible" }); + if (!shareReplay) { + const cameraBackground = editor + .getByText("Background", { exact: true }) + .locator(".."); + await cameraBackground.locator("button").click(); + await editor.getByRole("option", { name: "Remove Background" }).click(); + await cameraBackground.getByText("Remove Background").waitFor({ + state: "visible", + }); + } + await editor.getByRole("button", { name: "Download", exact: true }).click(); + await editor.getByRole("button", { name: "Back to editor" }).waitFor({ + state: "visible", + timeout: 20_000, + }); + await editor.getByRole("button", { name: "Export to File" }).waitFor({ + state: "visible", + timeout: 20_000, + }); + try { + await editor.getByRole("img", { name: "Export preview" }).waitFor({ + state: "visible", + timeout: 20_000, + }); + } catch (cause) { + throw new Error( + `Export preview failed: ${JSON.stringify({ pageErrors, failedResponses, pageText: (await editor.locator("body").innerText()).slice(0, 2_000) })}`, + { cause }, + ); + } + assert.ok(relativeImagePreviewRequests > 0); + const exportScreenshot = await page.screenshot(); + if (process.env.CAP_EDITOR_UI_EXPORT_SCREENSHOT_PATH) + await writeFile( + process.env.CAP_EDITOR_UI_EXPORT_SCREENSHOT_PATH, + exportScreenshot, + ); + if (cursorMovReplay) { + await editor.getByRole("button", { name: "Advanced" }).click(); + const cursorOnly = editor.getByRole("group", { + name: "Export cursor only", + }); + await cursorOnly.locator(".cap-toggle").click(); + assert.equal( + await cursorOnly.getByRole("switch").getAttribute("aria-checked"), + "true", + ); + await editor + .getByText("Exports as a transparent MOV.", { exact: false }) + .waitFor({ state: "visible" }); + const destination = editor.getByRole("radio", { + name: "File", + exact: true, + }); + await destination.click(); + assert.equal(await destination.getAttribute("aria-checked"), "true"); + const downloading = page.waitForEvent("download", { + timeout: 120_000, + }); + await editor.getByRole("button", { name: "Export to File" }).click(); + let download: Download; + try { + download = await downloading; + } catch (cause) { + let workerStatus: { status: number; body: string } | string | null = + null; + if (renderedExportId && sessionId) { + try { + const response = await app.request( + `/editor/sessions/${sessionId}/exports/${renderedExportId}`, + { headers }, + ); + workerStatus = { + status: response.status, + body: (await response.text()).slice(0, 800), + }; + } catch (error) { + workerStatus = String(error); + } + } + let pageText: string; + try { + pageText = (await editor.locator("body").innerText()).slice(0, 2_000); + } catch (error) { + pageText = String(error); + } + throw new Error( + `Cursor MOV download failed: ${JSON.stringify({ + renderedExportId, + cursorMovTicketRequests, + workerStatus, + exportResponses, + dialogMessages, + pageErrors, + failedResponses, + browserPages: page + .context() + .pages() + .map((browserPage) => browserPage.url().split("?")[0]), + pageText, + })}`, + { cause }, + ); + } + await editor + .getByText("Cursor track exported to file", { exact: false }) + .waitFor({ state: "visible" }); + assert.ok(download.suggestedFilename().endsWith(".mov")); + const downloadPath = await download.path(); + assert.ok(downloadPath); + const downloadedBytes = await readFile(downloadPath); + assert.ok(downloadedBytes.length > 20_000); + assert.equal(cursorMovTicketRequests, 1); + const workerDigest = requireCursorMovWorkerDigest(); + const digest = createHash("sha256").update(downloadedBytes).digest("hex"); + assert.equal(downloadedBytes.length, workerDigest.size); + assert.equal(digest, workerDigest.sha256); + const probe = spawnSync("ffprobe", [ + "-v", + "error", + "-show_entries", + "stream=codec_name,pix_fmt", + "-of", + "json", + downloadPath, + ]); + assert.equal(probe.status, 0); + const streams = JSON.parse(probe.stdout.toString()) as { + streams: Array<{ codec_name?: string; pix_fmt?: string }>; + }; + assert.equal(streams.streams[0]?.codec_name, "prores"); + assert.ok(streams.streams[0]?.pix_fmt?.startsWith("yuva")); + const decoder = spawnSync( + "ffmpeg", + [ + "-v", + "error", + "-i", + downloadPath, + "-vf", + "fps=2,scale=320:180", + "-pix_fmt", + "rgba", + "-f", + "rawvideo", + "-", + ], + { maxBuffer: 4 * 1024 * 1024 }, + ); + assert.equal(decoder.status, 0); + const pixelsPerFrame = 320 * 180; + const frameBytes = pixelsPerFrame * 4; + assert.equal(decoder.stdout.length % frameBytes, 0); + let visibleFrames = 0; + for (let frame = 0; frame < decoder.stdout.length / frameBytes; frame++) { + let visiblePixels = 0; + for (let pixel = 0; pixel < pixelsPerFrame; pixel++) { + if (decoder.stdout[frame * frameBytes + pixel * 4 + 3] > 0) + visiblePixels++; + } + if (visiblePixels > 0) visibleFrames++; + assert.ok(visiblePixels < pixelsPerFrame / 20); + } + assert.ok(visibleFrames >= 2); + assert.ok(cursorMovDownloadUrl); + const consumedUrl = new URL(cursorMovDownloadUrl); + assert.equal( + (await app.request(consumedUrl.pathname + consumedUrl.search)).status, + 404, + ); + cursorMovVerification = { + sha256: digest, + size: downloadedBytes.length, + visibleFrames, + }; + } + if (shareReplay) { + const destination = editor.getByRole("radio", { + name: "Reupload", + exact: true, + }); + assert.equal(await destination.isDisabled(), false); + await destination.click(); + assert.equal(await destination.getAttribute("aria-checked"), "true"); + await editor + .getByRole("button", { name: "Reupload to same link" }) + .click(); + await editor.getByText("Reupload complete", { exact: true }).waitFor({ + state: "visible", + timeout: 120_000, + }); + await editor.getByRole("button", { name: "Copy Link" }).waitFor({ + state: "visible", + }); + assert.equal(multipartInitiations, 1); + assert.ok(exportChunkRequests > 0); + assert.ok(uploadedParts.size > 0); + assert.ok(completedShare); + } + const bridgeError = await page.evaluate( + () => + (window as typeof window & { capTestEditorError?: string }) + .capTestEditorError ?? null, + ); + const invalidFrameCloses = await editor + .locator("body") + .evaluate( + () => + (window as typeof window & { capTestInvalidFrameCloses?: number }) + .capTestInvalidFrameCloses ?? 0, + ); + assert.equal(bridgeError, null); + assert.equal(invalidFrameCloses, 0); + await Promise.all(failedResponseReads); + if (cursorMovReplay) { + assert.ok(sessionId); + assert.ok(renderedExportId); + const expectedStatus = `404 ${base}/api/editor/sessions/${encodeURIComponent(sessionId)}/exports/${encodeURIComponent(renderedExportId)}?videoId=${videoId}:`; + for (let index = failedResponses.length - 1; index >= 0; index--) { + if (failedResponses[index]?.startsWith(expectedStatus)) { + failedResponses.splice(index, 1); + cursorMovExpectedStatus404++; + } + } + } + assert.deepEqual(failedResponses, []); + assert.deepEqual(pageErrors, []); + process.stdout.write( + `${JSON.stringify({ + videoId, + browserEngine: browserEngine.name(), + proCaptions, + coldMount, + delayedSkeletonRequests, + delayedEditorRequests, + separateCameraTabEnabled: true, + cropFrameSize, + cropFrameLoadMs, + cropRatiosAndThemesVerified: !shareReplay && !cursorMovReplay, + recordingBundleActionAbsent: bundleTicketRequests === 0, + freeCaptionsUpgradeVisible: !proCaptions, + proCaptionGenerationVisible: proCaptions, + proCaptionGenerationApplied: proCaptions && captionRequests === 1, + captionPlanChangesWithoutReload: true, + playbackAdvanced, + localModelDownloadsAbsent: true, + imageOverlayImported: imageImports === 1 && imagePreviewRequests > 0, + cameraControlsVisible: true, + cameraBackgroundRemovalSelectable: !shareReplay, + exportPreviewVisible: true, + cursorMovReplay, + canvasFallbackReplay, + canvasFallbackPreviewPixel, + firefoxColorPath, + nativePreviewParity, + cursorMovVerification, + cursorMovExpectedStatus404, + cursorMovExpectedConsole404, + shareReplay, + multipartInitiations, + exportChunkRequests, + completedShare, + screenshotSha256: createHash("sha256").update(screenshot).digest("hex"), + exportScreenshotSha256: createHash("sha256") + .update(exportScreenshot) + .digest("hex"), + pageErrors, + failedResponses, + rendererFallbacks: rendererFallbacks.length, + invalidFrameCloses, + })}\n`, + ); + await page.evaluate(() => { + ( + window as typeof window & { capTestBridge?: { dispose: () => void } } + ).capTestBridge?.dispose(); + }); + await page.close(); + const faultPage = await browser.newPage(); + let blockedEditorChunks = 0; + await faultPage.route( + /\/editor-solid\/assets\/Editor-[^/]+\.js(?:\?.*)?$/, + async (route) => { + blockedEditorChunks++; + await route.abort("failed"); + }, + ); + await faultPage.goto(`${base}/test-editor`); + await faultPage.locator("#editor").evaluate(async (frame) => { + const iframe = frame as HTMLIFrameElement; + if (iframe.contentDocument?.readyState === "complete") return; + await new Promise((resolve) => + iframe.addEventListener("load", () => resolve(), { once: true }), + ); + }); + const mountFailure = await faultPage.evaluate( + async ({ recordingId, editorSession, ownerId }) => { + const { EditorHostBridge } = await import( + new URL("/test-host.js", location.origin).href + ); + const iframe = document.getElementById("editor") as HTMLIFrameElement; + const bridge = new EditorHostBridge( + recordingId, + editorSession, + ownerId, + () => undefined, + () => undefined, + ); + try { + await bridge.connect(iframe); + return null; + } catch (cause) { + return cause instanceof Error ? cause.message : String(cause); + } finally { + bridge.dispose(); + } + }, + { recordingId: videoId, editorSession: sessionId, ownerId: userId }, + ); + assert.ok(blockedEditorChunks > 0); + assert.equal(mountFailure, "Editor could not load"); + await faultPage.close(); + process.stdout.write( + `${JSON.stringify({ editorChunkFailureVisible: true, blockedEditorChunks })}\n`, + ); + } +} finally { + if (sessionId) + await app.request(`/editor/sessions/${sessionId}`, { + method: "DELETE", + headers, + }); + await browser?.close(); + socketServer?.stop(true); + server?.stop(true); + if (previousSecret === undefined) + delete process.env.MEDIA_SERVER_WEBHOOK_SECRET; + else process.env.MEDIA_SERVER_WEBHOOK_SECRET = previousSecret; + if (previousAllowHttp === undefined) + delete process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + else process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = previousAllowHttp; + if (previousPublicOrigin === undefined) + delete process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN; + else process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN = previousPublicOrigin; +} diff --git a/apps/media-server/src/__tests__/routes/editor.test.ts b/apps/media-server/src/__tests__/routes/editor.test.ts new file mode 100644 index 00000000000..534d497d908 --- /dev/null +++ b/apps/media-server/src/__tests__/routes/editor.test.ts @@ -0,0 +1,2002 @@ +import { afterAll, expect, spyOn, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + CAP_BUNDLE_HEADER_BYTES, + parseCapBundleManifest, + readCapBundleManifestLength, +} from "@cap/editor-cap-bundle"; +import app from "../../editor-worker-app"; +import { getEditorExport } from "../../lib/editor-exports"; +import { getEditorSession } from "../../lib/editor-sessions"; +import { editorWallpaperDirectory } from "../../lib/editor-wallpapers"; +import { + editorWebSocketHandler, + handleEditorSocketUpgrade, +} from "../../lib/editor-websocket"; + +const secret = "editor-route-test-secret"; +process.env.MEDIA_SERVER_WEBHOOK_SECRET = secret; + +afterAll(() => { + delete process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA; + delete process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN; +}); + +test("the editor session endpoint rejects unauthenticated media requests", async () => { + const response = await app.request("/editor/preparations", { + method: "POST", + body: JSON.stringify({}), + }); + expect(response.status).toBe(401); +}); + +test("the editor session endpoint rejects incomplete paired sources", async () => { + const response = await app.request("/editor/preparations", { + method: "POST", + headers: { + "x-media-server-secret": secret, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + videoId: "test", + title: "Test", + display: { + url: "https://cap.so/display.webm", + contentType: "video/webm", + size: 100, + fps: 30, + }, + camera: { + url: "https://cap.so/camera.webm", + contentType: "video/webm", + size: 100, + fps: 25, + offsetMs: 30_001, + }, + }), + }); + expect(response.status).toBe(400); + const duplicatedCamera = await app.request("/editor/preparations", { + method: "POST", + headers: { + "x-media-server-secret": secret, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + videoId: "test", + title: "Test", + display: { + url: "https://cap.so/original.mp4", + contentType: "video/mp4", + size: 100, + fps: 30, + }, + camera: { + url: "https://cap.so/camera.webm", + contentType: "video/webm", + size: 100, + fps: 25, + offsetMs: 0, + }, + legacyEditSpec: { + version: 1, + sourceDuration: 4, + keepRanges: [{ start: 0, end: 2 }], + }, + }), + }); + expect(duplicatedCamera.status).toBe(400); + const invalidStudioSound = await app.request("/editor/preparations", { + method: "POST", + headers: { + "x-media-server-secret": secret, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + videoId: "test", + title: "Test", + display: { + url: "https://cap.so/display.webm", + contentType: "video/webm", + size: 100, + fps: 30, + }, + audioDefault: { enabledByDefault: true, isolation: "invalid" }, + }), + }); + expect(invalidStudioSound.status).toBe(400); +}); + +const hasNativeBinaries = + !!process.env.CAP_WEB_EDITOR_PREPARE_BIN && + !!process.env.CAP_WEB_EDITOR_SERVICE_BIN; + +test.skipIf(!hasNativeBinaries)( + "a paired source reaches native preview and is released after closing", + async () => { + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA = "1"; + const root = await mkdtemp(join(tmpdir(), "cap-editor-route-test-")); + const displayPath = join(root, "display.webm"); + const cameraPath = join(root, "camera.webm"); + const importedAudioPath = join(root, "imported.mp3"); + const importedImagePath = join(root, "overlay.png"); + const importedVideoPath = join(root, "overlay.mp4"); + let server: ReturnType | null = null; + let socketServer: ReturnType | null = null; + let socket: WebSocket | null = null; + let fastStartSocket: WebSocket | null = null; + let commandSocket: WebSocket | null = null; + let sessionId: string | null = null; + try { + for (const [path, size, fps] of [ + [displayPath, "640x360", "30"], + [cameraPath, "320x180", "25"], + ]) { + const result = Bun.spawn( + [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + `testsrc2=size=${size}:rate=${fps}:duration=3`, + ...(path === displayPath + ? [ + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=48000:duration=3", + "-c:a", + "libopus", + ] + : []), + "-c:v", + "libvpx", + "-b:v", + "500k", + path, + ], + { stderr: "pipe" }, + ); + const [exitCode, stderr] = await Promise.all([ + result.exited, + new Response(result.stderr).text(), + ]); + if (exitCode !== 0) { + throw new Error(stderr || "FFmpeg fixture generation failed"); + } + } + const importedAudioFixture = Bun.spawn( + [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-f", + "lavfi", + "-i", + "sine=frequency=1200:sample_rate=48000:duration=3", + "-c:a", + "libmp3lame", + importedAudioPath, + ], + { stderr: "pipe" }, + ); + const [audioFixtureExit, audioFixtureError] = await Promise.all([ + importedAudioFixture.exited, + new Response(importedAudioFixture.stderr).text(), + ]); + if (audioFixtureExit !== 0) { + throw new Error(audioFixtureError || "Audio fixture generation failed"); + } + const imageFixture = spawnSync("ffmpeg", [ + "-v", + "error", + "-f", + "lavfi", + "-i", + "color=c=magenta:s=120x80:duration=1", + "-frames:v", + "1", + "-threads", + "1", + importedImagePath, + ]); + expect(imageFixture.status).toBe(0); + const videoFixture = spawnSync("ffmpeg", [ + "-v", + "error", + "-f", + "lavfi", + "-i", + "color=c=green:s=320x180:r=30:d=3", + "-f", + "lavfi", + "-i", + "sine=frequency=660:sample_rate=48000:duration=3", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + importedVideoPath, + ]); + expect(videoFixture.status).toBe(0); + server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + const path = new URL(request.url).pathname; + if (path === "/display.webm") + return new Response(Bun.file(displayPath)); + if (path === "/camera.webm") + return new Response(Bun.file(cameraPath)); + if (path === "/imported.mp3") + return new Response(Bun.file(importedAudioPath)); + if (path === "/overlay.png") + return new Response(Bun.file(importedImagePath)); + if (path === "/overlay.mp4") + return new Response(Bun.file(importedVideoPath)); + return new Response("Not found", { status: 404 }); + }, + }); + const headers = { + "x-media-server-secret": secret, + "Content-Type": "application/json", + }; + const sources = { + videoId: "test-paired", + title: "Paired test", + audioDefault: { + enabledByDefault: true, + isolation: "strong", + }, + display: { + url: `http://127.0.0.1:${server.port}/display.webm`, + contentType: "video/webm", + size: (await stat(displayPath)).size, + fps: 30, + }, + camera: { + url: `http://127.0.0.1:${server.port}/camera.webm`, + contentType: "video/webm", + size: (await stat(cameraPath)).size, + fps: 25, + offsetMs: -120, + }, + }; + const response = await app.request("/editor/preparations", { + method: "POST", + headers, + body: JSON.stringify(sources), + }); + expect(response.status).toBe(202); + const created = (await response.json()) as { id: string }; + let readySessionId: string | null = null; + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + const pending = await app.request( + `/editor/preparations/${created.id}`, + { headers }, + ); + const state = (await pending.json()) as { + status: string; + sessionId?: string; + }; + if (state.status === "ready") { + readySessionId = state.sessionId ?? null; + break; + } + if (state.status === "error") + throw new Error("Native preparation failed"); + await Bun.sleep(50); + } + expect(readySessionId).not.toBeNull(); + if (!readySessionId) throw new Error("Native preparation has no session"); + sessionId = readySessionId; + const directNative = getEditorSession(sessionId); + expect(directNative).not.toBeNull(); + if (!directNative) + throw new Error("Native editor session is unavailable"); + expect((await fetch(`${directNative.origin}/health`)).status).toBe(401); + expect( + ( + await fetch(`${directNative.origin}/seek`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + frameNumber: 0, + fps: 30, + resolutionBase: { x: 640, y: 360 }, + }), + }) + ).status, + ).toBe(401); + expect((await directNative.request("/health")).status).toBe(200); + const directSocket = new WebSocket( + `${directNative.origin.replace(/^http/, "ws")}/frames`, + ); + await new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error("Direct native socket rejection timed out")), + 5000, + ); + directSocket.onopen = () => { + clearTimeout(timer); + reject(new Error("Unauthenticated native socket opened")); + }; + directSocket.onerror = () => { + clearTimeout(timer); + resolve(); + }; + directSocket.onclose = () => { + clearTimeout(timer); + resolve(); + }; + }); + const instance = await app.request( + `/editor/sessions/${sessionId}/instance`, + { headers }, + ); + expect(instance.status).toBe(200); + const data = (await instance.json()) as { + recordingDuration: number; + path: string; + savedProjectConfig: { + audio: { improve: boolean; isolation: string }; + camera: { + mirror: boolean; + backgroundBlur: { mode: "off" | "light" | "heavy" | "remove" }; + }; + background: { + source: { type: string; path?: string }; + }; + timeline: { + audioSegments: Array<{ + start: number; + end: number; + track: number; + path: string; + name: string; + enabled: boolean; + trimStart: number; + volumeDb: number; + fadeIn: number; + fadeOut: number; + duration: number; + }>; + imageSegments: Array<{ + start: number; + end: number; + track: number; + path: string; + name: string; + enabled: boolean; + center: { x: number; y: number }; + size: { x: number; y: number }; + opacity: number; + rotation: number; + flipX: boolean; + flipY: boolean; + lockAspect: boolean; + }>; + }; + }; + }; + expect(data.recordingDuration).toBeGreaterThan(2.8); + expect(data.savedProjectConfig.audio).toMatchObject({ + improve: true, + isolation: "strong", + }); + const titlePath = `/editor/sessions/${sessionId}/meta`; + const initialTitle = await app.request(titlePath, { headers }); + expect(initialTitle.status).toBe(200); + expect( + ((await initialTitle.json()) as { pretty_name: string }).pretty_name, + ).toBe("Paired test"); + for (const prettyName of ["Tiny", " Trimmed title", "Bad\u0007title"]) { + const invalidTitle = await app.request(titlePath, { + method: "PUT", + headers, + body: JSON.stringify({ prettyName }), + }); + expect(invalidTitle.status).toBe(400); + } + const renamed = await app.request(titlePath, { + method: "PUT", + headers, + body: JSON.stringify({ prettyName: "Renamed paired recording" }), + }); + expect(renamed.status).toBe(204); + const renamedTitle = await app.request(titlePath, { headers }); + expect(renamedTitle.status).toBe(200); + expect( + ((await renamedTitle.json()) as { pretty_name: string }).pretty_name, + ).toBe("Renamed paired recording"); + const persistedTitle = JSON.parse( + await Bun.file(join(data.path, "recording-meta.json")).text(), + ) as { pretty_name: string }; + expect(persistedTitle.pretty_name).toBe("Renamed paired recording"); + const configPath = join(data.path, "project-config.json"); + const diskBefore = await Bun.file(configPath).text(); + const paidConfig = { + ...data.savedProjectConfig, + captions: { segments: [], settings: { enabled: true } }, + }; + for (const suffix of ["config", "config/memory"]) { + const paidWrite = await app.request( + `/editor/sessions/${sessionId}/${suffix}`, + { + method: "PUT", + headers, + body: JSON.stringify(paidConfig), + }, + ); + expect(paidWrite.status).toBe(403); + } + expect(await Bun.file(configPath).text()).toBe(diskBefore); + const preview = await app.request( + `/editor/sessions/${sessionId}/preview`, + { + method: "POST", + headers, + body: JSON.stringify({ + frameNumber: 30, + fps: 30, + resolutionBase: { x: 640, y: 360 }, + }), + }, + ); + expect(preview.status).toBe(200); + const initialPreviewBytes = Buffer.from(await preview.arrayBuffer()); + expect(initialPreviewBytes.length).toBeGreaterThan(600_000); + if (process.env.CAP_WEB_EDITOR_ENABLE_CAMERA_REMOVAL === "1") { + const removalConfig = structuredClone(data.savedProjectConfig); + removalConfig.camera.backgroundBlur.mode = "remove"; + const removalUpdate = await app.request( + `/editor/sessions/${sessionId}/config/memory`, + { + method: "PUT", + headers, + body: JSON.stringify(removalConfig), + }, + ); + expect(removalUpdate.status).toBe(204); + const removalPreview = await app.request( + `/editor/sessions/${sessionId}/preview`, + { + method: "POST", + headers, + body: JSON.stringify({ + frameNumber: 30, + fps: 30, + resolutionBase: { x: 640, y: 360 }, + }), + }, + ); + expect(removalPreview.status).toBe(200); + const removalBytes = Buffer.from(await removalPreview.arrayBuffer()); + expect(removalBytes.equals(initialPreviewBytes)).toBe(false); + const restore = await app.request( + `/editor/sessions/${sessionId}/config/memory`, + { + method: "PUT", + headers, + body: JSON.stringify(data.savedProjectConfig), + }, + ); + expect(restore.status).toBe(204); + } + const wallpaperId = + "cap-web-wallpaper://assets/backgrounds/macOS/tahoe-dusk-min.jpg"; + const wallpaperConfig = structuredClone(data.savedProjectConfig); + wallpaperConfig.background.source = { + type: "wallpaper", + path: wallpaperId, + }; + const rejectedConfig = structuredClone(wallpaperConfig); + rejectedConfig.background.source = { + type: "image", + path: "/tmp/worker-local.jpg", + }; + const rejectedMemory = await app.request( + `/editor/sessions/${sessionId}/config/memory`, + { + method: "PUT", + headers, + body: JSON.stringify(rejectedConfig), + }, + ); + expect(rejectedMemory.status).toBe(502); + const rejectedSave = await app.request( + `/editor/sessions/${sessionId}/config`, + { + method: "PUT", + headers, + body: JSON.stringify(rejectedConfig), + }, + ); + expect(rejectedSave.status).toBe(502); + expect(await Bun.file(configPath).text()).toBe(diskBefore); + const wallpaperMemory = await app.request( + `/editor/sessions/${sessionId}/config/memory`, + { + method: "PUT", + headers, + body: JSON.stringify(wallpaperConfig), + }, + ); + expect(wallpaperMemory.status).toBe(204); + const wallpaperView = await app.request( + `/editor/sessions/${sessionId}/config`, + { headers }, + ); + expect( + ((await wallpaperView.json()) as typeof wallpaperConfig).background + .source.path, + ).toBe(wallpaperId); + const nativeWallpaper = + await getEditorSession(sessionId)?.request("/config"); + expect(nativeWallpaper?.status).toBe(200); + expect( + ((await nativeWallpaper?.json()) as typeof wallpaperConfig).background + .source.path, + ).toBe(join(editorWallpaperDirectory(), "macOS/tahoe-dusk-min.jpg")); + const wallpaperPreview = await app.request( + `/editor/sessions/${sessionId}/preview`, + { + method: "POST", + headers, + body: JSON.stringify({ + frameNumber: 30, + fps: 30, + resolutionBase: { x: 640, y: 360 }, + }), + }, + ); + expect(wallpaperPreview.status).toBe(200); + expect((await wallpaperPreview.arrayBuffer()).byteLength).toBeGreaterThan( + 600_000, + ); + socketServer = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request, listener) { + const upgrade = handleEditorSocketUpgrade(request, listener); + return upgrade === null ? app.fetch(request) : upgrade; + }, + websocket: editorWebSocketHandler, + }); + process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN = `http://127.0.0.1:${socketServer.port}`; + const bundlePath = `/editor/sessions/${sessionId}/project-bundle`; + const unauthorizedBundle = await app.request( + `${bundlePath}/download-ticket`, + { method: "POST", body: "{}" }, + ); + expect(unauthorizedBundle.status).toBe(401); + const bundleTicket = await app.request(`${bundlePath}/download-ticket`, { + method: "POST", + headers, + body: JSON.stringify({ fileName: "Paired test.capbundle" }), + }); + expect(bundleTicket.status).toBe(200); + const bundleUrl = new URL( + ((await bundleTicket.json()) as { url: string }).url, + ); + expect(bundleUrl.pathname).toBe(`${bundlePath}/download`); + const downloadedBundle = await app.request( + `${bundleUrl.pathname}${bundleUrl.search}`, + ); + expect(downloadedBundle.status).toBe(200); + expect(downloadedBundle.headers.get("Content-Disposition")).toContain( + "Paired test.capbundle", + ); + const bundleBytes = new Uint8Array(await downloadedBundle.arrayBuffer()); + const manifestLength = readCapBundleManifestLength( + bundleBytes.subarray(0, CAP_BUNDLE_HEADER_BYTES), + ); + expect(manifestLength).not.toBeNull(); + if (manifestLength === null) + throw new Error("Invalid editor bundle header"); + const bundleManifest = parseCapBundleManifest( + bundleBytes.subarray( + CAP_BUNDLE_HEADER_BYTES, + CAP_BUNDLE_HEADER_BYTES + manifestLength, + ), + bundleBytes.byteLength, + ); + expect(bundleManifest?.files.map((file) => file.path)).toContain( + "content/segments/segment-0/display.webm", + ); + expect(bundleManifest?.files.map((file) => file.path)).toContain( + "content/segments/segment-0/camera.webm", + ); + expect( + (await app.request(`${bundleUrl.pathname}${bundleUrl.search}`)).status, + ).toBe(404); + const ticketResponse = await app.request( + `/editor/sessions/${sessionId}/sockets`, + { + method: "POST", + headers, + body: JSON.stringify({ origin: "http://127.0.0.1:3000" }), + }, + ); + expect(ticketResponse.status).toBe(200); + const ticketData = (await ticketResponse.json()) as { + videoId: string; + sockets: { + frames: { url: string; ticket: string }; + commands: { url: string; ticket: string }; + }; + }; + expect(ticketData.videoId).toBe("test-paired"); + const frameSocket = ticketData.sockets.frames; + const BunWebSocket = WebSocket as unknown as new ( + url: string, + options: Bun.WebSocketOptions, + ) => WebSocket; + socket = new BunWebSocket(frameSocket.url, { + protocols: ["cap-editor-v1", `cap-editor-ticket.${frameSocket.ticket}`], + headers: { Origin: "http://127.0.0.1:3000" }, + }); + socket.binaryType = "arraybuffer"; + const frame = await new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error("Editor frame socket timed out")), + 5000, + ); + if (!socket) return reject(new Error("Missing editor frame socket")); + socket.onmessage = (event: MessageEvent) => { + clearTimeout(timer); + if (event.data instanceof ArrayBuffer) resolve(event.data); + else reject(new Error("Editor frame was not binary")); + }; + socket.onerror = () => { + clearTimeout(timer); + reject(new Error("Editor frame socket failed")); + }; + }); + expect(frame.byteLength).toBeGreaterThan(20_000); + expect(frame.byteLength).toBeLessThan(600_000); + const compressedFrame = new Uint8Array(frame); + expect(new TextDecoder().decode(compressedFrame.subarray(0, 8))).toBe( + "CAPPNG01", + ); + expect(compressedFrame.subarray(8, 16)).toEqual( + new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10]), + ); + expect(socket.protocol).toBe("cap-editor-v1"); + const fastStartTicketResponse = await app.request( + `/editor/sessions/${sessionId}/sockets`, + { + method: "POST", + headers, + body: JSON.stringify({ origin: "http://127.0.0.1:3000" }), + }, + ); + expect(fastStartTicketResponse.status).toBe(200); + const fastStartTicket = ( + (await fastStartTicketResponse.json()) as { + sockets: { frames: { url: string; ticket: string } }; + } + ).sockets.frames; + fastStartSocket = new BunWebSocket(fastStartTicket.url, { + protocols: [ + "cap-editor-v1", + "cap-editor-h264-v1", + `cap-editor-ticket.${fastStartTicket.ticket}`, + ], + headers: { Origin: "http://127.0.0.1:3000" }, + }); + fastStartSocket.binaryType = "arraybuffer"; + let fastStartPngFrames = 0; + await new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error("Fast-start editor socket timed out")), + 5000, + ); + if (!fastStartSocket) + return reject(new Error("Missing fast-start editor socket")); + fastStartSocket.onopen = () => { + fastStartSocket?.send(JSON.stringify({ bitrate: "low" })); + }; + fastStartSocket.onmessage = (event: MessageEvent) => { + if (!(event.data instanceof ArrayBuffer)) return; + const magic = new TextDecoder().decode( + new Uint8Array(event.data, 0, 8), + ); + if (magic !== "CAPPNG01") return; + fastStartPngFrames++; + clearTimeout(timer); + resolve(); + }; + fastStartSocket.onerror = () => { + clearTimeout(timer); + reject(new Error("Fast-start editor socket failed")); + }; + }); + const nativeSession = getEditorSession(sessionId); + if (!nativeSession) + throw new Error("Native editor session is unavailable"); + const nativeMetrics = await nativeSession.request("/metrics"); + expect(nativeMetrics.status).toBe(200); + expect( + ((await nativeMetrics.json()) as { h264Viewers: number }).h264Viewers, + ).toBe(1); + fastStartSocket.send(JSON.stringify({ mode: "h264" })); + await Bun.sleep(150); + expect(fastStartPngFrames).toBe(1); + expect(fastStartSocket.protocol).toBe("cap-editor-v1"); + const commandTicket = ticketData.sockets.commands; + commandSocket = new BunWebSocket(commandTicket.url, { + protocols: [ + "cap-editor-v1", + `cap-editor-ticket.${commandTicket.ticket}`, + ], + headers: { Origin: "http://127.0.0.1:3000" }, + }); + await new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error("Editor command socket timed out")), + 5000, + ); + if (!commandSocket) + return reject(new Error("Missing editor command socket")); + commandSocket.onopen = () => { + clearTimeout(timer); + resolve(); + }; + commandSocket.onerror = () => { + clearTimeout(timer); + reject(new Error("Editor command socket failed")); + }; + }); + if (!commandSocket) throw new Error("Missing editor command socket"); + const attachedFuture = Date.now() + 5 * 60 * 1000; + const attachedClock = spyOn(Date, "now").mockReturnValue(attachedFuture); + try { + expect(getEditorSession(sessionId)).not.toBeNull(); + } finally { + attachedClock.mockRestore(); + } + const currentCommandSocket = commandSocket; + type CommandReply = { + kind: string; + id: number; + value?: unknown; + error?: string; + }; + const pendingReplies = new Map< + number, + { + resolve: (reply: CommandReply) => void; + reject: (error: Error) => void; + timer: ReturnType; + } + >(); + const estimateUpdates: unknown[] = []; + currentCommandSocket.onmessage = (event: MessageEvent) => { + if (typeof event.data !== "string") return; + const reply = JSON.parse(event.data) as CommandReply; + if (reply.kind === "channel" && reply.id === 55) { + estimateUpdates.push(reply.value); + return; + } + const pending = pendingReplies.get(reply.id); + if (!pending) return; + pendingReplies.delete(reply.id); + clearTimeout(pending.timer); + pending.resolve(reply); + }; + const sendCommand = (id: number, name: string, args: unknown[]) => + new Promise((resolve, reject) => { + const timer = setTimeout(() => { + pendingReplies.delete(id); + reject(new Error("Editor command reply timed out")); + }, 5000); + pendingReplies.set(id, { resolve, reject, timer }); + currentCommandSocket.send( + JSON.stringify({ kind: "invoke", id, name, args }), + ); + }); + const captionAccessPath = `/editor/sessions/${sessionId}/caption-access`; + const unauthenticatedAccess = await app.request(captionAccessPath, { + method: "PUT", + body: JSON.stringify({ captionsEnabled: true }), + }); + expect(unauthenticatedAccess.status).toBe(401); + const invalidAccess = await app.request(captionAccessPath, { + method: "PUT", + headers, + body: JSON.stringify({ captionsEnabled: "true" }), + }); + expect(invalidAccess.status).toBe(400); + const grantAccess = await app.request(captionAccessPath, { + method: "PUT", + headers, + body: JSON.stringify({ captionsEnabled: true }), + }); + expect(grantAccess.status).toBe(204); + const liveBefore = await app.request( + `/editor/sessions/${sessionId}/config`, + { headers }, + ); + expect(liveBefore.status).toBe(200); + const captionLiveConfig = (await liveBefore.json()) as Record< + string, + unknown + >; + const paidLiveConfig = { + ...captionLiveConfig, + captions: paidConfig.captions, + }; + expect( + await sendCommand(100, "checkUpgradedAndUpdate", []), + ).toMatchObject({ + kind: "result", + value: true, + }); + for (const suffix of ["config", "config/memory"]) { + const paidWrite = await app.request( + `/editor/sessions/${sessionId}/${suffix}`, + { + method: "PUT", + headers, + body: JSON.stringify(paidLiveConfig), + }, + ); + expect(paidWrite.status).toBe(204); + } + expect(await sendCommand(101, "loadCaptions", [])).toMatchObject({ + kind: "result", + value: paidLiveConfig.captions, + }); + expect( + await sendCommand(102, "updateProjectConfigInMemory", [ + paidLiveConfig, + null, + null, + null, + ]), + ).toMatchObject({ kind: "result" }); + const nativeBeforeRevoke = getEditorSession(sessionId); + if (!nativeBeforeRevoke) throw new Error("Missing native editor session"); + const restoreConfig = await nativeBeforeRevoke.request("/config", { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: diskBefore, + }); + expect(restoreConfig.status).toBe(204); + expect(await Bun.file(configPath).text()).toBe(diskBefore); + const restoreMemory = await app.request( + `/editor/sessions/${sessionId}/config/memory`, + { + method: "PUT", + headers, + body: JSON.stringify(captionLiveConfig), + }, + ); + expect(restoreMemory.status).toBe(204); + const revokeAccess = await app.request(captionAccessPath, { + method: "PUT", + headers, + body: JSON.stringify({ captionsEnabled: false }), + }); + expect(revokeAccess.status).toBe(204); + expect( + await sendCommand(103, "checkUpgradedAndUpdate", []), + ).toMatchObject({ + kind: "result", + value: false, + }); + expect(await sendCommand(104, "loadCaptions", [])).toMatchObject({ + kind: "result", + value: null, + }); + for (const suffix of ["config", "config/memory"]) { + const paidWrite = await app.request( + `/editor/sessions/${sessionId}/${suffix}`, + { + method: "PUT", + headers, + body: JSON.stringify(paidLiveConfig), + }, + ); + expect(paidWrite.status).toBe(403); + } + expect( + await sendCommand(105, "updateProjectConfigInMemory", [ + paidLiveConfig, + null, + null, + null, + ]), + ).toMatchObject({ kind: "error" }); + const browserInstance = await sendCommand(1, "createEditorInstance", []); + expect(browserInstance.kind).toBe("result"); + expect(browserInstance.value).toMatchObject({ + path: `cap-web-editor://session/${sessionId}`, + framesSocketUrl: "", + savedProjectConfig: { + background: { source: { path: wallpaperId } }, + }, + }); + const fonts = await sendCommand(18, "tauri:list_system_fonts", [null]); + expect(fonts.kind).toBe("result"); + expect(Array.isArray(fonts.value)).toBe(true); + const families = fonts.value as string[]; + expect(families.length).toBeGreaterThan(0); + expect(families).toEqual([...new Set(families)].sort()); + expect( + families.every( + (family) => family.length > 0 && !family.startsWith("."), + ), + ).toBe(true); + const musicCatalog = await sendCommand(14, "listAudioLibrary", []); + expect(musicCatalog.kind).toBe("result"); + expect(musicCatalog.value).toHaveLength(10); + expect((musicCatalog.value as Array<{ name: string }>)[0]?.name).toBe( + "Lofi Beats", + ); + const randomGradient = await sendCommand( + 16, + "randomAnimatedGradient", + [], + ); + expect(randomGradient.kind).toBe("result"); + expect(randomGradient.value).toMatchObject({ + colorStops: expect.any(Array), + seed: expect.any(Number), + motionSpeed: expect.any(Number), + }); + const clipThumbnail = await sendCommand(17, "getClipThumbnail", [0, 1.5]); + expect(clipThumbnail.kind).toBe("result"); + expect(clipThumbnail.value).toMatch(/^data:image\/jpeg;base64,/); + const thumbnailBytes = Buffer.from( + (clipThumbnail.value as string).slice("data:image/jpeg;base64,".length), + "base64", + ); + expect(thumbnailBytes.length).toBeGreaterThan(1000); + const thumbnailPath = join(root, "clip-thumbnail.jpg"); + await writeFile(thumbnailPath, thumbnailBytes); + const thumbnailProbe = spawnSync("ffprobe", [ + "-v", + "error", + "-show_entries", + "stream=width,height", + "-of", + "json", + thumbnailPath, + ]); + expect(thumbnailProbe.status).toBe(0); + expect(JSON.parse(thumbnailProbe.stdout.toString())).toMatchObject({ + streams: [{ width: 240, height: 135 }], + }); + const invalidThumbnail = await sendCommand( + 18, + "getClipThumbnail", + [-1, 1.5], + ); + expect(invalidThumbnail.kind).toBe("error"); + expect(invalidThumbnail.error).toBe("Invalid clip thumbnail request"); + const musicTrack = await sendCommand(15, "addAudioLibraryTrack", [ + "lofi-beats-mirostar", + ]); + expect(musicTrack.kind).toBe("result"); + expect(musicTrack.value).toMatchObject({ + path: "assets/audio/library-lofi-beats-mirostar.mp3", + name: "Lofi Beats", + }); + expect( + (musicTrack.value as { duration: number }).duration, + ).toBeGreaterThan(30); + const importedAudioAsset = { + path: `assets/audio/import-${randomUUID()}.mp3`, + name: "Imported tone", + url: `http://127.0.0.1:${server.port}/imported.mp3`, + size: (await stat(importedAudioPath)).size, + contentType: "audio/mpeg", + objectIdentity: null, + }; + const audioImport = await app.request( + `/editor/sessions/${sessionId}/audio-assets`, + { + method: "POST", + headers, + body: JSON.stringify(importedAudioAsset), + }, + ); + expect(audioImport.status).toBe(200); + const importedTrack = (await audioImport.json()) as { + path: string; + name: string; + duration: number; + }; + expect(importedTrack.path).toBe(importedAudioAsset.path); + expect(importedTrack.duration).toBeGreaterThan(2.9); + expect((await stat(join(data.path, importedTrack.path))).size).toBe( + importedAudioAsset.size, + ); + const importedImageAsset = { + path: `content/images/${randomUUID()}.png`, + name: "Overlay", + url: `http://127.0.0.1:${server.port}/overlay.png`, + size: (await stat(importedImagePath)).size, + contentType: "image/png", + objectIdentity: null, + }; + const imageImport = await app.request( + `/editor/sessions/${sessionId}/image-assets`, + { + method: "POST", + headers, + body: JSON.stringify(importedImageAsset), + }, + ); + expect(imageImport.status).toBe(200); + const importedImage = (await imageImport.json()) as { + path: string; + name: string; + width: number; + height: number; + }; + expect(importedImage).toEqual({ + path: importedImageAsset.path, + name: "Overlay", + width: 120, + height: 80, + }); + const importedVideoAsset = { + path: `content/videos/${randomUUID()}.mp4`, + name: "Green overlay", + url: `http://127.0.0.1:${server.port}/overlay.mp4`, + size: (await stat(importedVideoPath)).size, + contentType: "video/mp4", + objectIdentity: null, + }; + const videoImport = await app.request( + `/editor/sessions/${sessionId}/video-assets`, + { + method: "POST", + headers, + body: JSON.stringify(importedVideoAsset), + }, + ); + expect(videoImport.status).toBe(202); + const videoJobId = ((await videoImport.json()) as { id: string }).id; + type ImportedVideoFixture = { + path: string; + name: string; + duration: number; + fps: number; + width: number; + height: number; + hasAudio: boolean; + }; + let importedVideo: ImportedVideoFixture | null = null; + const videoImportDeadline = Date.now() + 10_000; + while (Date.now() < videoImportDeadline) { + const response = await app.request( + `/editor/sessions/${sessionId}/video-assets/${videoJobId}`, + { headers }, + ); + expect(response.status).toBe(200); + const status = (await response.json()) as { + status: string; + result: ImportedVideoFixture | null; + error: string | null; + }; + if (status.status === "ready") { + importedVideo = status.result; + break; + } + if (status.status === "error") + throw new Error(status.error ?? "Video import failed"); + await Bun.sleep(50); + } + if (!importedVideo) throw new Error("Imported video did not stage"); + expect(importedVideo).toMatchObject({ + path: importedVideoAsset.path, + name: "Green overlay", + fps: 30, + width: 320, + height: 180, + hasAudio: true, + }); + expect(importedVideo.duration).toBeGreaterThan(2.9); + expect((await stat(join(data.path, importedVideo.path))).size).toBe( + importedVideoAsset.size, + ); + const imageBackgroundConfig = structuredClone(data.savedProjectConfig); + imageBackgroundConfig.background.source = { + type: "image", + path: importedImage.path, + }; + const imageBackgroundUpdate = await app.request( + `/editor/sessions/${sessionId}/config/memory`, + { + method: "PUT", + headers, + body: JSON.stringify(imageBackgroundConfig), + }, + ); + expect(imageBackgroundUpdate.status).toBe(204); + const nativeImageBackground = + await getEditorSession(sessionId)?.request("/config"); + expect( + ((await nativeImageBackground?.json()) as typeof imageBackgroundConfig) + .background.source.path, + ).toBe(join(data.path, importedImage.path)); + const browserImageBackground = await app.request( + `/editor/sessions/${sessionId}/config`, + { headers }, + ); + expect( + ((await browserImageBackground.json()) as typeof imageBackgroundConfig) + .background.source.path, + ).toBe(importedImage.path); + const imageBackgroundPreview = await app.request( + `/editor/sessions/${sessionId}/preview`, + { + method: "POST", + headers, + body: JSON.stringify({ + frameNumber: 30, + fps: 30, + resolutionBase: { x: 640, y: 360 }, + }), + }, + ); + expect(imageBackgroundPreview.status).toBe(200); + expect( + (await imageBackgroundPreview.arrayBuffer()).byteLength, + ).toBeGreaterThan(600_000); + const restoreWallpaper = await app.request( + `/editor/sessions/${sessionId}/config/memory`, + { + method: "PUT", + headers, + body: JSON.stringify(wallpaperConfig), + }, + ); + expect(restoreWallpaper.status).toBe(204); + expect( + (await sendCommand(16, "addAudioLibraryTrack", ["../foreign"])).kind, + ).toBe("error"); + const exportPreview = await sendCommand(9, "generateExportPreviewFast", [ + 1.5, + { + fps: 30, + resolution_base: { x: 640, y: 360 }, + compression_bpp: 0.15, + cursor_only: false, + }, + ]); + expect(exportPreview.kind).toBe("result"); + const previewResult = exportPreview.value as { + jpeg_base64: string; + actual_width: number; + actual_height: number; + estimated_size_mb: number; + total_frames: number; + }; + expect(previewResult.jpeg_base64.startsWith("/9j/")).toBe(true); + expect(previewResult.actual_width).toBe(640); + expect(previewResult.actual_height).toBeGreaterThanOrEqual(350); + expect(previewResult.estimated_size_mb).toBeGreaterThan(0); + expect(previewResult.total_frames).toBeGreaterThan(80); + const estimateSettings = { + format: "Mp4", + fps: 30, + resolution_base: { x: 640, y: 360 }, + compression: "Web", + custom_bpp: null, + }; + const estimate = await sendCommand(10, "getExportEstimates", [ + `cap-web-editor://session/${sessionId}`, + estimateSettings, + "__CHANNEL__:55", + ]); + expect(estimate.kind).toBe("result"); + const estimateResult = estimate.value as { + duration_seconds: number; + estimated_time_seconds: number; + estimated_size_mb: number; + }; + expect(estimateResult.duration_seconds).toBeGreaterThan(2); + expect(estimateResult.estimated_time_seconds).toBeGreaterThan(0); + expect(estimateResult.estimated_size_mb).toBeGreaterThan(0); + const repeatedEstimate = await sendCommand(11, "getExportEstimates", [ + `cap-web-editor://session/${sessionId}`, + estimateSettings, + "__CHANNEL__:55", + ]); + expect(repeatedEstimate.kind).toBe("result"); + expect(estimateUpdates.length).toBeGreaterThan(0); + expect(estimateUpdates.at(-1)).toEqual(estimateResult); + expect(await sendCommand(12, "cancelExportEstimates", [])).toMatchObject({ + kind: "result", + value: null, + }); + expect( + ( + await sendCommand(13, "getExportEstimates", [ + "cap-web-editor://session/foreign", + estimateSettings, + "__CHANNEL__:55", + ]) + ).kind, + ).toBe("error"); + const systemWaveforms = await sendCommand( + 5, + "getSystemAudioWaveforms", + [], + ); + expect(systemWaveforms.kind).toBe("result"); + const systemPeaks = systemWaveforms.value as number[][]; + expect(systemPeaks).toHaveLength(1); + expect(systemPeaks[0]?.length).toBeGreaterThan(20); + expect(systemPeaks[0]?.some((peak) => peak > -60)).toBe(true); + const micWaveforms = await sendCommand(6, "getMicWaveforms", []); + expect(micWaveforms).toMatchObject({ kind: "result", value: [[]] }); + const native = getEditorSession(readySessionId); + if (!native) throw new Error("Missing native editor session"); + const originalRequest = native.request; + native.request = (path, init) => + path === "/waveforms/system" + ? (async () => { + await Bun.sleep(1500); + return originalRequest(path, init); + })() + : originalRequest(path, init); + try { + const slowStarted = performance.now(); + const slowWaveform = sendCommand(7, "getSystemAudioWaveforms", []); + await Bun.sleep(20); + const seekStarted = performance.now(); + const seek = await sendCommand(8, "seekTo", [30]); + expect(seek.kind).toBe("result"); + expect(performance.now() - seekStarted).toBeLessThan(1000); + expect((await slowWaveform).kind).toBe("result"); + expect(performance.now() - slowStarted).toBeGreaterThan(1400); + } finally { + native.request = originalRequest; + } + const changedConfig = structuredClone(data.savedProjectConfig); + changedConfig.camera.mirror = !changedConfig.camera.mirror; + changedConfig.background.source = { + type: "wallpaper", + path: wallpaperId, + }; + const selectedMusic = musicTrack.value as { + path: string; + name: string; + duration: number; + }; + changedConfig.timeline.audioSegments.push({ + start: 0, + end: 2.5, + track: 0, + path: selectedMusic.path, + name: selectedMusic.name, + enabled: true, + trimStart: 0, + volumeDb: 0, + fadeIn: 0, + fadeOut: 0, + duration: selectedMusic.duration, + }); + changedConfig.timeline.audioSegments.push({ + start: 0, + end: 2.5, + track: 1, + path: importedTrack.path, + name: importedTrack.name, + enabled: true, + trimStart: 0, + volumeDb: 0, + fadeIn: 0, + fadeOut: 0, + duration: importedTrack.duration, + }); + changedConfig.timeline.imageSegments ??= []; + changedConfig.timeline.imageSegments.push({ + start: 0, + end: 3, + track: 0, + path: importedImage.path, + name: importedImage.name, + enabled: true, + center: { x: 0.5, y: 0.5 }, + size: { x: 0.5, y: 0.5 }, + opacity: 1, + rotation: 0, + flipX: false, + flipY: false, + lockAspect: true, + }); + const memoryUpdate = await sendCommand(2, "updateProjectConfigInMemory", [ + changedConfig, + 30, + 30, + { x: 640, y: 360 }, + ]); + expect(memoryUpdate.kind).toBe("result"); + const memoryConfig = await app.request( + `/editor/sessions/${sessionId}/config`, + { headers }, + ); + const configuredImage = (await memoryConfig.json()) as { + background: { source: { path: string } }; + timeline: { imageSegments: Array<{ path: string }> }; + }; + expect(configuredImage.background.source.path).toBe(wallpaperId); + expect(configuredImage.timeline.imageSegments[0]?.path).toBe( + importedImage.path, + ); + const centerMagentaScore = (bytes: Buffer) => { + const stride = bytes.readUInt32LE(bytes.length - 24); + const offset = 180 * stride + 320 * 4; + return bytes[offset] + bytes[offset + 2] - 2 * bytes[offset + 1]; + }; + let imageScore = 0; + for (let attempt = 0; attempt < 10; attempt++) { + const imagePreview = await app.request( + `/editor/sessions/${sessionId}/preview`, + { + method: "POST", + headers, + body: JSON.stringify({ + frameNumber: 31 + attempt, + fps: 30, + resolutionBase: { x: 640, y: 360 }, + }), + }, + ); + expect(imagePreview.status).toBe(200); + imageScore = centerMagentaScore( + Buffer.from(await imagePreview.arrayBuffer()), + ); + if (imageScore > 400) break; + await Bun.sleep(50); + } + expect(imageScore).toBeGreaterThan(400); + expect(await Bun.file(configPath).text()).toBe(diskBefore); + const stockConfig = await sendCommand(3, "getDefaultProjectConfig", []); + expect(stockConfig.value).toMatchObject({ + background: { padding: 10, rounding: 7.5 }, + camera: { mirror: data.savedProjectConfig.camera.mirror }, + }); + const liveConfig = await app.request( + `/editor/sessions/${sessionId}/config`, + { headers }, + ); + expect(liveConfig.status).toBe(200); + expect(await liveConfig.json()).toMatchObject({ + camera: { mirror: changedConfig.camera.mirror }, + }); + const deniedSave = await sendCommand(4, "setProjectConfig", [ + changedConfig, + ]); + expect(deniedSave.kind).toBe("error"); + const saved = await app.request(`/editor/sessions/${sessionId}/config`, { + method: "PUT", + headers, + body: JSON.stringify(changedConfig), + }); + expect(saved.status).toBe(204); + expect(await Bun.file(configPath).text()).not.toBe(diskBefore); + const invalidExport = await app.request( + `/editor/sessions/${sessionId}/exports`, + { + method: "POST", + headers, + body: JSON.stringify({ format: "Mp4", fps: 0 }), + }, + ); + expect(invalidExport.status).toBe(400); + const mp4Settings = { + format: "Mp4", + fps: 30, + resolution_base: { x: 640, y: 360 }, + compression: "Social", + custom_bpp: null, + force_ffmpeg_decoder: true, + optimize_filesize: false, + }; + const startedExport = await app.request( + `/editor/sessions/${sessionId}/exports`, + { + method: "POST", + headers, + body: JSON.stringify(mp4Settings), + }, + ); + expect(startedExport.status).toBe(202); + const exportId = ((await startedExport.json()) as { id: string }).id; + const duplicateExport = await app.request( + `/editor/sessions/${sessionId}/exports`, + { + method: "POST", + headers, + body: JSON.stringify(mp4Settings), + }, + ); + expect(duplicateExport.status).toBe(503); + const waitForExport = async (id: string) => { + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + const pending = await app.request( + `/editor/sessions/${sessionId}/exports/${id}`, + { headers }, + ); + const state = (await pending.json()) as { + status: string; + progress: { rendered_count: number; total_frames: number } | null; + error: string | null; + size: number | null; + mediaMetadata: { + duration: number; + width: number; + height: number; + fps: number; + } | null; + }; + if (state.status === "ready") return state; + if (state.status === "error") + throw new Error(state.error || "Native editor export failed"); + await Bun.sleep(50); + } + throw new Error("Native editor export timed out"); + }; + const exportedMp4 = await waitForExport(exportId); + expect(exportedMp4.progress?.rendered_count).toBeGreaterThan(80); + expect(exportedMp4.progress?.total_frames).toBeGreaterThan(80); + expect(exportedMp4.size).toBeGreaterThan(40_000); + expect(exportedMp4.mediaMetadata?.width).toBe(640); + expect(exportedMp4.mediaMetadata?.height).toBeGreaterThanOrEqual(350); + expect(exportedMp4.mediaMetadata?.height).toBeLessThanOrEqual(360); + expect(exportedMp4.mediaMetadata?.duration).toBeGreaterThan(2); + const mp4File = await app.request( + `/editor/sessions/${sessionId}/exports/${exportId}/file`, + { headers }, + ); + expect(mp4File.status).toBe(200); + expect(mp4File.headers.get("Content-Type")).toBe("video/mp4"); + const mp4Bytes = await mp4File.arrayBuffer(); + expect(mp4Bytes.byteLength).toBeGreaterThan(40_000); + expect(exportedMp4.size).toBe(mp4Bytes.byteLength); + const exportChunkPath = `/editor/sessions/${sessionId}/exports/${exportId}/chunk`; + for (const [offset, length] of [ + [0, 16 * 1024], + [mp4Bytes.byteLength - 16 * 1024, 16 * 1024], + ]) { + const chunk = await app.request( + `${exportChunkPath}?offset=${offset}&length=${length}`, + { headers }, + ); + expect(chunk.status).toBe(206); + expect(chunk.headers.get("Content-Range")).toBe( + `bytes ${offset}-${offset + length - 1}/${mp4Bytes.byteLength}`, + ); + expect(Buffer.from(await chunk.arrayBuffer())).toEqual( + Buffer.from(mp4Bytes.slice(offset, offset + length)), + ); + } + const invalidChunk = await app.request( + `${exportChunkPath}?offset=0&length=${16 * 1024 * 1024 + 1}`, + { headers }, + ); + expect(invalidChunk.status).toBe(400); + const downloadedMp4 = join(root, "downloaded-export.mp4"); + await writeFile(downloadedMp4, Buffer.from(mp4Bytes)); + const mp4Probe = spawnSync("ffprobe", [ + "-v", + "error", + "-show_entries", + "stream=codec_name,codec_type,width,height,nb_frames", + "-of", + "json", + downloadedMp4, + ]); + expect(mp4Probe.status).toBe(0); + const mp4Streams = JSON.parse(mp4Probe.stdout.toString()) as { + streams: Array<{ + codec_name: string; + codec_type: string; + width?: number; + height?: number; + nb_frames?: string; + }>; + }; + const videoStream = mp4Streams.streams.find( + (stream) => stream.codec_type === "video", + ); + expect(videoStream?.codec_name).toBe("h264"); + expect(videoStream?.width).toBe(640); + expect(exportedMp4.mediaMetadata?.height).toBe(videoStream?.height); + expect(videoStream?.height).toBeGreaterThanOrEqual(350); + expect(videoStream?.height).toBeLessThanOrEqual(360); + expect( + mp4Streams.streams.some((stream) => stream.codec_type === "audio"), + ).toBe(true); + const filteredAudio = spawnSync( + "ffmpeg", + [ + "-v", + "error", + "-i", + downloadedMp4, + "-af", + "highpass=f=1000", + "-f", + "s16le", + "-ac", + "1", + "-ar", + "16000", + "-", + ], + { maxBuffer: 1024 * 1024 }, + ); + expect(filteredAudio.status).toBe(0); + const rms = (from: number, to: number) => { + let total = 0; + const start = Math.floor(from * 16000); + const end = Math.min( + Math.floor(to * 16000), + filteredAudio.stdout.length / 2, + ); + for (let sample = start; sample < end; sample++) { + const value = filteredAudio.stdout.readInt16LE(sample * 2); + total += value * value; + } + return Math.sqrt(total / (end - start)); + }; + expect(rms(0.5, 1.5)).toBeGreaterThan(rms(2.7, 2.95) * 1.5); + const toneLevel = (from: number, to: number) => { + const start = Math.floor(from * 16000); + const end = Math.min( + Math.floor(to * 16000), + filteredAudio.stdout.length / 2, + ); + let cosine = 0; + let sine = 0; + for (let sample = start; sample < end; sample++) { + const value = filteredAudio.stdout.readInt16LE(sample * 2); + const phase = (2 * Math.PI * 1200 * sample) / 16000; + cosine += value * Math.cos(phase); + sine += value * Math.sin(phase); + } + return Math.hypot(cosine, sine) / (end - start); + }; + expect(toneLevel(0.5, 1.5)).toBeGreaterThan(toneLevel(2.7, 2.95) * 10); + const removedMp4 = await app.request( + `/editor/sessions/${sessionId}/exports/${exportId}`, + { method: "DELETE", headers }, + ); + expect(removedMp4.status).toBe(204); + expect( + ( + await app.request( + `/editor/sessions/${sessionId}/exports/${exportId}/file`, + { headers }, + ) + ).status, + ).toBe(404); + const gifStart = await app.request( + `/editor/sessions/${sessionId}/exports`, + { + method: "POST", + headers, + body: JSON.stringify({ + format: "Gif", + fps: 10, + resolution_base: { x: 320, y: 180 }, + quality: { quality: 80, fast: true }, + }), + }, + ); + expect(gifStart.status).toBe(202); + const gifId = ((await gifStart.json()) as { id: string }).id; + const exportedGif = await waitForExport(gifId); + expect(exportedGif.progress?.total_frames).toBeGreaterThan(20); + const gifFile = await app.request( + `/editor/sessions/${sessionId}/exports/${gifId}/file`, + { headers }, + ); + expect(gifFile.status).toBe(200); + expect(gifFile.headers.get("Content-Type")).toBe("image/gif"); + const gifBytes = await gifFile.arrayBuffer(); + expect(Buffer.from(gifBytes).toString("ascii", 0, 4)).toBe("GIF8"); + const downloadTicketResponse = await app.request( + `/editor/sessions/${sessionId}/exports/${gifId}/download-ticket`, + { + method: "POST", + headers, + body: JSON.stringify({ fileName: "Paired test.gif" }), + }, + ); + expect(downloadTicketResponse.status).toBe(200); + const downloadTicket = (await downloadTicketResponse.json()) as { + url: string; + }; + const downloadUrl = new URL(downloadTicket.url); + expect( + (await app.request(`${downloadUrl.pathname}?ticket=wrong-token`)) + .status, + ).toBe(404); + const directDownload = await app.request( + `${downloadUrl.pathname}${downloadUrl.search}`, + ); + expect(directDownload.status).toBe(200); + expect(directDownload.headers.get("Content-Type")).toBe("image/gif"); + expect(directDownload.headers.get("Content-Disposition")).toContain( + `filename="Paired test.gif"`, + ); + expect(directDownload.headers.get("Content-Length")).toBe( + String(gifBytes.byteLength), + ); + expect(Buffer.from(await directDownload.arrayBuffer())).toEqual( + Buffer.from(gifBytes), + ); + expect( + (await app.request(`${downloadUrl.pathname}${downloadUrl.search}`)) + .status, + ).toBe(404); + expect( + ( + await app.request( + `/editor/sessions/${sessionId}/exports/${gifId}/file`, + { headers }, + ) + ).status, + ).toBe(404); + const closeExportStart = await app.request( + `/editor/sessions/${sessionId}/exports`, + { + method: "POST", + headers, + body: JSON.stringify({ + format: "Gif", + fps: 10, + resolution_base: { x: 320, y: 180 }, + quality: { quality: 80, fast: true }, + }), + }, + ); + expect(closeExportStart.status).toBe(202); + const closeExportId = ((await closeExportStart.json()) as { id: string }) + .id; + await waitForExport(closeExportId); + const closeExportFile = await app.request( + `/editor/sessions/${sessionId}/exports/${closeExportId}/file`, + { headers }, + ); + expect(closeExportFile.status).toBe(200); + const closeExportBytes = Buffer.from(await closeExportFile.arrayBuffer()); + const closeDownloadTicket = await app.request( + `/editor/sessions/${sessionId}/exports/${closeExportId}/download-ticket`, + { + method: "POST", + headers, + body: JSON.stringify({ fileName: "After page close.gif" }), + }, + ); + expect(closeDownloadTicket.status).toBe(200); + const closeDownloadUrl = new URL( + ((await closeDownloadTicket.json()) as { url: string }).url, + ); + commandSocket.close(); + commandSocket = null; + socket.close(); + socket = null; + const expectSocketRejected = async ( + credential: { url: string; ticket: string }, + origin: string, + ) => { + const attempt = new BunWebSocket(credential.url, { + protocols: [ + "cap-editor-v1", + `cap-editor-ticket.${credential.ticket}`, + ], + headers: { Origin: origin }, + }); + try { + await new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error("Unauthorized socket did not terminate")), + 3000, + ); + attempt.onopen = () => { + clearTimeout(timer); + reject(new Error("Unauthorized editor socket opened")); + }; + attempt.onerror = () => { + clearTimeout(timer); + resolve(); + }; + attempt.onclose = () => { + clearTimeout(timer); + resolve(); + }; + }); + } finally { + attempt.close(); + } + }; + await expectSocketRejected(frameSocket, "http://127.0.0.1:3000"); + const wrongOriginTicketResponse = await app.request( + `/editor/sessions/${sessionId}/sockets`, + { + method: "POST", + headers, + body: JSON.stringify({ origin: "http://127.0.0.1:3000" }), + }, + ); + expect(wrongOriginTicketResponse.status).toBe(200); + const wrongOriginTickets = (await wrongOriginTicketResponse.json()) as { + sockets: { commands: { url: string; ticket: string } }; + }; + await expectSocketRejected( + wrongOriginTickets.sockets.commands, + "http://127.0.0.1:3001", + ); + await expectSocketRejected( + wrongOriginTickets.sockets.commands, + "http://127.0.0.1:3000", + ); + const closed = await app.request(`/editor/sessions/${sessionId}`, { + method: "DELETE", + headers, + }); + expect(closed.status).toBe(204); + sessionId = null; + const downloadAfterClose = await app.request( + `${closeDownloadUrl.pathname}${closeDownloadUrl.search}`, + ); + expect(downloadAfterClose.status).toBe(200); + expect(downloadAfterClose.headers.get("Content-Disposition")).toContain( + `filename="After page close.gif"`, + ); + expect(Buffer.from(await downloadAfterClose.arrayBuffer())).toEqual( + closeExportBytes, + ); + expect( + ( + await app.request( + `${closeDownloadUrl.pathname}${closeDownloadUrl.search}`, + ) + ).status, + ).toBe(404); + const reopen = await app.request("/editor/preparations", { + method: "POST", + headers, + body: JSON.stringify({ + ...sources, + projectConfig: changedConfig, + audioAssets: [importedAudioAsset], + imageAssets: [importedImageAsset], + videoAssets: [importedVideoAsset], + }), + }); + expect(reopen.status).toBe(202); + const reopenedPreparation = (await reopen.json()) as { id: string }; + const reopenDeadline = Date.now() + 10_000; + while (Date.now() < reopenDeadline) { + const pending = await app.request( + `/editor/preparations/${reopenedPreparation.id}`, + { headers }, + ); + const state = (await pending.json()) as { + status: string; + sessionId?: string; + }; + if (state.status === "ready") { + sessionId = state.sessionId ?? null; + break; + } + if (state.status === "error") + throw new Error("Saved editor project could not reopen"); + await Bun.sleep(50); + } + expect(sessionId).not.toBeNull(); + if (!sessionId) throw new Error("Saved editor project did not reopen"); + const reopenedInstance = await app.request( + `/editor/sessions/${sessionId}/instance`, + { headers }, + ); + expect(reopenedInstance.status).toBe(200); + const restored = (await reopenedInstance.json()) as { + savedProjectConfig: { + camera: { mirror: boolean }; + background: { source: { path: string } }; + timeline: { + audioSegments: Array<{ path: string }>; + imageSegments: Array<{ path: string }>; + }; + }; + }; + expect(restored.savedProjectConfig.camera.mirror).toBe( + changedConfig.camera.mirror, + ); + expect(restored.savedProjectConfig.background.source.path).toBe( + wallpaperId, + ); + expect(restored.savedProjectConfig.timeline.audioSegments[0]?.path).toBe( + selectedMusic.path, + ); + expect(restored.savedProjectConfig.timeline.audioSegments[1]?.path).toBe( + importedTrack.path, + ); + expect(restored.savedProjectConfig.timeline.imageSegments[0]?.path).toBe( + importedImage.path, + ); + const reopenedProjectPath = getEditorSession(sessionId)?.projectPath; + if (!reopenedProjectPath) + throw new Error("Missing reopened editor project"); + expect( + (await stat(join(reopenedProjectPath, selectedMusic.path))).size, + ).toBeGreaterThan(100_000); + expect( + (await stat(join(reopenedProjectPath, importedTrack.path))).size, + ).toBe(importedAudioAsset.size); + expect( + (await stat(join(reopenedProjectPath, importedImage.path))).size, + ).toBe(importedImageAsset.size); + expect( + (await stat(join(reopenedProjectPath, importedVideo.path))).size, + ).toBe(importedVideoAsset.size); + const unclaimedSessionId = sessionId; + const expiredFuture = Date.now() + 5 * 60 * 1000; + const expiredClock = spyOn(Date, "now").mockReturnValue(expiredFuture); + let screenOnly: Response | null = null; + try { + screenOnly = await app.request("/editor/preparations", { + method: "POST", + headers, + body: JSON.stringify({ + videoId: "test-screen-only", + title: "Screen only", + display: sources.display, + }), + }); + } finally { + expiredClock.mockRestore(); + } + if (!screenOnly) throw new Error("Screen-only editor preparation failed"); + expect(screenOnly.status).toBe(202); + expect(getEditorSession(unclaimedSessionId)).toBeNull(); + const expiredPreparation = await app.request( + `/editor/preparations/${reopenedPreparation.id}`, + { headers }, + ); + expect( + ((await expiredPreparation.json()) as { status: string }).status, + ).toBe("closed"); + sessionId = null; + const screenPreparation = (await screenOnly.json()) as { id: string }; + const screenDeadline = Date.now() + 10_000; + while (Date.now() < screenDeadline) { + const pending = await app.request( + `/editor/preparations/${screenPreparation.id}`, + { headers }, + ); + const state = (await pending.json()) as { + status: string; + sessionId?: string; + }; + if (state.status === "ready") { + sessionId = state.sessionId ?? null; + break; + } + if (state.status === "error") + throw new Error("Screen-only editor project could not prepare"); + await Bun.sleep(50); + } + expect(sessionId).not.toBeNull(); + if (!sessionId) + throw new Error("Screen-only editor project has no session"); + const screenInstance = await app.request( + `/editor/sessions/${sessionId}/instance`, + { headers }, + ); + expect(screenInstance.status).toBe(200); + const screenPreview = await app.request( + `/editor/sessions/${sessionId}/preview`, + { + method: "POST", + headers, + body: JSON.stringify({ + frameNumber: 30, + fps: 30, + resolutionBase: { x: 640, y: 360 }, + }), + }, + ); + expect(screenPreview.status).toBe(200); + expect((await screenPreview.arrayBuffer()).byteLength).toBeGreaterThan( + 1_000, + ); + const backgroundExport = await app.request( + `/editor/sessions/${sessionId}/exports`, + { + method: "POST", + headers, + body: JSON.stringify(mp4Settings), + }, + ); + expect(backgroundExport.status).toBe(202); + const backgroundExportId = ( + (await backgroundExport.json()) as { id: string } + ).id; + expect(getEditorExport(sessionId, backgroundExportId)?.status).toBe( + "running", + ); + const exportStartedAt = Date.now(); + const laterClock = spyOn(Date, "now").mockReturnValue(exportStartedAt); + try { + for (let minute = 2; minute <= 12; minute += 2) { + laterClock.mockReturnValue(exportStartedAt + minute * 60 * 1000); + const runningAfterIdle = await app.request( + `/editor/sessions/${sessionId}/exports/${backgroundExportId}`, + { headers }, + ); + expect(runningAfterIdle.status).toBe(200); + } + laterClock.mockReturnValue(exportStartedAt + 12 * 60 * 1000); + const exportDeadline = performance.now() + 10_000; + let completedExport = false; + while (performance.now() < exportDeadline) { + const state = getEditorExport(sessionId, backgroundExportId); + if (state?.status === "ready") { + completedExport = true; + break; + } + if (state?.status === "error") { + throw new Error(state.error || "Background editor export failed"); + } + await Bun.sleep(50); + } + expect(completedExport).toBe(true); + laterClock.mockReturnValue(exportStartedAt + 21 * 60 * 1000 + 30_000); + const stillAvailable = await app.request( + `/editor/sessions/${sessionId}/exports/${backgroundExportId}`, + { headers }, + ); + expect(stillAvailable.status).toBe(200); + expect(getEditorSession(sessionId)).not.toBeNull(); + } finally { + laterClock.mockRestore(); + } + const screenClose = await app.request(`/editor/sessions/${sessionId}`, { + method: "DELETE", + headers, + }); + expect(screenClose.status).toBe(204); + sessionId = null; + } finally { + commandSocket?.close(); + fastStartSocket?.close(); + socket?.close(); + socketServer?.stop(true); + if (sessionId) { + await app.request(`/editor/sessions/${sessionId}`, { + method: "DELETE", + headers: { "x-media-server-secret": secret }, + }); + } + server?.stop(true); + await rm(root, { recursive: true, force: true }); + } + }, + 30_000, +); diff --git a/apps/media-server/src/__tests__/routes/video.test.ts b/apps/media-server/src/__tests__/routes/video.test.ts index d2643b3bf7f..2f55dd579dd 100644 --- a/apps/media-server/src/__tests__/routes/video.test.ts +++ b/apps/media-server/src/__tests__/routes/video.test.ts @@ -278,6 +278,124 @@ describe("POST /video/thumbnail", () => { }); }); +describe("POST /video/preview-assets", () => { + beforeEach(() => { + mock.restore(); + spyOn(jobManager, "canAcceptNewVideoProcess").mockReturnValue(true); + }); + + test("returns 401 without media server secret", async () => { + const response = await app.fetch( + unauthenticatedVideoPostRequest("/video/preview-assets", { + videoUrl: "https://example.com/video.mp4", + previewGifPresignedUrl: "https://storage.example/preview.gif", + }), + ); + + expect(response.status).toBe(401); + }); + + test("returns 400 when no asset is requested", async () => { + const response = await app.fetch( + videoPostRequest("/video/preview-assets", { + videoUrl: "https://example.com/video.mp4", + }), + ); + + expect(response.status).toBe(400); + const data = await response.json(); + expect(data.code).toBe("INVALID_REQUEST"); + }); + + test("returns 503 when video capacity is exhausted", async () => { + spyOn(jobManager, "canAcceptNewVideoProcess").mockReturnValue(false); + + const response = await app.fetch( + videoPostRequest("/video/preview-assets", { + videoUrl: "https://example.com/video.mp4", + previewGifPresignedUrl: "https://storage.example/preview.gif", + }), + ); + + expect(response.status).toBe(503); + expect(response.headers.get("Retry-After")).toBe("15"); + }); + + test("generates both assets from the video URL and uploads them", async () => { + const thumbnail = new Uint8Array([0xff, 0xd8, 0xff]); + const uploads: Array<{ url: string; contentType: string }> = []; + const cleanup = mock(async () => {}); + + mock.module("../../lib/media-probe", () => ({ + ...mediaProbe, + probeVideo: async () => ({ duration: 12 }), + })); + mock.module("../../lib/media-video", () => ({ + ...mediaVideo, + generateThumbnail: async (input: string, duration: number) => { + expect(input).toBe("https://example.com/video.mp4"); + expect(duration).toBe(12); + return thumbnail; + }, + generatePreviewGif: async (input: string, duration: number) => { + expect(input).toBe("https://example.com/video.mp4"); + expect(duration).toBe(12); + return { path: "/tmp/preview.gif", cleanup }; + }, + uploadToS3: async (data: Uint8Array, url: string, type: string) => { + expect(data).toBe(thumbnail); + uploads.push({ url, contentType: type }); + }, + uploadFileToS3: async (_path: string, url: string, type: string) => { + uploads.push({ url, contentType: type }); + return {}; + }, + })); + + const { default: appWithMock } = await import("../../app"); + const response = await appWithMock.fetch( + videoPostRequest("/video/preview-assets", { + videoUrl: "https://example.com/video.mp4", + thumbnailPresignedUrl: "https://storage.example/screenshot.jpg", + previewGifPresignedUrl: "https://storage.example/preview.gif", + }), + ); + + expect(response.status).toBe(200); + expect(uploads).toEqual([ + { + url: "https://storage.example/screenshot.jpg", + contentType: "image/jpeg", + }, + { url: "https://storage.example/preview.gif", contentType: "image/gif" }, + ]); + expect(cleanup).toHaveBeenCalledTimes(1); + }); + + test("fails when a requested asset cannot be generated", async () => { + mock.module("../../lib/media-probe", () => ({ + ...mediaProbe, + probeVideo: async () => ({ duration: 12 }), + })); + mock.module("../../lib/media-video", () => ({ + ...mediaVideo, + generatePreviewGif: async () => { + throw new Error("broken input"); + }, + })); + + const { default: appWithMock } = await import("../../app"); + const response = await appWithMock.fetch( + videoPostRequest("/video/preview-assets", { + videoUrl: "https://example.com/video.mp4", + previewGifPresignedUrl: "https://storage.example/preview.gif", + }), + ); + + expect(response.status).toBe(500); + }); +}); + describe("POST /video/convert", () => { beforeEach(() => { mock.restore(); diff --git a/apps/media-server/src/editor-worker-app.ts b/apps/media-server/src/editor-worker-app.ts new file mode 100644 index 00000000000..14ba723bf7b --- /dev/null +++ b/apps/media-server/src/editor-worker-app.ts @@ -0,0 +1,34 @@ +import { access } from "node:fs/promises"; +import { join } from "node:path"; +import { Hono } from "hono"; +import { nativeEditorBinary } from "./lib/editor-native"; +import { publicEditorOrigin } from "./lib/editor-socket-tickets"; +import { editorWallpaperDirectory } from "./lib/editor-wallpapers"; +import editor from "./routes/editor"; + +const app = new Hono(); + +app.get("/health", async (c) => { + try { + const checks = [ + access(nativeEditorBinary("prepare")), + access(nativeEditorBinary("service")), + access(join(editorWallpaperDirectory(), "macOS/tahoe-dusk-min.jpg")), + ]; + if (process.env.CAP_WEB_EDITOR_ENABLE_CAMERA_REMOVAL === "1") { + const runtimePath = process.env.ORT_DYLIB_PATH; + if (!runtimePath) throw new Error("ONNX Runtime path is unavailable"); + checks.push(access(runtimePath)); + } + await Promise.all(checks); + publicEditorOrigin(); + if (!Bun.which("ffprobe")) throw new Error("ffprobe is unavailable"); + return c.json({ status: "ok" }); + } catch { + return c.json({ status: "unavailable" }, 503); + } +}); + +app.route("/editor", editor); + +export default app; diff --git a/apps/media-server/src/editor-worker.ts b/apps/media-server/src/editor-worker.ts new file mode 100644 index 00000000000..e646a67fbfb --- /dev/null +++ b/apps/media-server/src/editor-worker.ts @@ -0,0 +1,32 @@ +import app from "./editor-worker-app"; +import { closeAllEditorSessions } from "./lib/editor-sessions"; +import { + type EditorSocketConnection, + editorWebSocketHandler, + handleEditorSocketUpgrade, +} from "./lib/editor-websocket"; + +const port = Number(process.env.PORT) || 3457; +let shuttingDown = false; + +async function shutdown() { + if (shuttingDown) return; + shuttingDown = true; + await closeAllEditorSessions(); + process.exit(0); +} + +for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) { + process.on(signal, () => { + void shutdown(); + }); +} + +export default { + port, + fetch(request: Request, server: Bun.Server) { + const upgrade = handleEditorSocketUpgrade(request, server); + return upgrade === null ? app.fetch(request) : upgrade; + }, + websocket: editorWebSocketHandler, +}; diff --git a/apps/media-server/src/lib/editor-assets.ts b/apps/media-server/src/lib/editor-assets.ts new file mode 100644 index 00000000000..fa12a8b9a8a --- /dev/null +++ b/apps/media-server/src/lib/editor-assets.ts @@ -0,0 +1,107 @@ +import { runEditorFile as runFile } from "./editor-process"; +import { stageSignedEditorAsset } from "./editor-signed-assets"; + +const MAX_AUDIO_BYTES = 32 * 1024 * 1024; +const ASSET_PATH = + /^assets\/audio\/import-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.(ogg|m4a|mp3|wav|aac|flac)$/; +const CONTENT_TYPES: Record = { + ogg: "audio/ogg", + m4a: "audio/mp4", + mp3: "audio/mpeg", + wav: "audio/wav", + aac: "audio/aac", + flac: "audio/flac", +}; + +export type EditorAudioAsset = { + path: string; + name: string; + url: string; + size: number; + contentType: string; + objectIdentity?: string | null; +}; + +export function validateEditorAudioAsset(asset: EditorAudioAsset) { + const url = new URL(asset.url); + const match = ASSET_PATH.exec(asset.path); + if ( + !match || + CONTENT_TYPES[match[1] ?? ""] !== asset.contentType || + asset.name.length < 1 || + asset.name.length > 100 || + asset.name + .split("") + .some( + (character) => + character.charCodeAt(0) < 32 || character.charCodeAt(0) === 127, + ) || + !Number.isSafeInteger(asset.size) || + asset.size < 1 || + asset.size > MAX_AUDIO_BYTES || + (url.protocol !== "https:" && + !( + url.protocol === "http:" && + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA === "1" + )) || + url.username || + url.password || + (asset.objectIdentity != null && + (asset.objectIdentity.length < 1 || asset.objectIdentity.length > 256)) + ) { + throw new Error("Invalid editor audio asset"); + } +} + +async function probeAudioDuration(path: string) { + const { stdout } = await runFile( + "ffprobe", + [ + "-v", + "error", + "-select_streams", + "a:0", + "-show_entries", + "stream=codec_type:format=duration", + "-of", + "json", + path, + ], + { timeout: 15_000, maxBuffer: 64 * 1024 }, + ).catch(() => { + throw new Error("Imported file has no audio stream"); + }); + const data: unknown = JSON.parse(stdout); + if ( + typeof data !== "object" || + data === null || + !("streams" in data) || + !Array.isArray(data.streams) || + data.streams[0]?.codec_type !== "audio" || + !("format" in data) || + typeof data.format !== "object" || + data.format === null || + !("duration" in data.format) + ) { + throw new Error("Imported file has no audio stream"); + } + const duration = Number(data.format.duration); + if (!Number.isFinite(duration) || duration <= 0 || duration > 24 * 60 * 60) { + throw new Error("Imported audio duration is invalid"); + } + return duration; +} + +export async function stageSignedEditorAudioAsset( + projectPath: string, + asset: EditorAudioAsset, + abortSignal?: AbortSignal, +) { + validateEditorAudioAsset(asset); + return stageSignedEditorAsset( + projectPath, + asset, + async (path) => ({ duration: await probeAudioDuration(path) }), + abortSignal, + ); +} diff --git a/apps/media-server/src/lib/editor-audio-library.ts b/apps/media-server/src/lib/editor-audio-library.ts new file mode 100644 index 00000000000..b1d1a8f3b4f --- /dev/null +++ b/apps/media-server/src/lib/editor-audio-library.ts @@ -0,0 +1,133 @@ +import { randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { copyFile, lstat, mkdir, rename, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { runEditorFile as runFile } from "./editor-process"; + +const MAX_TRACK_BYTES = 32 * 1024 * 1024; +const LOCAL_MUSIC_DIR = fileURLToPath( + new URL("../../../../apps/desktop/src/assets/music/", import.meta.url), +); +const tracks = [ + { id: "lofi-beats-mirostar", name: "Lofi Beats", category: "Lo-Fi" }, + { id: "raindrops-lofi-sleep-bluelike", name: "Raindrops", category: "Lo-Fi" }, + { + id: "sunday-mood-lofi-cafe-upbeat-bluelike", + name: "Sunday Mood", + category: "Lo-Fi", + }, + { + id: "good-night-lofi-cozy-chill-fassounds", + name: "Good Night", + category: "Lo-Fi", + }, + { + id: "ambient-trap-empty-streets-dreamstate-openmindaudio", + name: "Empty Streets", + category: "Lo-Fi", + }, + { + id: "lofi-study-calm-peaceful-chill-hop-fassounds", + name: "Study", + category: "Lo-Fi", + }, + { id: "lofi-cinematic-pulsebox", name: "Cinematic", category: "Lo-Fi" }, + { id: "lofi-hip-hop-leberch", name: "Hip Hop", category: "Lo-Fi" }, + { id: "cassette-retrositive", name: "Cassette", category: "Lo-Fi" }, + { id: "lofi-smooth-pulsebox", name: "Smooth", category: "Lo-Fi" }, +] as const; + +function musicDirectory() { + return process.env.CAP_WEB_EDITOR_MUSIC_DIR ?? LOCAL_MUSIC_DIR; +} + +function libraryTrack(id: unknown) { + if (typeof id !== "string") throw new Error("Invalid library track"); + const track = tracks.find((item) => item.id === id); + if (!track) throw new Error("Unknown library track"); + return track; +} + +export function listEditorAudioLibrary() { + return tracks.map((track) => ({ ...track })); +} + +async function stageLibraryTrack(projectPath: string, id: string) { + const track = libraryTrack(id); + const source = join(musicDirectory(), `${track.id}.mp3`); + const sourceStats = await lstat(source); + if ( + !sourceStats.isFile() || + sourceStats.size < 1 || + sourceStats.size > MAX_TRACK_BYTES + ) { + throw new Error("Bundled library track is unavailable"); + } + const audioDir = join(projectPath, "assets", "audio"); + await mkdir(audioDir, { recursive: true, mode: 0o700 }); + const fileName = `library-${track.id}.mp3`; + const destination = join(audioDir, fileName); + const existing = await lstat(destination).catch(() => null); + if (!existing?.isFile() || existing.size !== sourceStats.size) { + const temporary = join(audioDir, `${randomUUID()}.mp3`); + try { + await copyFile(source, temporary, constants.COPYFILE_EXCL); + await rename(temporary, destination); + } finally { + await rm(temporary, { force: true }); + } + } + return { track, destination, path: `assets/audio/${fileName}` }; +} + +export async function addEditorAudioLibraryTrack( + projectPath: string, + id: unknown, +) { + const staged = await stageLibraryTrack(projectPath, libraryTrack(id).id); + let duration = 0; + try { + const { stdout } = await runFile( + "ffprobe", + [ + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + staged.destination, + ], + { timeout: 10_000, maxBuffer: 1024 }, + ); + const parsed = Number(stdout.trim()); + if (Number.isFinite(parsed) && parsed > 0) duration = parsed; + } catch { + duration = 0; + } + return { path: staged.path, name: staged.track.name, duration }; +} + +export async function stageSavedEditorAudioLibrary( + projectPath: string, + projectConfig?: Record, +) { + const timeline = projectConfig?.timeline; + if (typeof timeline !== "object" || timeline === null) return; + if (!("audioSegments" in timeline)) return; + const segments = timeline.audioSegments; + if (!Array.isArray(segments)) return; + const ids = new Set(); + for (const segment of segments) { + if (typeof segment !== "object" || segment === null) continue; + if (!("path" in segment) || typeof segment.path !== "string") continue; + const match = /^assets\/audio\/library-([a-z0-9-]+)\.mp3$/.exec( + segment.path, + ); + if (match && tracks.some((track) => track.id === match[1])) { + ids.add(match[1]); + } + } + await Promise.all([...ids].map((id) => stageLibraryTrack(projectPath, id))); +} diff --git a/apps/media-server/src/lib/editor-cap-assets.ts b/apps/media-server/src/lib/editor-cap-assets.ts new file mode 100644 index 00000000000..ab5d4ad303a --- /dev/null +++ b/apps/media-server/src/lib/editor-cap-assets.ts @@ -0,0 +1,76 @@ +import { rm } from "node:fs/promises"; +import { join } from "node:path"; +import { + CAP_BUNDLE_CONTENT_TYPE, + MAX_CAP_BUNDLE_BYTES, +} from "@cap/editor-cap-bundle"; +import { extractEditorCapBundle } from "./editor-cap-bundle"; +import { + type SignedEditorAsset, + stageSignedEditorAsset, +} from "./editor-signed-assets"; + +const CAP_BUNDLE_PATH = + /^content\/imports\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.capbundle$/; +const CAP_BUNDLE_DOWNLOAD_TIMEOUT_MS = 30 * 60 * 1000; + +export type EditorCapAsset = SignedEditorAsset; + +export function validateEditorCapAsset(asset: EditorCapAsset) { + const url = new URL(asset.url); + if ( + !CAP_BUNDLE_PATH.test(asset.path) || + asset.contentType !== CAP_BUNDLE_CONTENT_TYPE || + !Number.isSafeInteger(asset.size) || + asset.size <= 0 || + asset.size > MAX_CAP_BUNDLE_BYTES || + asset.name.length < 1 || + asset.name.length > 100 || + asset.name.split("").some((character) => { + const code = character.charCodeAt(0); + return ( + code < 32 || code === 127 || character === "/" || character === "\\" + ); + }) || + (url.protocol !== "https:" && + !( + url.protocol === "http:" && + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA === "1" + )) || + url.username || + url.password || + (asset.objectIdentity != null && + (asset.objectIdentity.length < 1 || asset.objectIdentity.length > 256)) + ) { + throw new Error("Invalid editor Cap project asset"); + } +} + +export async function stageSignedEditorCapAsset( + projectPath: string, + asset: EditorCapAsset, + abortSignal?: AbortSignal, +) { + validateEditorCapAsset(asset); + const bundlePath = join(projectPath, asset.path); + await stageSignedEditorAsset( + projectPath, + asset, + async () => ({}), + abortSignal, + CAP_BUNDLE_DOWNLOAD_TIMEOUT_MS, + ); + try { + const extracted = await extractEditorCapBundle(bundlePath, abortSignal); + return { + path: extracted.path, + cleanup: async () => { + await extracted.cleanup(); + await rm(bundlePath, { force: true }); + }, + }; + } catch (error) { + await rm(bundlePath, { force: true }); + throw error; + } +} diff --git a/apps/media-server/src/lib/editor-cap-bundle.ts b/apps/media-server/src/lib/editor-cap-bundle.ts new file mode 100644 index 00000000000..f841aef5110 --- /dev/null +++ b/apps/media-server/src/lib/editor-cap-bundle.ts @@ -0,0 +1,116 @@ +import type { FileHandle } from "node:fs/promises"; +import { lstat, mkdir, mkdtemp, open, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { + CAP_BUNDLE_HEADER_BYTES, + MAX_CAP_BUNDLE_BYTES, + parseCapBundleManifest, + readCapBundleManifestLength, +} from "@cap/editor-cap-bundle"; + +const COPY_CHUNK_BYTES = 1024 * 1024; + +async function readExact(handle: FileHandle, bytes: number, position: number) { + const data = Buffer.allocUnsafe(bytes); + let read = 0; + while (read < bytes) { + const result = await handle.read(data, read, bytes - read, position + read); + if (result.bytesRead === 0) { + throw new Error("Cap project bundle ended unexpectedly"); + } + read += result.bytesRead; + } + return data; +} + +export async function extractEditorCapBundle( + bundlePath: string, + abortSignal?: AbortSignal, +) { + const sourceStats = await lstat(bundlePath); + if ( + !sourceStats.isFile() || + sourceStats.size < CAP_BUNDLE_HEADER_BYTES || + sourceStats.size > MAX_CAP_BUNDLE_BYTES + ) { + throw new Error("Invalid Cap project bundle file"); + } + const source = await open(bundlePath, "r"); + let root: string | null = null; + try { + const openedStats = await source.stat(); + if ( + openedStats.size !== sourceStats.size || + openedStats.ino !== sourceStats.ino || + openedStats.dev !== sourceStats.dev + ) { + throw new Error("Cap project bundle changed before extraction"); + } + const header = await readExact(source, CAP_BUNDLE_HEADER_BYTES, 0); + const manifestLength = readCapBundleManifestLength(header); + if (manifestLength === null) { + throw new Error("Invalid Cap project bundle header"); + } + const manifest = parseCapBundleManifest( + await readExact(source, manifestLength, CAP_BUNDLE_HEADER_BYTES), + sourceStats.size, + ); + if (!manifest) { + throw new Error("Invalid Cap project bundle manifest"); + } + if (abortSignal?.aborted) { + throw new Error("Cap project import canceled"); + } + root = await mkdtemp(join(tmpdir(), "cap-editor-import-")); + const projectPath = join(root, "source.cap"); + await mkdir(projectPath, { mode: 0o700 }); + const buffer = Buffer.allocUnsafe(COPY_CHUNK_BYTES); + const dataStart = CAP_BUNDLE_HEADER_BYTES + manifestLength; + for (const entry of manifest.files) { + const destinationPath = join(projectPath, entry.path); + await mkdir(dirname(destinationPath), { recursive: true, mode: 0o700 }); + const destination = await open(destinationPath, "wx", 0o600); + try { + let copied = 0; + while (copied < entry.size) { + if (abortSignal?.aborted) { + throw new Error("Cap project import canceled"); + } + const length = Math.min(buffer.byteLength, entry.size - copied); + const result = await source.read( + buffer, + 0, + length, + dataStart + entry.offset + copied, + ); + if (result.bytesRead === 0) { + throw new Error("Cap project bundle ended unexpectedly"); + } + await destination.writeFile(buffer.subarray(0, result.bytesRead)); + copied += result.bytesRead; + } + } finally { + await destination.close(); + } + } + const finalStats = await source.stat(); + if ( + finalStats.size !== openedStats.size || + finalStats.mtimeMs !== openedStats.mtimeMs || + finalStats.ctimeMs !== openedStats.ctimeMs + ) { + throw new Error("Cap project bundle changed during extraction"); + } + const extractedRoot = root; + return { + path: projectPath, + cleanup: () => rm(extractedRoot, { recursive: true, force: true }), + }; + } catch (error) { + if (root) await rm(root, { recursive: true, force: true }); + throw error; + } finally { + await source.close(); + } +} diff --git a/apps/media-server/src/lib/editor-cap-captions.ts b/apps/media-server/src/lib/editor-cap-captions.ts new file mode 100644 index 00000000000..7c364834b06 --- /dev/null +++ b/apps/media-server/src/lib/editor-cap-captions.ts @@ -0,0 +1,383 @@ +import { chmod, lstat, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { extname, join } from "node:path"; +import { validCapBundlePath } from "@cap/editor-cap-bundle"; +import { + type EditorCapAsset, + stageSignedEditorCapAsset, + validateEditorCapAsset, +} from "./editor-cap-assets"; +import { nativeEditorBinary } from "./editor-native"; +import { runEditorFile as runFile } from "./editor-process"; + +const MAX_AUDIO_GENERATION_MS = 30 * 60 * 1000; +const AUDIO_EXTENSIONS = new Set([ + ".aac", + ".flac", + ".m4a", + ".mp3", + ".mp4", + ".ogg", + ".wav", + ".webm", +]); +const VIDEO_EXTENSIONS = new Set([ + ".avi", + ".flv", + ".m4v", + ".mkv", + ".mov", + ".mp4", + ".webm", + ".wmv", +]); + +type ExpectedSegment = { + mediaDurationMs: number; + segmentDurationMs: number; + hasAudio: boolean; +}; +type SourceTrack = { + path: string; + start_time?: number | null; +}; +type SourceSegment = { + display: SourceTrack; + camera?: SourceTrack | null; + mic?: SourceTrack | null; + system_audio?: SourceTrack | null; +}; + +function record(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function validExpectedSegments(segments: readonly ExpectedSegment[]) { + return ( + segments.length >= 1 && + segments.length <= 1000 && + segments.every( + (segment) => + Number.isSafeInteger(segment.mediaDurationMs) && + segment.mediaDurationMs > 0 && + segment.mediaDurationMs <= 86_400_000 && + Number.isSafeInteger(segment.segmentDurationMs) && + segment.segmentDurationMs >= segment.mediaDurationMs && + segment.segmentDurationMs <= 86_400_000 && + typeof segment.hasAudio === "boolean", + ) + ); +} + +function sourceTrack(value: unknown): SourceTrack | null { + if ( + !record(value) || + typeof value.path !== "string" || + !validCapBundlePath(value.path) || + (value.start_time !== undefined && + value.start_time !== null && + (typeof value.start_time !== "number" || + !Number.isFinite(value.start_time))) + ) { + return null; + } + return { + path: value.path, + start_time: typeof value.start_time === "number" ? value.start_time : null, + }; +} + +function parseSourceSegments(value: unknown): SourceSegment[] { + if ( + !record(value) || + !Array.isArray(value.segments) || + value.segments.length < 1 || + value.segments.length > 1000 + ) { + throw new Error("Cap source audio inspection was invalid"); + } + return value.segments.map((item: unknown) => { + if (!record(item)) { + throw new Error("Cap source audio segment was invalid"); + } + const display = sourceTrack(item.display); + if ( + !display || + !VIDEO_EXTENSIONS.has(extname(display.path).toLowerCase()) + ) { + throw new Error("Cap source display media was invalid"); + } + const camera = + item.camera === null || item.camera === undefined + ? null + : sourceTrack(item.camera); + const mic = + item.mic === null || item.mic === undefined + ? null + : sourceTrack(item.mic); + const systemAudio = + item.system_audio === null || item.system_audio === undefined + ? null + : sourceTrack(item.system_audio); + if ( + (item.camera !== null && item.camera !== undefined && !camera) || + (item.mic !== null && item.mic !== undefined && !mic) || + (item.system_audio !== null && + item.system_audio !== undefined && + !systemAudio) || + (camera && !VIDEO_EXTENSIONS.has(extname(camera.path).toLowerCase())) || + (mic && !AUDIO_EXTENSIONS.has(extname(mic.path).toLowerCase())) || + (systemAudio && + !AUDIO_EXTENSIONS.has(extname(systemAudio.path).toLowerCase())) + ) { + throw new Error("Cap source voice media was invalid"); + } + return { + display, + camera, + mic, + system_audio: systemAudio, + }; + }); +} + +async function checkedMediaPath(projectPath: string, path: string) { + const absolute = join(projectPath, path); + const metadata = await lstat(absolute); + if (!metadata.isFile() || metadata.size <= 0) { + throw new Error("Cap source media is unavailable"); + } + return absolute; +} + +async function probeDuration(path: string, signal?: AbortSignal) { + const { stdout } = await runFile( + "ffprobe", + [ + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=nw=1:nk=1", + path, + ], + { timeout: 60_000, maxBuffer: 64 * 1024, signal }, + ); + const seconds = Number(stdout.trim()); + if (!Number.isFinite(seconds) || seconds <= 0 || seconds > 86_400) { + throw new Error("Cap source media duration is invalid"); + } + return seconds; +} + +function trackOffset(display: SourceTrack, audio: SourceTrack) { + if ( + typeof display.start_time !== "number" || + typeof audio.start_time !== "number" + ) { + return 0; + } + const offset = audio.start_time - display.start_time; + if (!Number.isFinite(offset) || Math.abs(offset) > 30) { + throw new Error("Cap source voice timing is invalid"); + } + return offset; +} + +async function renderSegmentAudio( + projectPath: string, + segment: SourceSegment, + expected: ExpectedSegment, + outputPath: string, + signal?: AbortSignal, +) { + const displayPath = await checkedMediaPath(projectPath, segment.display.path); + const displayDurationMs = Math.round( + (await probeDuration(displayPath, signal)) * 1000, + ); + if (Math.abs(displayDurationMs - expected.mediaDurationMs) > 2000) { + throw new Error("Cap source display duration changed"); + } + const durationSeconds = (expected.segmentDurationMs / 1000).toFixed(3); + const voiceTracks = expected.hasAudio + ? [segment.mic, segment.system_audio] + .filter( + (track): track is SourceTrack => + track !== null && track !== undefined, + ) + .filter( + (track, index, tracks) => + tracks.findIndex((candidate) => candidate.path === track.path) === + index, + ) + : []; + if (expected.hasAudio && voiceTracks.length === 0) { + throw new Error("Cap source voice tracks changed"); + } + const inputArguments: string[] = []; + const filters: string[] = []; + for (const [index, track] of voiceTracks.entries()) { + const path = await checkedMediaPath(projectPath, track.path); + inputArguments.push("-i", path); + const offset = trackOffset(segment.display, track); + let filter = `[${index}:a:0]aresample=16000,aformat=channel_layouts=mono`; + if (offset < 0) filter += `,atrim=start=${(-offset).toFixed(6)}`; + filter += ",asetpts=PTS-STARTPTS"; + if (offset > 0) filter += `,adelay=${Math.round(offset * 1000)}`; + filters.push(`${filter}[voice${index}]`); + } + if (voiceTracks.length === 0) { + inputArguments.push( + "-f", + "lavfi", + "-i", + "anullsrc=channel_layout=mono:sample_rate=16000", + ); + filters.push( + `[0:a:0]atrim=duration=${durationSeconds},asetpts=PTS-STARTPTS[out]`, + ); + } else { + const labels = voiceTracks.map((_, index) => `[voice${index}]`).join(""); + const mix = + voiceTracks.length === 1 + ? labels + : `${labels}amix=inputs=${voiceTracks.length}:duration=longest:normalize=0,`; + filters.push( + `${mix}apad,atrim=duration=${durationSeconds},asetpts=PTS-STARTPTS[out]`, + ); + } + await runFile( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + ...inputArguments, + "-filter_complex", + filters.join(";"), + "-map", + "[out]", + "-vn", + "-c:a", + "aac", + "-b:a", + "64k", + "-ar", + "16000", + "-ac", + "1", + "-n", + outputPath, + ], + { + timeout: MAX_AUDIO_GENERATION_MS, + maxBuffer: 64 * 1024, + signal, + }, + ); +} + +export async function prepareEditorCapCaptionAudio( + asset: EditorCapAsset, + expectedSegments: readonly ExpectedSegment[], + signal?: AbortSignal, +) { + validateEditorCapAsset(asset); + if (!validExpectedSegments(expectedSegments)) { + throw new Error("Cap caption source timing is invalid"); + } + const root = await mkdtemp(join(tmpdir(), "cap-editor-cap-captions-")); + await chmod(root, 0o700); + const cleanup = () => rm(root, { recursive: true, force: true }); + try { + const staged = await stageSignedEditorCapAsset(root, asset, signal); + const { stdout } = await runFile( + nativeEditorBinary("prepare"), + ["inspect-cap-audio", staged.path], + { + timeout: 60_000, + maxBuffer: 2 * 1024 * 1024, + signal, + }, + ); + const sourceSegments = parseSourceSegments(JSON.parse(stdout)); + if (sourceSegments.length !== expectedSegments.length) { + throw new Error("Cap caption source clip count changed"); + } + for (const [index, source] of sourceSegments.entries()) { + const expected = expectedSegments[index]; + if (!expected) throw new Error("Cap caption source timing changed"); + await renderSegmentAudio( + staged.path, + source, + expected, + join(root, `clip-${index}.m4a`), + signal, + ); + } + await staged.cleanup(); + const concatPath = join(root, "clips.ffconcat"); + await writeFile( + concatPath, + `ffconcat version 1.0\n${expectedSegments + .map( + (segment, index) => + `file 'clip-${index}.m4a'\nduration ${( + segment.segmentDurationMs / 1000 + ).toFixed(3)}\n`, + ) + .join("")}`, + { flag: "wx", mode: 0o600 }, + ); + const outputPath = join(root, "captions.m4a"); + await runFile( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-f", + "concat", + "-safe", + "0", + "-i", + concatPath, + "-vn", + "-c:a", + "aac", + "-b:a", + "64k", + "-ar", + "16000", + "-ac", + "1", + "-n", + outputPath, + ], + { + timeout: MAX_AUDIO_GENERATION_MS, + maxBuffer: 64 * 1024, + signal, + }, + ); + const expectedMs = expectedSegments.reduce( + (total, segment) => total + segment.segmentDurationMs, + 0, + ); + const actualMs = Math.round( + (await probeDuration(outputPath, signal)) * 1000, + ); + if (Math.abs(expectedMs - actualMs) > 250) { + throw new Error("Cap caption audio timing changed"); + } + const metadata = await lstat(outputPath); + if (!metadata.isFile() || metadata.size <= 0) { + throw new Error("Cap caption audio is unavailable"); + } + return { path: outputPath, size: metadata.size, cleanup }; + } catch (error) { + await cleanup(); + throw error; + } +} diff --git a/apps/media-server/src/lib/editor-cap-imports.ts b/apps/media-server/src/lib/editor-cap-imports.ts new file mode 100644 index 00000000000..1b85c94a106 --- /dev/null +++ b/apps/media-server/src/lib/editor-cap-imports.ts @@ -0,0 +1,184 @@ +import { randomUUID } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { + type EditorCapAsset, + stageSignedEditorCapAsset, + validateEditorCapAsset, +} from "./editor-cap-assets"; +import { mapEditorConfigPaths } from "./editor-config-paths"; +import { nativeEditorBinary } from "./editor-native"; +import { runEditorFile as runFile } from "./editor-process"; + +const IMPORT_TIMEOUT_MS = 30 * 60 * 1000; + +type ImportStatus = "staging" | "ready" | "error" | "canceled"; +type ImportResult = { + path: string; + name: string; + clipCount: number; + projectConfig: Record; +}; +type CapImport = { + id: string; + sessionId: string; + asset: EditorCapAsset; + status: ImportStatus; + result: ImportResult | null; + error: string | null; + controller: AbortController; + task: Promise; +}; + +const imports = new Map(); +const activeBySession = new Map(); + +export class EditorCapImportBusyError extends Error {} + +function clipCount(stdout: string | Buffer) { + let value: unknown; + try { + value = JSON.parse(stdout.toString()); + } catch { + throw new Error("Cap project import returned invalid clip metadata"); + } + if ( + typeof value !== "object" || + value === null || + !("clipCount" in value) || + typeof value.clipCount !== "number" || + !Number.isSafeInteger(value.clipCount) || + value.clipCount < 1 || + value.clipCount > 1000 + ) { + throw new Error("Cap project import returned invalid clip count"); + } + return value.clipCount; +} + +export function beginEditorCapImport( + sessionId: string, + projectPath: string, + asset: EditorCapAsset, +) { + validateEditorCapAsset(asset); + for (const job of imports.values()) { + if (job.sessionId !== sessionId || job.asset.path !== asset.path) continue; + if ( + job.asset.size !== asset.size || + job.asset.objectIdentity !== asset.objectIdentity || + job.asset.contentType !== asset.contentType + ) { + throw new Error("Imported Cap project identity changed"); + } + if (job.status === "ready" || job.status === "staging") return job.id; + imports.delete(job.id); + } + if (activeBySession.has(sessionId)) { + throw new EditorCapImportBusyError("A Cap project is still importing"); + } + const job: CapImport = { + id: randomUUID(), + sessionId, + asset, + status: "staging", + result: null, + error: null, + controller: new AbortController(), + task: Promise.resolve(), + }; + imports.set(job.id, job); + activeBySession.set(sessionId, job.id); + job.task = (async () => { + const staged = await stageSignedEditorCapAsset( + projectPath, + asset, + job.controller.signal, + ); + try { + const { stdout } = await runFile( + nativeEditorBinary("prepare"), + ["append-cap", projectPath, staged.path], + { + timeout: IMPORT_TIMEOUT_MS, + maxBuffer: 64 * 1024, + signal: job.controller.signal, + }, + ); + const savedConfig = await readFile( + join(projectPath, "project-config.json"), + "utf8", + ); + if (Buffer.byteLength(savedConfig, "utf8") > 8 * 1024 * 1024) { + throw new Error("Imported Cap project settings are too large"); + } + const nativeConfig: unknown = JSON.parse(savedConfig); + if ( + typeof nativeConfig !== "object" || + nativeConfig === null || + Array.isArray(nativeConfig) + ) { + throw new Error("Imported Cap project settings are invalid"); + } + return { + path: asset.path, + name: asset.name, + clipCount: clipCount(stdout), + projectConfig: mapEditorConfigPaths( + nativeConfig as Record, + "browser", + projectPath, + ), + }; + } finally { + await staged.cleanup(); + } + })() + .then((result) => { + job.result = result; + job.status = "ready"; + }) + .catch((error) => { + job.status = job.controller.signal.aborted ? "canceled" : "error"; + job.error = + error instanceof Error + ? error.message + : "Cap project could not be imported"; + }) + .finally(() => { + if (activeBySession.get(sessionId) === job.id) { + activeBySession.delete(sessionId); + } + }); + return job.id; +} + +export function getEditorCapImport(sessionId: string, id: string) { + const job = imports.get(id); + if (!job || job.sessionId !== sessionId) return null; + return { + id: job.id, + status: job.status, + result: job.result, + error: job.error, + }; +} + +export async function cancelEditorCapImport(sessionId: string, id: string) { + const job = imports.get(id); + if (!job || job.sessionId !== sessionId) return false; + job.controller.abort(); + await job.task; + imports.delete(id); + return true; +} + +export async function closeEditorCapImports(sessionId: string) { + const jobs = [...imports.values()].filter( + (job) => job.sessionId === sessionId, + ); + for (const job of jobs) job.controller.abort(); + await Promise.all(jobs.map((job) => job.task)); + for (const job of jobs) imports.delete(job.id); + activeBySession.delete(sessionId); +} diff --git a/apps/media-server/src/lib/editor-clip-thumbnails.ts b/apps/media-server/src/lib/editor-clip-thumbnails.ts new file mode 100644 index 00000000000..2b9c6dccad0 --- /dev/null +++ b/apps/media-server/src/lib/editor-clip-thumbnails.ts @@ -0,0 +1,60 @@ +import type { startNativeEditorSession } from "./editor-native"; + +type NativeEditorSession = Awaited>; + +const MAX_THUMBNAIL_BYTES = 256 * 1024; + +export async function renderEditorClipThumbnail( + native: NativeEditorSession, + recordingSegment: unknown, + time: unknown, + abortSignal?: AbortSignal, +) { + if ( + typeof recordingSegment !== "number" || + !Number.isInteger(recordingSegment) || + recordingSegment < 0 || + recordingSegment > 1024 || + typeof time !== "number" || + !Number.isFinite(time) || + time < 0 || + time > 43_200 + ) { + throw new Error("Invalid clip thumbnail request"); + } + const timeMs = Math.round(time * 1000); + const response = await native.request( + `/clip-thumbnail/${recordingSegment}/${timeMs}`, + { + signal: AbortSignal.any([ + abortSignal ?? new AbortController().signal, + AbortSignal.timeout(10_000), + ]), + }, + ); + if (!response.ok) { + throw new Error(`Editor thumbnail request failed: ${response.status}`); + } + if (response.headers.get("content-type") !== "image/jpeg") { + throw new Error("Editor thumbnail has an invalid format"); + } + const contentLength = Number(response.headers.get("content-length")); + if ( + !Number.isSafeInteger(contentLength) || + contentLength < 4 || + contentLength > MAX_THUMBNAIL_BYTES + ) { + throw new Error("Editor thumbnail has an invalid size"); + } + const bytes = Buffer.from(await response.arrayBuffer()); + if ( + bytes.length !== contentLength || + bytes[0] !== 0xff || + bytes[1] !== 0xd8 || + bytes[bytes.length - 2] !== 0xff || + bytes[bytes.length - 1] !== 0xd9 + ) { + throw new Error("Editor thumbnail is damaged"); + } + return `data:image/jpeg;base64,${bytes.toString("base64")}`; +} diff --git a/apps/media-server/src/lib/editor-command-socket.ts b/apps/media-server/src/lib/editor-command-socket.ts new file mode 100644 index 00000000000..9c527ac7f0b --- /dev/null +++ b/apps/media-server/src/lib/editor-command-socket.ts @@ -0,0 +1,279 @@ +import { z } from "zod"; +import { hasEditorCaptionContent } from "../../../web/lib/editor-caption-access"; +import { + createEditorCaptionCache, + type EditorCaptionCache, + restoreEditorCaptionConfig, +} from "../../../web/lib/editor-caption-transport"; +import { listEditorAudioLibrary } from "./editor-audio-library"; + +const MAX_CONFIG_BYTES = 8 * 1024 * 1024; +const MAX_FRAME_NUMBER = 1_000_000; + +const resolutionSchema = z.object({ + x: z.number().int().min(1).max(3840), + y: z.number().int().min(1).max(2160), +}); +const frameSchema = z.object({ + frameNumber: z.number().int().min(0).max(MAX_FRAME_NUMBER), + fps: z.number().int().min(1).max(60), + resolutionBase: resolutionSchema, +}); +const keyboardGenerationSchema = z.object({ + groupingThresholdMs: z.number().finite().min(0).max(60_000), + lingerDurationMs: z.number().finite().min(0).max(60_000), + showModifiers: z.boolean(), + showSpecialKeys: z.boolean(), +}); +const autoZoomSchema = z.object({ + zoomAmount: z.number().finite().min(0.1).max(10), +}); +const renderEventSchema = z.object({ + frame_number: z.number().int().min(0).max(MAX_FRAME_NUMBER), + fps: z.number().int().min(1).max(60), + resolution_base: resolutionSchema, +}); +const socketRequestSchema = z.object({ + kind: z.enum(["invoke", "emit"]), + id: z.number().int().min(1).max(1_000_000_000), + name: z.string().min(1).max(100), + args: z.array(z.unknown()).max(4), +}); + +export type EditorSocketRequest = z.infer; +export type EditorCommandState = { + frameNumber: number; + fps: number; + resolutionBase: { x: number; y: number }; + captionsEnabled?: boolean; + captionCache?: EditorCaptionCache | null; +}; +export type NativeEditorRequest = ( + path: string, + method: "GET" | "PUT" | "POST" | "DELETE", + body?: unknown, + eventOnly?: boolean, +) => Promise; +export type EditorSocketReply = + | { kind: "result"; id: number; value: unknown } + | { kind: "error"; id: number; error: string }; + +export function parseEditorSocketRequest(value: unknown) { + const result = socketRequestSchema.safeParse(value); + return result.success ? result.data : null; +} + +function asObject(value: unknown, cache: EditorCaptionCache | null) { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error("Invalid editor project configuration"); + } + const restored = restoreEditorCaptionConfig( + value as Record, + cache, + ); + if (!restored) throw new Error("Caption payload cache is unavailable"); + const json = JSON.stringify(restored); + if (Buffer.byteLength(json, "utf8") > MAX_CONFIG_BYTES) { + throw new Error("Editor project configuration is too large"); + } + return restored; +} + +function asInstance(value: unknown, sessionId: string) { + if ( + typeof value !== "object" || + value === null || + Array.isArray(value) || + !("instanceId" in value) || + typeof value.instanceId !== "string" + ) { + throw new Error("Editor instance is unavailable"); + } + return { + ...value, + path: `cap-web-editor://session/${sessionId}`, + framesSocketUrl: "", + audioSocketUrl: "", + eventsSocketUrl: "", + }; +} + +function asFrameNumber(value: unknown) { + return frameSchema.shape.frameNumber.parse(value); +} + +async function executeEditorCommand( + sessionId: string, + request: EditorSocketRequest, + state: EditorCommandState, + native: NativeEditorRequest, +) { + const args = request.args; + if (request.kind === "emit") { + if (request.name !== "renderFrameEvent") { + throw new Error(`Unsupported editor event: ${request.name}`); + } + const event = renderEventSchema.parse(args[0]); + state.frameNumber = event.frame_number; + state.fps = event.fps; + state.resolutionBase = event.resolution_base; + await native( + "/preview", + "POST", + { + frameNumber: event.frame_number, + fps: event.fps, + resolutionBase: event.resolution_base, + }, + true, + ); + return null; + } + switch (request.name) { + case "getEditorProjectPath": + return `cap-web-editor://session/${sessionId}`; + case "getRecordingMetaByPath": + if (args[0] !== `cap-web-editor://session/${sessionId}`) { + throw new Error("Invalid editor project path"); + } + return native("/meta", "GET"); + case "getEditorMeta": + return native("/meta", "GET"); + case "getVideoMetadata": { + if (args[0] !== `cap-web-editor://session/${sessionId}`) + throw new Error("Invalid editor project path"); + const instance = await native("/instance", "GET"); + if ( + typeof instance !== "object" || + instance === null || + !("recordingDuration" in instance) || + typeof instance.recordingDuration !== "number" || + !Number.isFinite(instance.recordingDuration) || + instance.recordingDuration <= 0 + ) { + throw new Error("Recording duration is unavailable"); + } + return { + duration: instance.recordingDuration, + size: (8_192_000 * instance.recordingDuration) / (8 * 1024 * 1024), + }; + } + case "checkUpgradedAndUpdate": + return state.captionsEnabled === true; + case "createEditorInstance": + return asInstance(await native("/instance", "GET"), sessionId); + case "getDefaultProjectConfig": + return native("/default-config", "GET"); + case "tauri:list_system_fonts": + return native("/fonts", "GET"); + case "animatedGradientCatalog": + return native("/animated-gradients", "GET"); + case "randomAnimatedGradient": + return native("/animated-gradients/random", "GET"); + case "loadCaptions": { + if (state.captionsEnabled === false) return null; + const config = await native("/config", "GET"); + return typeof config === "object" && + config !== null && + "captions" in config + ? config.captions + : null; + } + case "getMicWaveforms": + return native("/waveforms/mic", "GET"); + case "getSystemAudioWaveforms": + return native("/waveforms/system", "GET"); + case "generateKeyboardSegments": { + const settings = keyboardGenerationSchema.parse({ + groupingThresholdMs: args[0], + lingerDurationMs: args[1], + showModifiers: args[2], + showSpecialKeys: args[3], + }); + return native("/keyboard-segments", "POST", settings); + } + case "generateZoomSegmentsFromClicks": { + const settings = autoZoomSchema.parse({ zoomAmount: args[0] }); + return native("/auto-zoom-segments", "POST", settings); + } + case "listAudioLibrary": + return listEditorAudioLibrary(); + case "setWindowTransparent": + case "tauri:get_recording_recovery_success": + return request.name === "setWindowTransparent" ? null : false; + case "setProjectConfig": + throw new Error("Editor project saves require the authenticated web API"); + case "updateProjectConfigInMemory": { + const reused = + typeof args[0] === "object" && + args[0] !== null && + "webCaptionRef" in args[0]; + const config = asObject(args[0], state.captionCache ?? null); + if (state.captionsEnabled === false && hasEditorCaptionContent(config)) { + throw new Error("Cap Pro is required for captions"); + } + await native("/config/memory", "PUT", config); + if (!reused) state.captionCache = await createEditorCaptionCache(config); + if (args[1] !== null && args[2] !== null && args[3] !== null) { + const frame = frameSchema.parse({ + frameNumber: args[1], + fps: args[2], + resolutionBase: args[3], + }); + state.frameNumber = frame.frameNumber; + state.fps = frame.fps; + state.resolutionBase = frame.resolutionBase; + await native("/preview", "POST", frame, true); + } + return null; + } + case "startPlayback": { + const frame = frameSchema.parse({ + frameNumber: state.frameNumber, + fps: args[0], + resolutionBase: args[1], + }); + state.fps = frame.fps; + state.resolutionBase = frame.resolutionBase; + await native("/playback", "POST", frame); + return null; + } + case "stopPlayback": + await native("/playback", "DELETE"); + return null; + case "seekTo": + case "setPlayheadPosition": { + const frameNumber = asFrameNumber(args[0]); + await native("/seek", "PUT", { + frameNumber, + fps: state.fps, + resolutionBase: state.resolutionBase, + }); + state.frameNumber = frameNumber; + return null; + } + default: + throw new Error(`Unsupported editor command: ${request.name}`); + } +} + +export async function dispatchEditorSocketCommand( + sessionId: string, + request: EditorSocketRequest, + state: EditorCommandState, + native: NativeEditorRequest, +): Promise { + try { + return { + kind: "result", + id: request.id, + value: await executeEditorCommand(sessionId, request, state, native), + }; + } catch (cause) { + return { + kind: "error", + id: request.id, + error: cause instanceof Error ? cause.message : "Editor command failed", + }; + } +} diff --git a/apps/media-server/src/lib/editor-config-paths.ts b/apps/media-server/src/lib/editor-config-paths.ts new file mode 100644 index 00000000000..c4189f82549 --- /dev/null +++ b/apps/media-server/src/lib/editor-config-paths.ts @@ -0,0 +1,163 @@ +import { isAbsolute, relative, resolve } from "node:path"; +import { + editorWallpaperDirectory, + mapEditorWallpaperConfig, +} from "./editor-wallpapers"; + +const assetId = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"; +const uploadedWallpaperPath = new RegExp( + `^content/images/${assetId}\\.(?:png|jpg|webp|gif|bmp|tiff)$`, +); +const imagePath = new RegExp( + `^(?:content/images/${assetId}\\.(?:png|jpg|webp|gif|bmp|tiff)|content/segments/segment-[0-9]+/display\\.(?:png|jpg|webp|gif|bmp|tiff))$`, +); +const audioPath = new RegExp( + `^assets/audio/(?:library-[a-z0-9-]+\\.mp3|import-${assetId}\\.(?:ogg|m4a|mp3|wav|aac|flac))$`, +); +const videoPath = new RegExp( + `^content/videos/${assetId}\\.(?:mp4|mov|avi|mkv|webm|wmv|m4v|flv)$`, +); +const wallpaperPath = + /^cap-web-wallpaper:\/\/assets\/backgrounds\/(macOS|blue|purple|cities|dark|orange)\/[a-z0-9-]+\.jpg$/; + +function asRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function mapBackground( + background: unknown, + direction: "native" | "browser", + projectPath: string, + wallpaperDirectory: string, +) { + if (!asRecord(background) || !asRecord(background.source)) return background; + const source = background.source; + const path = source.path; + if (path === null || path === undefined || path === "") return background; + if (typeof path !== "string") + throw new Error("Invalid editor background path"); + if (source.type === "wallpaper") { + const uploadedPath = + direction === "native" + ? uploadedWallpaperPath.test(path) + ? resolve(projectPath, path) + : null + : isAbsolute(path) + ? relative(projectPath, path) + : path; + if ( + uploadedPath && + uploadedWallpaperPath.test(direction === "native" ? path : uploadedPath) + ) { + return uploadedPath === path + ? background + : { ...background, source: { ...source, path: uploadedPath } }; + } + if (direction === "native" && !wallpaperPath.test(path)) { + throw new Error("Invalid editor wallpaper path"); + } + const mapped = mapEditorWallpaperConfig( + { background }, + direction, + wallpaperDirectory, + ) as { background: Record }; + const mappedSource = mapped.background.source; + if ( + direction === "browser" && + (!asRecord(mappedSource) || + !wallpaperPath.test(String(mappedSource.path))) + ) { + throw new Error("Invalid editor wallpaper path"); + } + return mapped.background; + } + if (source.type !== "image") return background; + const mappedPath = + direction === "native" + ? imagePath.test(path) + ? resolve(projectPath, path) + : null + : isAbsolute(path) + ? relative(projectPath, path) + : path; + if (!mappedPath || (direction === "browser" && !imagePath.test(mappedPath))) { + throw new Error("Invalid editor background image path"); + } + if (mappedPath === path) return background; + return { ...background, source: { ...source, path: mappedPath } }; +} + +function validateSegments(timeline: Record) { + for (const [name, allowed] of [ + ["audioSegments", audioPath], + ["imageSegments", imagePath], + ["videoSegments", videoPath], + ] as const) { + const segments = timeline[name]; + if (segments === undefined) continue; + if (!Array.isArray(segments)) throw new Error("Invalid editor media track"); + for (const segment of segments) { + if ( + !asRecord(segment) || + typeof segment.path !== "string" || + (segment.path !== "" && !allowed.test(segment.path)) + ) { + throw new Error(`Invalid editor ${name} path`); + } + } + } +} + +export function mapEditorConfigPaths( + value: unknown, + direction: "native" | "browser", + projectPath: string, + wallpaperDirectory = editorWallpaperDirectory(), +) { + if (!asRecord(value)) throw new Error("Invalid editor project configuration"); + const background = mapBackground( + value.background, + direction, + projectPath, + wallpaperDirectory, + ); + let timeline = value.timeline; + if (asRecord(timeline)) { + validateSegments(timeline); + if (Array.isArray(timeline.styleSegments)) { + const styleSegments = timeline.styleSegments.map((segment) => { + if (!asRecord(segment) || !asRecord(segment.overrides)) return segment; + const overrides = segment.overrides; + const mapped = mapBackground( + overrides.background, + direction, + projectPath, + wallpaperDirectory, + ); + return mapped === overrides.background + ? segment + : { ...segment, overrides: { ...overrides, background: mapped } }; + }); + timeline = { ...timeline, styleSegments }; + } + } + if (background === value.background && timeline === value.timeline) + return value; + return { ...value, background, timeline }; +} + +export function mapEditorInstanceConfigPaths( + value: unknown, + direction: "native" | "browser", + projectPath: string, +) { + if (!asRecord(value) || !asRecord(value.savedProjectConfig)) return value; + return { + ...value, + savedProjectConfig: mapEditorConfigPaths( + value.savedProjectConfig, + direction, + projectPath, + ), + }; +} diff --git a/apps/media-server/src/lib/editor-export-estimates.ts b/apps/media-server/src/lib/editor-export-estimates.ts new file mode 100644 index 00000000000..b5c04f4e0dd --- /dev/null +++ b/apps/media-server/src/lib/editor-export-estimates.ts @@ -0,0 +1,192 @@ +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createInterface } from "node:readline"; +import { editorExportSettingsSchema } from "./editor-exports"; +import { + nativeEditorBinary, + type startNativeEditorSession, +} from "./editor-native"; +import { editorProcessEnv } from "./editor-process"; + +type NativeSession = Awaited>; +export type EditorExportEstimate = { + duration_seconds: number; + estimated_time_seconds: number; + estimated_size_mb: number; + time_range_seconds: [number, number]; + size_range_mb: [number, number]; +}; +type ActiveEstimate = { controller: AbortController; done: Promise }; + +const active = new Map(); +const cache = new Map< + string, + { result: EditorExportEstimate; expiresAt: number } +>(); +const ESTIMATE_TIMEOUT_MS = 30_000; +const CACHE_TTL_MS = 5 * 60 * 1000; +const MAX_CACHED_ESTIMATES = 64; + +function isEstimate(value: unknown): value is EditorExportEstimate { + if (typeof value !== "object" || value === null) return false; + const candidate = value as Record; + const fields = [ + "duration_seconds", + "estimated_time_seconds", + "estimated_size_mb", + ] as const; + if ( + fields.some((field) => { + const item = candidate[field]; + return typeof item !== "number" || !Number.isFinite(item) || item < 0; + }) || + !("time_range_seconds" in value) || + !("size_range_mb" in value) + ) { + return false; + } + return [value.time_range_seconds, value.size_range_mb].every( + (range) => + Array.isArray(range) && + range.length === 2 && + range.every( + (item) => + typeof item === "number" && Number.isFinite(item) && item >= 0, + ), + ); +} + +function channelId(value: unknown) { + if (typeof value !== "string") return null; + const match = /^__CHANNEL__:(\d+)$/.exec(value); + if (!match) return null; + const id = Number(match[1]); + return Number.isSafeInteger(id) ? id : null; +} + +export function cancelEditorExportEstimate(sessionId: string) { + active.get(sessionId)?.controller.abort(); +} + +export async function getEditorExportEstimate( + sessionId: string, + native: NativeSession, + args: unknown[], + socketSignal: AbortSignal, + onEstimate: (channel: number, value: EditorExportEstimate) => void, +) { + const [projectPath, settingsInput, channel] = args; + const id = channelId(channel); + if (projectPath !== `cap-web-editor://session/${sessionId}` || id === null) { + throw new Error("Invalid editor export estimate request"); + } + const settings = editorExportSettingsSchema.parse(settingsInput); + const previous = active.get(sessionId); + previous?.controller.abort(); + const controller = new AbortController(); + let finish = () => {}; + const done = new Promise((resolve) => { + finish = resolve; + }); + const current = { controller, done }; + active.set(sessionId, current); + const signal = AbortSignal.any([ + controller.signal, + socketSignal, + AbortSignal.timeout(ESTIMATE_TIMEOUT_MS), + ]); + let root: string | null = null; + try { + await previous?.done; + if (signal.aborted) throw new Error("Export estimate canceled"); + const config = await readFile( + join(native.projectPath, "project-config.json"), + ); + const key = createHash("sha256") + .update(native.projectPath) + .update("\0") + .update(config) + .update("\0") + .update(JSON.stringify(settings)) + .digest("hex"); + const cached = cache.get(key); + if (cached && cached.expiresAt > Date.now()) { + onEstimate(id, cached.result); + return cached.result; + } + root = await mkdtemp(join(tmpdir(), "cap-editor-estimate-")); + await chmod(root, 0o700); + const configPath = join(root, "project-config.json"); + const settingsPath = join(root, "settings.json"); + await Promise.all([ + writeFile(configPath, config, { flag: "wx", mode: 0o600 }), + writeFile(settingsPath, JSON.stringify(settings), { + flag: "wx", + mode: 0o600, + }), + ]); + if (signal.aborted) throw new Error("Export estimate canceled"); + const child = spawn( + nativeEditorBinary("prepare"), + ["estimate", native.projectPath, configPath, settingsPath], + { stdio: ["pipe", "pipe", "pipe"], env: editorProcessEnv() }, + ); + child.stdin.end(); + const abort = () => child.kill("SIGTERM"); + signal.addEventListener("abort", abort, { once: true }); + if (signal.aborted) abort(); + let stderr = ""; + child.stderr.on("data", (chunk: Buffer) => { + stderr = `${stderr}${chunk.toString("utf8")}`.slice(-4096); + }); + let result: EditorExportEstimate | null = null; + const lines = createInterface({ input: child.stdout }); + lines.on("line", (line) => { + let output: unknown; + try { + output = JSON.parse(line); + } catch { + return; + } + if ( + typeof output !== "object" || + output === null || + !("kind" in output) || + !("value" in output) || + !isEstimate(output.value) + ) { + return; + } + if (output.kind === "estimate" && !signal.aborted) { + onEstimate(id, output.value); + } else if (output.kind === "result") { + result = output.value; + } + }); + let spawnError: Error | null = null; + child.once("error", (cause) => { + spawnError = cause; + }); + const code = await new Promise((resolve) => { + child.once("close", resolve); + }); + signal.removeEventListener("abort", abort); + lines.close(); + if (signal.aborted) throw new Error("Export estimate canceled"); + if (spawnError || code !== 0 || !result) { + throw spawnError || new Error(stderr || "Native export estimate failed"); + } + cache.set(key, { result, expiresAt: Date.now() + CACHE_TTL_MS }); + if (cache.size > MAX_CACHED_ESTIMATES) { + cache.delete(cache.keys().next().value ?? ""); + } + return result; + } finally { + if (root) await rm(root, { recursive: true, force: true }); + finish(); + if (active.get(sessionId) === current) active.delete(sessionId); + } +} diff --git a/apps/media-server/src/lib/editor-export-previews.ts b/apps/media-server/src/lib/editor-export-previews.ts new file mode 100644 index 00000000000..b541178f7ec --- /dev/null +++ b/apps/media-server/src/lib/editor-export-previews.ts @@ -0,0 +1,170 @@ +import { createHash } from "node:crypto"; +import { + chmod, + lstat, + mkdtemp, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { z } from "zod"; +import { + nativeEditorBinary, + type startNativeEditorSession, +} from "./editor-native"; +import { runEditorFile as runFile } from "./editor-process"; + +const MAX_PREVIEW_BYTES = 8 * 1024 * 1024; +const PREVIEW_TIMEOUT_MS = 30_000; +const CACHE_TTL_MS = 60_000; +const MAX_CACHED_PREVIEWS = 32; + +const settingsSchema = z.object({ + fps: z.number().int().min(1).max(60), + resolution_base: z.object({ + x: z.number().int().min(1).max(3840), + y: z.number().int().min(1).max(2160), + }), + compression_bpp: z.number().finite().min(0).max(1), + cursor_only: z.boolean().optional(), +}); + +type NativeSession = Awaited>; +type PreviewResult = { + jpeg_base64: string; + estimated_size_mb: number; + actual_width: number; + actual_height: number; + frame_render_time_ms: number; + total_frames: number; +}; +const active = new Map(); +const cache = new Map(); + +function parsePreviewOutput(value: unknown) { + if ( + typeof value !== "object" || + value === null || + !("actualWidth" in value) || + !("actualHeight" in value) || + !("frameRenderTimeMs" in value) || + !("totalFrames" in value) || + !("estimatedSizeMb" in value) || + typeof value.actualWidth !== "number" || + typeof value.actualHeight !== "number" || + typeof value.frameRenderTimeMs !== "number" || + typeof value.totalFrames !== "number" || + typeof value.estimatedSizeMb !== "number" || + !Number.isSafeInteger(value.actualWidth) || + !Number.isSafeInteger(value.actualHeight) || + !Number.isSafeInteger(value.totalFrames) || + value.actualWidth < 1 || + value.actualHeight < 1 || + value.totalFrames < 1 || + !Number.isFinite(value.frameRenderTimeMs) || + value.frameRenderTimeMs < 0 || + !Number.isFinite(value.estimatedSizeMb) || + value.estimatedSizeMb < 0 + ) { + throw new Error("Native export preview returned invalid dimensions"); + } + return { + actual_width: value.actualWidth, + actual_height: value.actualHeight, + frame_render_time_ms: value.frameRenderTimeMs, + total_frames: value.totalFrames, + estimated_size_mb: value.estimatedSizeMb, + }; +} + +export async function renderEditorExportPreview( + sessionId: string, + native: NativeSession, + frameTime: unknown, + settingsInput: unknown, + socketSignal: AbortSignal, +): Promise { + if ( + typeof frameTime !== "number" || + !Number.isFinite(frameTime) || + frameTime < 0 || + frameTime > 36_000 + ) { + throw new Error("Invalid export preview frame time"); + } + const settings = settingsSchema.parse(settingsInput); + const projectConfigPath = join(native.projectPath, "project-config.json"); + const config = await readFile(projectConfigPath); + const key = createHash("sha256") + .update(native.projectPath) + .update("\0") + .update(config) + .update("\0") + .update(String(frameTime)) + .update("\0") + .update(JSON.stringify(settings)) + .digest("hex"); + active.get(sessionId)?.abort(); + const cached = cache.get(key); + if (cached && cached.expiresAt > Date.now()) return cached.result; + const controller = new AbortController(); + active.set(sessionId, controller); + const signal = AbortSignal.any([ + controller.signal, + socketSignal, + AbortSignal.timeout(PREVIEW_TIMEOUT_MS), + ]); + const root = await mkdtemp(join(tmpdir(), "cap-editor-preview-")); + try { + await chmod(root, 0o700); + const settingsPath = join(root, "settings.json"); + const configPath = join(root, "project-config.json"); + const outputPath = join(root, "preview.jpg"); + await Promise.all([ + writeFile(settingsPath, JSON.stringify(settings), { + flag: "wx", + mode: 0o600, + }), + writeFile(configPath, config, { flag: "wx", mode: 0o600 }), + ]); + const { stdout } = await runFile( + nativeEditorBinary("prepare"), + [ + "preview", + native.projectPath, + configPath, + String(frameTime), + settingsPath, + outputPath, + ], + { signal, maxBuffer: 64 * 1024 }, + ); + if (signal.aborted) throw new Error("Export preview was canceled"); + let metadata: unknown; + try { + metadata = JSON.parse(stdout.trim()); + } catch { + throw new Error("Native export preview returned invalid metadata"); + } + const dimensions = parsePreviewOutput(metadata); + const file = await lstat(outputPath); + if (!file.isFile() || file.size === 0 || file.size > MAX_PREVIEW_BYTES) { + throw new Error("Native export preview image was invalid"); + } + const image = await readFile(outputPath); + const result: PreviewResult = { + ...dimensions, + jpeg_base64: image.toString("base64"), + }; + cache.set(key, { result, expiresAt: Date.now() + CACHE_TTL_MS }); + if (cache.size > MAX_CACHED_PREVIEWS) { + cache.delete(cache.keys().next().value ?? ""); + } + return result; + } finally { + if (active.get(sessionId) === controller) active.delete(sessionId); + await rm(root, { recursive: true, force: true }); + } +} diff --git a/apps/media-server/src/lib/editor-exports.ts b/apps/media-server/src/lib/editor-exports.ts new file mode 100644 index 00000000000..07977d1df6b --- /dev/null +++ b/apps/media-server/src/lib/editor-exports.ts @@ -0,0 +1,476 @@ +import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process"; +import { randomBytes, randomUUID } from "node:crypto"; +import { + chmod, + copyFile, + link, + lstat, + mkdtemp, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { createInterface } from "node:readline"; +import { z } from "zod"; +import { + nativeEditorBinary, + type startNativeEditorSession, +} from "./editor-native"; +import { editorProcessEnv } from "./editor-process"; +import { publicEditorOrigin } from "./editor-socket-tickets"; +import { probeVideo } from "./editor-video-assets"; + +const resolutionSchema = z.object({ + x: z.number().int().min(1).max(3840), + y: z.number().int().min(1).max(2160), +}); +const baseSettings = z.object({ + fps: z.number().int().min(1).max(60), + resolution_base: resolutionSchema, +}); + +export const editorExportSettingsSchema = z.discriminatedUnion("format", [ + baseSettings.extend({ + format: z.literal("Mp4"), + compression: z.enum(["Maximum", "Social", "Web", "Potato"]), + custom_bpp: z.number().positive().max(1).nullable(), + force_ffmpeg_decoder: z.boolean().optional(), + optimize_filesize: z.boolean().optional(), + }), + baseSettings.extend({ + format: z.literal("Gif"), + quality: z + .object({ + quality: z.number().int().min(1).max(100).nullable().optional(), + fast: z.boolean().nullable().optional(), + }) + .nullable(), + }), + baseSettings.extend({ + format: z.literal("Mov"), + cursor_only: z.boolean().optional(), + }), +]); + +export type EditorExportSettings = z.infer; +type NativeSession = Awaited>; +type ExportStatus = "running" | "ready" | "error" | "canceled"; +type ExportProgress = { rendered_count: number; total_frames: number }; + +type ExportJob = { + id: string; + sessionId: string; + status: ExportStatus; + format: EditorExportSettings["format"]; + settingsPath: string; + outputPath: string; + progress: ExportProgress | null; + error: string | null; + downloadStartedAt: number | null; + size: number | null; + mediaMetadata: { + duration: number; + width: number; + height: number; + fps: number; + } | null; + child: ChildProcessWithoutNullStreams; + task: Promise; + startedAt: number; + finishedAt: number | null; + lastSeenAt: number; +}; + +const jobs = new Map(); +const starting = new Map>(); +const closingSessions = new Set(); +const EXPORT_TIMEOUT_MS = 20 * 60 * 1000; +const DOWNLOAD_TICKET_TTL_MS = 60_000; +const MAX_DOWNLOAD_TICKETS = 100; +const downloadTickets = new Map< + string, + { + sessionId: string; + exportId: string; + fileName: string; + format: EditorExportSettings["format"]; + path: string; + root: string; + expiresAt: number; + } +>(); + +export class EditorExportBusyError extends Error {} + +function extension(format: EditorExportSettings["format"]) { + return format === "Mp4" ? "mp4" : format === "Gif" ? "gif" : "mov"; +} + +function parseProgress(line: string): ExportProgress | null { + let value: unknown; + try { + value = JSON.parse(line); + } catch { + return null; + } + if ( + typeof value !== "object" || + value === null || + !("rendered_count" in value) || + !("total_frames" in value) || + typeof value.rendered_count !== "number" || + typeof value.total_frames !== "number" || + !Number.isSafeInteger(value.rendered_count) || + !Number.isSafeInteger(value.total_frames) || + value.rendered_count < 0 || + value.total_frames < 0 || + value.rendered_count > value.total_frames + ) { + return null; + } + return { + rendered_count: value.rendered_count, + total_frames: value.total_frames, + }; +} + +export async function beginEditorExport( + sessionId: string, + native: NativeSession, + settings: EditorExportSettings, +) { + if ( + closingSessions.has(sessionId) || + starting.has(sessionId) || + [...jobs.values()].some((job) => job.sessionId === sessionId) + ) { + throw new EditorExportBusyError( + "Finish or cancel the current editor export first", + ); + } + let settleStart = () => {}; + starting.set( + sessionId, + new Promise((resolve) => { + settleStart = resolve; + }), + ); + const id = randomUUID(); + const root = dirname(native.projectPath); + const settingsPath = join(root, `export-${id}.json`); + const outputPath = join(root, `export-${id}.${extension(settings.format)}`); + let child: ChildProcessWithoutNullStreams; + try { + await writeFile(settingsPath, JSON.stringify(settings), { + flag: "wx", + mode: 0o600, + }); + if (closingSessions.has(sessionId)) { + throw new Error("Editor session is closing"); + } + child = spawn( + nativeEditorBinary("prepare"), + [ + "export", + native.projectPath, + join(native.projectPath, "project-config.json"), + settingsPath, + outputPath, + ], + { stdio: ["pipe", "pipe", "pipe"], env: editorProcessEnv() }, + ); + } catch (cause) { + starting.delete(sessionId); + settleStart(); + await rm(settingsPath, { force: true }); + throw cause; + } + const job: ExportJob = { + id, + sessionId, + status: "running", + format: settings.format, + settingsPath, + outputPath, + progress: null, + error: null, + downloadStartedAt: null, + size: null, + mediaMetadata: null, + child, + task: Promise.resolve(), + startedAt: Date.now(), + finishedAt: null, + lastSeenAt: Date.now(), + }; + jobs.set(id, job); + child.stdin.end(); + const stdout = createInterface({ input: child.stdout }); + stdout.on("line", (line) => { + const progress = parseProgress(line); + if (progress) job.progress = progress; + }); + let stderr = ""; + child.stderr.on("data", (chunk: Buffer) => { + stderr = `${stderr}${chunk.toString("utf8")}`.slice(-4096); + }); + job.task = new Promise((resolve) => { + let failure: Error | null = null; + const timer = setTimeout(() => { + failure = new Error("Editor export timed out"); + child.kill("SIGKILL"); + }, EXPORT_TIMEOUT_MS); + timer.unref(); + child.once("error", (cause) => { + failure = cause; + }); + child.once("close", async (code) => { + clearTimeout(timer); + stdout.close(); + if (job.status === "canceled") { + job.finishedAt = Date.now(); + resolve(); + return; + } + if (failure || code !== 0) { + job.status = "error"; + job.error = failure?.message || stderr || "Editor export failed"; + await rm(job.outputPath, { force: true }); + job.finishedAt = Date.now(); + resolve(); + return; + } + try { + const output = await lstat(job.outputPath); + if (!output.isFile() || output.size === 0) { + throw new Error("Editor export did not produce a video file"); + } + if (output.size > 12 * 1024 * 1024 * 1024) { + throw new Error("Editor export exceeds the maximum recording size"); + } + job.size = output.size; + if (job.format === "Mp4") { + const video = await probeVideo(job.outputPath); + job.mediaMetadata = { + duration: video.duration, + width: video.width, + height: video.height, + fps: video.fps, + }; + } + job.status = "ready"; + } catch (cause) { + job.status = "error"; + job.error = + cause instanceof Error ? cause.message : "Editor export failed"; + await rm(job.outputPath, { force: true }); + } + job.finishedAt = Date.now(); + resolve(); + }); + }); + starting.delete(sessionId); + settleStart(); + return id; +} + +export function getEditorExport(sessionId: string, id: string) { + const job = jobs.get(id); + if (!job || job.sessionId !== sessionId) return null; + job.lastSeenAt = Date.now(); + return { + id: job.id, + status: job.status, + format: job.format, + progress: job.progress, + error: job.error, + downloadStartedAt: job.downloadStartedAt, + size: job.size, + mediaMetadata: job.mediaMetadata, + startedAt: job.startedAt, + }; +} + +/// When the session's export last showed signs of a client: a running export +/// counts only while someone polls it, so a closed browser doesn't hold the +/// worker until the export finishes. +export function editorExportActivity(sessionId: string) { + const job = [...jobs.values()].find( + (candidate) => candidate.sessionId === sessionId, + ); + if (!job) return null; + const running = job.status === "running" || job.finishedAt === null; + return { + at: running + ? job.lastSeenAt + : Math.max(job.finishedAt ?? 0, job.lastSeenAt), + running, + }; +} + +export function getEditorExportFile(sessionId: string, id: string) { + const job = jobs.get(id); + return job?.sessionId === sessionId && job.status === "ready" + ? job.outputPath + : null; +} + +export function markEditorExportDownloadStarted(sessionId: string, id: string) { + const job = jobs.get(id); + if (job?.sessionId === sessionId && job.status === "ready") { + job.downloadStartedAt = Date.now(); + } +} + +export async function createEditorExportDownloadTicket( + sessionId: string, + id: string, + requestedName?: string, +) { + const job = jobs.get(id); + if (!job || job.sessionId !== sessionId || job.status !== "ready") + return null; + if (downloadTickets.size >= MAX_DOWNLOAD_TICKETS) { + throw new Error("Editor download capacity is busy"); + } + const origin = publicEditorOrigin(); + const suffix = extension(job.format); + const requestedStem = requestedName?.endsWith(`.${suffix}`) + ? requestedName.slice(0, -suffix.length - 1) + : null; + const fileName = + requestedStem && + requestedStem.length <= 180 && + !requestedStem.includes("/") && + !requestedStem.includes("\\") && + [...requestedStem].every( + (character) => + character.charCodeAt(0) > 31 && character.charCodeAt(0) !== 127, + ) + ? `${requestedStem}.${suffix}` + : `cap-export-${id}.${suffix}`; + const root = await mkdtemp(join(tmpdir(), "cap-web-editor-download-")); + try { + await chmod(root, 0o700); + const path = join(root, fileName); + try { + await link(job.outputPath, path); + } catch (cause) { + if ( + typeof cause !== "object" || + cause === null || + !("code" in cause) || + cause.code !== "EXDEV" + ) { + throw cause; + } + await copyFile(job.outputPath, path); + } + if (jobs.get(id) !== job || job.status !== "ready") { + await rm(root, { recursive: true, force: true }); + return null; + } + const ticket = randomBytes(32).toString("base64url"); + downloadTickets.set(ticket, { + sessionId, + exportId: id, + fileName, + format: job.format, + path, + root, + expiresAt: Date.now() + DOWNLOAD_TICKET_TTL_MS, + }); + return { + url: `${origin}/editor/sessions/${encodeURIComponent(sessionId)}/exports/${encodeURIComponent(id)}/download?ticket=${ticket}`, + }; + } catch (cause) { + await rm(root, { recursive: true, force: true }); + throw cause; + } +} + +export function consumeEditorExportDownloadTicket( + sessionId: string, + id: string, + token: string | null, +) { + if (!token || !/^[A-Za-z0-9_-]{43}$/.test(token)) return null; + const ticket = downloadTickets.get(token); + downloadTickets.delete(token); + if ( + !ticket || + ticket.sessionId !== sessionId || + ticket.exportId !== id || + ticket.expiresAt < Date.now() + ) { + return null; + } + return ticket; +} + +export async function cancelEditorExport( + sessionId: string, + id: string, + preserveDownloadTickets = false, +) { + const job = jobs.get(id); + if (!job || job.sessionId !== sessionId) return false; + jobs.delete(id); + const ticketRoots: string[] = []; + for (const [token, ticket] of downloadTickets) { + if ( + !preserveDownloadTickets && + ticket.sessionId === sessionId && + ticket.exportId === id + ) { + downloadTickets.delete(token); + ticketRoots.push(ticket.root); + } + } + if (job.status === "running") { + job.status = "canceled"; + job.child.kill("SIGTERM"); + const stopped = await Promise.race([ + job.task.then(() => true), + Bun.sleep(5000).then(() => false), + ]); + if (!stopped) { + job.child.kill("SIGKILL"); + await job.task; + } + } + await Promise.all([ + rm(job.settingsPath, { force: true }), + rm(job.outputPath, { force: true }), + ...ticketRoots.map((root) => rm(root, { recursive: true, force: true })), + ]); + return true; +} + +const ticketSweep = setInterval(() => { + for (const [token, ticket] of downloadTickets) { + if (ticket.expiresAt < Date.now()) { + downloadTickets.delete(token); + void rm(ticket.root, { recursive: true, force: true }).catch((cause) => { + console.error("Expired editor download cleanup failed", cause); + }); + } + } +}, 30_000); +ticketSweep.unref(); + +export async function closeEditorExports(sessionId: string) { + closingSessions.add(sessionId); + try { + await starting.get(sessionId); + await Promise.all( + [...jobs.values()] + .filter((job) => job.sessionId === sessionId) + .map((job) => cancelEditorExport(sessionId, job.id, true)), + ); + } finally { + const release = setTimeout(() => closingSessions.delete(sessionId), 60_000); + release.unref(); + } +} diff --git a/apps/media-server/src/lib/editor-image-assets.ts b/apps/media-server/src/lib/editor-image-assets.ts new file mode 100644 index 00000000000..3fec2c8cdcc --- /dev/null +++ b/apps/media-server/src/lib/editor-image-assets.ts @@ -0,0 +1,97 @@ +import { extname } from "node:path"; +import { runEditorFile as runFile } from "./editor-process"; +import { + type SignedEditorAsset, + stageSignedEditorAsset, +} from "./editor-signed-assets"; + +const MAX_IMAGE_BYTES = 64 * 1024 * 1024; +const MAX_IMAGE_PIXELS = 16_777_216; +const ASSET_PATH = + /^content\/images\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.(png|jpg|webp|gif|bmp|tiff)$/; +const CONTENT_TYPES: Record = { + png: "image/png", + jpg: "image/jpeg", + webp: "image/webp", + gif: "image/gif", + bmp: "image/bmp", + tiff: "image/tiff", +}; +export type EditorImageAsset = SignedEditorAsset; + +export function validateEditorImageAsset(asset: EditorImageAsset) { + const url = new URL(asset.url); + const match = ASSET_PATH.exec(asset.path); + if ( + !match || + CONTENT_TYPES[match[1] ?? ""] !== asset.contentType || + asset.name.length < 1 || + asset.name.length > 100 || + asset.name.split("").some((character) => { + const code = character.charCodeAt(0); + return ( + code < 32 || code === 127 || character === "/" || character === "\\" + ); + }) || + !Number.isSafeInteger(asset.size) || + asset.size < 1 || + asset.size > MAX_IMAGE_BYTES || + (url.protocol !== "https:" && + !( + url.protocol === "http:" && + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA === "1" + )) || + url.username || + url.password || + (asset.objectIdentity != null && + (asset.objectIdentity.length < 1 || asset.objectIdentity.length > 256)) + ) { + throw new Error("Invalid editor image asset"); + } +} + +async function probeImage(path: string) { + const binary = process.env.CAP_WEB_EDITOR_PREPARE_BIN; + if (!binary) throw new Error("Native image inspector is unavailable"); + const { stdout } = await runFile(binary, ["inspect-image", path], { + timeout: 30_000, + maxBuffer: 64 * 1024, + }).catch(() => { + throw new Error("Imported file has no decodable image"); + }); + const data: unknown = JSON.parse(stdout); + const extension = extname(path).slice(1); + if ( + typeof data !== "object" || + data === null || + !("extension" in data) || + data.extension !== extension + ) { + throw new Error("Imported image format does not match its file name"); + } + const width = "width" in data ? data.width : null; + const height = "height" in data ? data.height : null; + if ( + typeof width !== "number" || + typeof height !== "number" || + !Number.isInteger(width) || + !Number.isInteger(height) || + width < 1 || + height < 1 + ) { + throw new Error("Imported file has no decodable image"); + } + if (width > 32_768 || height > 32_768 || width * height > MAX_IMAGE_PIXELS) { + throw new Error("Imported image dimensions are too large"); + } + return { width, height }; +} + +export async function stageSignedEditorImageAsset( + projectPath: string, + asset: EditorImageAsset, + abortSignal?: AbortSignal, +) { + validateEditorImageAsset(asset); + return stageSignedEditorAsset(projectPath, asset, probeImage, abortSignal); +} diff --git a/apps/media-server/src/lib/editor-media.ts b/apps/media-server/src/lib/editor-media.ts new file mode 100644 index 00000000000..8f5c1596a80 --- /dev/null +++ b/apps/media-server/src/lib/editor-media.ts @@ -0,0 +1,280 @@ +import { chmod, mkdtemp, open, rename, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { runEditorFile as runFile } from "./editor-process"; +import { fetchMedia } from "./media-transfer"; + +const MAX_SOURCE_BYTES = 12 * 1024 * 1024 * 1024; +const MAX_INPUT_EVENTS_BYTES = 64 * 1024 * 1024; +const DOWNLOAD_TIMEOUT_MS = 10 * 60 * 1000; + +export type EditorMediaSource = { + url: string; + contentType: + | "video/webm" + | "video/mp4" + | "audio/webm" + | "audio/mp4" + | "application/x-ndjson"; + size: number; + objectIdentity?: string | null; +}; + +function validateSource(source: EditorMediaSource) { + const url = new URL(source.url); + if ( + (url.protocol !== "https:" && + !( + url.protocol === "http:" && + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA === "1" + )) || + url.username || + url.password || + !( + [ + "video/webm", + "video/mp4", + "audio/webm", + "audio/mp4", + "application/x-ndjson", + ] as const + ).some((type) => type === source.contentType) || + !Number.isSafeInteger(source.size) || + source.size < 1 || + source.size > + (source.contentType === "application/x-ndjson" + ? MAX_INPUT_EVENTS_BYTES + : MAX_SOURCE_BYTES) + ) { + throw new Error("Invalid editor media source"); + } + if ( + source.objectIdentity != null && + (source.objectIdentity.length < 1 || source.objectIdentity.length > 256) + ) { + throw new Error("Invalid editor object identity"); + } +} + +async function readableWebmDuration(path: string, signal?: AbortSignal) { + const { stdout } = await runFile( + "ffprobe", + [ + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + path, + ], + { timeout: 30_000, maxBuffer: 64 * 1024, signal }, + ); + const duration = Number(stdout.trim()); + return Number.isFinite(duration) && duration > 0; +} + +async function finalizeWebmDuration(path: string, signal?: AbortSignal) { + if (await readableWebmDuration(path, signal)) return (await stat(path)).size; + const finalizedPath = `${path}.finalized.webm`; + try { + await runFile( + "ffmpeg", + [ + "-hide_banner", + "-loglevel", + "error", + "-nostdin", + "-i", + path, + "-map", + "0", + "-c", + "copy", + "-f", + "webm", + finalizedPath, + ], + { timeout: DOWNLOAD_TIMEOUT_MS, maxBuffer: 64 * 1024, signal }, + ); + if (!(await readableWebmDuration(finalizedPath, signal))) { + throw new Error("Editor WebM duration is unavailable"); + } + await chmod(finalizedPath, 0o600); + await rename(finalizedPath, path); + return (await stat(path)).size; + } finally { + await rm(finalizedPath, { force: true }); + } +} + +export async function downloadEditorMedia( + source: EditorMediaSource, + abortSignal?: AbortSignal, +) { + validateSource(source); + const root = await mkdtemp(join(tmpdir(), "cap-editor-source-")); + await chmod(root, 0o700); + const extension = + source.contentType === "application/x-ndjson" + ? ".ndjson" + : source.contentType.endsWith("/mp4") + ? ".mp4" + : ".webm"; + const path = join(root, `source${extension}`); + const cleanup = () => rm(root, { recursive: true, force: true }); + const handle = await open(path, "wx", 0o600); + let closed = false; + try { + const response = await fetchMedia(source.url, { + headers: source.objectIdentity + ? { "If-Match": source.objectIdentity } + : undefined, + signal: abortSignal + ? AbortSignal.any([ + abortSignal, + AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS), + ]) + : AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS), + redirect: "manual", + }); + if (!response.ok || !response.body) { + throw new Error(`Editor source download failed: ${response.status}`); + } + const responseSize = response.headers.get("content-length"); + if ( + responseSize != null && + (!Number.isSafeInteger(Number(responseSize)) || + Number(responseSize) !== source.size) + ) { + throw new Error("Editor source size changed before download"); + } + const responseIdentity = response.headers.get("etag"); + if (source.objectIdentity && responseIdentity !== source.objectIdentity) { + throw new Error("Editor source identity changed before download"); + } + let received = 0; + const reader = response.body.getReader(); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + received += value.byteLength; + if (received > source.size) { + throw new Error("Editor source exceeded expected size"); + } + await handle.writeFile(value); + } + } finally { + await reader.cancel().catch(() => undefined); + reader.releaseLock(); + } + if (received !== source.size) { + throw new Error("Editor source download was incomplete"); + } + await handle.sync(); + await handle.close(); + closed = true; + const size = source.contentType.endsWith("/webm") + ? await finalizeWebmDuration(path, abortSignal) + : received; + return { path, size, cleanup }; + } catch (error) { + if (!closed) await handle.close(); + await cleanup(); + throw error; + } +} + +function parseFrameRate(value: unknown) { + if (typeof value !== "string") return null; + const parts = value.split("/"); + if (parts.length !== 2) return null; + const numerator = Number(parts[0]); + const denominator = Number(parts[1]); + if (!Number.isFinite(numerator) || !Number.isFinite(denominator)) { + return null; + } + const rate = Math.round(numerator / denominator); + return rate >= 1 && rate <= 120 ? rate : null; +} + +export async function inspectEditorDisplayMedia( + path: string, + reportedFps?: number, +) { + const { stdout } = await runFile( + "ffprobe", + [ + "-v", + "error", + "-show_entries", + "stream=codec_type,avg_frame_rate,r_frame_rate:format=duration", + "-of", + "json", + path, + ], + { timeout: 30_000, maxBuffer: 64 * 1024 }, + ); + const data = JSON.parse(stdout) as { + format?: { duration?: string }; + streams?: Array<{ + codec_type?: string; + avg_frame_rate?: unknown; + r_frame_rate?: unknown; + }>; + }; + const video = data.streams?.find((stream) => stream.codec_type === "video"); + if (!video) throw new Error("Editor source has no video stream"); + const fps = + (Number.isInteger(reportedFps) && + reportedFps !== undefined && + reportedFps >= 1 && + reportedFps <= 120 + ? reportedFps + : null) ?? + parseFrameRate(video.avg_frame_rate) ?? + parseFrameRate(video.r_frame_rate); + if (!fps) throw new Error("Editor source frame rate is unavailable"); + return { + fps, + duration: Number(data.format?.duration), + hasAudio: + data.streams?.some((stream) => stream.codec_type === "audio") ?? false, + }; +} + +export async function resolveEditorFps(path: string, reported?: number) { + if ( + Number.isInteger(reported) && + reported !== undefined && + reported > 0 && + reported <= 120 + ) { + return reported; + } + const { stdout } = await runFile( + "ffprobe", + [ + "-v", + "error", + "-select_streams", + "v:0", + "-show_entries", + "stream=avg_frame_rate,r_frame_rate", + "-of", + "json", + path, + ], + { timeout: 30_000, maxBuffer: 64 * 1024 }, + ); + const data = JSON.parse(stdout) as { + streams?: Array<{ avg_frame_rate?: unknown; r_frame_rate?: unknown }>; + }; + const stream = data.streams?.[0]; + const fps = + parseFrameRate(stream?.avg_frame_rate) ?? + parseFrameRate(stream?.r_frame_rate); + if (!fps) throw new Error("Editor source frame rate is unavailable"); + return fps; +} diff --git a/apps/media-server/src/lib/editor-native.ts b/apps/media-server/src/lib/editor-native.ts new file mode 100644 index 00000000000..b3318466fe4 --- /dev/null +++ b/apps/media-server/src/lib/editor-native.ts @@ -0,0 +1,575 @@ +import { type ChildProcessByStdio, spawn } from "node:child_process"; +import { randomBytes, randomUUID } from "node:crypto"; +import { + chmod, + lstat, + mkdtemp, + realpath, + rename, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, extname, join } from "node:path"; +import type { Readable } from "node:stream"; +import { + type EditorAudioAsset, + stageSignedEditorAudioAsset, +} from "./editor-assets"; +import { stageSavedEditorAudioLibrary } from "./editor-audio-library"; +import { + type EditorCapAsset, + stageSignedEditorCapAsset, + validateEditorCapAsset, +} from "./editor-cap-assets"; +import { mapEditorConfigPaths } from "./editor-config-paths"; +import { + type EditorImageAsset, + stageSignedEditorImageAsset, +} from "./editor-image-assets"; +import { editorProcessEnv, runEditorFile } from "./editor-process"; +import { + type EditorVideoAsset, + stageSignedEditorVideoAsset, +} from "./editor-video-assets"; + +const STARTUP_TIMEOUT_MS = 30_000; +const STOP_TIMEOUT_MS = 5_000; +const CLIP_VIDEO_PATH = + /^content\/videos\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.(mp4|webm)$/; + +type MediaInput = { + path: string; + contentType: "video/webm" | "video/mp4"; + size: number; + fps: number; +}; + +type AudioMediaInput = { + path: string; + contentType: "audio/webm" | "audio/mp4"; + size: number; + offsetMs: number; +}; + +type InputEventsInput = { + path: string; + contentType: "application/x-ndjson"; + size: number; +}; + +export type NativeEditorClipInput = { + displayPath: string; + duration: number; + fps: number; + hasAudio: boolean; + cameraPath?: string; + cameraFps?: number; + cameraOffsetMs?: number; +}; + +export type NativeEditorImport = + | { kind: "clip"; clip: NativeEditorClipInput } + | { kind: "cap"; asset: EditorCapAsset; clipCount: number }; + +export type LegacyEditorEditSpec = { + version: 1; + sourceDuration: number; + keepRanges: Array<{ start: number; end: number }>; +}; + +export type NativeEditorInputs = { + title: string; + display: MediaInput; + camera?: MediaInput & { offsetMs: number }; + mic?: AudioMediaInput; + systemAudio?: AudioMediaInput; + inputEvents?: InputEventsInput; + mixedAudioInDisplay: boolean; + audioDefault?: { + enabledByDefault: boolean; + isolation: "light" | "balanced" | "strong"; + }; + projectConfig?: Record; + legacyEditSpec?: LegacyEditorEditSpec; + audioAssets?: EditorAudioAsset[]; + imageAssets?: EditorImageAsset[]; + videoAssets?: EditorVideoAsset[]; + clips?: NativeEditorClipInput[]; + imports?: NativeEditorImport[]; +}; + +export type NativeEditorProject = { + path: string; + cleanup: () => Promise; +}; + +export function nativeEditorBinary(name: "prepare" | "service") { + const value = + name === "prepare" + ? process.env.CAP_WEB_EDITOR_PREPARE_BIN + : process.env.CAP_WEB_EDITOR_SERVICE_BIN; + if (!value) throw new Error(`Native editor ${name} binary is unavailable`); + return value; +} + +async function validateInput(input: MediaInput) { + if ( + !Number.isSafeInteger(input.size) || + input.size <= 0 || + !Number.isSafeInteger(input.fps) || + input.fps <= 0 || + input.fps > 120 + ) { + throw new Error("Invalid editor media dimensions"); + } + const extension = input.contentType === "video/mp4" ? ".mp4" : ".webm"; + if (extname(input.path).toLowerCase() !== extension) { + throw new Error("Editor media extension does not match its content type"); + } + const metadata = await lstat(input.path); + if (!metadata.isFile() || metadata.size !== input.size) { + throw new Error("Editor media file changed before project preparation"); + } +} + +async function validateAudioInput(input: AudioMediaInput) { + if ( + !Number.isSafeInteger(input.size) || + input.size <= 0 || + !Number.isSafeInteger(input.offsetMs) || + Math.abs(input.offsetMs) > 30_000 + ) { + throw new Error("Invalid editor audio source"); + } + const extension = input.contentType === "audio/mp4" ? ".mp4" : ".webm"; + if (extname(input.path).toLowerCase() !== extension) { + throw new Error("Editor audio extension does not match its content type"); + } + const metadata = await lstat(input.path); + if (!metadata.isFile() || metadata.size !== input.size) { + throw new Error("Editor audio file changed before project preparation"); + } +} + +async function validateInputEvents(input: InputEventsInput) { + if ( + input.contentType !== "application/x-ndjson" || + !Number.isSafeInteger(input.size) || + input.size <= 0 || + input.size > 64 * 1024 * 1024 || + extname(input.path).toLowerCase() !== ".ndjson" + ) { + throw new Error("Invalid editor input event source"); + } + const metadata = await lstat(input.path); + if (!metadata.isFile() || metadata.size !== input.size) { + throw new Error("Editor input events changed before project preparation"); + } +} + +export async function prepareNativeEditorProject( + inputs: NativeEditorInputs, + abortSignal?: AbortSignal, +): Promise { + await Promise.all([ + validateInput(inputs.display), + ...(inputs.camera ? [validateInput(inputs.camera)] : []), + ...(inputs.mic ? [validateAudioInput(inputs.mic)] : []), + ...(inputs.systemAudio ? [validateAudioInput(inputs.systemAudio)] : []), + ...(inputs.inputEvents ? [validateInputEvents(inputs.inputEvents)] : []), + ]); + if ( + inputs.mixedAudioInDisplay && + (inputs.mic !== undefined || inputs.systemAudio !== undefined) + ) { + throw new Error("Mixed display audio cannot be added twice"); + } + if ( + inputs.camera && + (!Number.isSafeInteger(inputs.camera.offsetMs) || + Math.abs(inputs.camera.offsetMs) > 30_000) + ) { + throw new Error("Invalid webcam recording offset"); + } + const clips = + inputs.imports?.flatMap((item) => + item.kind === "clip" ? [item.clip] : [], + ) ?? + inputs.clips ?? + []; + const capImports = + inputs.imports?.filter((item) => item.kind === "cap") ?? []; + const assetPaths = new Set( + (inputs.videoAssets ?? []).map((asset) => asset.path), + ); + if ( + (inputs.imports !== undefined && inputs.clips !== undefined) || + (inputs.imports?.length ?? 0) > 100 || + clips.length > 49 || + capImports.reduce((total, item) => total + item.clipCount, clips.length) > + 1000 || + new Set(capImports.map((item) => item.asset.path)).size !== + capImports.length || + capImports.some( + (item) => + !Number.isSafeInteger(item.clipCount) || + item.clipCount < 1 || + item.clipCount > 1000, + ) || + new Set(clips.map((clip) => clip.displayPath)).size !== clips.length || + clips.some( + (clip) => + !CLIP_VIDEO_PATH.test(clip.displayPath) || + !assetPaths.has(clip.displayPath) || + !Number.isFinite(clip.duration) || + clip.duration <= 0 || + clip.duration > 86_400 || + !Number.isSafeInteger(clip.fps) || + clip.fps < 1 || + clip.fps > 120 || + typeof clip.hasAudio !== "boolean" || + (clip.cameraPath !== undefined && + (!CLIP_VIDEO_PATH.test(clip.cameraPath) || + clip.cameraPath === clip.displayPath || + !assetPaths.has(clip.cameraPath) || + !Number.isSafeInteger(clip.cameraFps) || + !Number.isSafeInteger(clip.cameraOffsetMs) || + clip.cameraFps === undefined || + clip.cameraFps < 1 || + clip.cameraFps > 120 || + clip.cameraOffsetMs === undefined || + Math.abs(clip.cameraOffsetMs) > 30_000)) || + (clip.cameraPath === undefined && + (clip.cameraFps !== undefined || clip.cameraOffsetMs !== undefined)), + ) + ) { + throw new Error("Invalid persisted editor recording clips"); + } + for (const item of capImports) validateEditorCapAsset(item.asset); + const root = await mkdtemp(join(await realpath(tmpdir()), "cap-web-editor-")); + await chmod(root, 0o700); + const projectPath = join(root, `${randomUUID()}.cap`); + const manifestPath = join(root, "manifest.json"); + const cleanup = () => rm(root, { recursive: true, force: true }); + try { + await writeFile( + manifestPath, + JSON.stringify({ + version: 1, + title: inputs.title, + displayPath: inputs.display.path, + displayFps: inputs.display.fps, + cameraPath: inputs.camera?.path ?? null, + cameraFps: inputs.camera?.fps ?? null, + cameraOffsetMs: inputs.camera?.offsetMs ?? null, + micPath: inputs.mic?.path ?? null, + micOffsetMs: inputs.mic?.offsetMs ?? null, + systemAudioPath: inputs.systemAudio?.path ?? null, + systemAudioOffsetMs: inputs.systemAudio?.offsetMs ?? null, + inputEventsPath: inputs.inputEvents?.path ?? null, + mixedAudioInDisplay: inputs.mixedAudioInDisplay, + audioDefault: inputs.audioDefault ?? null, + initialProjectConfig: + inputs.projectConfig && capImports.length === 0 + ? mapEditorConfigPaths(inputs.projectConfig, "native", projectPath) + : null, + legacyEditSpec: inputs.projectConfig + ? null + : (inputs.legacyEditSpec ?? null), + }), + { flag: "wx", mode: 0o600 }, + ); + await runEditorFile( + nativeEditorBinary("prepare"), + ["prepare", projectPath, manifestPath], + { + timeout: STARTUP_TIMEOUT_MS, + maxBuffer: 64 * 1024, + signal: abortSignal, + }, + ); + await stageSavedEditorAudioLibrary(projectPath, inputs.projectConfig); + for (const asset of inputs.audioAssets ?? []) { + await stageSignedEditorAudioAsset(projectPath, asset, abortSignal); + } + for (const asset of inputs.imageAssets ?? []) { + await stageSignedEditorImageAsset(projectPath, asset, abortSignal); + } + for (const asset of inputs.videoAssets ?? []) { + await stageSignedEditorVideoAsset(projectPath, asset, abortSignal); + } + if (inputs.imports) { + for (const [index, item] of inputs.imports.entries()) { + if (item.kind === "clip") { + const clipManifestPath = join(root, `clip-${index}.json`); + await writeFile( + clipManifestPath, + JSON.stringify({ + version: 1, + clips: [ + { + ...item.clip, + displayPath: join(projectPath, item.clip.displayPath), + ...(item.clip.cameraPath + ? { cameraPath: join(projectPath, item.clip.cameraPath) } + : {}), + }, + ], + }), + { flag: "wx", mode: 0o600 }, + ); + await runEditorFile( + nativeEditorBinary("prepare"), + ["append-clip", projectPath, clipManifestPath], + { timeout: 60_000, maxBuffer: 64 * 1024, signal: abortSignal }, + ); + } else { + const staged = await stageSignedEditorCapAsset( + projectPath, + item.asset, + abortSignal, + ); + try { + const { stdout } = await runEditorFile( + nativeEditorBinary("prepare"), + ["append-cap", projectPath, staged.path], + { + timeout: 30 * 60 * 1000, + maxBuffer: 64 * 1024, + signal: abortSignal, + }, + ); + const value: unknown = JSON.parse(stdout.toString()); + if ( + typeof value !== "object" || + value === null || + !("clipCount" in value) || + value.clipCount !== item.clipCount + ) { + throw new Error("Persisted Cap project clip count changed"); + } + } finally { + await staged.cleanup(); + } + } + } + if (capImports.length > 0 && inputs.projectConfig) { + const restoredConfigPath = join(root, "restored-project-config.json"); + await writeFile( + restoredConfigPath, + JSON.stringify( + mapEditorConfigPaths(inputs.projectConfig, "native", projectPath), + ), + { flag: "wx", mode: 0o600 }, + ); + await rename( + restoredConfigPath, + join(projectPath, "project-config.json"), + ); + } + } else if (inputs.clips?.length) { + const clipManifestPath = join(root, "clips-manifest.json"); + await writeFile( + clipManifestPath, + JSON.stringify({ + version: 1, + clips: inputs.clips.map((clip) => ({ + ...clip, + displayPath: join(projectPath, clip.displayPath), + ...(clip.cameraPath + ? { cameraPath: join(projectPath, clip.cameraPath) } + : {}), + })), + }), + { flag: "wx", mode: 0o600 }, + ); + await runEditorFile( + nativeEditorBinary("prepare"), + ["append-clips", projectPath, clipManifestPath], + { + timeout: 60_000, + maxBuffer: 64 * 1024, + signal: abortSignal, + }, + ); + } + return { path: projectPath, cleanup }; + } catch (error) { + await cleanup(); + throw error; + } +} + +function parseLoopbackAddress(input: string) { + let data: unknown; + try { + data = JSON.parse(input); + } catch { + throw new Error("Native editor returned an invalid startup message"); + } + if ( + typeof data !== "object" || + data === null || + !("address" in data) || + typeof data.address !== "string" + ) { + throw new Error("Native editor did not publish its listening address"); + } + const url = new URL(`http://${data.address}`); + if ( + url.hostname !== "127.0.0.1" || + !url.port || + !Number.isInteger(Number(url.port)) || + Number(url.port) < 1 || + Number(url.port) > 65_535 || + url.pathname !== "/" || + url.search || + url.hash + ) { + throw new Error("Native editor published a non-loopback address"); + } + return url.origin; +} + +function waitForStartup(child: ChildProcessByStdio) { + return new Promise((resolve, reject) => { + let output = ""; + let settled = false; + const finish = (error?: Error, address?: string) => { + if (settled) return; + settled = true; + clearTimeout(timer); + child.stdout.off("data", onData); + child.off("error", onError); + child.off("exit", onExit); + if (error) reject(error); + else resolve(address ?? ""); + }; + const onData = (chunk: Buffer) => { + output += chunk.toString("utf8"); + if (output.length > 4096) { + finish(new Error("Native editor startup message is too large")); + return; + } + const newline = output.indexOf("\n"); + if (newline < 0) return; + try { + finish(undefined, parseLoopbackAddress(output.slice(0, newline))); + } catch (error) { + finish(error instanceof Error ? error : new Error(String(error))); + } + }; + const onError = (error: Error) => finish(error); + const onExit = () => + finish(new Error("Native editor exited before startup")); + const timer = setTimeout( + () => finish(new Error("Native editor startup timed out")), + STARTUP_TIMEOUT_MS, + ); + child.stdout.on("data", onData); + child.once("error", onError); + child.once("exit", onExit); + }); +} + +function waitForExit(exit: Promise, timeoutMs: number) { + return new Promise((resolve) => { + const timer = setTimeout(() => resolve(false), timeoutMs); + exit.then(() => { + clearTimeout(timer); + resolve(true); + }); + }); +} + +export async function startNativeEditorSession(project: NativeEditorProject) { + const internalToken = randomBytes(32).toString("base64url"); + const child = spawn(nativeEditorBinary("service"), [project.path], { + stdio: ["ignore", "pipe", "pipe"], + env: editorProcessEnv(internalToken, dirname(project.path)), + }); + let stderr = ""; + child.stderr.on("data", (chunk: Buffer) => { + stderr = `${stderr}${chunk.toString("utf8")}`.slice(-4096); + }); + let closed = false; + const exit = new Promise((resolve) => + child.once("close", () => resolve()), + ); + let closePromise: Promise | null = null; + const close = () => { + closePromise ??= (async () => { + closed = true; + if (child.exitCode === null && child.signalCode === null) + child.kill("SIGTERM"); + if (!(await waitForExit(exit, STOP_TIMEOUT_MS))) { + child.kill("SIGKILL"); + if (!(await waitForExit(exit, STOP_TIMEOUT_MS))) { + throw new Error("Native editor did not exit after termination"); + } + } + await project.cleanup(); + })(); + return closePromise; + }; + try { + const origin = await waitForStartup(child); + const authorizedHeaders = (headers?: HeadersInit) => { + const result = new Headers(headers); + result.set("Authorization", `Bearer ${internalToken}`); + return result; + }; + const health = await fetch(`${origin}/health`, { + headers: authorizedHeaders(), + signal: AbortSignal.timeout(STARTUP_TIMEOUT_MS), + }); + if (!health.ok) + throw new Error(`Native editor health failed: ${health.status}`); + void exit + .then(() => close()) + .catch((error) => { + console.error("Native editor cleanup failed", error); + }); + return { + origin, + pid: child.pid ?? null, + projectPath: project.path, + request: (path: string, init?: RequestInit) => { + if (closed) throw new Error("Native editor session is closed"); + if (!path.startsWith("/") || path.startsWith("//")) { + throw new Error("Invalid native editor request path"); + } + return fetch(`${origin}${path}`, { + ...init, + headers: authorizedHeaders(init?.headers), + }); + }, + connectSocket: (url: string) => { + if (closed) throw new Error("Native editor session is closed"); + const upstream = new URL(url); + const expected = new URL(origin); + if ( + upstream.protocol !== "ws:" || + upstream.host !== expected.host || + upstream.hostname !== "127.0.0.1" + ) { + throw new Error("Invalid native editor socket address"); + } + const BunWebSocket = WebSocket as unknown as new ( + address: string, + options: Bun.WebSocketOptions, + ) => WebSocket; + return new BunWebSocket(url, { + headers: { Authorization: `Bearer ${internalToken}` }, + }); + }, + close, + }; + } catch (error) { + await close(); + throw new Error( + `Failed to start native editor: ${error instanceof Error ? error.message : String(error)}${stderr ? `; ${stderr}` : ""}`, + ); + } +} diff --git a/apps/media-server/src/lib/editor-process.ts b/apps/media-server/src/lib/editor-process.ts new file mode 100644 index 00000000000..a911e868021 --- /dev/null +++ b/apps/media-server/src/lib/editor-process.ts @@ -0,0 +1,60 @@ +import { + type ExecFileOptionsWithStringEncoding, + execFile, +} from "node:child_process"; +import { promisify } from "node:util"; + +const runFile = promisify(execFile); +const EDITOR_PROCESS_ENV_KEYS = [ + "PATH", + "TMPDIR", + "TEMP", + "TMP", + "LANG", + "LC_ALL", + "LD_LIBRARY_PATH", + "DYLD_LIBRARY_PATH", + "DYLD_FALLBACK_LIBRARY_PATH", + "FONTCONFIG_PATH", + "FONTCONFIG_FILE", + "XDG_CACHE_HOME", + "XDG_RUNTIME_DIR", + "VK_ICD_FILENAMES", + "VK_LAYER_PATH", + "ORT_DYLIB_PATH", + "CAP_RENDER_FORCE_SOFTWARE_ADAPTER", + "CAP_EDITOR_FORCE_FFMPEG_DECODER", + "CAP_EXPORT_DISABLE_ZERO_COPY", + "CAP_EXPORT_FORCE_SOFTWARE_ENCODER", + "CAP_DISABLE_ENCODER_SELF_TEST", +] as const; + +export function editorProcessEnv( + internalToken?: string, + projectRoot?: string, +): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = {}; + for (const name of EDITOR_PROCESS_ENV_KEYS) { + const value = process.env[name]; + if (value !== undefined) env[name] = value; + } + if (internalToken !== undefined) { + env.CAP_WEB_EDITOR_INTERNAL_TOKEN = internalToken; + } + if (projectRoot !== undefined) { + env.CAP_WEB_EDITOR_PROJECT_ROOT = projectRoot; + } + return env; +} + +export function runEditorFile( + file: string, + args: string[], + options: ExecFileOptionsWithStringEncoding = {}, +) { + return runFile(file, args, { + ...options, + encoding: "utf8", + env: editorProcessEnv(), + }); +} diff --git a/apps/media-server/src/lib/editor-project-bundles.ts b/apps/media-server/src/lib/editor-project-bundles.ts new file mode 100644 index 00000000000..82789e0d37b --- /dev/null +++ b/apps/media-server/src/lib/editor-project-bundles.ts @@ -0,0 +1,324 @@ +import { randomBytes } from "node:crypto"; +import { + copyFile, + link, + lstat, + mkdir, + mkdtemp, + open, + readdir, + rm, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { + CAP_BUNDLE_CONTENT_TYPE, + CAP_BUNDLE_HEADER_BYTES, + CAP_BUNDLE_MAGIC, + type CapBundleEntry, + MAX_CAP_BUNDLE_BYTES, + MAX_CAP_BUNDLE_FILES, + MAX_CAP_BUNDLE_MANIFEST_BYTES, + parseCapBundleManifest, + validCapBundlePath, +} from "@cap/editor-cap-bundle"; +import { publicEditorOrigin } from "./editor-socket-tickets"; + +type SnapshotFile = CapBundleEntry & { + dev: number; + ino: number; + mtimeMs: number; +}; + +type BundleTicket = { + sessionId: string; + root: string; + files: SnapshotFile[]; + manifestBytes: Uint8Array; + fileName: string; + size: number; + expiresAt: number; +}; + +const tickets = new Map(); +const encoder = new TextEncoder(); +const TICKET_TTL_MS = 60_000; +const MAX_TICKETS = 100; +const READ_CHUNK_BYTES = 1024 * 1024; + +function sameFile( + left: { + dev: number | bigint; + ino: number | bigint; + size: number | bigint; + mtimeMs: number | bigint; + }, + right: typeof left, +) { + return ( + left.dev === right.dev && + left.ino === right.ino && + left.size === right.size && + left.mtimeMs === right.mtimeMs + ); +} + +export async function scanEditorProject(projectPath: string) { + const project = await lstat(projectPath); + if (!project.isDirectory()) throw new Error("Invalid editor project folder"); + const files: { + path: string; + source: string; + stats: Awaited>; + }[] = []; + const pending = [{ directory: projectPath, prefix: "" }]; + while (pending.length > 0) { + const current = pending.pop(); + if (!current) break; + for (const entry of await readdir(current.directory, { + withFileTypes: true, + })) { + const path = current.prefix + ? `${current.prefix}/${entry.name}` + : entry.name; + const source = join(current.directory, entry.name); + const stats = await lstat(source); + if (stats.isDirectory()) { + pending.push({ directory: source, prefix: path }); + continue; + } + if (!stats.isFile()) + throw new Error("Editor project contains an unsupported file"); + if (!validCapBundlePath(path)) { + if ( + path.startsWith("content/") || + path.startsWith("output/") || + path.startsWith("screenshots/") || + path.startsWith("assets/audio/") + ) + throw new Error("Editor project contains an invalid media path"); + continue; + } + files.push({ path, source, stats }); + if (files.length > MAX_CAP_BUNDLE_FILES) + throw new Error("Editor project has too many files"); + } + } + files.sort((left, right) => left.path.localeCompare(right.path)); + if (!files.some((file) => file.path === "recording-meta.json")) + throw new Error("Editor project is missing recording metadata"); + return files; +} + +function bundleFileName(requestedName?: string) { + const stem = requestedName?.endsWith(".capbundle") + ? requestedName.slice(0, -".capbundle".length) + : null; + return stem && + stem.length <= 180 && + !stem.includes("/") && + !stem.includes("\\") && + [...stem].every((character) => { + const code = character.charCodeAt(0); + return code > 31 && code !== 127; + }) + ? `${stem}.capbundle` + : "Cap Recording.capbundle"; +} + +export async function createEditorProjectBundleDownloadTicket( + sessionId: string, + projectPath: string, + requestedName?: string, +) { + if (tickets.size >= MAX_TICKETS) + throw new Error("Editor bundle download capacity is busy"); + const origin = publicEditorOrigin(); + const sources = await scanEditorProject(projectPath); + const root = await mkdtemp(join(tmpdir(), "cap-web-editor-bundle-")); + try { + const files: SnapshotFile[] = []; + let offset = 0; + for (const source of sources) { + const destination = join(root, source.path); + await mkdir(dirname(destination), { recursive: true, mode: 0o700 }); + if ( + source.path === "recording-meta.json" || + source.path === "project-config.json" || + source.path === "recording-diagnostics.json" + ) { + await copyFile(source.source, destination); + } else { + await link(source.source, destination); + } + const [snapshot, current] = await Promise.all([ + lstat(destination), + lstat(source.source), + ]); + if ( + !snapshot.isFile() || + !current.isFile() || + !sameFile(current, source.stats) || + snapshot.size !== source.stats.size || + (snapshot.ino !== source.stats.ino && + source.path !== "recording-meta.json" && + source.path !== "project-config.json" && + source.path !== "recording-diagnostics.json") + ) { + throw new Error("Editor project changed during bundle snapshot"); + } + files.push({ + path: source.path, + size: snapshot.size, + offset, + dev: snapshot.dev, + ino: snapshot.ino, + mtimeMs: snapshot.mtimeMs, + }); + offset += snapshot.size; + if (!Number.isSafeInteger(offset) || offset > MAX_CAP_BUNDLE_BYTES) + throw new Error("Editor project bundle is too large"); + } + const manifestBytes = encoder.encode( + JSON.stringify({ + version: 1, + files: files.map(({ path, size, offset }) => ({ path, size, offset })), + }), + ); + const size = CAP_BUNDLE_HEADER_BYTES + manifestBytes.byteLength + offset; + if ( + manifestBytes.byteLength > MAX_CAP_BUNDLE_MANIFEST_BYTES || + size > MAX_CAP_BUNDLE_BYTES || + !parseCapBundleManifest(manifestBytes, size) + ) { + throw new Error("Editor project bundle manifest is invalid"); + } + const token = randomBytes(32).toString("base64url"); + tickets.set(token, { + sessionId, + root, + files, + manifestBytes, + fileName: bundleFileName(requestedName), + size, + expiresAt: Date.now() + TICKET_TTL_MS, + }); + return { + url: `${origin}/editor/sessions/${encodeURIComponent(sessionId)}/project-bundle/download?ticket=${token}`, + }; + } catch (error) { + await rm(root, { recursive: true, force: true }); + throw error; + } +} + +export function consumeEditorProjectBundleDownloadTicket( + sessionId: string, + token: string | null, +) { + if (!token || !/^[A-Za-z0-9_-]{43}$/.test(token)) return null; + const ticket = tickets.get(token); + tickets.delete(token); + if ( + !ticket || + ticket.sessionId !== sessionId || + ticket.expiresAt < Date.now() + ) { + if (ticket) + void rm(ticket.root, { recursive: true, force: true }).catch( + console.error, + ); + return null; + } + return ticket; +} + +async function* bundleChunks(ticket: BundleTicket) { + try { + const header = new Uint8Array(CAP_BUNDLE_HEADER_BYTES); + header.set(encoder.encode(CAP_BUNDLE_MAGIC)); + new DataView(header.buffer).setUint32( + CAP_BUNDLE_MAGIC.length, + ticket.manifestBytes.byteLength, + true, + ); + yield header; + yield ticket.manifestBytes; + for (const file of ticket.files) { + const handle = await open(join(ticket.root, file.path), "r"); + try { + const before = await handle.stat(); + if (!sameFile(before, file)) + throw new Error("Editor project bundle file changed"); + let position = 0; + while (position < file.size) { + const bytes = Buffer.allocUnsafe( + Math.min(READ_CHUNK_BYTES, file.size - position), + ); + const result = await handle.read( + bytes, + 0, + bytes.byteLength, + position, + ); + if (result.bytesRead === 0) + throw new Error("Editor project bundle file ended early"); + position += result.bytesRead; + yield bytes.subarray(0, result.bytesRead); + } + if (!sameFile(await handle.stat(), before)) + throw new Error("Editor project bundle file changed"); + } finally { + await handle.close(); + } + } + } finally { + await rm(ticket.root, { recursive: true, force: true }); + } +} + +export function editorProjectBundleResponse(ticket: BundleTicket) { + const iterator = bundleChunks(ticket); + const stream = new ReadableStream({ + async pull(controller) { + try { + const next = await iterator.next(); + if (next.done) controller.close(); + else controller.enqueue(next.value); + } catch (error) { + controller.error(error); + } + }, + async cancel() { + await iterator.return(undefined); + await rm(ticket.root, { recursive: true, force: true }); + }, + }); + const asciiName = ticket.fileName.replace(/[^\x20-\x7e]|["\\]/g, "_"); + const encodedName = encodeURIComponent(ticket.fileName).replace( + /['()*]/g, + (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`, + ); + return new Response(stream, { + headers: { + "Content-Type": CAP_BUNDLE_CONTENT_TYPE, + "Content-Length": String(ticket.size), + "Content-Disposition": `attachment; filename="${asciiName}"; filename*=UTF-8''${encodedName}`, + "Cache-Control": "private, no-store", + "Referrer-Policy": "no-referrer", + "X-Content-Type-Options": "nosniff", + }, + }); +} + +const ticketSweep = setInterval(() => { + for (const [token, ticket] of tickets) { + if (ticket.expiresAt < Date.now()) { + tickets.delete(token); + void rm(ticket.root, { recursive: true, force: true }).catch( + console.error, + ); + } + } +}, 30_000); +ticketSweep.unref(); diff --git a/apps/media-server/src/lib/editor-render-project.ts b/apps/media-server/src/lib/editor-render-project.ts new file mode 100644 index 00000000000..342d32adc43 --- /dev/null +++ b/apps/media-server/src/lib/editor-render-project.ts @@ -0,0 +1,138 @@ +import { readFile, realpath } from "node:fs/promises"; +import { isAbsolute, relative, resolve } from "node:path"; +import { scanEditorProject } from "./editor-project-bundles"; +import { editorWallpaperDirectory } from "./editor-wallpapers"; + +const MAX_RECORDING_META_BYTES = 4 * 1024 * 1024; +const MAX_RENDER_UPLOADS = 4000; +const UPLOAD_CONCURRENCY = 4; +const RENDER_SKIPPED_PATH = /^(?:output|screenshots)\//; +const WALLPAPER_FILE = + /^(macOS|blue|purple|cities|dark|orange)\/[a-z0-9-]+\.jpg$/; + +export type EditorRenderUploads = { + files: { path: string; url: string }[]; + wallpapers: { file: string; url: string }[]; +}; + +export async function listEditorRenderProject(projectPath: string) { + const files = await scanEditorProject(projectPath); + const meta = files.find((file) => file.path === "recording-meta.json"); + if (!meta || Number(meta.stats.size) > MAX_RECORDING_META_BYTES) { + throw new Error("Editor project recording metadata is unavailable"); + } + const recordingMeta: unknown = JSON.parse( + await readFile(meta.source, "utf8"), + ); + return { + files: files + .filter((file) => !RENDER_SKIPPED_PATH.test(file.path)) + .map((file) => ({ + path: file.path, + size: Number(file.stats.size), + inode: `${file.stats.dev}:${file.stats.ino}`, + })), + recordingMeta, + }; +} + +function validUploadUrl(value: string) { + const url = new URL(value); + return ( + (url.protocol === "https:" || + (url.protocol === "http:" && + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA === "1")) && + !url.username && + !url.password + ); +} + +async function putFile(url: string, source: string, signal?: AbortSignal) { + const file = Bun.file(source); + const size = file.size; + const response = await fetch(url, { + method: "PUT", + body: file, + headers: { "Content-Length": String(size) }, + redirect: "error", + signal, + }); + if (!response.ok) { + throw new Error(`Render project upload failed with ${response.status}`); + } + return size; +} + +async function wallpaperSource(file: string) { + if (!WALLPAPER_FILE.test(file)) throw new Error("Invalid editor wallpaper"); + const root = await realpath(editorWallpaperDirectory()); + const source = await realpath(resolve(root, file)); + const within = relative(root, source); + if (within.startsWith("..") || isAbsolute(within) || within !== file) { + throw new Error("Invalid editor wallpaper"); + } + return source; +} + +async function inBatches( + items: T[], + run: (item: T) => Promise, +): Promise { + const results: R[] = new Array(items.length); + let next = 0; + await Promise.all( + Array.from( + { length: Math.min(UPLOAD_CONCURRENCY, items.length) }, + async () => { + while (next < items.length) { + const index = next++; + results[index] = await run(items[index] as T); + } + }, + ), + ); + return results; +} + +export async function uploadEditorRenderProject( + projectPath: string, + uploads: EditorRenderUploads, + signal?: AbortSignal, +) { + if ( + uploads.files.length + uploads.wallpapers.length > MAX_RENDER_UPLOADS || + new Set(uploads.files.map((upload) => upload.path)).size !== + uploads.files.length || + [...uploads.files, ...uploads.wallpapers].some( + (upload) => !validUploadUrl(upload.url), + ) + ) { + throw new Error("Invalid render project uploads"); + } + const files = new Map( + (await scanEditorProject(projectPath)) + .filter((file) => !RENDER_SKIPPED_PATH.test(file.path)) + .map((file) => [file.path, file.source]), + ); + const sources = uploads.files.map((upload) => { + const source = files.get(upload.path); + if (!source) throw new Error("Render project file is missing"); + return { ...upload, source }; + }); + const wallpapers = await Promise.all( + uploads.wallpapers.map(async (upload) => ({ + ...upload, + source: await wallpaperSource(upload.file), + })), + ); + return { + files: await inBatches(sources, async (upload) => ({ + path: upload.path, + size: await putFile(upload.url, upload.source, signal), + })), + wallpapers: await inBatches(wallpapers, async (upload) => ({ + file: upload.file, + size: await putFile(upload.url, upload.source, signal), + })), + }; +} diff --git a/apps/media-server/src/lib/editor-sessions.ts b/apps/media-server/src/lib/editor-sessions.ts new file mode 100644 index 00000000000..78b37b35c0c --- /dev/null +++ b/apps/media-server/src/lib/editor-sessions.ts @@ -0,0 +1,550 @@ +import { randomUUID } from "node:crypto"; +import { stripEditorCaptionContent } from "../../../web/lib/editor-caption-access"; +import type { EditorAudioAsset } from "./editor-assets"; +import { closeEditorCapImports } from "./editor-cap-imports"; +import { closeEditorExports, editorExportActivity } from "./editor-exports"; +import type { EditorImageAsset } from "./editor-image-assets"; +import { + downloadEditorMedia, + type EditorMediaSource, + inspectEditorDisplayMedia, + resolveEditorFps, +} from "./editor-media"; +import { + type LegacyEditorEditSpec, + type NativeEditorClipInput, + type NativeEditorImport, + type NativeEditorInputs, + prepareNativeEditorProject, + startNativeEditorSession, +} from "./editor-native"; +import type { EditorVideoAsset } from "./editor-video-assets"; +import { closeEditorVideoImports } from "./editor-video-imports"; + +const IDLE_TIMEOUT_MS = 10 * 60 * 1000; +const SESSION_ATTACH_TIMEOUT_MS = 2 * 60 * 1000; +// A client polls a running export every few seconds, and a background tab's +// timers still run at least once a minute. +const EXPORT_LEASE_MS = 3 * 60 * 1000; +const DETACHED_TIMEOUT_MS = 2 * 60 * 1000; +const PREPARATION_RETENTION_MS = 30 * 60 * 1000; +const MAX_ACTIVE_SESSIONS = editorSessionLimit( + process.env.CAP_WEB_EDITOR_MAX_SESSIONS, +); + +/// Each session runs a native renderer, so one per worker unless the +/// deployment sizes the instance for more. +export function editorSessionLimit(value: string | undefined) { + if (value === undefined || value === "") return 1; + const limit = Number(value); + if (!Number.isInteger(limit) || limit < 1 || limit > 16) { + throw new Error("Invalid CAP_WEB_EDITOR_MAX_SESSIONS"); + } + return limit; +} + +function newEditorSessionId() { + const workerId = process.env.CAP_WEB_EDITOR_WORKER_ID; + if (!workerId) return randomUUID(); + if (!/^[a-z][a-z0-9-]{0,23}$/.test(workerId)) { + throw new Error("Invalid editor worker ID"); + } + return `${workerId}.${randomUUID()}`; +} + +type VideoSource = EditorMediaSource & { + contentType: "video/webm" | "video/mp4"; + fps?: number; +}; + +type AudioSource = EditorMediaSource & { + contentType: "audio/webm" | "audio/mp4"; + offsetMs: number; +}; + +type InputEventsSource = EditorMediaSource & { + contentType: "application/x-ndjson"; +}; + +export type EditorSessionInput = { + videoId: string; + title: string; + captionsEnabled: boolean; + display: VideoSource; + camera?: VideoSource & { offsetMs: number }; + mic?: AudioSource; + systemAudio?: AudioSource; + inputEvents?: InputEventsSource; + audioDefault?: { + enabledByDefault: boolean; + isolation: "light" | "balanced" | "strong"; + } | null; + projectConfig?: Record; + legacyEditSpec?: LegacyEditorEditSpec; + audioAssets?: EditorAudioAsset[]; + imageAssets?: EditorImageAsset[]; + videoAssets?: EditorVideoAsset[]; + clips?: NativeEditorClipInput[]; + imports?: NativeEditorImport[]; +}; + +type NativeSession = Awaited>; + +type Session = { + id: string; + videoId: string; + native: NativeSession & { captionsEnabled: boolean }; + lastActive: number; + readyAt: number; + attached: boolean; + sockets: number; + detachedAt: number | null; +}; + +const sessions = new Map(); +const closingSessions = new Map>(); + +export type EditorSessionActivity = { + lastActive: number; + readyAt: number; + attached: boolean; + sockets: number; + detachedAt: number | null; + exportRunning: boolean; +}; + +/// A session ends when nobody has used it for a while, when it was never +/// picked up, when the client stopped polling its running export, or when +/// every socket it had closed and no request followed. +export function editorSessionIdle(session: EditorSessionActivity, now: number) { + return ( + now - session.lastActive > + (session.exportRunning ? EXPORT_LEASE_MS : IDLE_TIMEOUT_MS) || + (!session.attached && now - session.readyAt > SESSION_ATTACH_TIMEOUT_MS) || + (session.sockets === 0 && + session.detachedAt !== null && + now - Math.max(session.lastActive, session.detachedAt) > + DETACHED_TIMEOUT_MS) + ); +} + +function editorSessionExpired(session: Session, now: number) { + const activity = editorExportActivity(session.id); + return editorSessionIdle( + { + lastActive: Math.max(session.lastActive, activity?.at ?? 0), + readyAt: session.readyAt, + attached: session.attached, + sockets: session.sockets, + detachedAt: session.detachedAt, + exportRunning: activity?.running ?? false, + }, + now, + ); +} + +async function reclaimExpiredEditorSessions() { + const now = Date.now(); + const expired = [...sessions.values()].filter((session) => + editorSessionExpired(session, now), + ); + await Promise.all([ + ...closingSessions.values(), + ...expired.map((session) => closeEditorSession(session.id)), + ]); +} + +type Preparation = { + id: string; + videoId: string; + status: "preparing" | "ready" | "error" | "canceled" | "closed"; + sessionId?: string; + controller: AbortController; + task?: Promise; + updatedAt: number; +}; +const preparations = new Map(); +let starting = 0; + +export class EditorSessionBusyError extends Error {} + +export async function createEditorSession( + input: EditorSessionInput, + abortSignal?: AbortSignal, +) { + if (sessions.size + starting >= MAX_ACTIVE_SESSIONS) { + throw new EditorSessionBusyError("Editor render capacity is busy"); + } + starting++; + try { + const [ + displayResult, + cameraResult, + micResult, + systemAudioResult, + inputEventsResult, + ] = await Promise.allSettled([ + downloadEditorMedia(input.display, abortSignal), + input.camera + ? downloadEditorMedia(input.camera, abortSignal) + : Promise.resolve(null), + input.mic + ? downloadEditorMedia(input.mic, abortSignal) + : Promise.resolve(null), + input.systemAudio + ? downloadEditorMedia(input.systemAudio, abortSignal) + : Promise.resolve(null), + input.inputEvents + ? downloadEditorMedia(input.inputEvents, abortSignal) + : Promise.resolve(null), + ]); + if ( + displayResult.status === "rejected" || + cameraResult.status === "rejected" || + micResult.status === "rejected" || + systemAudioResult.status === "rejected" || + inputEventsResult.status === "rejected" + ) { + await Promise.all([ + displayResult.status === "fulfilled" + ? displayResult.value.cleanup() + : Promise.resolve(), + cameraResult.status === "fulfilled" && cameraResult.value + ? cameraResult.value.cleanup() + : Promise.resolve(), + micResult.status === "fulfilled" && micResult.value + ? micResult.value.cleanup() + : Promise.resolve(), + systemAudioResult.status === "fulfilled" && systemAudioResult.value + ? systemAudioResult.value.cleanup() + : Promise.resolve(), + inputEventsResult.status === "fulfilled" && inputEventsResult.value + ? inputEventsResult.value.cleanup() + : Promise.resolve(), + ]); + throw displayResult.status === "rejected" + ? displayResult.reason + : cameraResult.status === "rejected" + ? cameraResult.reason + : micResult.status === "rejected" + ? micResult.reason + : systemAudioResult.status === "rejected" + ? systemAudioResult.reason + : inputEventsResult.status === "rejected" + ? inputEventsResult.reason + : new Error("Editor source unavailable"); + } + const display = displayResult.value; + const camera = cameraResult.value; + const mic = micResult.value; + const systemAudio = systemAudioResult.value; + const inputEvents = inputEventsResult.value; + let project: Awaited>; + try { + const [displayMedia, cameraFps] = await Promise.all([ + inspectEditorDisplayMedia(display.path, input.display.fps), + camera + ? resolveEditorFps(camera.path, input.camera?.fps) + : Promise.resolve(null), + ]); + if ( + input.legacyEditSpec && + (!Number.isFinite(displayMedia.duration) || + Math.abs( + displayMedia.duration - input.legacyEditSpec.sourceDuration, + ) > 0.75) + ) { + throw new Error( + "The preserved edit source does not match its timeline", + ); + } + const inputs: NativeEditorInputs = { + title: input.title, + display: { + path: display.path, + contentType: input.display.contentType, + size: display.size, + fps: displayMedia.fps, + }, + ...(camera && input.camera + ? { + camera: { + path: camera.path, + contentType: input.camera.contentType, + size: camera.size, + fps: cameraFps ?? displayMedia.fps, + offsetMs: input.camera.offsetMs, + }, + } + : {}), + ...(mic && input.mic + ? { + mic: { + path: mic.path, + contentType: input.mic.contentType, + size: mic.size, + offsetMs: input.mic.offsetMs, + }, + } + : {}), + ...(systemAudio && input.systemAudio + ? { + systemAudio: { + path: systemAudio.path, + contentType: input.systemAudio.contentType, + size: systemAudio.size, + offsetMs: input.systemAudio.offsetMs, + }, + } + : {}), + ...(inputEvents && input.inputEvents + ? { + inputEvents: { + path: inputEvents.path, + contentType: input.inputEvents.contentType, + size: inputEvents.size, + }, + } + : {}), + mixedAudioInDisplay: displayMedia.hasAudio && !mic && !systemAudio, + ...(input.audioDefault ? { audioDefault: input.audioDefault } : {}), + ...(input.legacyEditSpec + ? { legacyEditSpec: input.legacyEditSpec } + : {}), + ...(input.projectConfig + ? { + projectConfig: input.captionsEnabled + ? input.projectConfig + : stripEditorCaptionContent(input.projectConfig), + } + : {}), + ...(input.audioAssets ? { audioAssets: input.audioAssets } : {}), + ...(input.imageAssets ? { imageAssets: input.imageAssets } : {}), + ...(input.videoAssets ? { videoAssets: input.videoAssets } : {}), + ...(input.clips ? { clips: input.clips } : {}), + ...(input.imports ? { imports: input.imports } : {}), + }; + if (abortSignal?.aborted) throw new Error("Editor preparation canceled"); + project = await prepareNativeEditorProject(inputs, abortSignal); + } finally { + await Promise.all([ + display.cleanup(), + camera?.cleanup() ?? Promise.resolve(), + mic?.cleanup() ?? Promise.resolve(), + systemAudio?.cleanup() ?? Promise.resolve(), + inputEvents?.cleanup() ?? Promise.resolve(), + ]); + } + if (abortSignal?.aborted) { + await project.cleanup(); + throw new Error("Editor preparation canceled"); + } + const nativeSession = await startNativeEditorSession(project); + const native = { + ...nativeSession, + captionsEnabled: input.captionsEnabled, + }; + if (abortSignal?.aborted) { + await native.close(); + throw new Error("Editor preparation canceled"); + } + const readyAt = Date.now(); + const session: Session = { + id: newEditorSessionId(), + videoId: input.videoId, + native, + lastActive: readyAt, + readyAt, + attached: false, + sockets: 0, + detachedAt: null, + }; + sessions.set(session.id, session); + return session.id; + } finally { + starting--; + } +} + +export async function beginEditorPreparation(input: EditorSessionInput) { + await reclaimExpiredEditorSessions(); + if (sessions.size + starting >= MAX_ACTIVE_SESSIONS) { + throw new EditorSessionBusyError("Editor render capacity is busy"); + } + const preparation: Preparation = { + id: newEditorSessionId(), + videoId: input.videoId, + status: "preparing", + controller: new AbortController(), + updatedAt: Date.now(), + }; + preparations.set(preparation.id, preparation); + preparation.task = createEditorSession(input, preparation.controller.signal) + .then(async (sessionId) => { + if (preparation.controller.signal.aborted) { + await closeEditorSession(sessionId); + preparation.status = "canceled"; + } else { + preparation.status = "ready"; + preparation.sessionId = sessionId; + } + preparation.updatedAt = Date.now(); + }) + .catch((error) => { + if (!preparation.controller.signal.aborted) { + console.error("Editor preparation failed", error); + preparation.status = "error"; + } + preparation.updatedAt = Date.now(); + }); + return preparation.id; +} + +export function getEditorPreparation(id: string) { + const preparation = preparations.get(id); + if (!preparation) return null; + if ( + preparation.status === "ready" && + preparation.sessionId && + !getEditorSessionVideoId(preparation.sessionId) + ) { + preparation.status = "closed"; + preparation.updatedAt = Date.now(); + } + return { + videoId: preparation.videoId, + status: preparation.status, + ...(preparation.sessionId ? { sessionId: preparation.sessionId } : {}), + }; +} + +export function getEditorSessionVideoId(id: string) { + const session = sessions.get(id); + if (!session) return null; + if (editorSessionExpired(session, Date.now())) { + void closeEditorSession(id).catch((error) => { + console.error("Expired editor session cleanup failed", error); + }); + return null; + } + return session.videoId; +} + +export function cancelEditorPreparation(id: string) { + const preparation = preparations.get(id); + if (!preparation) return false; + preparation.controller.abort(); + preparation.updatedAt = Date.now(); + if (preparation.sessionId) { + void closeEditorSession(preparation.sessionId).catch((error) => { + console.error("Editor session cancellation failed", error); + }); + } + preparation.status = "canceled"; + return true; +} + +export function getEditorSession(id: string) { + const session = sessions.get(id); + if (!session) return null; + const now = Date.now(); + if (editorSessionExpired(session, now)) { + void closeEditorSession(id).catch((error) => { + console.error("Expired editor session cleanup failed", error); + }); + return null; + } + session.lastActive = now; + return session.native; +} + +export function attachEditorSession(id: string) { + if (!getEditorSession(id)) return false; + const session = sessions.get(id); + if (!session) return false; + session.attached = true; + return true; +} + +export function openEditorSessionSocket(id: string) { + if (!attachEditorSession(id)) return false; + const session = sessions.get(id); + if (!session) return false; + session.sockets++; + session.detachedAt = null; + return true; +} + +export function closeEditorSessionSocket(id: string) { + const session = sessions.get(id); + if (!session || session.sockets === 0) return; + session.sockets--; + if (session.sockets === 0) session.detachedAt = Date.now(); +} + +export async function closeEditorSession(id: string) { + const closing = closingSessions.get(id); + if (closing) return closing; + const session = sessions.get(id); + if (!session) return false; + sessions.delete(id); + for (const preparation of preparations.values()) { + if (preparation.sessionId === id && preparation.status === "ready") { + preparation.status = "closed"; + preparation.updatedAt = Date.now(); + } + } + const task = (async () => { + try { + await Promise.all([ + closeEditorExports(id), + closeEditorVideoImports(id), + closeEditorCapImports(id), + ]); + } finally { + try { + await session.native.close(); + } finally { + closingSessions.delete(id); + } + } + return true; + })(); + closingSessions.set(id, task); + return task; +} + +export async function closeAllEditorSessions() { + for (const preparation of preparations.values()) { + preparation.controller.abort(); + } + await Promise.all( + [...preparations.values()].map( + (preparation) => preparation.task ?? Promise.resolve(), + ), + ); + await Promise.all([ + ...closingSessions.values(), + ...[...sessions.keys()].map(closeEditorSession), + ]); +} + +const sweep = setInterval(() => { + const now = Date.now(); + for (const preparation of preparations.values()) { + if (now - preparation.updatedAt > PREPARATION_RETENTION_MS) { + if (preparation.status === "preparing") { + preparation.controller.abort(); + } + preparations.delete(preparation.id); + } + } + for (const session of sessions.values()) { + if (editorSessionExpired(session, now)) { + void closeEditorSession(session.id).catch((error) => { + console.error("Editor session cleanup failed", error); + }); + } + } +}, 60_000); +sweep.unref(); diff --git a/apps/media-server/src/lib/editor-signed-assets.ts b/apps/media-server/src/lib/editor-signed-assets.ts new file mode 100644 index 00000000000..c74c83e692e --- /dev/null +++ b/apps/media-server/src/lib/editor-signed-assets.ts @@ -0,0 +1,222 @@ +import { createHash, randomUUID } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { + lstat, + mkdir, + open, + readFile, + rename, + rm, + writeFile, +} from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { fetchMedia } from "./media-transfer"; + +export type SignedEditorAsset = { + path: string; + name: string; + url: string; + size: number; + contentType: string; + objectIdentity?: string | null; +}; + +const DOWNLOAD_TIMEOUT_MS = 2 * 60 * 1000; + +function assetReceiptPath(projectPath: string, assetPath: string) { + return join( + projectPath, + ".editor-asset-receipts", + `${createHash("sha256").update(assetPath).digest("hex")}.json`, + ); +} + +async function fileDigest(path: string) { + const digest = createHash("sha256"); + for await (const bytes of createReadStream(path)) digest.update(bytes); + return digest.digest("hex"); +} + +async function confirmsExistingAsset( + destination: string, + receiptPath: string, + asset: SignedEditorAsset, +) { + if (!asset.objectIdentity) return false; + const receiptFile = await lstat(receiptPath).catch(() => null); + if (!receiptFile?.isFile() || receiptFile.size > 1024) return false; + let receipt: unknown; + try { + receipt = JSON.parse(await readFile(receiptPath, "utf8")); + } catch { + return false; + } + if ( + typeof receipt !== "object" || + receipt === null || + !("identity" in receipt) || + receipt.identity !== asset.objectIdentity || + !("size" in receipt) || + receipt.size !== asset.size || + !("sha256" in receipt) || + typeof receipt.sha256 !== "string" || + !/^[0-9a-f]{64}$/.test(receipt.sha256) + ) { + return false; + } + return (await fileDigest(destination).catch(() => null)) === receipt.sha256; +} + +async function saveAssetReceipt( + receiptPath: string, + identity: string, + size: number, + sha256: string, +) { + const directory = dirname(receiptPath); + await mkdir(directory, { recursive: true, mode: 0o700 }); + if (!(await lstat(directory)).isDirectory()) { + throw new Error("Invalid editor asset receipt folder"); + } + const temporary = join(directory, `${randomUUID()}.tmp`); + try { + await writeFile(temporary, JSON.stringify({ identity, size, sha256 }), { + flag: "wx", + mode: 0o600, + }); + await rename(temporary, receiptPath); + } finally { + await rm(temporary, { force: true }); + } +} + +async function stageSignedEditorAssetUnlocked< + T extends Record, +>( + projectPath: string, + asset: SignedEditorAsset, + probe: (path: string) => Promise, + abortSignal?: AbortSignal, + downloadTimeoutMs = DOWNLOAD_TIMEOUT_MS, +) { + const destination = join(projectPath, asset.path); + const receiptPath = assetReceiptPath(projectPath, asset.path); + await mkdir(dirname(destination), { recursive: true, mode: 0o700 }); + const existing = await lstat(destination).catch(() => null); + if (existing) { + if (!existing.isFile() || existing.size !== asset.size) { + throw new Error("Imported editor asset changed"); + } + if (await confirmsExistingAsset(destination, receiptPath, asset)) { + return { + path: asset.path, + name: asset.name, + ...(await probe(destination)), + }; + } + await rm(destination, { force: true }); + } + const handle = await open(destination, "wx", 0o600); + const digest = asset.objectIdentity ? createHash("sha256") : null; + let closed = false; + try { + const response = await fetchMedia(asset.url, { + headers: asset.objectIdentity + ? { "If-Match": asset.objectIdentity } + : undefined, + signal: abortSignal + ? AbortSignal.any([abortSignal, AbortSignal.timeout(downloadTimeoutMs)]) + : AbortSignal.timeout(downloadTimeoutMs), + redirect: "manual", + }); + if (!response.ok || !response.body) { + throw new Error(`Editor asset download failed: ${response.status}`); + } + const length = response.headers.get("content-length"); + if (length && Number(length) !== asset.size) { + throw new Error("Editor asset size changed before download"); + } + const identity = response.headers.get("etag"); + if (asset.objectIdentity && identity !== asset.objectIdentity) { + throw new Error("Editor asset identity changed before download"); + } + let received = 0; + const reader = response.body.getReader(); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + received += value.byteLength; + if (received > asset.size) { + throw new Error("Editor asset exceeded expected size"); + } + digest?.update(value); + await handle.writeFile(value); + } + } finally { + await reader.cancel().catch(() => undefined); + reader.releaseLock(); + } + if (received !== asset.size) { + throw new Error("Editor asset download was incomplete"); + } + await handle.sync(); + await handle.close(); + closed = true; + const details = await probe(destination); + if (asset.objectIdentity && digest) { + await saveAssetReceipt( + receiptPath, + asset.objectIdentity, + asset.size, + digest.digest("hex"), + ); + } + return { + path: asset.path, + name: asset.name, + ...details, + }; + } catch (error) { + if (!closed) await handle.close(); + await rm(destination, { force: true }); + throw error; + } +} + +const pendingStages = new Map>(); + +export async function stageSignedEditorAsset>( + projectPath: string, + asset: SignedEditorAsset, + probe: (path: string) => Promise, + abortSignal?: AbortSignal, + downloadTimeoutMs = DOWNLOAD_TIMEOUT_MS, +) { + if (!Number.isSafeInteger(downloadTimeoutMs) || downloadTimeoutMs < 1000) { + throw new Error("Invalid editor asset timeout"); + } + const destination = join(projectPath, asset.path); + const previous = pendingStages.get(destination); + let release: () => void = () => undefined; + const turn = new Promise((resolve) => { + release = () => resolve(); + }); + pendingStages.set(destination, turn); + try { + if (previous) await previous; + if (abortSignal?.aborted) + throw new Error("Editor asset staging was canceled"); + return await stageSignedEditorAssetUnlocked( + projectPath, + asset, + probe, + abortSignal, + downloadTimeoutMs, + ); + } finally { + release(); + if (pendingStages.get(destination) === turn) + pendingStages.delete(destination); + } +} diff --git a/apps/media-server/src/lib/editor-socket-tickets.ts b/apps/media-server/src/lib/editor-socket-tickets.ts new file mode 100644 index 00000000000..79a6a61b00b --- /dev/null +++ b/apps/media-server/src/lib/editor-socket-tickets.ts @@ -0,0 +1,121 @@ +import { randomBytes } from "node:crypto"; +import { getEditorSession, getEditorSessionVideoId } from "./editor-sessions"; + +const TICKET_TTL_MS = 30_000; +const MAX_TICKETS = 300; + +export type EditorSocketScope = "frames" | "audio" | "events" | "commands"; + +type Ticket = { + sessionId: string; + scope: EditorSocketScope; + origin: string; + expiresAt: number; +}; + +const tickets = new Map(); + +export function publicEditorOrigin() { + const configured = process.env.CAP_WEB_EDITOR_PUBLIC_ORIGIN; + if (!configured) + throw new Error("Public editor socket origin is unavailable"); + const url = new URL(configured); + if ( + (url.protocol !== "https:" && + !( + url.protocol === "http:" && + ["localhost", "127.0.0.1"].includes(url.hostname) + )) || + url.username || + url.password || + url.pathname !== "/" || + url.search || + url.hash + ) { + throw new Error("Invalid public editor socket origin"); + } + return url.origin; +} + +function publicSocketBase() { + const url = new URL(publicEditorOrigin()); + url.protocol = url.protocol === "https:" ? "wss:" : "ws:"; + return url.origin; +} + +export function createEditorSocketTickets(sessionId: string, origin: string) { + const page = new URL(origin); + if ( + (page.protocol !== "https:" && + !( + page.protocol === "http:" && + ["localhost", "127.0.0.1"].includes(page.hostname) + )) || + page.username || + page.password || + page.origin !== origin + ) { + throw new Error("Invalid editor page origin"); + } + const session = getEditorSession(sessionId); + const videoId = getEditorSessionVideoId(sessionId); + if (!session || !videoId) return null; + if (tickets.size + 4 > MAX_TICKETS) { + throw new Error("Editor socket ticket capacity is busy"); + } + const base = publicSocketBase(); + const sockets = Object.fromEntries( + (["frames", "audio", "events", "commands"] as const).map((scope) => { + const ticket = randomBytes(32).toString("base64url"); + tickets.set(ticket, { + sessionId, + scope, + origin, + expiresAt: Date.now() + TICKET_TTL_MS, + }); + return [ + scope, + { + url: `${base}/editor/sessions/${encodeURIComponent(sessionId)}/${scope}`, + ticket, + }, + ]; + }), + ) as Record; + return { videoId, sockets }; +} + +export function consumeEditorSocketTicket( + sessionId: string, + scope: EditorSocketScope, + origin: string | null, + protocols: string | null, +) { + const protocol = protocols + ?.split(",") + .map((value) => value.trim()) + .find((value) => value.startsWith("cap-editor-ticket.")); + if (!protocol) return null; + const token = protocol.slice("cap-editor-ticket.".length); + const ticket = tickets.get(token); + tickets.delete(token); + if ( + !ticket || + !origin || + ticket.expiresAt < Date.now() || + ticket.sessionId !== sessionId || + ticket.scope !== scope || + ticket.origin !== origin || + !protocols?.split(",").some((value) => value.trim() === "cap-editor-v1") + ) { + return null; + } + return getEditorSession(sessionId); +} + +const sweep = setInterval(() => { + for (const [token, ticket] of tickets) { + if (ticket.expiresAt < Date.now()) tickets.delete(token); + } +}, 30_000); +sweep.unref(); diff --git a/apps/media-server/src/lib/editor-video-assets.ts b/apps/media-server/src/lib/editor-video-assets.ts new file mode 100644 index 00000000000..765a7ca557d --- /dev/null +++ b/apps/media-server/src/lib/editor-video-assets.ts @@ -0,0 +1,138 @@ +import { runEditorFile as runFile } from "./editor-process"; +import { + type SignedEditorAsset, + stageSignedEditorAsset, +} from "./editor-signed-assets"; + +const MAX_VIDEO_BYTES = 12 * 1024 * 1024 * 1024; +const MAX_VIDEO_PIXELS = 33_554_432; +const VIDEO_DOWNLOAD_TIMEOUT_MS = 30 * 60 * 1000; +const assetPath = + /^content\/videos\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.(mp4|mov|avi|mkv|webm|wmv|m4v|flv)$/; +const contentTypes: Record = { + mp4: "video/mp4", + mov: "video/quicktime", + avi: "video/x-msvideo", + mkv: "video/x-matroska", + webm: "video/webm", + wmv: "video/x-ms-wmv", + m4v: "video/mp4", + flv: "video/x-flv", +}; + +export type EditorVideoAsset = SignedEditorAsset; + +export function validateEditorVideoAsset(asset: EditorVideoAsset) { + const match = assetPath.exec(asset.path); + const url = new URL(asset.url); + if ( + !match || + contentTypes[match[1] ?? ""] !== asset.contentType || + asset.name.length < 1 || + asset.name.length > 100 || + asset.name.split("").some((character) => { + const code = character.charCodeAt(0); + return ( + code < 32 || code === 127 || character === "/" || character === "\\" + ); + }) || + !Number.isSafeInteger(asset.size) || + asset.size < 1 || + asset.size > MAX_VIDEO_BYTES || + (url.protocol !== "https:" && + !( + url.protocol === "http:" && + process.env.CAP_WEB_EDITOR_ALLOW_HTTP_MEDIA === "1" + )) || + url.username || + url.password || + (asset.objectIdentity != null && + (asset.objectIdentity.length < 1 || asset.objectIdentity.length > 256)) + ) { + throw new Error("Invalid editor video asset"); + } +} + +function asRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function videoFrameRate(value: unknown) { + if (typeof value !== "string") return 30; + const [numerator, denominator] = value.split("/").map(Number); + const rate = numerator / denominator; + return Number.isFinite(rate) && rate >= 1 && rate <= 240 + ? Math.round(rate) + : 30; +} + +export async function probeVideo(path: string) { + const { stdout } = await runFile( + "ffprobe", + [ + "-v", + "error", + "-show_entries", + "stream=codec_type,width,height,avg_frame_rate,duration:format=duration", + "-of", + "json", + path, + ], + { timeout: 30_000, maxBuffer: 64 * 1024 }, + ).catch(() => { + throw new Error("Imported file has no decodable video"); + }); + const data: unknown = JSON.parse(stdout); + if (!asRecord(data) || !Array.isArray(data.streams)) { + throw new Error("Imported file has no video track"); + } + const video = data.streams.find( + (stream: unknown) => asRecord(stream) && stream.codec_type === "video", + ); + if (!asRecord(video)) throw new Error("Imported file has no video track"); + const width = Number(video.width); + const height = Number(video.height); + if ( + !Number.isSafeInteger(width) || + !Number.isSafeInteger(height) || + width < 1 || + height < 1 || + width > 16_384 || + height > 16_384 || + width * height > MAX_VIDEO_PIXELS + ) { + throw new Error("Imported video dimensions are too large"); + } + const duration = Number( + asRecord(data.format) && data.format.duration + ? data.format.duration + : video.duration, + ); + if (!Number.isFinite(duration) || duration <= 0) { + throw new Error("Cannot determine imported video duration"); + } + return { + duration, + fps: videoFrameRate(video.avg_frame_rate), + width, + height, + hasAudio: data.streams.some( + (stream: unknown) => asRecord(stream) && stream.codec_type === "audio", + ), + }; +} + +export async function stageSignedEditorVideoAsset( + projectPath: string, + asset: EditorVideoAsset, + abortSignal?: AbortSignal, +) { + validateEditorVideoAsset(asset); + return stageSignedEditorAsset( + projectPath, + asset, + probeVideo, + abortSignal, + VIDEO_DOWNLOAD_TIMEOUT_MS, + ); +} diff --git a/apps/media-server/src/lib/editor-video-imports.ts b/apps/media-server/src/lib/editor-video-imports.ts new file mode 100644 index 00000000000..1019f01a569 --- /dev/null +++ b/apps/media-server/src/lib/editor-video-imports.ts @@ -0,0 +1,112 @@ +import { randomUUID } from "node:crypto"; +import { + type EditorVideoAsset, + stageSignedEditorVideoAsset, + validateEditorVideoAsset, +} from "./editor-video-assets"; + +type ImportedVideo = Awaited>; +type ImportStatus = "staging" | "ready" | "error" | "canceled"; + +type VideoImport = { + id: string; + sessionId: string; + asset: EditorVideoAsset; + status: ImportStatus; + result: ImportedVideo | null; + error: string | null; + controller: AbortController; + task: Promise; +}; + +const imports = new Map(); +const activeBySession = new Map(); + +export class EditorVideoImportBusyError extends Error {} + +export function beginEditorVideoImport( + sessionId: string, + projectPath: string, + asset: EditorVideoAsset, +) { + validateEditorVideoAsset(asset); + for (const job of imports.values()) { + if (job.sessionId !== sessionId || job.asset.path !== asset.path) continue; + if ( + job.asset.size !== asset.size || + job.asset.objectIdentity !== asset.objectIdentity || + job.asset.contentType !== asset.contentType + ) { + throw new Error("Imported video identity changed"); + } + if (job.status === "ready" || job.status === "staging") return job.id; + imports.delete(job.id); + } + if (activeBySession.has(sessionId)) { + throw new EditorVideoImportBusyError("An imported video is still staging"); + } + const job: VideoImport = { + id: randomUUID(), + sessionId, + asset, + status: "staging", + result: null, + error: null, + controller: new AbortController(), + task: Promise.resolve(), + }; + imports.set(job.id, job); + activeBySession.set(sessionId, job.id); + job.task = stageSignedEditorVideoAsset( + projectPath, + asset, + job.controller.signal, + ) + .then((result) => { + job.result = result; + job.status = "ready"; + }) + .catch((error) => { + job.status = job.controller.signal.aborted ? "canceled" : "error"; + job.error = + error instanceof Error + ? error.message + : "Imported video could not be staged"; + }) + .finally(() => { + if (activeBySession.get(sessionId) === job.id) { + activeBySession.delete(sessionId); + } + }); + return job.id; +} + +export function getEditorVideoImport(sessionId: string, id: string) { + const job = imports.get(id); + if (!job || job.sessionId !== sessionId) return null; + return { + id: job.id, + status: job.status, + result: job.result, + error: job.error, + }; +} + +export async function cancelEditorVideoImport(sessionId: string, id: string) { + const job = imports.get(id); + if (!job || job.sessionId !== sessionId) return false; + job.controller.abort(); + await job.task; + imports.delete(id); + return true; +} + +export async function closeEditorVideoImports(sessionId: string) { + const jobs = [...imports.values()].filter( + (job) => job.sessionId === sessionId, + ); + for (const job of jobs) job.controller.abort(); + await Promise.all(jobs.map((job) => job.task)); + for (const job of jobs) imports.delete(job.id); + activeBySession.delete(sessionId); +} diff --git a/apps/media-server/src/lib/editor-wallpapers.ts b/apps/media-server/src/lib/editor-wallpapers.ts new file mode 100644 index 00000000000..6c6150087ae --- /dev/null +++ b/apps/media-server/src/lib/editor-wallpapers.ts @@ -0,0 +1,86 @@ +import { lstatSync, realpathSync } from "node:fs"; +import { isAbsolute, relative, resolve } from "node:path"; + +const logicalPrefix = "cap-web-wallpaper://assets/backgrounds/"; +const wallpaperFile = + /^(macOS|blue|purple|cities|dark|orange)\/[a-z0-9-]+\.jpg$/; +const nativePaths = new Map(); + +function asRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function editorWallpaperDirectory() { + return ( + process.env.CAP_WEB_EDITOR_WALLPAPER_DIR ?? + resolve(import.meta.dir, "../../../desktop/src-tauri/assets/backgrounds") + ); +} + +function nativeWallpaperPath(logical: string, directory: string) { + const file = logical.slice(logicalPrefix.length); + if (!wallpaperFile.test(file)) { + throw new Error("Invalid editor wallpaper path"); + } + const key = `${directory}\0${file}`; + const cached = nativePaths.get(key); + if (cached) return cached; + const root = realpathSync(directory); + const native = realpathSync(resolve(root, file)); + const within = relative(root, native); + if ( + within.startsWith("..") || + isAbsolute(within) || + !wallpaperFile.test(within) || + !lstatSync(native).isFile() + ) { + throw new Error("Invalid editor wallpaper path"); + } + nativePaths.set(key, native); + return native; +} + +export function mapEditorWallpaperConfig( + value: unknown, + direction: "native" | "browser", + directory = editorWallpaperDirectory(), +) { + if (!asRecord(value) || !asRecord(value.background)) return value; + const background = value.background; + if (!asRecord(background.source)) return value; + const source = background.source; + if (source.type !== "wallpaper" || typeof source.path !== "string") { + return value; + } + let path = source.path; + if (direction === "native" && path.startsWith(logicalPrefix)) { + path = nativeWallpaperPath(path, directory); + } else if (direction === "browser" && isAbsolute(path)) { + const file = relative(realpathSync(directory), path); + if (wallpaperFile.test(file)) path = `${logicalPrefix}${file}`; + } + if (path === source.path) return value; + return { + ...value, + background: { + ...background, + source: { ...source, path }, + }, + }; +} + +export function mapEditorInstanceWallpaper( + value: unknown, + direction: "native" | "browser", + directory = editorWallpaperDirectory(), +) { + if (!asRecord(value) || !asRecord(value.savedProjectConfig)) return value; + return { + ...value, + savedProjectConfig: mapEditorWallpaperConfig( + value.savedProjectConfig, + direction, + directory, + ), + }; +} diff --git a/apps/media-server/src/lib/editor-websocket.ts b/apps/media-server/src/lib/editor-websocket.ts new file mode 100644 index 00000000000..19b385ab645 --- /dev/null +++ b/apps/media-server/src/lib/editor-websocket.ts @@ -0,0 +1,494 @@ +import { randomBytes } from "node:crypto"; +import { addEditorAudioLibraryTrack } from "./editor-audio-library"; +import { renderEditorClipThumbnail } from "./editor-clip-thumbnails"; +import { + dispatchEditorSocketCommand, + type EditorCommandState, + type EditorSocketReply, + parseEditorSocketRequest, +} from "./editor-command-socket"; +import { + mapEditorConfigPaths, + mapEditorInstanceConfigPaths, +} from "./editor-config-paths"; +import { + cancelEditorExportEstimate, + getEditorExportEstimate, +} from "./editor-export-estimates"; +import { renderEditorExportPreview } from "./editor-export-previews"; +import { + closeEditorSessionSocket, + getEditorSession, + openEditorSessionSocket, +} from "./editor-sessions"; +import { + consumeEditorSocketTicket, + type EditorSocketScope, +} from "./editor-socket-tickets"; + +export type EditorSocketConnection = { + upstreamUrl: string; + scope: EditorSocketScope; + sessionId: string; + upstream: WebSocket | null; + commandState: EditorCommandState; + commandQueue: Promise; + pendingCommands: number; + abort: AbortController; + frameMode: "png" | "h264" | "png-fallback"; + reducedBitrate: boolean; + bandwidthProbeUsed: boolean; + sessionSocketOpen: boolean; +}; + +const SOCKET_PATH = + /^\/editor\/sessions\/((?:[a-z][a-z0-9-]{0,23}\.)?[0-9a-f-]{36})\/(frames|audio|events|commands)$/; +const FRAME_BACKPRESSURE_BYTES = 4 * 1024 * 1024; +const MAX_COMMAND_BYTES = 8 * 1024 * 1024 + 64 * 1024; +const MAX_CROP_JPEG_BYTES = 2 * 1024 * 1024; +const MAX_PENDING_COMMANDS = 64; +const H264_MODE = JSON.stringify({ mode: "h264" }); +const PNG_MODE = JSON.stringify({ mode: "png" }); +const LOW_BITRATE = JSON.stringify({ bitrate: "low" }); +const HIGH_BITRATE = JSON.stringify({ bitrate: "high" }); +const BANDWIDTH_PROBE_REQUEST = JSON.stringify({ probe: "bandwidth" }); +const BANDWIDTH_PROBE = Buffer.concat([ + Buffer.from("CAPBAND1"), + randomBytes(512 * 1024), +]); + +export function handleEditorSocketUpgrade( + request: Request, + server: Bun.Server, +) { + const match = new URL(request.url).pathname.match(SOCKET_PATH); + if (!match) return null; + const sessionId = match[1]; + const scope = match[2] as EditorSocketScope; + const native = consumeEditorSocketTicket( + sessionId, + scope, + request.headers.get("origin"), + request.headers.get("sec-websocket-protocol"), + ); + if (!native) return new Response("Unauthorized", { status: 401 }); + const h264Hint = + scope === "frames" && + request.headers + .get("sec-websocket-protocol") + ?.split(",") + .some((value) => value.trim() === "cap-editor-h264-v1") === true; + const upstream = new URL(`/${scope}`, native.origin); + upstream.protocol = "ws:"; + if (h264Hint) upstream.pathname = "/frames-h264"; + const accepted = server.upgrade(request, { + data: { + upstreamUrl: upstream.toString(), + scope, + sessionId, + upstream: null, + commandState: { + frameNumber: 0, + fps: 60, + resolutionBase: { x: 960, y: 540 }, + captionsEnabled: native.captionsEnabled, + }, + commandQueue: Promise.resolve(), + pendingCommands: 0, + abort: new AbortController(), + frameMode: h264Hint ? "h264" : "png", + reducedBitrate: h264Hint, + bandwidthProbeUsed: false, + sessionSocketOpen: false, + }, + headers: { "Sec-WebSocket-Protocol": "cap-editor-v1" }, + }); + return accepted + ? undefined + : new Response("WebSocket upgrade failed", { status: 400 }); +} + +function connectEditorUpstream( + ws: Bun.ServerWebSocket, + url: string, +) { + const previous = ws.data.upstream; + const native = getEditorSession(ws.data.sessionId); + if (!native) { + ws.close(1011, "Editor session closed"); + return; + } + const upstream = native.connectSocket(url); + upstream.binaryType = "arraybuffer"; + ws.data.upstream = upstream; + if (previous && previous.readyState < 2) previous.close(); + upstream.onopen = () => { + if ( + ws.data.upstream === upstream && + ws.data.frameMode === "h264" && + ws.data.reducedBitrate + ) { + upstream.send(LOW_BITRATE); + } + }; + upstream.onmessage = (event: MessageEvent) => { + if (ws.data.upstream !== upstream || ws.readyState !== 1) return; + if (!getEditorSession(ws.data.sessionId)) { + ws.close(1011, "Editor session closed"); + return; + } + if ( + ws.data.scope === "frames" && + ws.data.frameMode === "h264" && + !ws.data.reducedBitrate && + ws.getBufferedAmount() > 1024 * 1024 + ) { + ws.data.reducedBitrate = true; + if (upstream.readyState === WebSocket.OPEN) upstream.send(LOW_BITRATE); + } + if (ws.getBufferedAmount() > FRAME_BACKPRESSURE_BYTES) { + if (ws.data.scope === "frames") { + if (ws.data.frameMode === "h264") { + ws.data.frameMode = "png-fallback"; + if (upstream.readyState === WebSocket.OPEN) upstream.send(PNG_MODE); + ws.send( + JSON.stringify({ + kind: "cap-h264-unavailable", + error: "Editor preview connection is congested", + }), + ); + } + return; + } + ws.close(1013, "Editor socket is congested"); + return; + } + const packet = event.data; + if ( + typeof packet === "string" || + packet instanceof ArrayBuffer || + packet instanceof Uint8Array || + packet instanceof Blob + ) { + ws.send(packet); + } + }; + upstream.onclose = () => { + if (ws.data.upstream === upstream && ws.readyState === 1) + ws.close(1011, "Editor renderer disconnected"); + }; + upstream.onerror = () => { + if (ws.data.upstream === upstream && ws.readyState === 1) + ws.close(1011, "Editor renderer unavailable"); + }; +} + +export const editorWebSocketHandler: Bun.WebSocketHandler = + { + data: {} as EditorSocketConnection, + message(ws, message) { + if (ws.data.scope === "frames") { + if (typeof message !== "string") { + ws.close(1008, "Invalid editor frame mode"); + return; + } + if (message === H264_MODE && ws.data.frameMode === "png") { + ws.data.frameMode = "h264"; + const upstream = new URL(ws.data.upstreamUrl); + upstream.pathname = "/frames-h264"; + connectEditorUpstream(ws, upstream.toString()); + return; + } + if (message === H264_MODE && ws.data.frameMode === "h264") { + return; + } + if ( + message === BANDWIDTH_PROBE_REQUEST && + !ws.data.bandwidthProbeUsed + ) { + ws.data.bandwidthProbeUsed = true; + ws.send(BANDWIDTH_PROBE); + return; + } + if (message === LOW_BITRATE && ws.data.frameMode === "h264") { + ws.data.reducedBitrate = true; + if (ws.data.upstream?.readyState === WebSocket.OPEN) + ws.data.upstream.send(message); + return; + } + if (message === HIGH_BITRATE && ws.data.frameMode === "h264") { + ws.data.reducedBitrate = false; + if (ws.data.upstream?.readyState === WebSocket.OPEN) + ws.data.upstream.send(message); + return; + } + if ( + message === PNG_MODE && + (ws.data.frameMode === "h264" || ws.data.frameMode === "png-fallback") + ) { + if (ws.data.frameMode === "png-fallback") return; + ws.data.frameMode = "png-fallback"; + if (ws.data.upstream?.readyState === WebSocket.OPEN) + ws.data.upstream.send(message); + return; + } + ws.close(1008, "Invalid editor frame mode"); + return; + } + if (ws.data.scope !== "commands") { + ws.close(1008, "Editor sockets are read-only"); + return; + } + let value: unknown; + try { + value = JSON.parse( + typeof message === "string" + ? message + : Buffer.from(message).toString("utf8"), + ); + } catch { + ws.close(1008, "Invalid editor command"); + return; + } + const command = parseEditorSocketRequest(value); + if (!command) { + ws.close(1008, "Invalid editor command"); + return; + } + if (ws.data.pendingCommands >= MAX_PENDING_COMMANDS) { + ws.close(1013, "Editor command queue is full"); + return; + } + ws.data.pendingCommands++; + const parallelRead = + command.kind === "invoke" && + (command.name === "getMicWaveforms" || + command.name === "getSystemAudioWaveforms" || + command.name === "getClipThumbnail" || + command.name === "generateExportPreviewFast" || + command.name === "getExportEstimates" || + command.name === "cancelExportEstimates"); + const commandTask = ( + parallelRead ? Promise.resolve() : ws.data.commandQueue + ) + .then(async () => { + const native = getEditorSession(ws.data.sessionId); + if (!native) { + ws.close(1011, "Editor session closed"); + return; + } + ws.data.commandState.captionsEnabled = native.captionsEnabled; + let reply: EditorSocketReply; + try { + if ( + command.kind === "invoke" && + command.name === "getClipThumbnail" + ) { + if (command.args.length !== 2) + throw new Error("Invalid clip thumbnail request"); + reply = { + kind: "result", + id: command.id, + value: await renderEditorClipThumbnail( + native, + command.args[0], + command.args[1], + ws.data.abort.signal, + ), + }; + } else if ( + command.kind === "invoke" && + command.name === "getDisplayFrameForCropping" + ) { + const fps = command.args[0]; + if ( + command.args.length !== 1 || + typeof fps !== "number" || + !Number.isInteger(fps) || + fps < 1 || + fps > 60 + ) { + throw new Error("Invalid crop frame rate"); + } + const response = await native.request(`/crop-frame?fps=${fps}`, { + signal: AbortSignal.any([ + ws.data.abort.signal, + AbortSignal.timeout(30_000), + ]), + }); + if ( + !response.ok || + response.headers.get("content-type") !== "image/jpeg" + ) { + throw new Error( + `Crop frame request failed: ${response.status}`, + ); + } + const contentLength = response.headers.get("content-length"); + if ( + contentLength && + Number(contentLength) > MAX_CROP_JPEG_BYTES + ) { + throw new Error("Crop frame is too large"); + } + const jpeg = Buffer.from(await response.arrayBuffer()); + if (jpeg.length < 4 || jpeg.length > MAX_CROP_JPEG_BYTES) { + throw new Error("Crop frame is unavailable or too large"); + } + reply = { + kind: "result", + id: command.id, + value: { jpegBase64: jpeg.toString("base64") }, + }; + } else if ( + command.kind === "invoke" && + command.name === "generateExportPreviewFast" + ) { + reply = { + kind: "result", + id: command.id, + value: await renderEditorExportPreview( + ws.data.sessionId, + native, + command.args[0], + command.args[1], + ws.data.abort.signal, + ), + }; + } else if ( + command.kind === "invoke" && + command.name === "getExportEstimates" + ) { + reply = { + kind: "result", + id: command.id, + value: await getEditorExportEstimate( + ws.data.sessionId, + native, + command.args, + ws.data.abort.signal, + (id, value) => { + if (ws.readyState === 1) + ws.send(JSON.stringify({ kind: "channel", id, value })); + }, + ), + }; + } else if ( + command.kind === "invoke" && + command.name === "addAudioLibraryTrack" + ) { + if (command.args.length !== 1) + throw new Error("Invalid library track request"); + reply = { + kind: "result", + id: command.id, + value: await addEditorAudioLibraryTrack( + native.projectPath, + command.args[0], + ), + }; + } else if ( + command.kind === "invoke" && + command.name === "cancelExportEstimates" + ) { + cancelEditorExportEstimate(ws.data.sessionId); + reply = { kind: "result", id: command.id, value: null }; + } else { + reply = await dispatchEditorSocketCommand( + ws.data.sessionId, + command, + ws.data.commandState, + async (path, method, body, eventOnly) => { + const nativeBody = + path === "/config/memory" && method === "PUT" + ? mapEditorConfigPaths(body, "native", native.projectPath) + : body; + const headers = new Headers(); + if (nativeBody !== undefined) + headers.set("Content-Type", "application/json"); + if (eventOnly) + headers.set("Accept", "application/vnd.cap.editor-event"); + const response = await native.request(path, { + method, + headers, + ...(nativeBody === undefined + ? {} + : { body: JSON.stringify(nativeBody) }), + signal: AbortSignal.any([ + ws.data.abort.signal, + AbortSignal.timeout(parallelRead ? 60_000 : 10_000), + ]), + }); + if (!response.ok) { + throw new Error( + `Editor renderer request failed: ${response.status}`, + ); + } + if (response.status === 204) return null; + if (eventOnly) { + await response.body?.cancel(); + return null; + } + const value: unknown = await response.json(); + if (path === "/instance") + return mapEditorInstanceConfigPaths( + value, + "browser", + native.projectPath, + ); + if (path === "/config") + return mapEditorConfigPaths( + value, + "browser", + native.projectPath, + ); + return value; + }, + ); + } + } catch (cause) { + reply = { + kind: "error", + id: command.id, + error: + cause instanceof Error + ? cause.message + : "Editor command failed", + }; + } + if (ws.readyState === 1) ws.send(JSON.stringify(reply)); + }) + .catch((error) => { + if (ws.readyState !== 1) return; + console.error("Editor socket command failed", error); + ws.close(1011, "Editor command unavailable"); + }) + .finally(() => { + ws.data.pendingCommands--; + }); + if (!parallelRead) ws.data.commandQueue = commandTask; + }, + open(ws) { + if (!openEditorSessionSocket(ws.data.sessionId)) { + ws.close(1011, "Editor session closed"); + return; + } + ws.data.sessionSocketOpen = true; + if (ws.data.scope === "commands") return; + if (ws.data.upstream) return; + connectEditorUpstream(ws, ws.data.upstreamUrl); + }, + close(ws) { + if (ws.data.sessionSocketOpen) { + ws.data.sessionSocketOpen = false; + closeEditorSessionSocket(ws.data.sessionId); + } + ws.data.abort.abort(); + const upstream = ws.data.upstream; + ws.data.upstream = null; + if (upstream && upstream.readyState < 2) upstream.close(); + }, + maxPayloadLength: MAX_COMMAND_BYTES, + backpressureLimit: FRAME_BACKPRESSURE_BYTES, + closeOnBackpressureLimit: false, + }; diff --git a/apps/media-server/src/lib/media-probe.ts b/apps/media-server/src/lib/media-probe.ts index ef6ee8ace8f..a9016b3a0f5 100644 --- a/apps/media-server/src/lib/media-probe.ts +++ b/apps/media-server/src/lib/media-probe.ts @@ -1,4 +1,9 @@ -import type { AudioCodec, MediaCodec, VideoCodec } from "mediabunny"; +import { + type AudioCodec, + IsobmffInputFormat, + type MediaCodec, + type VideoCodec, +} from "mediabunny"; import type { VideoMetadata } from "./job-manager"; import { createMediaInput, @@ -78,10 +83,14 @@ async function probeMedia(path: string): Promise { } const audioTrack = await input.getPrimaryAudioTrack(); + // A fragmented MP4, as the browser recorder writes, declares only its + // first fragment's length. MP4 packet tables make the real end cheap. const duration = - (await input.getDurationFromMetadata(undefined, { - skipLiveWait: true, - })) ?? + ((await input.getFormat()) instanceof IsobmffInputFormat + ? null + : await input.getDurationFromMetadata(undefined, { + skipLiveWait: true, + })) ?? (await input.computeDuration(undefined, { skipLiveWait: true, })); diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.ts index 8f87ca0cec3..ea549cc1bb7 100644 --- a/apps/media-server/src/lib/media-video.ts +++ b/apps/media-server/src/lib/media-video.ts @@ -28,8 +28,13 @@ import { type TempFileHandle, } from "./temp-files"; +// Keep audio in runs of up to a second rather than a slice per video frame: +// the editor reads a video's audio alone for its waveform, which interleaving +// at every frame turns into reading the whole file. +const AUDIO_RUN_ARGS = ["-chunk_duration", "1000000"]; + const PROCESS_TIMEOUT_PER_SECOND_MS = 20_000; -const MAX_PROCESS_TIMEOUT_MS = 2 * 60 * 60 * 1000; +export const MAX_PROCESS_TIMEOUT_MS = 2 * 60 * 60 * 1000; // HLS/DASH sources are pulled as many sequential segment requests rather than // one streamed fetch, so per-request overhead scales with video length. A // flat 10-minute budget is enough for typical short recordings but not for a @@ -1285,13 +1290,26 @@ export async function downloadVideoToTemp( } } +/** The size limits turned to match the video, so portrait video keeps its resolution. */ +function getOrientedLimits( + metadata: VideoMetadata, + options: VideoProcessingOptions, +) { + const maxWidth = options.maxWidth ?? DEFAULT_OPTIONS.maxWidth; + const maxHeight = options.maxHeight ?? DEFAULT_OPTIONS.maxHeight; + const longEdge = Math.max(maxWidth, maxHeight); + const shortEdge = Math.min(maxWidth, maxHeight); + return metadata.height > metadata.width + ? { maxWidth: shortEdge, maxHeight: longEdge } + : { maxWidth: longEdge, maxHeight: shortEdge }; +} + function needsVideoTranscode( metadata: VideoMetadata, options: VideoProcessingOptions, sourceH264Level: number | null, ): boolean { - const maxWidth = options.maxWidth ?? DEFAULT_OPTIONS.maxWidth; - const maxHeight = options.maxHeight ?? DEFAULT_OPTIONS.maxHeight; + const { maxWidth, maxHeight } = getOrientedLimits(metadata, options); return ( metadata.width > maxWidth || metadata.height > maxHeight || @@ -1393,8 +1411,7 @@ function getTargetVideoDimensions( metadata: VideoMetadata, options: VideoProcessingOptions, ) { - const maxWidth = options.maxWidth ?? DEFAULT_OPTIONS.maxWidth; - const maxHeight = options.maxHeight ?? DEFAULT_OPTIONS.maxHeight; + const { maxWidth, maxHeight } = getOrientedLimits(metadata, options); return { width: Math.min(metadata.width, maxWidth), @@ -1407,12 +1424,14 @@ export function pickMobileSafeH264Level( options: VideoProcessingOptions = {}, ): H264Level { const { width, height } = getTargetVideoDimensions(metadata, options); + const longEdge = Math.max(width, height); + const shortEdge = Math.min(width, height); - if (width <= MAX_LEVEL_4_2_WIDTH && height <= MAX_LEVEL_4_2_HEIGHT) { + if (longEdge <= MAX_LEVEL_4_2_WIDTH && shortEdge <= MAX_LEVEL_4_2_HEIGHT) { return { value: 42, ffmpegValue: "4.2" }; } - if (width <= MAX_LEVEL_5_1_WIDTH && height <= MAX_LEVEL_5_1_HEIGHT) { + if (longEdge <= MAX_LEVEL_5_1_WIDTH && shortEdge <= MAX_LEVEL_5_1_HEIGHT) { return { value: 51, ffmpegValue: "5.1" }; } @@ -1500,13 +1519,14 @@ export async function processVideo( ? await probeH264Level(inputPath, abortSignal) : null; const targetH264Level = pickMobileSafeH264Level(metadata, opts); + const limits = getOrientedLimits(metadata, opts); const normalizeH264Level = opts.normalizeH264Level && metadata.videoCodec === "h264" && sourceH264Level !== null && sourceH264Level > targetH264Level.value && - metadata.width <= opts.maxWidth && - metadata.height <= opts.maxHeight; + metadata.width <= limits.maxWidth && + metadata.height <= limits.maxHeight; const videoTranscode = remuxOnly ? false : needsVideoTranscode(metadata, opts, sourceH264Level); @@ -1546,7 +1566,7 @@ export async function processVideo( "-crf", opts.crf.toString(), "-vf", - `scale='min(${opts.maxWidth},iw)':'min(${opts.maxHeight},ih)':force_original_aspect_ratio=decrease,scale=trunc(iw/2)*2:trunc(ih/2)*2`, + `scale='min(${limits.maxWidth},iw)':'min(${limits.maxHeight},ih)':force_original_aspect_ratio=decrease,scale=trunc(iw/2)*2:trunc(ih/2)*2`, "-pix_fmt", "yuv420p", "-level:v", @@ -1573,6 +1593,7 @@ export async function processVideo( ffmpegArgs.push( "-movflags", "+faststart", + ...(metadata.audioCodec ? AUDIO_RUN_ARGS : []), ...extraOutputArgs, "-progress", "pipe:2", @@ -2574,6 +2595,7 @@ export async function muxMediaTracksToMp4( "1000000", "-movflags", "+faststart", + ...AUDIO_RUN_ARGS, outputPath, ] : [ diff --git a/apps/media-server/src/routes/editor.ts b/apps/media-server/src/routes/editor.ts new file mode 100644 index 00000000000..d8d8bfc0c69 --- /dev/null +++ b/apps/media-server/src/routes/editor.ts @@ -0,0 +1,1089 @@ +import { rm } from "node:fs/promises"; +import { + CAP_BUNDLE_CONTENT_TYPE, + MAX_CAP_BUNDLE_BYTES, +} from "@cap/editor-cap-bundle"; +import { Hono } from "hono"; +import { z } from "zod"; +import { hasEditorCaptionContent } from "../../../web/lib/editor-caption-access"; +import { validWebEditorTitle } from "../../../web/lib/editor-recording-title"; +import { validateMediaServerSecret } from "../lib/auth"; +import { stageSignedEditorAudioAsset } from "../lib/editor-assets"; +import { prepareEditorCapCaptionAudio } from "../lib/editor-cap-captions"; +import { + beginEditorCapImport, + cancelEditorCapImport, + EditorCapImportBusyError, + getEditorCapImport, +} from "../lib/editor-cap-imports"; +import { + mapEditorConfigPaths, + mapEditorInstanceConfigPaths, +} from "../lib/editor-config-paths"; +import { + beginEditorExport, + cancelEditorExport, + consumeEditorExportDownloadTicket, + createEditorExportDownloadTicket, + EditorExportBusyError, + editorExportSettingsSchema, + getEditorExport, + getEditorExportFile, + markEditorExportDownloadStarted, +} from "../lib/editor-exports"; +import { stageSignedEditorImageAsset } from "../lib/editor-image-assets"; +import { + consumeEditorProjectBundleDownloadTicket, + createEditorProjectBundleDownloadTicket, + editorProjectBundleResponse, +} from "../lib/editor-project-bundles"; +import { + listEditorRenderProject, + uploadEditorRenderProject, +} from "../lib/editor-render-project"; +import { + attachEditorSession, + beginEditorPreparation, + cancelEditorPreparation, + closeEditorSession, + EditorSessionBusyError, + getEditorPreparation, + getEditorSession, + getEditorSessionVideoId, +} from "../lib/editor-sessions"; +import { createEditorSocketTickets } from "../lib/editor-socket-tickets"; +import { + beginEditorVideoImport, + cancelEditorVideoImport, + EditorVideoImportBusyError, + getEditorVideoImport, +} from "../lib/editor-video-imports"; + +const MAX_CONFIG_BYTES = 8 * 1024 * 1024; + +const legacyEditSpecSchema = z.object({ + version: z.literal(1), + sourceDuration: z.number().finite().positive().max(86_400), + keepRanges: z + .array( + z.object({ + start: z.number().finite().nonnegative(), + end: z.number().finite().positive(), + }), + ) + .min(1) + .max(1_000), +}); + +const mediaSchema = z.object({ + url: z.string().url(), + contentType: z.enum(["video/webm", "video/mp4"]), + size: z + .number() + .int() + .positive() + .max(12 * 1024 * 1024 * 1024), + fps: z.number().int().min(1).max(120).optional(), + objectIdentity: z.string().min(1).max(256).nullable().optional(), +}); + +const audioMediaSchema = z.object({ + url: z.string().url(), + contentType: z.enum(["audio/webm", "audio/mp4"]), + size: z + .number() + .int() + .positive() + .max(12 * 1024 * 1024 * 1024), + objectIdentity: z.string().min(1).max(256).nullable().optional(), + offsetMs: z.number().int().min(-30_000).max(30_000), +}); + +const inputEventsMediaSchema = z.object({ + url: z.string().url(), + contentType: z.literal("application/x-ndjson"), + size: z + .number() + .int() + .positive() + .max(64 * 1024 * 1024), + objectIdentity: z.string().min(1).max(256).nullable().optional(), +}); + +const audioAssetSchema = z.object({ + path: z + .string() + .regex( + /^assets\/audio\/import-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.(ogg|m4a|mp3|wav|aac|flac)$/, + ), + name: z.string().min(1).max(100), + url: z.string().url(), + size: z + .number() + .int() + .positive() + .max(32 * 1024 * 1024), + contentType: z.string().min(1).max(100), + objectIdentity: z.string().min(1).max(256).nullable().optional(), +}); + +const imageAssetSchema = z.object({ + path: z + .string() + .regex( + /^content\/images\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.(png|jpg|webp|gif|bmp|tiff)$/, + ), + name: z.string().min(1).max(100), + url: z.string().url(), + size: z + .number() + .int() + .positive() + .max(64 * 1024 * 1024), + contentType: z.string().min(1).max(100), + objectIdentity: z.string().min(1).max(256).nullable().optional(), +}); + +const videoAssetSchema = z.object({ + path: z + .string() + .regex( + /^content\/videos\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.(mp4|mov|avi|mkv|webm|wmv|m4v|flv)$/, + ), + name: z.string().min(1).max(100), + url: z.string().url(), + size: z + .number() + .int() + .positive() + .max(12 * 1024 * 1024 * 1024), + contentType: z.string().min(1).max(100), + objectIdentity: z.string().min(1).max(256).nullable().optional(), +}); + +const capAssetSchema = z.object({ + path: z + .string() + .regex( + /^content\/imports\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.capbundle$/, + ), + name: z.string().min(1).max(100), + url: z.string().url(), + size: z.number().int().positive().max(MAX_CAP_BUNDLE_BYTES), + contentType: z.literal(CAP_BUNDLE_CONTENT_TYPE), + objectIdentity: z.string().min(1).max(256).nullable().optional(), +}); + +const capCaptionRequestSchema = z.object({ + asset: capAssetSchema, + segments: z + .array( + z + .object({ + mediaDurationMs: z.number().int().positive().max(86_400_000), + segmentDurationMs: z.number().int().positive().max(86_400_000), + hasAudio: z.boolean(), + }) + .refine( + (segment) => segment.segmentDurationMs >= segment.mediaDurationMs, + ), + ) + .min(1) + .max(1000), +}); + +const clipSchema = z + .object({ + displayPath: z + .string() + .regex( + /^content\/videos\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.(mp4|webm)$/, + ), + duration: z.number().positive().max(86_400), + fps: z.number().int().min(1).max(120), + hasAudio: z.boolean(), + cameraPath: z + .string() + .regex( + /^content\/videos\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.(mp4|webm)$/, + ) + .optional(), + cameraFps: z.number().int().min(1).max(120).optional(), + cameraOffsetMs: z.number().int().min(-30_000).max(30_000).optional(), + }) + .refine( + (clip) => + (clip.cameraPath === undefined && + clip.cameraFps === undefined && + clip.cameraOffsetMs === undefined) || + (clip.cameraPath !== undefined && + clip.cameraPath !== clip.displayPath && + clip.cameraFps !== undefined && + clip.cameraOffsetMs !== undefined), + ); + +const importSchema = z.union([ + z.object({ kind: z.literal("clip"), clip: clipSchema }), + z.object({ + kind: z.literal("cap"), + asset: capAssetSchema, + clipCount: z.number().int().min(1).max(1000), + }), +]); + +const sessionSchema = z + .object({ + videoId: z.string().min(1).max(100), + title: z.string().min(1).max(255), + captionsEnabled: z.boolean().default(false), + display: mediaSchema, + camera: mediaSchema + .extend({ offsetMs: z.number().int().min(-30_000).max(30_000) }) + .optional(), + mic: audioMediaSchema.optional(), + systemAudio: audioMediaSchema.optional(), + inputEvents: inputEventsMediaSchema.optional(), + audioDefault: z + .object({ + enabledByDefault: z.boolean(), + isolation: z.enum(["light", "balanced", "strong"]), + }) + .nullable() + .optional(), + projectConfig: z + .record(z.unknown()) + .refine( + (config) => + Buffer.byteLength(JSON.stringify(config), "utf8") <= MAX_CONFIG_BYTES, + ) + .optional(), + legacyEditSpec: legacyEditSpecSchema.optional(), + audioAssets: z + .array(audioAssetSchema) + .max(100) + .refine( + (assets) => + assets.reduce((bytes, asset) => bytes + asset.size, 0) <= + 512 * 1024 * 1024, + ) + .optional(), + imageAssets: z + .array(imageAssetSchema) + .max(100) + .refine( + (assets) => + assets.reduce((bytes, asset) => bytes + asset.size, 0) <= + 512 * 1024 * 1024, + ) + .optional(), + videoAssets: z + .array(videoAssetSchema) + .max(100) + .refine( + (assets) => + assets.reduce((bytes, asset) => bytes + asset.size, 0) <= + 12 * 1024 * 1024 * 1024, + ) + .optional(), + clips: z.array(clipSchema).max(49).optional(), + imports: z.array(importSchema).max(100).optional(), + }) + .refine((input) => { + if (input.clips && input.imports) return false; + if (input.legacyEditSpec && input.camera) return false; + const clips = + input.imports?.flatMap((item) => + item.kind === "clip" ? [item.clip] : [], + ) ?? + input.clips ?? + []; + const capCount = + input.imports + ?.filter((item) => item.kind === "cap") + .reduce((total, item) => total + item.clipCount, 0) ?? 0; + return ( + clips.length <= 49 && + clips.length + capCount <= 1000 && + clips.every( + (clip) => + input.videoAssets?.some((asset) => asset.path === clip.displayPath) && + (!clip.cameraPath || + input.videoAssets.some((asset) => asset.path === clip.cameraPath)), + ) + ); + }); + +const previewSchema = z.object({ + frameNumber: z.number().int().nonnegative(), + fps: z.number().int().min(1).max(60), + resolutionBase: z.object({ + x: z.number().int().min(1).max(3840), + y: z.number().int().min(1).max(2160), + }), +}); + +const editor = new Hono(); + +editor.use("*", async (c, next) => { + if ( + c.req.method === "GET" && + /^\/editor\/sessions\/(?:[a-z][a-z0-9-]{0,23}\.)?[0-9a-f-]{36}\/(?:exports\/[0-9a-f-]{36}|project-bundle)\/download$/.test( + c.req.path, + ) + ) { + await next(); + return; + } + if (!validateMediaServerSecret(c)) { + return c.json({ error: "Unauthorized" }, 401); + } + await next(); +}); + +editor.post("/caption-cap-audio", async (c) => { + let body: unknown; + try { + body = await c.req.json(); + } catch { + return c.json({ error: "Invalid Cap caption source" }, 400); + } + const parsed = capCaptionRequestSchema.safeParse(body); + if (!parsed.success) { + return c.json({ error: "Invalid Cap caption source" }, 400); + } + let prepared: Awaited>; + try { + prepared = await prepareEditorCapCaptionAudio( + parsed.data.asset, + parsed.data.segments, + c.req.raw.signal, + ); + } catch (error) { + console.error("Editor Cap caption audio failed", error); + return c.json({ error: "Cap caption audio is unavailable" }, 502); + } + const reader = Bun.file(prepared.path).stream().getReader(); + let cleaned = false; + const cleanup = async () => { + if (cleaned) return; + cleaned = true; + await prepared.cleanup(); + }; + const stream = new ReadableStream({ + async pull(controller) { + try { + const { done, value } = await reader.read(); + if (done) { + controller.close(); + await cleanup(); + } else { + controller.enqueue(value); + } + } catch (error) { + controller.error(error); + await cleanup(); + } + }, + async cancel() { + await reader.cancel(); + await cleanup(); + }, + }); + return c.body(stream, 200, { + "Content-Type": "audio/mp4", + "Content-Length": String(prepared.size), + "Cache-Control": "no-store", + }); +}); + +editor.post("/preparations", async (c) => { + let body: unknown; + try { + body = await c.req.json(); + } catch { + return c.json({ error: "Invalid editor session request" }, 400); + } + const parsed = sessionSchema.safeParse(body); + if (!parsed.success) { + return c.json({ error: "Invalid editor session request" }, 400); + } + try { + const id = await beginEditorPreparation(parsed.data); + return c.json({ id, status: "preparing" as const }, 202); + } catch (error) { + if (error instanceof EditorSessionBusyError) { + return c.json({ error: error.message }, 503); + } + console.error("Editor session creation failed", error); + return c.json({ error: "Editor session could not start" }, 502); + } +}); + +editor.get("/preparations/:id", (c) => { + const preparation = getEditorPreparation(c.req.param("id")); + return preparation + ? c.json(preparation) + : c.json({ error: "Not found" }, 404); +}); + +editor.delete("/preparations/:id", (c) => { + const canceled = cancelEditorPreparation(c.req.param("id")); + return canceled ? c.body(null, 204) : c.json({ error: "Not found" }, 404); +}); + +editor.delete("/sessions/:id", async (c) => { + const closed = await closeEditorSession(c.req.param("id")); + return closed ? c.body(null, 204) : c.json({ error: "Not found" }, 404); +}); + +editor.get("/sessions/:id", (c) => { + const videoId = getEditorSessionVideoId(c.req.param("id")); + return videoId ? c.json({ videoId }) : c.json({ error: "Not found" }, 404); +}); + +editor.put("/sessions/:id/caption-access", async (c) => { + const session = getEditorSession(c.req.param("id")); + if (!session) return c.json({ error: "Not found" }, 404); + const text = await c.req.text(); + if (Buffer.byteLength(text, "utf8") > 256) + return c.json({ error: "Invalid caption access" }, 413); + let body: unknown; + try { + body = JSON.parse(text); + } catch { + return c.json({ error: "Invalid caption access" }, 400); + } + const parsed = z + .object({ captionsEnabled: z.boolean() }) + .strict() + .safeParse(body); + if (!parsed.success) return c.json({ error: "Invalid caption access" }, 400); + session.captionsEnabled = parsed.data.captionsEnabled; + return c.body(null, 204); +}); + +editor.post("/sessions/:id/sockets", async (c) => { + let body: unknown; + try { + body = await c.req.json(); + } catch { + return c.json({ error: "Invalid editor socket request" }, 400); + } + const parsed = z.object({ origin: z.string().url() }).safeParse(body); + if (!parsed.success) { + return c.json({ error: "Invalid editor socket request" }, 400); + } + try { + const result = createEditorSocketTickets( + c.req.param("id"), + parsed.data.origin, + ); + return result ? c.json(result) : c.json({ error: "Not found" }, 404); + } catch (error) { + console.error("Editor socket ticket creation failed", error); + return c.json({ error: "Editor sockets unavailable" }, 503); + } +}); + +editor.post("/sessions/:id/audio-assets", async (c) => { + const session = getEditorSession(c.req.param("id")); + if (!session) return c.json({ error: "Not found" }, 404); + let body: unknown; + try { + body = await c.req.json(); + } catch { + return c.json({ error: "Invalid editor audio asset" }, 400); + } + const parsed = audioAssetSchema.safeParse(body); + if (!parsed.success) + return c.json({ error: "Invalid editor audio asset" }, 400); + try { + const imported = await stageSignedEditorAudioAsset( + session.projectPath, + parsed.data, + c.req.raw.signal, + ); + return c.json(imported); + } catch (error) { + console.error("Editor audio import failed", error); + return c.json({ error: "Editor audio could not be imported" }, 502); + } +}); + +editor.post("/sessions/:id/image-assets", async (c) => { + const session = getEditorSession(c.req.param("id")); + if (!session) return c.json({ error: "Not found" }, 404); + let body: unknown; + try { + body = await c.req.json(); + } catch { + return c.json({ error: "Invalid editor image asset" }, 400); + } + const parsed = imageAssetSchema.safeParse(body); + if (!parsed.success) + return c.json({ error: "Invalid editor image asset" }, 400); + try { + const imported = await stageSignedEditorImageAsset( + session.projectPath, + parsed.data, + c.req.raw.signal, + ); + return c.json(imported); + } catch (error) { + console.error("Editor image import failed", error); + return c.json({ error: "Editor image could not be imported" }, 502); + } +}); + +editor.post("/sessions/:id/video-assets", async (c) => { + const sessionId = c.req.param("id"); + const session = getEditorSession(sessionId); + if (!session) return c.json({ error: "Not found" }, 404); + let body: unknown; + try { + body = await c.req.json(); + } catch { + return c.json({ error: "Invalid editor video asset" }, 400); + } + const parsed = videoAssetSchema.safeParse(body); + if (!parsed.success) + return c.json({ error: "Invalid editor video asset" }, 400); + try { + const id = beginEditorVideoImport( + sessionId, + session.projectPath, + parsed.data, + ); + return c.json({ id, status: "staging" as const }, 202); + } catch (error) { + if (error instanceof EditorVideoImportBusyError) { + return c.json({ error: error.message }, 503); + } + console.error("Editor video import failed", error); + return c.json({ error: "Editor video could not be imported" }, 502); + } +}); + +editor.get("/sessions/:id/video-assets/:jobId", (c) => { + const sessionId = c.req.param("id"); + if (!getEditorSession(sessionId)) return c.json({ error: "Not found" }, 404); + const job = getEditorVideoImport(sessionId, c.req.param("jobId")); + return job ? c.json(job) : c.json({ error: "Not found" }, 404); +}); + +editor.delete("/sessions/:id/video-assets/:jobId", async (c) => { + const sessionId = c.req.param("id"); + if (!getEditorSession(sessionId)) return c.json({ error: "Not found" }, 404); + const canceled = await cancelEditorVideoImport( + sessionId, + c.req.param("jobId"), + ); + return canceled ? c.body(null, 204) : c.json({ error: "Not found" }, 404); +}); + +editor.post("/sessions/:id/cap-assets", async (c) => { + const sessionId = c.req.param("id"); + const session = getEditorSession(sessionId); + if (!session) return c.json({ error: "Not found" }, 404); + let body: unknown; + try { + body = await c.req.json(); + } catch { + return c.json({ error: "Invalid editor Cap project asset" }, 400); + } + const parsed = capAssetSchema.safeParse(body); + if (!parsed.success) + return c.json({ error: "Invalid editor Cap project asset" }, 400); + try { + const id = beginEditorCapImport( + sessionId, + session.projectPath, + parsed.data, + ); + return c.json({ id, status: "staging" as const }, 202); + } catch (error) { + if (error instanceof EditorCapImportBusyError) { + return c.json({ error: error.message }, 503); + } + console.error("Editor Cap project import failed", error); + return c.json({ error: "Cap project could not be imported" }, 502); + } +}); + +editor.get("/sessions/:id/cap-assets/:jobId", (c) => { + const sessionId = c.req.param("id"); + if (!getEditorSession(sessionId)) return c.json({ error: "Not found" }, 404); + const job = getEditorCapImport(sessionId, c.req.param("jobId")); + return job ? c.json(job) : c.json({ error: "Not found" }, 404); +}); + +editor.delete("/sessions/:id/cap-assets/:jobId", async (c) => { + const sessionId = c.req.param("id"); + if (!getEditorSession(sessionId)) return c.json({ error: "Not found" }, 404); + const canceled = await cancelEditorCapImport(sessionId, c.req.param("jobId")); + return canceled ? c.body(null, 204) : c.json({ error: "Not found" }, 404); +}); + +function nativeSession(id: string) { + return getEditorSession(id); +} + +async function forward( + id: string, + path: string, + method: "GET" | "PUT" | "POST" | "DELETE", + body?: unknown, +) { + const session = nativeSession(id); + if (!session) return new Response("Not found", { status: 404 }); + if ( + (path === "/config" || path === "/config/memory") && + method === "PUT" && + !session.captionsEnabled && + hasEditorCaptionContent(body) + ) { + return new Response("Cap Pro is required for captions", { status: 403 }); + } + try { + const nativeBody = + (path === "/config" || path === "/config/memory") && method === "PUT" + ? mapEditorConfigPaths(body, "native", session.projectPath) + : body; + const response = await session.request(path, { + method, + ...(nativeBody !== undefined + ? { + body: JSON.stringify(nativeBody), + headers: { "Content-Type": "application/json" }, + } + : {}), + signal: AbortSignal.timeout(10_000), + }); + const headers = new Headers(response.headers); + headers.set("Cache-Control", "private, no-store"); + if ( + response.ok && + method === "GET" && + (path === "/instance" || path === "/config") + ) { + const value: unknown = await response.json(); + const browserValue = + path === "/instance" + ? mapEditorInstanceConfigPaths(value, "browser", session.projectPath) + : mapEditorConfigPaths(value, "browser", session.projectPath); + headers.delete("Content-Length"); + return new Response(JSON.stringify(browserValue), { + status: response.status, + headers, + }); + } + return new Response(response.body, { status: response.status, headers }); + } catch (error) { + console.error("Editor session request failed", error); + return new Response("Editor session unavailable", { status: 502 }); + } +} + +editor.get("/sessions/:id/instance", (c) => + forward(c.req.param("id"), "/instance", "GET"), +); +editor.get("/sessions/:id/meta", (c) => + forward(c.req.param("id"), "/meta", "GET"), +); +editor.put("/sessions/:id/meta", async (c) => { + const text = await c.req.text(); + if (Buffer.byteLength(text, "utf8") > 1024) + return c.json({ error: "Recording title is too large" }, 413); + let body: unknown; + try { + body = JSON.parse(text); + } catch { + return c.json({ error: "Invalid recording title" }, 400); + } + const parsed = z.object({ prettyName: z.string() }).safeParse(body); + if (!parsed.success || !validWebEditorTitle(parsed.data.prettyName)) + return c.json({ error: "Invalid recording title" }, 400); + return forward(c.req.param("id"), "/meta", "PUT", parsed.data); +}); +editor.get("/sessions/:id/config", (c) => + forward(c.req.param("id"), "/config", "GET"), +); +editor.get("/sessions/:id/animated-gradients", (c) => + forward(c.req.param("id"), "/animated-gradients", "GET"), +); +async function configBody(c: { req: { text: () => Promise } }) { + const text = await c.req.text(); + if (Buffer.byteLength(text, "utf8") > MAX_CONFIG_BYTES) { + return new Response("Editor config is too large", { status: 413 }); + } + try { + return JSON.parse(text) as unknown; + } catch { + return new Response("Invalid editor config", { status: 400 }); + } +} + +editor.put("/sessions/:id/config", async (c) => { + const body = await configBody(c); + return body instanceof Response + ? body + : forward(c.req.param("id"), "/config", "PUT", body); +}); +editor.put("/sessions/:id/config/memory", async (c) => { + const body = await configBody(c); + return body instanceof Response + ? body + : forward(c.req.param("id"), "/config/memory", "PUT", body); +}); + +async function previewBody(c: { + req: { json: () => Promise; param: (name: string) => string }; +}) { + let body: unknown; + try { + body = await c.req.json(); + } catch { + return new Response("Invalid preview request", { status: 400 }); + } + const parsed = previewSchema.safeParse(body); + if (!parsed.success) { + return new Response("Invalid preview request", { status: 400 }); + } + return parsed.data; +} + +editor.post("/sessions/:id/preview", async (c) => { + const body = await previewBody(c); + return body instanceof Response + ? body + : forward(c.req.param("id"), "/preview", "POST", body); +}); +editor.post("/sessions/:id/playback", async (c) => { + const body = await previewBody(c); + return body instanceof Response + ? body + : forward(c.req.param("id"), "/playback", "POST", body); +}); +editor.delete("/sessions/:id/playback", (c) => + forward(c.req.param("id"), "/playback", "DELETE"), +); +editor.put("/sessions/:id/seek", async (c) => { + const body = await previewBody(c); + return body instanceof Response + ? body + : forward(c.req.param("id"), "/seek", "PUT", body); +}); + +editor.post("/sessions/:id/exports", async (c) => { + let body: unknown; + try { + body = await c.req.json(); + } catch { + return c.json({ error: "Invalid editor export request" }, 400); + } + const settings = editorExportSettingsSchema.safeParse(body); + if (!settings.success) + return c.json({ error: "Invalid editor export settings" }, 400); + const sessionId = c.req.param("id"); + const native = getEditorSession(sessionId); + if (!native) return c.json({ error: "Not found" }, 404); + try { + const id = await beginEditorExport(sessionId, native, settings.data); + attachEditorSession(sessionId); + return c.json({ id, status: "running" as const }, 202); + } catch (cause) { + if (cause instanceof EditorExportBusyError) { + return c.json({ error: cause.message }, 503); + } + console.error("Editor export could not start", cause); + return c.json({ error: "Editor export unavailable" }, 503); + } +}); + +editor.get("/sessions/:id/exports/:exportId", (c) => { + const sessionId = c.req.param("id"); + if (!getEditorSession(sessionId)) return c.json({ error: "Not found" }, 404); + const job = getEditorExport(sessionId, c.req.param("exportId")); + return job ? c.json(job) : c.json({ error: "Not found" }, 404); +}); + +editor.get("/sessions/:id/exports/:exportId/file", (c) => { + const sessionId = c.req.param("id"); + if (!getEditorSession(sessionId)) return c.json({ error: "Not found" }, 404); + const exportId = c.req.param("exportId"); + const job = getEditorExport(sessionId, exportId); + const path = getEditorExportFile(sessionId, exportId); + if (!job || !path) return c.json({ error: "Not found" }, 404); + const extension = + job.format === "Mp4" ? "mp4" : job.format === "Gif" ? "gif" : "mov"; + const contentType = + job.format === "Mp4" + ? "video/mp4" + : job.format === "Gif" + ? "image/gif" + : "video/quicktime"; + return new Response(Bun.file(path), { + headers: { + "Content-Type": contentType, + "Content-Disposition": `attachment; filename="cap-export-${exportId}.${extension}"`, + "Cache-Control": "private, no-store", + }, + }); +}); + +editor.get("/sessions/:id/exports/:exportId/chunk", (c) => { + const sessionId = c.req.param("id"); + if (!getEditorSession(sessionId)) return c.json({ error: "Not found" }, 404); + const exportId = c.req.param("exportId"); + const job = getEditorExport(sessionId, exportId); + const path = getEditorExportFile(sessionId, exportId); + if ( + !job || + !path || + job.format !== "Mp4" || + !job.size || + !job.mediaMetadata + ) { + return c.json({ error: "Not found" }, 404); + } + const offsetText = c.req.query("offset") ?? ""; + const lengthText = c.req.query("length") ?? ""; + if ( + !/^(0|[1-9][0-9]*)$/.test(offsetText) || + !/^[1-9][0-9]*$/.test(lengthText) + ) + return c.json({ error: "Invalid export range" }, 400); + const offset = Number(offsetText); + const length = Number(lengthText); + if ( + !Number.isSafeInteger(offset) || + !Number.isSafeInteger(length) || + length > 16 * 1024 * 1024 || + offset + length > job.size + ) { + return c.json({ error: "Invalid export range" }, 400); + } + const file = Bun.file(path); + if (file.size !== job.size) + return c.json({ error: "Editor export changed" }, 503); + return new Response(file.slice(offset, offset + length), { + status: 206, + headers: { + "Content-Type": "video/mp4", + "Content-Length": String(length), + "Content-Range": `bytes ${offset}-${offset + length - 1}/${job.size}`, + "Cache-Control": "private, no-store", + }, + }); +}); + +editor.post("/sessions/:id/exports/:exportId/download-ticket", async (c) => { + const sessionId = c.req.param("id"); + if (!getEditorSession(sessionId)) return c.json({ error: "Not found" }, 404); + let fileName: string | undefined; + try { + const body: unknown = await c.req.json(); + if ( + typeof body === "object" && + body !== null && + "fileName" in body && + typeof body.fileName === "string" && + body.fileName.length <= 200 + ) { + fileName = body.fileName; + } + } catch { + fileName = undefined; + } + try { + const result = await createEditorExportDownloadTicket( + sessionId, + c.req.param("exportId"), + fileName, + ); + return result ? c.json(result) : c.json({ error: "Not found" }, 404); + } catch (cause) { + console.error("Editor download ticket unavailable", cause); + return c.json({ error: "Editor download unavailable" }, 503); + } +}); + +editor.get("/sessions/:id/exports/:exportId/download", (c) => { + const sessionId = c.req.param("id"); + const exportId = c.req.param("exportId"); + const ticket = consumeEditorExportDownloadTicket( + sessionId, + exportId, + c.req.query("ticket") ?? null, + ); + if (!ticket) return c.json({ error: "Not found" }, 404); + markEditorExportDownloadStarted(sessionId, exportId); + const contentType = + ticket.format === "Mp4" + ? "video/mp4" + : ticket.format === "Gif" + ? "image/gif" + : "application/octet-stream"; + const file = Bun.file(ticket.path); + const asciiName = ticket.fileName.replace(/[^\x20-\x7e]|["\\]/g, "_"); + const encodedName = encodeURIComponent(ticket.fileName).replace( + /['()*]/g, + (character) => `%${character.charCodeAt(0).toString(16).toUpperCase()}`, + ); + const reader = file.stream().getReader(); + let finished = false; + const finish = async () => { + if (finished) return; + finished = true; + await Promise.allSettled([ + rm(ticket.root, { recursive: true, force: true }), + cancelEditorExport(sessionId, exportId), + ]); + }; + const body = new ReadableStream({ + async pull(controller) { + try { + const packet = await reader.read(); + if (packet.done) { + controller.close(); + await finish(); + } else { + controller.enqueue(packet.value); + } + } catch (cause) { + await finish(); + controller.error(cause); + } + }, + async cancel() { + await reader.cancel(); + await finish(); + }, + }); + return new Response(body, { + headers: { + "Content-Type": contentType, + "Content-Length": String(file.size), + "Content-Disposition": `attachment; filename="${asciiName}"; filename*=UTF-8''${encodedName}`, + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", + "Referrer-Policy": "no-referrer", + }, + }); +}); + +editor.post("/sessions/:id/project-bundle/download-ticket", async (c) => { + const sessionId = c.req.param("id"); + const session = getEditorSession(sessionId); + if (!session) return c.json({ error: "Not found" }, 404); + let fileName: string | undefined; + try { + const body: unknown = await c.req.json(); + if ( + typeof body === "object" && + body !== null && + "fileName" in body && + typeof body.fileName === "string" && + body.fileName.length <= 200 + ) { + fileName = body.fileName; + } + } catch { + fileName = undefined; + } + try { + return c.json( + await createEditorProjectBundleDownloadTicket( + sessionId, + session.projectPath, + fileName, + ), + ); + } catch (cause) { + console.error("Editor project bundle unavailable", cause); + return c.json({ error: "Editor project bundle is unavailable" }, 503); + } +}); + +editor.get("/sessions/:id/project-bundle/download", (c) => { + const ticket = consumeEditorProjectBundleDownloadTicket( + c.req.param("id"), + c.req.query("ticket") ?? null, + ); + return ticket + ? editorProjectBundleResponse(ticket) + : c.json({ error: "Not found" }, 404); +}); + +const renderUploadsSchema = z + .object({ + files: z + .array( + z + .object({ + path: z.string().min(1).max(1024), + url: z.string().url().max(8192), + }) + .strict(), + ) + .max(4000), + wallpapers: z + .array( + z + .object({ + file: z.string().min(1).max(200), + url: z.string().url().max(8192), + }) + .strict(), + ) + .max(16), + }) + .strict(); + +editor.get("/sessions/:id/render-project", async (c) => { + const session = getEditorSession(c.req.param("id")); + if (!session) return c.json({ error: "Not found" }, 404); + try { + return c.json(await listEditorRenderProject(session.projectPath)); + } catch (cause) { + console.error("Editor render project unavailable", cause); + return c.json({ error: "Editor render project is unavailable" }, 503); + } +}); + +editor.post("/sessions/:id/render-project/uploads", async (c) => { + const sessionId = c.req.param("id"); + const session = getEditorSession(sessionId); + if (!session) return c.json({ error: "Not found" }, 404); + let body: unknown; + try { + body = await c.req.json(); + } catch { + return c.json({ error: "Invalid render project uploads" }, 400); + } + const parsed = renderUploadsSchema.safeParse(body); + if (!parsed.success) { + return c.json({ error: "Invalid render project uploads" }, 400); + } + try { + const result = await uploadEditorRenderProject( + session.projectPath, + parsed.data, + c.req.raw.signal, + ); + attachEditorSession(sessionId); + return c.json(result); + } catch (cause) { + console.error("Editor render project upload failed", cause); + return c.json({ error: "Editor render project upload failed" }, 502); + } +}); + +editor.delete("/sessions/:id/exports/:exportId", async (c) => { + const sessionId = c.req.param("id"); + if (!getEditorSession(sessionId)) return c.json({ error: "Not found" }, 404); + const canceled = await cancelEditorExport(sessionId, c.req.param("exportId")); + return canceled ? c.body(null, 204) : c.json({ error: "Not found" }, 404); +}); + +export default editor; diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.ts index cd2b44e68b5..3ad3f1acce6 100644 --- a/apps/media-server/src/routes/video.ts +++ b/apps/media-server/src/routes/video.ts @@ -54,6 +54,7 @@ import { downloadVideoToTemp, generatePreviewGif, generateThumbnail, + MAX_PROCESS_TIMEOUT_MS, muxMediaTracksToMp4, processVideo, repairContainer, @@ -106,6 +107,16 @@ const thumbnailSchema = z.object({ quality: z.number().min(1).max(100).optional(), }); +const previewAssetsSchema = z + .object({ + videoUrl: z.string().url(), + thumbnailPresignedUrl: z.string().url().optional(), + previewGifPresignedUrl: z.string().url().optional(), + }) + .refine((body) => body.thumbnailPresignedUrl || body.previewGifPresignedUrl, { + message: "No preview asset requested", + }); + const convertSchema = z.object({ videoUrl: z.string().url(), inputExtension: z.string().optional(), @@ -537,6 +548,103 @@ video.post("/thumbnail", async (c) => { } }); +video.post("/preview-assets", async (c) => { + if (!validateMediaServerSecret(c)) { + return c.json({ error: "Unauthorized" }, 401); + } + + const result = previewAssetsSchema.safeParse( + await c.req.json().catch(() => null), + ); + if (!result.success) { + return c.json( + { + error: "Invalid request", + code: "INVALID_REQUEST", + details: result.error.message, + }, + 400, + ); + } + + const slot = tryAcquireDirectVideoProcessSlot(canAcceptNewVideoProcess); + if (!slot) { + c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString()); + return c.json(getBusyResponseBody(getVideoCapacitySnapshot()), 503); + } + + const { videoUrl, thumbnailPresignedUrl, previewGifPresignedUrl } = + result.data; + const signal = c.req.raw.signal; + try { + const metadata = await probeVideo(videoUrl); + if (thumbnailPresignedUrl) { + const thumbnail = await generateThumbnail( + videoUrl, + metadata.duration, + {}, + signal, + ); + await uploadToS3(thumbnail, thumbnailPresignedUrl, "image/jpeg", signal); + } + if (previewGifPresignedUrl) { + const previewGif = await generatePreviewGif( + videoUrl, + metadata.duration, + {}, + signal, + ); + try { + await uploadFileToS3( + previewGif.path, + previewGifPresignedUrl, + "image/gif", + signal, + ); + } finally { + await previewGif.cleanup(); + } + } + return c.json({ success: true }); + } catch (err) { + console.error("[video/preview-assets] Error:", err); + + if (isBusyError(err)) { + c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString()); + return c.json( + { + error: "Server is busy", + code: "SERVER_BUSY", + details: "Too many concurrent requests, please retry later", + }, + 503, + ); + } + + if (isTimeoutError(err)) { + return c.json( + { + error: "Request timed out", + code: "TIMEOUT", + details: err instanceof Error ? err.message : String(err), + }, + 504, + ); + } + + return c.json( + { + error: "Failed to generate preview assets", + code: MEDIA_ENGINE_ERROR_CODE, + details: err instanceof Error ? err.message : String(err), + }, + 500, + ); + } finally { + slot.release(); + } +}); + video.post("/convert", async (c) => { if (!validateMediaServerSecret(c)) { return c.json({ error: "Unauthorized" }, 401); @@ -1327,8 +1435,14 @@ async function processVideoAsync( const isWebm = isWebmInput(options.inputExtension); if (isWebm) { updateJob(jobId, { message: "Checking the original recording..." }); + // Streamed recordings often carry no duration to size this by, so it + // gets the longest time a transcode of the same file may take. await withJobHeartbeat(jobId, () => - validateVideoInput(inputTempFile.path, abortController.signal), + validateVideoInput( + inputTempFile.path, + abortController.signal, + MAX_PROCESS_TIMEOUT_MS, + ), ); } diff --git a/apps/render-farm/Dockerfile b/apps/render-farm/Dockerfile index 0d9562c8891..8bdafe1fc21 100644 --- a/apps/render-farm/Dockerfile +++ b/apps/render-farm/Dockerfile @@ -49,6 +49,10 @@ ENV NVIDIA_VISIBLE_DEVICES=all \ RF_REQUIRE_GPU=1 COPY --from=bun /usr/local/bin/bun /usr/local/bin/bun COPY --from=engine /out/cap-render-farm /usr/local/bin/cap-render-farm +# Wallpapers and library music, the same files the editor worker ships, so a +# recording the farm prepares itself renders with them (see src/prepare.ts). +COPY apps/desktop/src-tauri/assets/backgrounds /opt/cap/backgrounds +COPY apps/desktop/src/assets/music /opt/cap/music WORKDIR /app COPY apps/render-farm/src ./src CMD ["bun", "/app/src/boot.ts"] diff --git a/apps/render-farm/Dockerfile.dockerignore b/apps/render-farm/Dockerfile.dockerignore index 3886a6b917c..802971d8e3b 100644 --- a/apps/render-farm/Dockerfile.dockerignore +++ b/apps/render-farm/Dockerfile.dockerignore @@ -5,6 +5,7 @@ !vendor !apps/cli !apps/desktop/src-tauri +!apps/desktop/src/assets/music !apps/render-farm/src **/target **/node_modules diff --git a/apps/render-farm/src/boxes.test-util.ts b/apps/render-farm/src/boxes.test-util.ts index 62a01fda8cc..e9ab47ed23a 100644 --- a/apps/render-farm/src/boxes.test-util.ts +++ b/apps/render-farm/src/boxes.test-util.ts @@ -1,3 +1,6 @@ +import { initSegment, segmentHeader } from "./fmp4"; +import { avcC, box, build, fullBox } from "./mp4"; + export type ParsedBox = { type: string; start: number; @@ -79,3 +82,141 @@ export const ANNEX_B_PARAMETER_SETS = Uint8Array.of( 0x22, 0xc0, ); + +// A complete x264 SPS (High, 4:2:0 8-bit, VUI with timing, no bitstream +// restriction) + PPS, for code that reads the SPS itself. +export const X264_PARAMETER_SETS = Uint8Array.of( + 0x00, + 0x00, + 0x00, + 0x01, + 0x67, + 0x64, + 0x00, + 0x0a, + 0xac, + 0xb2, + 0x04, + 0x17, + 0xf2, + 0xe0, + 0x22, + 0x00, + 0x00, + 0x03, + 0x00, + 0x02, + 0x00, + 0x00, + 0x03, + 0x00, + 0x78, + 0x1e, + 0x24, + 0x4c, + 0x90, + 0x00, + 0x00, + 0x00, + 0x01, + 0x68, + 0xeb, + 0xc3, + 0xcb, + 0x22, + 0xc0, +); + +/** + * A fragmented MP4 like a browser records: one fragment per GOP, video then + * audio samples in each mdat, and optionally an mfra listing the moofs. + * Video sample `n` is `size(n)` bytes of `n % 251 + 1`. + */ +export function fragmentedRecording(options: { + frames: number; + gop: number; + audio?: boolean; + mfra?: boolean; +}) { + const size = (frame: number) => 50 + ((frame * 37) % 200); + const parts: Uint8Array[] = []; + let offset = 0; + const push = (part: Uint8Array) => { + parts.push(part); + offset += part.byteLength; + }; + push( + initSegment({ + width: 128, + height: 72, + fps: 30, + avcC: avcC(X264_PARAMETER_SETS), + asc: options.audio ? Uint8Array.of(0x11, 0x90) : null, + }), + ); + const moofs: [number, number][] = []; + for (let first = 0; first < options.frames; first += options.gop) { + const frames = Array.from( + { length: Math.min(options.gop, options.frames - first) }, + (_, index) => first + index, + ); + const audioSizes = options.audio ? frames.map(() => 9) : []; + // segmentHeader leads with a 24-byte styp. + moofs.push([first * 1000, offset + 24]); + push( + segmentHeader({ + sequence: moofs.length, + firstFrame: first, + videoSizes: frames.map(size), + firstPacket: first, + audioSizes, + }), + ); + for (const frame of frames) { + push(new Uint8Array(size(frame)).fill((frame % 251) + 1)); + } + for (const bytes of audioSizes) push(new Uint8Array(bytes).fill(0xaa)); + } + if (options.mfra) { + const tfra = fullBox( + "tfra", + 1, + 0, + build((writer) => { + writer.u32(1); + writer.u32(0); + writer.u32(moofs.length); + for (const [time, moof] of moofs) { + writer.u64(time); + writer.u64(moof); + writer.u8(1); + writer.u8(1); + writer.u8(1); + } + }), + ); + const mfroSize = 16; + const mfraSize = 8 + tfra.byteLength + mfroSize; + push( + box( + "mfra", + tfra, + fullBox( + "mfro", + 0, + 0, + build((writer) => writer.u32(mfraSize)), + ), + ), + ); + } + const bytes = new Uint8Array( + parts.reduce((sum, part) => sum + part.byteLength, 0), + ); + let at = 0; + for (const part of parts) { + bytes.set(part, at); + at += part.byteLength; + } + return { bytes, size, moofs: moofs.map(([, moof]) => moof) }; +} diff --git a/apps/render-farm/src/coordinator.test.ts b/apps/render-farm/src/coordinator.test.ts index 661af17e1fa..64bdd3f982a 100644 --- a/apps/render-farm/src/coordinator.test.ts +++ b/apps/render-farm/src/coordinator.test.ts @@ -6,9 +6,10 @@ import { timingSafeEqual, } from "node:crypto"; import { readFileSync } from "node:fs"; -import { ANNEX_B_PARAMETER_SETS } from "./boxes.test-util"; +import { ANNEX_B_PARAMETER_SETS, fragmentedRecording } from "./boxes.test-util"; import type { Job, TaskState } from "./coordinator"; import * as fmp4 from "./fmp4"; +import * as fragmentIndex from "./fragment-index"; import * as hls from "./hls"; import * as mp4 from "./mp4"; import * as planning from "./planning"; @@ -152,6 +153,7 @@ function harness(env: Record = {}) { createHmac, ...validate, ...fmp4, + ...fragmentIndex, ...hls, ...mp4, ...planning, @@ -168,6 +170,12 @@ function harness(env: Record = {}) { mediaS3ConfigFromEnv: () => ({}), ProbeEngine: class {}, Engine: class {}, + prepareSupport: () => new Promise(() => {}), + prepareRecording: async () => false, + builtinPath: () => null, + builtinSize: () => { + throw new Error("no built-in assets in tests"); + }, process: { env: { RF_TOKEN: "test", @@ -618,6 +626,26 @@ describe("assembly", () => { await expect(h.onVideoDone(j, state, padded)).rejects.toThrow("add up"); expect(j.videoResults.size).toBe(0); }); + + test("a result whose stash was never stored is refused", async () => { + const h = harness(); + const j = job(); + const state = videoState(j); + const lost = stored(h, j, 0, 220_000, 1); + h.objects.delete(lost.stash.key); + await expect(h.onVideoDone(j, state, lost)).rejects.toThrow( + "is not stored", + ); + expect(j.videoResults.size).toBe(0); + expect(state.state).toBe("running"); + h.objects.set(lost.stash.key, new Uint8Array(lost.stash.bytes - 1)); + await expect(h.onVideoDone(j, state, lost)).rejects.toThrow( + "is not stored", + ); + h.objects.set(lost.stash.key, new Uint8Array(lost.stash.bytes)); + await h.onVideoDone(j, state, lost); + expect(j.videoResults.size).toBe(1); + }); }); describe("coordinator recovery", () => { @@ -694,8 +722,10 @@ describe("coordinator recovery", () => { h.gate(gate.promise); const first = result(original); const second = result(hedge); + h.objects.set(first.stash.key, new Uint8Array(first.stash.bytes)); const accepted = h.onVideoDone(j, original, first); const duplicate = h.onVideoDone(j, hedge, second); + await Bun.sleep(0); expect(j.videoResults.size).toBe(0); expect(h.writes).toEqual(["jobs/job/v/0.json"]); gate.resolve(); @@ -908,6 +938,141 @@ function call( ); } +describe("cursor jobs", () => { + const request = { + sourceRoot: "owner/video/", + source: "owner/video/raw-upload.webm", + outputPrefix: "owner/video/.recording/cursor/run1/", + callbackUrl: "https://app.cap.test/api/render-farm/callback", + reference: "cursor:video", + }; + const display = `${request.outputPrefix}display.mp4`; + const inputEvents = `${request.outputPrefix}input-events.ndjson`; + + test("are queued once, wait for chunk work and settle with a signed callback", async () => { + const h = harness(); + const created = (await (await call(h, "/cursor-jobs", request)).json()) as { + id: string; + status: string; + }; + expect(created).toMatchObject({ status: "queued", display, inputEvents }); + const again = (await (await call(h, "/cursor-jobs", request)).json()) as { + id: string; + }; + expect(again.id).toBe(created.id); + const work = (await ( + await call(h, "/work", { + worker: "gpu-a", + slots: 1, + cpus: 8, + kinds: ["video"], + }) + ).json()) as { task: protocol.CursorTask }; + expect(work.task).toMatchObject({ + kind: "cursor", + source: request.source, + outputPrefix: request.outputPrefix, + attempt: 1, + }); + await call(h, "/heartbeat", { + worker: "gpu-a", + slots: 1, + cpus: 8, + running: [ + { + taskId: work.task.taskId, + attempt: 1, + frames: 420, + total: 1000, + elapsedMs: 10, + }, + ], + }); + expect( + await (await call(h, `/cursor-jobs/${created.id}`)).json(), + ).toMatchObject({ status: "running", progress: 0.42 }); + const done = `/cursor-jobs/${created.id}/done`; + const report = { worker: "gpu-a", attempt: 1 }; + expect((await call(h, done, { ...report, worker: "gpu-b" })).status).toBe( + 409, + ); + h.objects.set(display, new Uint8Array(10)); + expect((await call(h, done, report)).status).toBe(409); + h.objects.set(inputEvents, new Uint8Array(10)); + expect((await call(h, done, report)).status).toBe(200); + expect( + await (await call(h, `/cursor-jobs/${created.id}`)).json(), + ).toMatchObject({ + status: "ready", + progress: 1, + bytes: { display: 10, inputEvents: 10 }, + }); + const callback = h.callbacks.find( + (entry) => entry.url === request.callbackUrl, + ); + const body = String(callback?.init.body); + expect(JSON.parse(body)).toMatchObject({ + id: created.id, + reference: request.reference, + status: "ready", + display, + inputEvents, + }); + const signature = createHmac("sha256", "test").update(body).digest("hex"); + expect( + (callback?.init.headers as Record)[ + "x-render-farm-signature" + ], + ).toBe(`sha256=${signature}`); + }); + + test("a failed dispatch is retried once after a backoff, then reported as an error", async () => { + const h = harness(); + const created = (await (await call(h, "/cursor-jobs", request)).json()) as { + id: string; + }; + for (const attempt of [1, 2]) { + const work = (await ( + await call(h, "/work", { worker: "gpu-a", slots: 1, cpus: 8 }) + ).json()) as { task: protocol.CursorTask }; + expect(work.task.attempt).toBe(attempt); + await call(h, `/cursor-jobs/${created.id}/fail`, { + worker: "gpu-a", + attempt, + error: "service exited 1", + }); + if (attempt === 1) { + expect( + await (await call(h, `/cursor-jobs/${created.id}`)).json(), + ).toMatchObject({ status: "queued" }); + await Bun.sleep(2100); + } + } + expect( + await (await call(h, `/cursor-jobs/${created.id}`)).json(), + ).toMatchObject({ status: "error", error: "service exited 1" }); + const callback = h.callbacks.find( + (entry) => entry.url === request.callbackUrl, + ); + expect(JSON.parse(String(callback?.init.body))).toMatchObject({ + status: "error", + }); + }, 10_000); + + test("reject sources and outputs outside the recording folder", async () => { + const h = harness(); + for (const invalid of [ + { ...request, source: "other/video/raw-upload.webm" }, + { ...request, outputPrefix: "other/video/cursor/" }, + { ...request, outputPrefix: "owner/video/.recording/cursor/run1" }, + { ...request, outputPrefix: "owner/video/" }, + { ...request, callbackUrl: "https://evil.example/callback" }, + ]) { + expect((await call(h, "/cursor-jobs", invalid)).status).toBe(400); + } + }); +}); + describe("transcodes", () => { const request = { sourceRoot: "owner/video/", @@ -1146,6 +1311,307 @@ describe("transcodes", () => { }); }); +describe("placements", () => { + const root = "owner/video/"; + const source = `${root}raw-upload.mp4`; + const output = `${root}.recording/render/sources/display.mp4`; + const prefixKey = `${root}.recording/render/sources/display.prefix-v1`; + let folder = 0; + + function project(h: ReturnType) { + const prefix = `${root}.recording/render/${folder++}/project`; + h.objects.set( + `${prefix}/recording-meta.json`, + new TextEncoder().encode("{}"), + ); + h.objects.set( + `${prefix}/manifest.json`, + new TextEncoder().encode( + JSON.stringify({ + files: [ + { path: "recording-meta.json", size: 2 }, + { path: "display.mp4", key: output, transcodeFrom: source }, + ], + }), + ), + ); + return prefix; + } + + function worker(h: ReturnType, features?: string[]) { + // The poll waits for work; registering is all these tests need. + void call(h, "/work", { + worker: `gpu-${features ? "new" : "old"}`, + slots: 1, + cpus: 8, + kinds: ["video"], + ...(features ? { features } : {}), + }); + } + + const sourceReads = (h: ReturnType) => + h.ranges.filter((range) => range.startsWith(`${source}:`)).length; + + test("a fragmented recording is read in place, with no transcode", async () => { + const h = harness(); + worker(h, ["prefix"]); + const recording = fragmentedRecording({ + frames: 300, + gop: 30, + audio: true, + mfra: true, + }); + h.objects.set(source, recording.bytes); + const index = (await h.sourceIndex(project(h), root)) as { + manifest: { files: { path: string; size: number }[] }; + mediaMeta: Map< + string, + { + key: string; + size: number; + prefix?: { key: string; size: number }; + index: { sizes: Uint32Array; offsets: Float64Array }; + } + >; + }; + const meta = index.mediaMeta.get("display.mp4"); + const prefix = h.objects.get(prefixKey) as Uint8Array; + expect(meta).toMatchObject({ + key: source, + size: prefix.byteLength + recording.bytes.byteLength, + prefix: { key: prefixKey, size: prefix.byteLength }, + }); + expect(meta?.index.sizes.length).toBe(300); + expect(meta?.index.offsets[0]).toBeGreaterThan(prefix.byteLength); + expect(h.objects.has(output)).toBe(false); + expect( + await ( + await call(h, "/transcodes", { sourceRoot: root, source, output }) + ).json(), + ).toMatchObject({ status: "ready", placed: true }); + + // Another Save reuses the placement without touching the recording. + const reads = sourceReads(h); + await h.sourceIndex(project(h), root); + expect(sourceReads(h)).toBe(reads); + + // So does a restarted coordinator, from the stored prefix. + const restarted = harness(); + worker(restarted, ["prefix"]); + restarted.objects.set(source, recording.bytes); + restarted.objects.set(prefixKey, prefix); + const again = (await restarted.sourceIndex( + project(restarted), + root, + )) as typeof index; + expect(sourceReads(restarted)).toBe(0); + expect([ + ...(again.mediaMeta.get("display.mp4")?.index.offsets ?? []), + ]).toEqual([...(meta?.index.offsets ?? [])]); + }); + + function prefixedTask(h: ReturnType) { + const j = job(); + h.jobs.set(j.id, j); + const state = videoState(j); + state.state = "queued"; + state.worker = undefined; + state.attempts = 0; + (state.task as protocol.VideoTask).files = [ + { + path: "display.mp4", + key: source, + size: 1100, + ranges: [[0, 600]], + prefix: { key: prefixKey, size: 100 }, + }, + ]; + h.queue.push(state); + return { j, state }; + } + + const answered = (poll: Promise) => + Promise.race([ + poll.then(() => "answered"), + Bun.sleep(20).then(() => "waiting"), + ]); + + test("a task with a prefix only goes to a worker that takes prefixes", async () => { + const h = harness(); + // Planned while every live worker took prefixes (this one only runs + // audio, so it leaves the chunk queued)... + void call(h, "/work", { + worker: "gpu-new-audio", + slots: 1, + cpus: 8, + kinds: ["audio"], + features: ["prefix"], + }); + const { state } = prefixedTask(h); + // ...then an older worker registers and asks for work. + const old = call(h, "/work", { + worker: "gpu-old", + slots: 1, + cpus: 8, + kinds: ["video"], + }); + expect(await answered(old)).toBe("waiting"); + expect(state.state).toBe("queued"); + const work = (await ( + await call(h, "/work", { + worker: "gpu-new", + slots: 1, + cpus: 8, + kinds: ["video"], + features: ["prefix"], + }) + ).json()) as { task: protocol.VideoTask }; + expect(work.task.taskId).toBe(state.task.taskId); + expect(state.worker).toBe("gpu-new"); + }); + + test("an older worker is not sent a hedge of a task with a prefix", async () => { + const h = harness(); + const j = job(); + h.jobs.set(j.id, j); + for (let index = 0; index < 3; index++) { + j.taskStats.push({ kind: "video", frames: 30, engineRenderMs: 1000 }); + } + const original = videoState(j); + (original.task as protocol.VideoTask).files = [ + { + path: "display.mp4", + key: source, + size: 1100, + ranges: [[0, 600]], + prefix: { key: prefixKey, size: 100 }, + }, + ]; + original.progress = { + frames: 27, + total: 30, + elapsedMs: 900, + at: 0, + advancedAt: Number.NEGATIVE_INFINITY, + }; + const old = call(h, "/work", { + worker: "gpu-old", + slots: 1, + cpus: 8, + kinds: ["video"], + }); + expect(await answered(old)).toBe("waiting"); + expect(original.duplicated).toBeFalsy(); + const work = (await ( + await call(h, "/work", { + worker: "gpu-new", + slots: 1, + cpus: 8, + kinds: ["video"], + features: ["prefix"], + }) + ).json()) as { task: protocol.VideoTask }; + expect(work.task.chunk).toBe( + original.task.kind === "video" ? original.task.chunk : -1, + ); + expect(original.duplicated).toBe(true); + }); + + async function runWatchdogs(h: ReturnType) { + for (const watchdog of h.watchdogs) { + try { + await watchdog(); + } catch {} + } + } + + test("with no worker left that takes prefixes, the job fails for a retry", async () => { + const h = harness({ RF_UNSERVABLE_GRACE_MS: "0" }); + void call(h, "/work", { + worker: "gpu-old", + slots: 1, + cpus: 8, + kinds: ["video"], + }); + const { j } = prefixedTask(h); + await runWatchdogs(h); + expect(j.status).toBe("error"); + expect(j.error).toContain("in-place sources"); + expect(h.queue).toHaveLength(0); + + // A worker that takes prefixes and video keeps the job going instead. + const kept = harness({ RF_UNSERVABLE_GRACE_MS: "0" }); + const poll = call(kept, "/work", { + worker: "gpu-new", + slots: 1, + cpus: 8, + kinds: ["video"], + features: ["prefix"], + }); + const waiting = prefixedTask(kept); + await runWatchdogs(kept); + expect(waiting.j.status).toBe("rendering"); + await poll; + expect(waiting.state.worker).toBe("gpu-new"); + }); + + test("a worker that takes prefixes only for audio can't serve a video task", async () => { + const h = harness({ RF_UNSERVABLE_GRACE_MS: "0" }); + void call(h, "/work", { + worker: "gpu-new-audio", + slots: 1, + cpus: 8, + kinds: ["audio"], + features: ["prefix"], + }); + const { j, state } = prefixedTask(h); + await runWatchdogs(h); + expect(state.worker).toBeUndefined(); + expect(j.status).toBe("error"); + expect(j.error).toContain("in-place sources"); + }); + + test("older workers, other sources and finished transcodes keep the transcode", async () => { + const recording = fragmentedRecording({ frames: 60, gop: 30, mfra: true }); + const request = { sourceRoot: root, source, output }; + // Transcoding: queued, or already handed to the registered worker. + const transcoding = async (h: ReturnType) => + ( + (await (await call(h, "/transcodes", request)).json()) as { + status: string; + } + ).status; + + const old = harness(); + worker(old); + old.objects.set(source, recording.bytes); + expect(["queued", "running"]).toContain(await transcoding(old)); + expect(old.objects.has(prefixKey)).toBe(false); + + const webm = harness(); + worker(webm, ["prefix"]); + webm.objects.set( + source, + Uint8Array.of(0x1a, 0x45, 0xdf, 0xa3, 0, 0, 0, 0, 0), + ); + expect(["queued", "running"]).toContain(await transcoding(webm)); + + const done = harness(); + worker(done, ["prefix"]); + done.objects.set(source, recording.bytes); + done.objects.set(output, new Uint8Array(42)); + expect( + await (await call(done, "/transcodes", request)).json(), + ).toMatchObject({ status: "ready", size: 42 }); + expect(done.objects.has(prefixKey)).toBe(false); + + const off = harness({ RF_FRAGMENT_INDEX: "0" }); + worker(off, ["prefix"]); + off.objects.set(source, recording.bytes); + expect(["queued", "running"]).toContain(await transcoding(off)); + }); +}); + describe("jobs for the product", () => { test("cached indexes must satisfy each job's source scope", async () => { const h = harness(); diff --git a/apps/render-farm/src/coordinator.ts b/apps/render-farm/src/coordinator.ts index 2a0b544f7ae..80d13f5acc6 100644 --- a/apps/render-farm/src/coordinator.ts +++ b/apps/render-farm/src/coordinator.ts @@ -8,6 +8,15 @@ import { mkdirSync, rmSync } from "node:fs"; import { join } from "node:path"; import { Engine } from "./engine"; import { initSegment, playlist } from "./fmp4"; +import { + buildPrefix, + PREFIX_VERSION, + parseFragmentedInit, + prefixProvenance, + remuxBlocker, + scanFragments, + UnsupportedSource, +} from "./fragment-index"; import { checkSegmentReport, segmentCuts, segmentKey } from "./hls"; import { type FileSpec, ProjectCache } from "./materialize"; import { @@ -24,11 +33,19 @@ import { MIN_RANGE_PARTS, planChunkBoundaries, } from "./planning"; +import { + builtinPath, + builtinSize, + prepareRecording, + prepareSupport, +} from "./prepare"; import { ProbeEngine } from "./probe-engine"; import { type AudioResultMeta, type AudioTask, COMPRESSION_BPP, + CURSOR_OUTPUTS, + type CursorTask, type JobRequest, MIN_PART, type SegmentReport, @@ -52,6 +69,7 @@ import { checkManifestBounds, isKey, sourceLimitsFromEnv, + validateCursorJobRequest, validateJobRequest, } from "./validate"; @@ -84,6 +102,15 @@ const probeEngine = new ProbeEngine( () => new Engine(ENGINE_BIN, {}, "coordinator-engine"), ); +// Advertised only once this engine can prepare projects, so the web app keeps +// preparing on editor workers until a farm build that can is running. +let prepareCapability: Awaited> | null = null; +prepareSupport((op, body) => probeEngine.request(body, op)) + .then((capability) => { + prepareCapability = capability; + }) + .catch((error) => console.warn(`project preparation unavailable: ${error}`)); + // Audio sections (Studio Sound) are CPU work. These lanes render them on the // coordinator's cores, pulling from the same queue as workers' audio lanes. const LOCAL_AUDIO_SLOTS = Number(process.env.RF_LOCAL_AUDIO_SLOTS ?? 2); @@ -228,6 +255,9 @@ type Worker = { service: string; audioSlots?: number; usage?: Usage | null; + /** What the worker's build understands, e.g. "prefix" file specs. */ + features?: string[]; + kinds?: string[] | "any"; }; type Section = { @@ -301,6 +331,7 @@ type Job = { headerStashes?: Map>; /** Summary frozen when the job ends; the job's media data is released then. */ final?: ReturnType; + unservableSince?: number; hls?: HlsState; id: string; request: JobRequest; @@ -413,6 +444,8 @@ type Manifest = { key?: string; /** Transcode this source to `key` first (see TranscodeTask). */ transcodeFrom?: string; + /** A wallpaper or library track this image ships (see prepare.ts). */ + builtin?: string; }[]; }; @@ -549,6 +582,7 @@ type SourceIndex = { index: TrackIndex; size: number; key: string; + prefix?: { key: string; size: number }; } >; }; @@ -603,6 +637,14 @@ function checkManifest( if (file.transcodeFrom !== undefined && file.key === undefined) { throw new Error(`manifest transcode for ${file.path} names no key`); } + if ( + file.builtin !== undefined && + (file.key !== undefined || + file.transcodeFrom !== undefined || + builtinPath(file.builtin) === null) + ) { + throw new Error(`manifest built-in for ${file.path} is invalid`); + } } } @@ -611,8 +653,17 @@ async function sourceIndex( sourceRoot?: string, onManifest?: (manifest: Manifest) => void, ): Promise { - const cached = + let cached = process.env.RF_INDEX_CACHE !== "0" ? sourceIndexes.get(prefix) : undefined; + // An index of in-place sources is only usable while every worker is. + if ( + cached && + !workersTakePrefixes() && + [...cached.mediaMeta.values()].some((meta) => meta.prefix) + ) { + sourceIndexes.delete(prefix); + cached = undefined; + } if (cached) { checkManifest(cached.manifest, prefix, sourceRoot); onManifest?.(cached.manifest); @@ -624,6 +675,9 @@ async function sourceIndex( ), ) as Manifest; checkManifest(manifest, prefix, sourceRoot); + for (const file of manifest.files) { + if (file.builtin !== undefined) file.size = builtinSize(file.builtin).size; + } onManifest?.(manifest); const keyOf = (file: { path: string; key?: string }) => file.key ?? `${prefix}/${file.path}`; @@ -642,9 +696,16 @@ async function sourceIndex( SOURCE_LIMITS, ); if (sourceBounds) throw new Error(sourceBounds); + const placed = new Map(); await Promise.all( manifest.files.map(async (file) => { if (file.transcodeFrom === undefined) return; + const placement = await placeSource(file.transcodeFrom, keyOf(file)); + if (placement) { + placed.set(file.path, placement); + file.size = placement.prefix.size + placement.sourceSize; + return; + } file.size = await awaitTranscode( await ensureTranscode(file.transcodeFrom, keyOf(file)), ); @@ -664,6 +725,18 @@ async function sourceIndex( ...manifest.files .filter((file) => file.path.endsWith(".mp4")) .map(async (file) => { + const placement = placed.get(file.path); + if (placement) { + mediaMeta.set(file.path, { + head: [0, placement.prefix.size], + moov: [0, placement.prefix.size], + index: placement.index, + size: file.size, + key: placement.source, + prefix: placement.prefix, + }); + return; + } const meta = await withIndexRead(() => mp4MetaRanges(keyOf(file), file.size), ); @@ -709,25 +782,67 @@ function baseFileSpecs( key: keyOf(file), size: file.size, ranges: "all" as const, + ...(file.builtin !== undefined + ? { local: builtinSize(file.builtin).path } + : {}), })), - ...audioFiles.map((file) => ({ - path: file.path, - key: keyOf(file), - size: file.size, - ranges: - audioMode === "all" - ? ("all" as const) - : mergeRanges([ - [0, Math.min(file.size, 256 * 1024)], - [Math.max(0, file.size - 256 * 1024), file.size], - ]), - })), + ...audioFiles.map((file) => + file.builtin !== undefined + ? { + path: file.path, + key: "", + size: file.size, + ranges: "all" as const, + local: builtinSize(file.builtin).path, + } + : { + path: file.path, + key: keyOf(file), + size: file.size, + ranges: + audioMode === "all" + ? ("all" as const) + : mergeRanges([ + [0, Math.min(file.size, 256 * 1024)], + [Math.max(0, file.size - 256 * 1024), file.size], + ]), + }, + ), ]; } async function planJob(job: Job) { const request = job.request; const prefix = request.recording.replace(/\/$/, ""); + if (request.prepare) { + await prepareRecording( + prefix, + request.prepare, + join(WORK_DIR, `${job.id}-prepare`), + { + getBounded, + put: (key, body, contentType) => s3.put(key, body, contentType), + presignGet: (key) => s3.presignFresh("GET", key, 6 * 3600), + inScope: (key) => + isKey(key) && + (key.startsWith(`${prefix}/`) || + (request.sourceRoot !== undefined && + key.startsWith(request.sourceRoot))), + engine: (op, body) => probeEngine.request(body, op), + // Measuring a never-edited recording reads its display video + // through, as the transcodes do, so they run side by side. + onManifest: (manifest) => { + checkManifest(manifest as Manifest, prefix, request.sourceRoot); + for (const file of (manifest as Manifest).files) { + if (file.transcodeFrom !== undefined && file.key !== undefined) { + warmSource(file.transcodeFrom, file.key); + } + } + }, + }, + ); + job.t.prepared = now(); + } const probeDir = join(WORK_DIR, job.id); mkdirSync(probeDir, { recursive: true }); const cache = new ProjectCache(s3, probeDir); @@ -760,6 +875,7 @@ async function planJob(job: Job) { key: meta.key, size: meta.size, ranges: mergeRanges([meta.head, meta.moov, ...extra]), + ...(meta.prefix ? { prefix: meta.prefix } : {}), }; }; @@ -1456,7 +1572,10 @@ function enqueue(job: Job, task: Task, duplicateOf?: string) { queue.push(state); } -function pickQueued(accepts: (kind: string) => boolean) { +function pickQueued( + accepts: (kind: string) => boolean, + canTake?: (state: TaskState) => boolean, +) { for (const state of [...queue]) { const job = jobs.get(state.task.jobId); if (job && taskAccepted(job, state.task)) retireTask(state); @@ -1479,6 +1598,7 @@ function pickQueued(accepts: (kind: string) => boolean) { headChunks: HEAD_CHUNKS, fifo: process.env.RF_SCHEDULER === "fifo", now: Date.now(), + canTake, }); } @@ -1546,6 +1666,209 @@ function requeue(state: TaskState, reason: string) { if (!queue.includes(state)) queue.unshift(state); } +// -------------------------------------------------------------- placements --- + +// A fragmented recording that the transcode would only have remuxed is read +// in place instead: its moofs become a regular moov, stored as a prefix next +// to where the transcode would have gone (see fragment-index.ts), and chunks +// fetch the samples they need from the recording itself. Anything else, or a +// source already transcoded, keeps the transcode. RF_FRAGMENT_INDEX=0 turns +// it off. +const FRAGMENT_INDEX = process.env.RF_FRAGMENT_INDEX !== "0"; + +type Placement = { + source: string; + sourceSize: number; + prefix: { key: string; size: number }; + index: TrackIndex; +}; + +/** By output key, like transcodes; null means transcode instead. */ +const placements = new Map>(); +const PLACEMENT_CACHE_ENTRIES = 64; + +const prefixKeyFor = (output: string) => + `${output.replace(/\.mp4$/, "")}.prefix-v${PREFIX_VERSION}`; + +/** + * Worker features a task's inputs need. A worker from before prefixed files + * would read their ranges from the raw recording at the wrong offsets. + */ +function taskNeeds(task: Task): string[] { + return "files" in task && task.files.some((file) => file.prefix) + ? ["prefix"] + : []; +} + +/** + * Whether a slot polling for `kinds` on a worker with `features` can run the + * task. Dispatch and the unservable check both ask this, so a worker counts + * as able to serve a task only if it would be handed it. + */ +function canRun( + kinds: readonly string[] | "any" | undefined, + features: readonly string[] | undefined, + task: Task, +) { + return ( + kinds !== undefined && + (kinds === "any" || kinds.includes(task.kind)) && + taskNeeds(task).every((need) => features?.includes(need)) + ); +} + +// Long enough for a rolling restart to bring a capable worker back. +const UNSERVABLE_GRACE_MS = Number( + process.env.RF_UNSERVABLE_GRACE_MS ?? 60_000, +); + +/** + * Fails a job whose queued work needs a feature no live worker has (the + * fleet was rolled back under it) once that has lasted a grace period, so it + * doesn't sit until the stall timeout. Its retry plans with the transcode, + * since placements need every live worker to take prefixes. + */ +function failUnservable(job: Job) { + const cutoff = Date.now() - 30_000; + const live = [...workers.values()].filter( + (worker) => worker.lastSeen >= cutoff, + ); + const stuck = [...job.tasks.values()].some( + (state) => + state.state === "queued" && + taskNeeds(state.task).length > 0 && + !live.some((worker) => canRun(worker.kinds, worker.features, state.task)), + ); + if (!stuck) { + job.unservableSince = undefined; + return; + } + job.unservableSince ??= Date.now(); + if (Date.now() - job.unservableSince >= UNSERVABLE_GRACE_MS) { + failJob( + job, + new Error("no live worker can read this job's in-place sources"), + ); + } +} + +/** Workers from before prefixed files would write the source at offset 0. */ +function workersTakePrefixes() { + const cutoff = Date.now() - 30_000; + let live = 0; + for (const worker of workers.values()) { + if (worker.lastSeen < cutoff) continue; + if (!worker.features?.includes("prefix")) return false; + live++; + } + return live > 0; +} + +function placeSource(source: string, output: string) { + if (!FRAGMENT_INDEX || !workersTakePrefixes()) return Promise.resolve(null); + const existing = placements.get(output); + if (existing) { + placements.delete(output); + placements.set(output, existing); + return existing; + } + const pending = buildPlacement(source, output).catch((error) => { + // A source that can't be placed stays that way; a failed read may not. + if (!(error instanceof UnsupportedSource)) placements.delete(output); + console.warn(`placing ${source}: ${String(error)}`); + return null; + }); + placements.set(output, pending); + if (placements.size > PLACEMENT_CACHE_ENTRIES) { + placements.delete(placements.keys().next().value as string); + } + return pending; +} + +/** Starts whatever a later render of this source will wait for. */ +function warmSource(source: string, output: string) { + void placeSource(source, output).then((placement) => { + if (!placement) void ensureTranscode(source, output); + }); +} + +async function buildPlacement( + source: string, + output: string, +): Promise { + const started = performance.now(); + const prefixKey = prefixKeyFor(output); + const [sourceSize, storedPrefix, transcoded] = await Promise.all([ + transcodeSourceSize(source), + s3.head(prefixKey), + storedTranscodeSize(output), + ]); + if (storedPrefix) { + if (storedPrefix.size > SOURCE_LIMITS.moovBytes) { + throw new UnsupportedSource(`${prefixKey} is ${storedPrefix.size} bytes`); + } + const bytes = await s3.get(prefixKey); + const provenance = prefixProvenance(bytes); + const where = locateMoov(bytes, bytes.byteLength); + if ( + provenance?.version === PREFIX_VERSION && + provenance.source === source && + provenance.size === sourceSize && + where && + "start" in where && + where.start !== undefined + ) { + return { + source, + sourceSize, + prefix: { key: prefixKey, size: bytes.byteLength }, + index: indexVideoTrack( + bytes.subarray(where.start, where.start + where.size), + ), + }; + } + } + // Already remuxed: reading that is as quick as building a prefix. + if (transcoded !== null) return null; + const headEnd = Math.min(sourceSize, 128 * 1024); + const head = await s3.getRange(source, 0, headEnd - 1); + const where = locateMoov(head, sourceSize); + if (!where || !("start" in where) || where.start === undefined) { + throw new UnsupportedSource("no moov up front"); + } + if (where.size > SOURCE_LIMITS.moovBytes) { + throw new UnsupportedSource(`a ${where.size} byte moov`); + } + const moov = + where.start + where.size <= head.byteLength + ? head.subarray(where.start, where.start + where.size) + : await s3.getRange(source, where.start, where.start + where.size - 1); + const init = parseFragmentedInit(moov, where.start); + if (!init) throw new UnsupportedSource("not fragmented"); + const scan = await scanFragments( + (start, end) => s3.getRange(source, start, end), + sourceSize, + init, + ); + const blocker = remuxBlocker(init, scan.samples); + if (blocker) throw new UnsupportedSource(blocker); + const { bytes, index } = buildPrefix(init, scan.samples, { + version: PREFIX_VERSION, + source, + size: sourceSize, + }); + await s3.put(prefixKey, bytes, "application/octet-stream"); + console.log( + `placed ${source}: ${scan.samples.count} frames in ${scan.fragments} fragments, ${scan.usedMfra ? "mfra" : "streamed"}, ${scan.requests} reads, ${(scan.bytes / 2 ** 20).toFixed(1)} MB read, ${bytes.byteLength} byte prefix, ${Math.round(performance.now() - started)} ms`, + ); + return { + source, + sourceSize, + prefix: { key: prefixKey, size: bytes.byteLength }, + index, + }; +} + // -------------------------------------------------------------- transcodes --- const TRANSCODE_KEYFRAME_SECONDS = 1; @@ -1719,6 +2042,140 @@ setInterval(() => { } }, 5_000); +// ------------------------------------------------------------ cursor jobs --- + +const CURSOR_ATTEMPTS = 2; + +type CursorJob = { + id: string; + task: CursorTask; + state: "queued" | "running" | "ready" | "error"; + worker?: string; + attempts: number; + lastReportedAt?: number; + notBefore?: number; + /** Thousandths of the whole job, from the worker's heartbeat. */ + progress: number; + error?: string; + callbackUrl?: string; + reference?: string; + settledAt?: number; + bytes?: { display: number; inputEvents: number }; +}; + +/** By output prefix: one reconstruction per target. */ +const cursorJobs = new Map(); + +function ensureCursorJob(request: { + source: string; + outputPrefix: string; + callbackUrl?: string; + reference?: string; +}) { + const existing = cursorJobs.get(request.outputPrefix); + if (existing && existing.state !== "error") return existing; + const id = createHash("sha256") + .update(request.outputPrefix) + .digest("hex") + .slice(0, 16); + const job: CursorJob = { + id, + task: { + kind: "cursor", + taskId: `cr:${id}`, + source: request.source, + outputPrefix: request.outputPrefix, + }, + state: "queued", + attempts: 0, + progress: 0, + callbackUrl: request.callbackUrl, + reference: request.reference, + }; + cursorJobs.set(request.outputPrefix, job); + dispatch(); + return job; +} + +function nextCursorJob() { + const at = Date.now(); + for (const job of cursorJobs.values()) { + if (job.state === "queued" && (job.notBefore ?? 0) <= at) return job; + } +} + +function cursorJobById(id: string) { + for (const job of cursorJobs.values()) { + if (job.id === id) return job; + } +} + +function cursorJobSummary(job: CursorJob) { + return { + id: job.id, + reference: job.reference, + status: job.state, + progress: job.progress / 1000, + error: job.error, + display: `${job.task.outputPrefix}${CURSOR_OUTPUTS.display}`, + inputEvents: `${job.task.outputPrefix}${CURSOR_OUTPUTS.inputEvents}`, + bytes: job.bytes, + }; +} + +function settleCursorJob( + job: CursorJob, + state: "ready" | "error", + error?: string, +) { + job.state = state; + job.worker = undefined; + job.error = error; + if (state === "ready") job.progress = 1000; + job.settledAt = Date.now(); + if (job.callbackUrl) { + postSigned( + job.callbackUrl, + JSON.stringify(cursorJobSummary(job)), + `cursor job ${job.id}`, + ); + } +} + +function requeueCursorJob(job: CursorJob, reason: string) { + console.warn(`requeue ${job.task.taskId}: ${reason}`); + if (job.attempts >= CURSOR_ATTEMPTS) { + settleCursorJob(job, "error", reason); + return; + } + const delay = 1000 * 2 ** job.attempts; + job.state = "queued"; + job.worker = undefined; + job.notBefore = Date.now() + delay; + setTimeout(dispatch, delay + 50).unref(); +} + +setInterval(() => { + const cutoff = Date.now() - 30_000; + for (const [prefix, job] of cursorJobs) { + if (job.state === "running") { + const worker = job.worker ? workers.get(job.worker) : null; + if ( + !worker || + worker.lastSeen < cutoff || + (job.lastReportedAt ?? 0) < cutoff + ) { + requeueCursorJob(job, "worker stopped reporting it"); + } + } else if ( + job.settledAt !== undefined && + Date.now() - job.settledAt > JOB_RETENTION_MS + ) { + cursorJobs.delete(prefix); + } + } +}, 5_000); + function dispatch() { pumpLocalAudio(); for (let p = 0; p < pollers.length; ) { @@ -1738,14 +2195,34 @@ function dispatch() { poller.resolve({ ...transcode.task, attempt: transcode.attempts }); continue; } - const next = pickQueued(accepts); + const canTake = (candidate: TaskState) => + canRun( + poller.kinds ?? "any", + workers.get(poller.worker)?.features, + candidate.task, + ); + const next = pickQueued(accepts, canTake); let state: TaskState | undefined; if (next >= 0) { state = queue.splice(next, 1)[0]; } else if (accepts("video") && !poller.prefetch) { - state = straggler(); + state = straggler(canTake); } if (!state) { + // Reconstruction is CPU work that no export waits on, so it only + // takes a slot nothing else wants. + const cursorJob = + accepts("video") && !poller.prefetch ? nextCursorJob() : undefined; + if (cursorJob) { + pollers.splice(p, 1); + cursorJob.state = "running"; + cursorJob.worker = poller.worker; + cursorJob.attempts++; + cursorJob.lastReportedAt = Date.now(); + cursorJob.progress = 0; + poller.resolve({ ...cursorJob.task, attempt: cursorJob.attempts }); + continue; + } p++; continue; } @@ -1788,7 +2265,9 @@ function dispatch() { */ const FROZEN_MS = 5_000; -function straggler(): TaskState | undefined { +function straggler( + canTake: (state: TaskState) => boolean = () => true, +): TaskState | undefined { let best: { state: TaskState; gain: number } | undefined; for (const job of jobs.values()) { if (job.status !== "rendering" || job.request.duplicateStragglers === false) @@ -1815,6 +2294,7 @@ function straggler(): TaskState | undefined { state.state !== "running" || state.duplicated || state.duplicateOf || + !canTake(state) || (state.prefetched && !state.progress) || job.videoResults.has(state.task.chunk) ) { @@ -1932,6 +2412,8 @@ setInterval(() => { ); continue; } + failUnservable(job); + if (job.status !== "rendering") continue; for (const state of job.tasks.values()) { if ( state.state !== "running" || @@ -2093,14 +2575,14 @@ function callbackPayload(job: Job) { }; } -/** Posts the outcome, signed `sha256=`, with retries. */ -function notify( - job: Job, - body = JSON.stringify(callbackPayload(job)), - attempt = 0, -) { +function notify(job: Job) { const url = job.request.callbackUrl; if (!url) return; + postSigned(url, JSON.stringify(callbackPayload(job)), `job ${job.id}`); +} + +/** Posts a callback body, signed `sha256=`, with retries. */ +function postSigned(url: string, body: string, label: string, attempt = 0) { const signature = createHmac("sha256", CALLBACK_SECRET) .update(body) .digest("hex"); @@ -2119,11 +2601,11 @@ function notify( }) .catch((error) => { if (attempt >= 8) { - console.error(`job ${job.id} callback gave up: ${error}`); + console.error(`${label} callback gave up: ${error}`); return; } setTimeout( - () => notify(job, body, attempt + 1), + () => postSigned(url, body, label, attempt + 1), Math.min(60_000, 1000 * 2 ** attempt), ).unref(); }); @@ -2195,6 +2677,12 @@ async function acceptVideo(job: Job, state: TaskState, result: VideoResult) { ) : "result is not from this chunk's dispatch"; if (problem) throw new Error(`chunk ${chunk}: ${problem}`); + // A worker that lost its stash upload must not be credited with the chunk: + // assembly could only fail later, with nothing left to re-render it. + const stored = await journalS3.head(result.stash.key); + if (stored?.size !== result.stash.bytes) { + throw new Error(`chunk ${chunk}: stash ${result.stash.key} is not stored`); + } await journalPut( journalKey(job.id, `v/${chunk}.json`), JSON.stringify(result), @@ -2751,7 +3239,11 @@ Bun.serve({ async fetch(request) { const url = new URL(request.url); if (url.pathname === "/health") - return Response.json({ ok: true, ...liveSlots() }); + return Response.json({ + ok: true, + ...liveSlots(), + ...(prepareCapability ? { prepare: prepareCapability } : {}), + }); if (!authorized(request)) return new Response("unauthorized", { status: 401 }); @@ -2765,6 +3257,7 @@ Bun.serve({ audioSlots?: number; prefetch?: boolean; draining?: boolean; + features?: string[]; }; const worker = workers.get(body.worker) ?? { id: body.worker, @@ -2776,6 +3269,11 @@ Bun.serve({ worker.lastSeen = Date.now(); worker.slots = body.slots; worker.audioSlots = body.audioSlots ?? 0; + worker.features = body.features ?? []; + worker.kinds = + !body.kinds || worker.kinds === "any" + ? "any" + : [...new Set([...(worker.kinds ?? []), ...body.kinds])]; supersede(body.worker); workers.set(body.worker, worker); if (body.draining) @@ -2824,6 +3322,20 @@ Bun.serve({ }; const staleTranscodes: string[] = []; for (const entry of body.running ?? []) { + if (entry.taskId.startsWith("cr:")) { + const job = cursorJobById(entry.taskId.slice(3)); + if ( + job?.state === "running" && + job.worker === body.worker && + job.attempts === entry.attempt + ) { + job.lastReportedAt = Date.now(); + job.progress = Math.min(999, Math.max(0, entry.frames)); + } else { + staleTranscodes.push(entry.taskId); + } + continue; + } if (entry.taskId.startsWith("tc:")) { const transcode = transcodeById(entry.taskId.slice(3)); if ( @@ -2931,6 +3443,68 @@ Bun.serve({ return Response.json({ ok: true }); } + if (url.pathname === "/cursor-jobs" && request.method === "POST") { + const parsed = validateCursorJobRequest( + await request.json().catch(() => null), + ); + if (typeof parsed === "string") + return new Response(parsed, { status: 400 }); + if (parsed.callbackUrl && !callbackAllowed(parsed.callbackUrl)) { + return new Response("callbackUrl host is not in RF_CALLBACK_HOSTS", { + status: 400, + }); + } + return Response.json(cursorJobSummary(ensureCursorJob(parsed))); + } + const cursorMatch = url.pathname.match( + /^\/cursor-jobs\/([0-9a-f]{16})(?:\/(done|fail))?$/, + ); + if (cursorMatch) { + const job = cursorJobById(cursorMatch[1] ?? ""); + if (!job) return new Response("unknown cursor job", { status: 404 }); + if (!cursorMatch[2] && request.method === "GET") { + return Response.json(cursorJobSummary(job)); + } + if (request.method !== "POST") { + return new Response("method not allowed", { status: 405 }); + } + const report = (await request.json()) as { + worker: string; + attempt?: number; + error?: string; + }; + const current = () => + cursorJobs.get(job.task.outputPrefix) === job && + job.state === "running" && + job.worker === report.worker && + job.attempts === report.attempt; + if (cursorMatch[2] === "done") { + if (job.state === "ready") return Response.json({ ok: true }); + if (!current()) { + return new Response("stale cursor job report", { status: 409 }); + } + const summary = cursorJobSummary(job); + const [display, inputEvents] = await Promise.all([ + s3.head(summary.display), + s3.head(summary.inputEvents), + ]); + if (!current()) { + return new Response("stale cursor job report", { status: 409 }); + } + if (!display?.size || !inputEvents?.size) { + return new Response("cursor job outputs are missing", { + status: 409, + }); + } + job.bytes = { display: display.size, inputEvents: inputEvents.size }; + settleCursorJob(job, "ready"); + dispatch(); + } else if (current()) { + requeueCursorJob(job, report.error ?? "failed"); + } + return Response.json({ ok: true }); + } + if (url.pathname === "/transcodes" && request.method === "POST") { const { source, output, sourceRoot } = (await request.json()) as Record< string, @@ -2950,6 +3524,18 @@ Bun.serve({ { status: 400 }, ); } + // Editors and uploads warm sources this way: a placed source needs + // no transcode at all. + const placement = await placeSource(source, output); + if (placement) { + return Response.json({ + status: "ready", + source, + output, + placed: true, + size: placement.prefix.size + placement.sourceSize, + }); + } return Response.json( transcodeSummary(await ensureTranscode(source, output)), ); diff --git a/apps/render-farm/src/cursor.test.ts b/apps/render-farm/src/cursor.test.ts new file mode 100644 index 00000000000..c1375b80e36 --- /dev/null +++ b/apps/render-farm/src/cursor.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, test } from "bun:test"; +import { cursorProgress } from "./cursor"; + +describe("cursorProgress", () => { + test("weights tracking and repair into one job-wide value", () => { + expect(cursorProgress('{"stage":"tracking","progress":0.5}')).toBe(350); + expect(cursorProgress('{"stage":"tracking","progress":1}')).toBe(700); + expect(cursorProgress('{"stage":"repairing","progress":0.5}')).toBe(845); + expect(cursorProgress('{"stage":"complete","progress":1}')).toBe(990); + expect(cursorProgress('{"stage":"tracking","progress":7}')).toBe(700); + }); + + test("ignores lines that are not progress reports", () => { + expect(cursorProgress("Error: Unsupported video")).toBeNull(); + expect(cursorProgress('{"stage":"rendering","progress":0}')).toBeNull(); + expect(cursorProgress('{"stage":"tracking"}')).toBeNull(); + }); +}); diff --git a/apps/render-farm/src/cursor.ts b/apps/render-farm/src/cursor.ts new file mode 100644 index 00000000000..30eab000a36 --- /dev/null +++ b/apps/render-farm/src/cursor.ts @@ -0,0 +1,17 @@ +/** Thousandths of the whole job: tracking takes about 70% of it. */ +export function cursorProgress(line: string) { + try { + const { stage, progress } = JSON.parse(line) as { + stage?: unknown; + progress?: unknown; + }; + if (typeof progress !== "number" || !Number.isFinite(progress)) return null; + const fraction = Math.min(1, Math.max(0, progress)); + if (stage === "tracking") return Math.floor(fraction * 700); + if (stage === "repairing") return 700 + Math.floor(fraction * 290); + if (stage === "complete") return 990; + return null; + } catch { + return null; + } +} diff --git a/apps/render-farm/src/fragment-index.test.ts b/apps/render-farm/src/fragment-index.test.ts new file mode 100644 index 00000000000..71ef6d5c9ac --- /dev/null +++ b/apps/render-farm/src/fragment-index.test.ts @@ -0,0 +1,292 @@ +import { describe, expect, test } from "bun:test"; +import { fragmentedRecording } from "./boxes.test-util"; +import { + buildPrefix, + type FragmentedInit, + FragmentSamples, + mfraMoofs, + parseFragmentedInit, + prefixProvenance, + readMoof, + readSps, + remuxBlocker, + scanFragments, + UnsupportedSource, +} from "./fragment-index"; +import { + box, + build, + byteRangeFor, + fullBox, + indexVideoTrack, + locateMoov, +} from "./mp4"; + +function reader(bytes: Uint8Array) { + const reads: [number, number][] = []; + const read = async (start: number, endInclusive: number) => { + reads.push([start, endInclusive]); + return bytes.slice(start, endInclusive + 1); + }; + return { read, reads }; +} + +function initOf(bytes: Uint8Array) { + const where = locateMoov(bytes, bytes.byteLength) as { + start: number; + size: number; + }; + return parseFragmentedInit( + bytes.subarray(where.start, where.start + where.size), + where.start, + ) as FragmentedInit; +} + +async function place(bytes: Uint8Array, options = {}) { + const init = initOf(bytes); + const { read, reads } = reader(bytes); + const scan = await scanFragments(read, bytes.byteLength, init, options); + const built = buildPrefix(init, scan.samples, { + version: 1, + source: "owner/video/raw-upload.mp4", + size: bytes.byteLength, + }); + return { init, scan, reads, ...built }; +} + +describe("fragment index", () => { + test("indexes every video sample where it sits in the recording", async () => { + const recording = fragmentedRecording({ + frames: 95, + gop: 30, + audio: true, + mfra: true, + }); + const { scan, bytes, index } = await place(recording.bytes); + expect(scan).toMatchObject({ fragments: 4, usedMfra: true }); + expect(index.sizes.length).toBe(95); + expect([...index.keyframes]).toEqual([0, 30, 60, 90]); + const joined = new Uint8Array( + bytes.byteLength + recording.bytes.byteLength, + ); + joined.set(bytes); + joined.set(recording.bytes, bytes.byteLength); + for (let frame = 0; frame < 95; frame++) { + const at = index.offsets[frame] as number; + const sample = joined.subarray(at, at + (index.sizes[frame] as number)); + expect(sample.byteLength).toBe(recording.size(frame)); + expect(sample.every((value) => value === (frame % 251) + 1)).toBe(true); + expect(index.times[frame]).toBeCloseTo(frame / 30, 9); + } + // The prefix is a regular MP4 header: its own moov indexes the same. + const where = locateMoov(bytes, bytes.byteLength) as { + start: number; + size: number; + }; + const reread = indexVideoTrack( + bytes.subarray(where.start, where.start + where.size), + ); + expect([...reread.offsets]).toEqual([...index.offsets]); + expect([...reread.sizes]).toEqual([...index.sizes]); + expect([...reread.keyframes]).toEqual([...index.keyframes]); + expect([...reread.times]).toEqual([...index.times]); + expect(prefixProvenance(bytes)).toEqual({ + version: 1, + source: "owner/video/raw-upload.mp4", + size: recording.bytes.byteLength, + }); + // The mdat header claims exactly the source that follows it. + const view = new DataView(bytes.buffer); + expect(Number(view.getBigUint64(bytes.byteLength - 8))).toBe( + 16 + recording.bytes.byteLength, + ); + const range = byteRangeFor(index, 1.5, 2.1) as { start: number }; + expect(range.start).toBe(index.offsets[30] as number); + }); + + test("with an mfra only the moofs are read; without one the file streams", async () => { + const withMfra = fragmentedRecording({ + frames: 3000, + gop: 300, + mfra: true, + }); + const listed = await place(withMfra.bytes, { + concurrency: 4, + tailBytes: 4096, + }); + // The tail, then one small read per moof. + const read = listed.reads.reduce( + (sum, [start, end]) => sum + end - start + 1, + 0, + ); + expect(listed.scan.usedMfra).toBe(true); + expect(listed.reads.length).toBe(11); + expect(read).toBeLessThan(withMfra.bytes.byteLength / 5); + + const bare = fragmentedRecording({ frames: 3000, gop: 300 }); + const streamed = await place(bare.bytes, { piece: 4096, windowPieces: 2 }); + expect(streamed.scan.usedMfra).toBe(false); + expect(streamed.scan.fragments).toBe(10); + expect([...streamed.index.sizes]).toEqual([...listed.index.sizes]); + expect([...streamed.index.keyframes]).toEqual([...listed.index.keyframes]); + }); + + test("fragments an mfra leaves out are found by walking the boxes", async () => { + const recording = fragmentedRecording({ frames: 120, gop: 30 }); + const init = initOf(recording.bytes); + // An mfra naming only the first and third moofs. + const listed = [recording.moofs[0], recording.moofs[2]] as number[]; + const tfra = fullBox( + "tfra", + 1, + 0, + build((writer) => { + writer.u32(1); + writer.u32(0); + writer.u32(listed.length); + for (const moof of listed) { + writer.u64(0); + writer.u64(moof); + writer.u8(1); + writer.u8(1); + writer.u8(1); + } + }), + ); + const mfra = box( + "mfra", + tfra, + fullBox( + "mfro", + 0, + 0, + build((writer) => writer.u32(8 + tfra.byteLength + 16)), + ), + ); + const bytes = new Uint8Array(recording.bytes.byteLength + mfra.byteLength); + bytes.set(recording.bytes); + bytes.set(mfra, recording.bytes.byteLength); + expect(mfraMoofs(bytes.subarray(bytes.byteLength - 4096), 1)).toMatchObject( + { moofs: listed }, + ); + const scan = await scanFragments( + async (start, end) => bytes.slice(start, end + 1), + bytes.byteLength, + init, + ); + expect(scan.samples.count).toBe(120); + expect(scan.fragments).toBe(4); + }); + + test("composition offsets become a ctts and presentation times", () => { + const recording = fragmentedRecording({ frames: 2, gop: 30 }); + const init = initOf(recording.bytes); + const samples = new FragmentSamples(); + // moof > traf > tfhd (base is moof, default size 10) + tfdt + trun + // with durations and signed composition offsets, no data offset. + const moof = box( + "moof", + fullBox( + "mfhd", + 0, + 0, + build((writer) => writer.u32(1)), + ), + box( + "traf", + fullBox( + "tfhd", + 0, + 0x20010, + build((writer) => { + writer.u32(1); + writer.u32(10); + }), + ), + fullBox( + "tfdt", + 1, + 0, + build((writer) => writer.u64(0)), + ), + fullBox( + "trun", + 1, + 0x900, + build((writer) => { + writer.u32(3); + for (const [duration, offset] of [ + [1000, 2000], + [1000, -1000], + [1000, 0], + ]) { + writer.u32(duration as number); + writer.u32(offset as number); + } + }), + ), + ), + ); + readMoof(moof, 5000, init, samples); + expect(samples.positions).toEqual([5000, 5010, 5020]); + expect(samples.offsets).toEqual([2000, -1000, 0]); + expect(remuxBlocker(init, samples)).toBe( + "presentation and decode times differ", + ); + const { bytes, index } = buildPrefix(init, samples, { + version: 1, + source: "s", + size: 6000, + }); + expect([...index.times]).toEqual([2 / 30, 0, 2 / 30]); + const text = new TextDecoder("latin1").decode(bytes); + expect(text.includes("ctts")).toBe(true); + }); + + test("sources a remux wouldn't copy as they are keep the transcode", () => { + const recording = fragmentedRecording({ frames: 300, gop: 150 }); + const init = initOf(recording.bytes); + const samples = new FragmentSamples(); + const start = (index: number) => ({ + dts: index * 1000, + sync: index % 150 === 0, + }); + for (let index = 0; index < 300; index++) { + samples.dts.push(start(index).dts); + samples.offsets.push(0); + samples.sizes.push(1); + samples.positions.push(index); + samples.sync.push(start(index).sync); + } + samples.firstDts = 0; + samples.lastDuration = 1000; + expect(remuxBlocker(init, samples)).toBe("keyframes too far apart"); + samples.sync = samples.sync.map((_, index) => index % 60 === 0); + expect(remuxBlocker(init, samples)).toBeNull(); + expect(remuxBlocker({ ...init, codec: "hvc1" }, samples)).toBe( + "codec hvc1", + ); + samples.firstDts = 1000; + expect(() => + buildPrefix(init, samples, { version: 1, source: "s", size: 300 }), + ).toThrow(UnsupportedSource); + }); + + test("reads the SPS fields ffmpeg reports", () => { + const recording = fragmentedRecording({ frames: 1, gop: 1 }); + const init = initOf(recording.bytes); + const avcC = init.avcC as Uint8Array; + const length = ((avcC[6] as number) << 8) | (avcC[7] as number); + expect(readSps(avcC.subarray(8, 8 + length))).toMatchObject({ + profile: 100, + chromaFormat: 1, + lumaDepth: 8, + chromaDepth: 8, + }); + }); + + test("a moov with no mvex is not a fragmented recording", () => { + const moov = box("moov", fullBox("mvhd", 0, 0, new Uint8Array(96))); + expect(parseFragmentedInit(moov, 0)).toBeNull(); + }); +}); diff --git a/apps/render-farm/src/fragment-index.ts b/apps/render-farm/src/fragment-index.ts new file mode 100644 index 00000000000..7ccb67bef3b --- /dev/null +++ b/apps/render-farm/src/fragment-index.ts @@ -0,0 +1,1046 @@ +import { + box, + build, + MAX_SAMPLES, + type Run, + sampleTable, + type TrackIndex, +} from "./mp4"; +import { MAX_REMUX_KEYFRAME_GAP_SECONDS } from "./transcode"; + +// Browser recordings are fragmented MP4: a moov with no samples, then +// moof + mdat pairs. The planner needs a sample index to fetch only the bytes +// each chunk decodes, and those files spread it over thousands of moofs, so +// they used to be remuxed first (download, rewrite, upload, read back). Here +// the moofs are read in place and turned into a regular moov for the same +// samples, laid out as a prefix: +// +// ftyp | free (provenance) | moov | mdat header | the source, untouched +// +// The source's sample offsets shift by the prefix size. Workers write the +// prefix, then the source ranges they need, and ffmpeg opens the result like +// the remuxed file, so the renderer needs no changes. + +export class UnsupportedSource extends Error {} + +export const PREFIX_VERSION = 1; + +type Box = { type: string; start: number; body: number; end: number }; + +function boxAt(view: DataView, offset: number, limit: number): Box | null { + if (offset + 8 > limit) return null; + let size = view.getUint32(offset); + let body = offset + 8; + if (size === 1) { + if (offset + 16 > limit) return null; + size = Number(view.getBigUint64(offset + 8)); + body = offset + 16; + } else if (size === 0) { + size = limit - offset; + } + const type = String.fromCharCode( + view.getUint8(offset + 4), + view.getUint8(offset + 5), + view.getUint8(offset + 6), + view.getUint8(offset + 7), + ); + if (size < body - offset) throw new UnsupportedSource(`bad ${type} box`); + return { type, start: offset, body, end: offset + size }; +} + +function childrenOf(view: DataView, parent: { body: number; end: number }) { + const out: Box[] = []; + let offset = parent.body; + while (offset + 8 <= parent.end) { + const child = boxAt(view, offset, parent.end); + if (!child || child.end > parent.end) { + throw new UnsupportedSource("box overruns its parent"); + } + out.push(child); + offset = child.end; + } + return out; +} + +function only(boxes: Box[], type: string) { + return boxes.find((candidate) => candidate.type === type); +} + +function need(boxes: Box[], type: string) { + const found = only(boxes, type); + if (!found) throw new UnsupportedSource(`no ${type}`); + return found; +} + +type TrackDefaults = { + descriptionIndex: number; + duration: number; + size: number; + flags: number; +}; + +export type FragmentedInit = { + /** Offset just past the moov: the first fragment starts at or after it. */ + moovEnd: number; + trackId: number; + timescale: number; + /** Sample description fourcc, e.g. avc1. */ + codec: string; + /** The avcC record, when the codec is H.264. */ + avcC: Uint8Array | null; + defaults: Map; + /** Boxes the built moov reuses as they are (durations patched). */ + reuse: { + mvhd: Uint8Array; + tkhd: Uint8Array; + mdhd: Uint8Array; + hdlr: Uint8Array; + mediaHeader: Uint8Array; + dinf: Uint8Array; + stsd: Uint8Array; + }; +}; + +/** + * The first video track of a fragmented file's moov (`moov` = the whole box, + * found at `moovStart`), or null when the file isn't fragmented. + */ +export function parseFragmentedInit( + moov: Uint8Array, + moovStart: number, +): FragmentedInit | null { + const view = new DataView(moov.buffer, moov.byteOffset, moov.byteLength); + const root = boxAt(view, 0, moov.byteLength); + if (!root || root.type !== "moov") throw new UnsupportedSource("no moov"); + const top = childrenOf(view, root); + const mvex = only(top, "mvex"); + if (!mvex) return null; + const defaults = new Map(); + for (const trex of childrenOf(view, mvex)) { + if (trex.type !== "trex" || trex.end - trex.body < 24) continue; + defaults.set(view.getUint32(trex.body + 4), { + descriptionIndex: view.getUint32(trex.body + 8), + duration: view.getUint32(trex.body + 12), + size: view.getUint32(trex.body + 16), + flags: view.getUint32(trex.body + 20), + }); + } + const slice = (found: Box) => moov.slice(found.start, found.end); + for (const trak of top.filter((candidate) => candidate.type === "trak")) { + const trakChildren = childrenOf(view, trak); + const mdia = need(trakChildren, "mdia"); + const mdiaChildren = childrenOf(view, mdia); + const hdlr = need(mdiaChildren, "hdlr"); + if (hdlr.body + 12 > hdlr.end) continue; + const handler = String.fromCharCode( + ...moov.subarray(hdlr.body + 8, hdlr.body + 12), + ); + if (handler !== "vide") continue; + // The first video track is the one ffmpeg's `-map 0:v:0` remux took. + if (only(trakChildren, "edts")) { + throw new UnsupportedSource("the video track has an edit list"); + } + const tkhd = need(trakChildren, "tkhd"); + const tkhdVersion = view.getUint8(tkhd.body); + const trackId = view.getUint32(tkhd.body + (tkhdVersion === 1 ? 20 : 12)); + const mdhd = need(mdiaChildren, "mdhd"); + const timescale = view.getUint32( + mdhd.body + (view.getUint8(mdhd.body) === 1 ? 20 : 12), + ); + if (!timescale) throw new UnsupportedSource("zero timescale"); + const minf = need(mdiaChildren, "minf"); + const minfChildren = childrenOf(view, minf); + const stbl = need(minfChildren, "stbl"); + const stblChildren = childrenOf(view, stbl); + const stsd = need(stblChildren, "stsd"); + const stsz = only(stblChildren, "stsz"); + if (stsz && view.getUint32(stsz.body + 8) !== 0) { + throw new UnsupportedSource("samples in the moov as well as fragments"); + } + const entries = childrenOf(view, { + body: stsd.body + 8, + end: stsd.end, + }); + const entry = entries[0]; + if (!entry || view.getUint32(stsd.body + 4) !== 1) { + throw new UnsupportedSource("not exactly one sample description"); + } + const codec = entry.type; + // VisualSampleEntry: 78 bytes of fields before its child boxes. + const avcC = + codec === "avc1" || codec === "avc3" + ? childrenOf(view, { body: entry.body + 78, end: entry.end }).find( + (candidate) => candidate.type === "avcC", + ) + : undefined; + const mediaHeader = only(minfChildren, "vmhd"); + const dinf = only(minfChildren, "dinf"); + const mvhd = need(top, "mvhd"); + if (!mediaHeader || !dinf) throw new UnsupportedSource("no vmhd/dinf"); + if (!defaults.has(trackId)) throw new UnsupportedSource("no trex"); + return { + moovEnd: moovStart + root.end, + trackId, + timescale, + codec, + avcC: avcC ? moov.slice(avcC.body, avcC.end) : null, + defaults, + reuse: { + mvhd: slice(mvhd), + tkhd: slice(tkhd), + mdhd: slice(mdhd), + hdlr: slice(hdlr), + mediaHeader: slice(mediaHeader), + dinf: slice(dinf), + stsd: slice(stsd), + }, + }; + } + throw new UnsupportedSource("no video track"); +} + +/** Video samples of a fragmented file, in decode order. */ +export class FragmentSamples { + dts: number[] = []; + /** Composition offsets: presentation time = decode time + offset. */ + offsets: number[] = []; + sizes: number[] = []; + positions: number[] = []; + sync: boolean[] = []; + lastDuration = 0; + /** Decode time the next fragment starts at when it carries no tfdt. */ + nextDts = 0; + firstDts: number | null = null; + + get count() { + return this.dts.length; + } +} + +const TFHD_BASE_OFFSET = 0x1; +const TFHD_DESCRIPTION = 0x2; +const TFHD_DURATION = 0x8; +const TFHD_SIZE = 0x10; +const TFHD_FLAGS = 0x20; +const TFHD_EMPTY = 0x10000; +const TFHD_BASE_IS_MOOF = 0x20000; +const TRUN_DATA_OFFSET = 0x1; +const TRUN_FIRST_FLAGS = 0x4; +const TRUN_DURATION = 0x100; +const TRUN_SIZE = 0x200; +const TRUN_FLAGS = 0x400; +const TRUN_COMPOSITION = 0x800; +// ffmpeg counts a sample as a keyframe when neither is set. +const SAMPLE_NON_SYNC = 0x10000; +const SAMPLE_DEPENDS_YES = 0x1000000; + +/** + * Appends the video track's samples from one moof (`moof` = the whole box, + * which starts at `position` in the file). Other tracks' runs are walked + * only to find where the next traf's data starts. + */ +export function readMoof( + moof: Uint8Array, + position: number, + init: FragmentedInit, + samples: FragmentSamples, +) { + const view = new DataView(moof.buffer, moof.byteOffset, moof.byteLength); + const root = boxAt(view, 0, moof.byteLength); + if (!root || root.type !== "moof") throw new UnsupportedSource("not a moof"); + let previousEnd = position; + for (const traf of childrenOf(view, root)) { + if (traf.type !== "traf") continue; + const trafChildren = childrenOf(view, traf); + const tfhd = need(trafChildren, "tfhd"); + const tfhdFlags = view.getUint32(tfhd.body) & 0xffffff; + const trackId = view.getUint32(tfhd.body + 4); + const trex = init.defaults.get(trackId); + if (!trex) + throw new UnsupportedSource(`fragment for unknown track ${trackId}`); + let at = tfhd.body + 8; + let base = previousEnd; + if (tfhdFlags & TFHD_BASE_OFFSET) { + base = Number(view.getBigUint64(at)); + at += 8; + } else if (tfhdFlags & TFHD_BASE_IS_MOOF) { + base = position; + } + let descriptionIndex = trex.descriptionIndex; + if (tfhdFlags & TFHD_DESCRIPTION) { + descriptionIndex = view.getUint32(at); + at += 4; + } + let defaultDuration = trex.duration; + if (tfhdFlags & TFHD_DURATION) { + defaultDuration = view.getUint32(at); + at += 4; + } + let defaultSize = trex.size; + if (tfhdFlags & TFHD_SIZE) { + defaultSize = view.getUint32(at); + at += 4; + } + let defaultFlags = trex.flags; + if (tfhdFlags & TFHD_FLAGS) defaultFlags = view.getUint32(at); + const video = trackId === init.trackId; + if (video && descriptionIndex !== 1) { + throw new UnsupportedSource("a fragment uses another sample description"); + } + const tfdt = only(trafChildren, "tfdt"); + let dts = samples.nextDts; + if (video && tfdt) { + dts = + view.getUint8(tfdt.body) === 1 + ? Number(view.getBigUint64(tfdt.body + 4)) + : view.getUint32(tfdt.body + 4); + } + let dataAt = base; + if (tfhdFlags & TFHD_EMPTY) { + previousEnd = base; + continue; + } + for (const trun of trafChildren) { + if (trun.type !== "trun") continue; + const version = view.getUint8(trun.body); + const flags = view.getUint32(trun.body) & 0xffffff; + const count = view.getUint32(trun.body + 4); + let cursor = trun.body + 8; + if (flags & TRUN_DATA_OFFSET) { + dataAt = base + view.getInt32(cursor); + cursor += 4; + } + let firstFlags: number | null = null; + if (flags & TRUN_FIRST_FLAGS) { + firstFlags = view.getUint32(cursor); + cursor += 4; + } + const entrySize = + 4 * + [TRUN_DURATION, TRUN_SIZE, TRUN_FLAGS, TRUN_COMPOSITION].filter( + (bit) => flags & bit, + ).length; + if (cursor + count * entrySize > trun.end) { + throw new UnsupportedSource("trun lists more samples than it holds"); + } + if (video && samples.count + count > MAX_SAMPLES) { + throw new UnsupportedSource(`more than ${MAX_SAMPLES} video samples`); + } + for (let index = 0; index < count; index++) { + let duration = defaultDuration; + let size = defaultSize; + let sampleFlags = + index === 0 && firstFlags !== null ? firstFlags : defaultFlags; + let composition = 0; + if (flags & TRUN_DURATION) { + duration = view.getUint32(cursor); + cursor += 4; + } + if (flags & TRUN_SIZE) { + size = view.getUint32(cursor); + cursor += 4; + } + if (flags & TRUN_FLAGS) { + sampleFlags = view.getUint32(cursor); + cursor += 4; + } + if (flags & TRUN_COMPOSITION) { + composition = + version === 0 ? view.getUint32(cursor) : view.getInt32(cursor); + cursor += 4; + } + if (video) { + samples.firstDts ??= dts; + samples.dts.push(dts); + samples.offsets.push(composition); + samples.sizes.push(size); + samples.positions.push(dataAt); + samples.sync.push( + (sampleFlags & (SAMPLE_NON_SYNC | SAMPLE_DEPENDS_YES)) === 0, + ); + samples.lastDuration = duration; + dts += duration; + } + dataAt += size; + } + } + if (video) samples.nextDts = dts; + previousEnd = dataAt; + } +} + +export type RangeReader = ( + start: number, + endInclusive: number, +) => Promise; + +/** Moof offsets for `trackId` from an `mfra` in the file's last bytes. */ +export function mfraMoofs(tail: Uint8Array, trackId: number) { + const view = new DataView(tail.buffer, tail.byteOffset, tail.byteLength); + if (tail.byteLength < 16) return null; + const mfro = tail.byteLength - 16; + if ( + view.getUint32(mfro) !== 16 || + String.fromCharCode(...tail.subarray(mfro + 4, mfro + 8)) !== "mfro" + ) { + return null; + } + const size = view.getUint32(mfro + 12); + const start = tail.byteLength - size; + if (size < 24 || start < 0) return { size }; + const mfra = boxAt(view, start, tail.byteLength); + if (!mfra || mfra.type !== "mfra") return null; + const moofs = new Set(); + for (const tfra of childrenOf(view, mfra)) { + if (tfra.type !== "tfra" || view.getUint32(tfra.body + 4) !== trackId) { + continue; + } + const version = view.getUint8(tfra.body); + const lengths = view.getUint32(tfra.body + 8); + const skip = + ((lengths >> 4) & 3) + + 1 + + (((lengths >> 2) & 3) + 1) + + ((lengths & 3) + 1); + const entries = view.getUint32(tfra.body + 12); + const entrySize = (version === 1 ? 16 : 8) + skip; + let cursor = tfra.body + 16; + if (cursor + entries * entrySize > tfra.end) { + throw new UnsupportedSource("tfra lists more entries than it holds"); + } + for (let index = 0; index < entries; index++) { + moofs.add( + version === 1 + ? Number(view.getBigUint64(cursor + 8)) + : view.getUint32(cursor + 4), + ); + cursor += entrySize; + } + } + return { size, moofs: [...moofs].sort((a, b) => a - b) }; +} + +type Region = { start: number; end: number; bytes: Promise }; + +/** + * Reads served from ranges fetched ahead: either small reads at the moofs an + * mfra lists, or (with no mfra) the whole file in large pieces, a window at a + * time, dropping what the walk has passed. + */ +class Prefetch { + private regions: Region[] = []; + private window: Map> | null = null; + requests = 0; + bytes = 0; + + constructor( + private read: RangeReader, + private size: number, + private options: { + concurrency: number; + piece: number; + windowPieces: number; + }, + ) {} + + private active = 0; + private waiting: (() => void)[] = []; + + private async fetch(start: number, end: number) { + if (this.active >= this.options.concurrency) { + await new Promise((resolve) => this.waiting.push(resolve)); + } else { + this.active++; + } + try { + const bytes = await this.read(start, end - 1); + if (bytes.byteLength !== end - start) { + throw new Error(`short read ${start}-${end}: ${bytes.byteLength}`); + } + this.requests++; + this.bytes += bytes.byteLength; + return bytes; + } finally { + const next = this.waiting.shift(); + if (next) next(); + else this.active--; + } + } + + /** Fetch these [start, end) ranges now, for reads that fall inside them. */ + hint(ranges: [number, number][]) { + for (const [start, end] of ranges) { + const clamped = Math.min(end, this.size); + if (clamped <= start) continue; + const bytes = this.fetch(start, clamped); + bytes.catch(() => {}); + this.regions.push({ start, end: clamped, bytes }); + } + this.regions.sort((a, b) => a.start - b.start); + } + + stream() { + this.window = new Map(); + } + + private piece(index: number) { + const window = this.window as Map>; + let bytes = window.get(index); + if (!bytes) { + const start = index * this.options.piece; + bytes = this.fetch( + start, + Math.min(this.size, start + this.options.piece), + ); + bytes.catch(() => {}); + window.set(index, bytes); + } + return bytes; + } + + async get(start: number, end: number): Promise { + end = Math.min(end, this.size); + if (this.window) { + const piece = this.options.piece; + const first = Math.floor(start / piece); + const last = Math.floor((end - 1) / piece); + for (const index of this.window.keys()) { + if (index < first) this.window.delete(index); + } + const pieces = Math.ceil(this.size / piece); + for ( + let index = first; + index < Math.min(pieces, last + 1 + this.options.windowPieces); + index++ + ) { + this.piece(index); + } + const out = new Uint8Array(end - start); + for (let index = first; index <= last; index++) { + const bytes = await this.piece(index); + const pieceStart = index * piece; + const from = Math.max(start, pieceStart); + const to = Math.min(end, pieceStart + bytes.byteLength); + out.set( + bytes.subarray(from - pieceStart, to - pieceStart), + from - start, + ); + } + return out; + } + // Regions are sorted and don't overlap: the last one starting at or + // before `start` is the only one that can cover the read. + let low = 0; + let high = this.regions.length - 1; + while (low <= high) { + const middle = (low + high) >> 1; + if ((this.regions[middle] as Region).start <= start) low = middle + 1; + else high = middle - 1; + } + const region = this.regions[high]; + if (region && region.end >= end) { + const bytes = await region.bytes; + return bytes.subarray(start - region.start, end - region.start); + } + return this.fetch(start, end); + } +} + +/** Bytes read per moof listed in an mfra: the moof and the next box header. */ +const MOOF_READ = 4096; + +export type FragmentScan = { + samples: FragmentSamples; + fragments: number; + usedMfra: boolean; + requests: number; + bytes: number; +}; + +/** + * Walks every top-level box after the moov and reads each moof. An mfra, when + * the recorder wrote one, says where the moofs are, so they're fetched in + * parallel small reads; otherwise the file streams through in large pieces. + */ +export async function scanFragments( + read: RangeReader, + size: number, + init: FragmentedInit, + options: { + concurrency?: number; + piece?: number; + windowPieces?: number; + tailBytes?: number; + } = {}, +): Promise { + // 128 small reads in flight index a 2 h recording's 3,600 moofs in under + // a second from the coordinator; 32 took 9 s, 64 about 1.5 s. + const prefetch = new Prefetch(read, size, { + concurrency: options.concurrency ?? 128, + piece: options.piece ?? 8 << 20, + windowPieces: options.windowPieces ?? 12, + }); + const tailStart = Math.max( + init.moovEnd, + size - (options.tailBytes ?? 256 << 10), + ); + if (size - tailStart >= 16) prefetch.hint([[tailStart, size]]); + let found = + size - tailStart >= 16 + ? mfraMoofs(await prefetch.get(tailStart, size), init.trackId) + : null; + if (found && !found.moofs && size - found.size >= init.moovEnd) { + found = mfraMoofs( + await prefetch.get(size - found.size, size), + init.trackId, + ); + } + const moofs = (found?.moofs ?? []).filter( + (offset) => offset >= init.moovEnd && offset < size, + ); + const usedMfra = moofs.length > 0; + if (usedMfra) { + // From a little before each moof (a styp may lead it) to past the + // mdat header after it; regions stay clear of the tail's. + const ranges: [number, number][] = []; + for (const offset of moofs) { + const last = ranges[ranges.length - 1]; + const start = Math.max(init.moovEnd, offset - 256, last?.[1] ?? 0); + const end = Math.min(tailStart, offset + MOOF_READ); + if (end <= start) continue; + if (last && start <= last[1] + MOOF_READ) last[1] = end; + else ranges.push([start, end]); + } + prefetch.hint(ranges); + } else { + prefetch.stream(); + } + const samples = new FragmentSamples(); + let fragments = 0; + let offset = init.moovEnd; + while (offset + 8 <= size) { + const header = await prefetch.get(offset, Math.min(size, offset + 16)); + const found = boxAt( + new DataView(header.buffer, header.byteOffset, header.byteLength), + 0, + header.byteLength, + ); + if (!found) break; + let length = found.end; + if (view32(header, 0) === 0) length = size - offset; + if (found.type === "moof") { + if (offset + length > size) throw new UnsupportedSource("truncated moof"); + readMoof( + await prefetch.get(offset, offset + length), + offset, + init, + samples, + ); + fragments++; + } else if (found.type === "mfra") { + break; + } + if (length < 8) throw new UnsupportedSource("zero-length box"); + offset += length; + } + return { + samples, + fragments, + usedMfra, + requests: prefetch.requests, + bytes: prefetch.bytes, + }; +} + +function view32(bytes: Uint8Array, at: number) { + return new DataView( + bytes.buffer, + bytes.byteOffset, + bytes.byteLength, + ).getUint32(at); +} + +function patchDuration( + boxBytes: Uint8Array, + at: { v0: number; v1: number }, + value: number, +) { + const out = boxBytes.slice(); + const view = new DataView(out.buffer); + // Full boxes: version at byte 8, after the 8-byte header. + if (view.getUint8(8) === 1) { + view.setBigUint64(at.v1, BigInt(value)); + } else { + if (value > 0xffffffff) throw new UnsupportedSource("duration overflows"); + view.setUint32(at.v0, value); + } + return out; +} + +export type Provenance = { + version: number; + source: string; + size: number; +}; + +/** + * The prefix for a fragmented source (see the top of this file) and the index + * the planner uses, with sample offsets counted in the prefixed file. + */ +export function buildPrefix( + init: FragmentedInit, + samples: FragmentSamples, + provenance: Provenance, +) { + const count = samples.count; + if (count === 0) throw new UnsupportedSource("no video samples"); + if (samples.firstDts !== 0) { + // A remux keeps that start with an edit list; not worth matching. + throw new UnsupportedSource("video doesn't start at time 0"); + } + const stts: [number, number][] = []; + for (let index = 0; index < count; index++) { + const delta = + index + 1 < count + ? (samples.dts[index + 1] as number) - (samples.dts[index] as number) + : samples.lastDuration; + if (delta < 0) throw new UnsupportedSource("decode times go backwards"); + const last = stts[stts.length - 1]; + if (last && last[1] === delta) last[0]++; + else stts.push([1, delta]); + } + const mediaDuration = + (samples.dts[count - 1] as number) + samples.lastDuration; + let ctts: [number, number][] | null = null; + if (samples.offsets.some((offset) => offset !== 0)) { + ctts = []; + for (const offset of samples.offsets) { + const last = ctts[ctts.length - 1]; + if (last && last[1] === offset) last[0]++; + else ctts.push([1, offset]); + } + } + const keyframes: number[] = []; + const runs: Run[] = []; + for (let index = 0; index < count; index++) { + if (samples.sync[index]) keyframes.push(index); + const position = samples.positions[index] as number; + const last = runs[runs.length - 1]; + if ( + last && + (samples.positions[index - 1] as number) + + (samples.sizes[index - 1] as number) === + position + ) { + last.count++; + } else { + runs.push({ first: index, count: 1, offset: position }); + } + } + const sizes = Uint32Array.from(samples.sizes); + const sourceEnd = samples.positions.reduce( + (end, position, index) => + Math.max(end, position + (samples.sizes[index] as number)), + 0, + ); + if (sourceEnd > provenance.size) { + throw new UnsupportedSource("samples point past the end of the file"); + } + + const movieTimescale = new DataView(init.reuse.mvhd.buffer).getUint32( + init.reuse.mvhd[8] === 1 ? 28 : 20, + ); + const movieDuration = Math.round( + (mediaDuration * movieTimescale) / init.timescale, + ); + const ftyp = box( + "ftyp", + build((writer) => { + writer.ascii("isom"); + writer.u32(0x200); + writer.ascii("isomiso2avc1mp41"); + }), + ); + const free = box( + "free", + new TextEncoder().encode(JSON.stringify(provenance)), + ); + const make = (shift: number) => + box( + "moov", + patchDuration(init.reuse.mvhd, { v0: 24, v1: 32 }, movieDuration), + box( + "trak", + patchDuration(init.reuse.tkhd, { v0: 28, v1: 36 }, movieDuration), + box( + "mdia", + patchDuration(init.reuse.mdhd, { v0: 24, v1: 32 }, mediaDuration), + init.reuse.hdlr, + box( + "minf", + init.reuse.mediaHeader, + init.reuse.dinf, + sampleTable( + { + sizes, + runs: runs.map((run) => ({ + ...run, + offset: run.offset + shift, + })), + }, + init.reuse.stsd, + stts, + Uint32Array.from(keyframes), + ctts, + ), + ), + ), + ), + ); + // co64 is fixed width, so the moov's size doesn't depend on the shift. + const shift = ftyp.byteLength + free.byteLength + make(0).byteLength + 16; + const moov = make(shift); + const mdat = new Uint8Array(16); + const mdatView = new DataView(mdat.buffer); + mdatView.setUint32(0, 1); + mdat.set(new TextEncoder().encode("mdat"), 4); + mdatView.setBigUint64(8, BigInt(16 + provenance.size)); + const bytes = new Uint8Array(shift); + let at = 0; + for (const part of [ftyp, free, moov, mdat]) { + bytes.set(part, at); + at += part.byteLength; + } + if (at !== shift) throw new Error("prefix layout mismatch"); + + const index: TrackIndex = { + timescale: init.timescale, + times: Float64Array.from( + samples.dts, + (dts, sample) => + (dts + (samples.offsets[sample] as number)) / init.timescale, + ), + offsets: Float64Array.from( + samples.positions, + (position) => position + shift, + ), + sizes, + keyframes: Uint32Array.from(keyframes), + }; + return { bytes, index }; +} + +/** Reads the provenance a prefix was built with, or null if it isn't one. */ +export function prefixProvenance(prefix: Uint8Array): Provenance | null { + const view = new DataView( + prefix.buffer, + prefix.byteOffset, + prefix.byteLength, + ); + const ftyp = boxAt(view, 0, prefix.byteLength); + if (ftyp?.type !== "ftyp") return null; + const free = boxAt(view, ftyp.end, prefix.byteLength); + if (free?.type !== "free" || free.end > prefix.byteLength) return null; + try { + const parsed = JSON.parse( + new TextDecoder().decode(prefix.subarray(free.body, free.end)), + ) as Provenance; + return typeof parsed.version === "number" ? parsed : null; + } catch { + return null; + } +} + +class Bits { + private bit = 0; + constructor(private bytes: Uint8Array) {} + + u(count: number) { + let value = 0; + for (let index = 0; index < count; index++) { + const byte = this.bytes[this.bit >> 3]; + if (byte === undefined) throw new UnsupportedSource("SPS ends early"); + value = value * 2 + ((byte >> (7 - (this.bit & 7))) & 1); + this.bit++; + } + return value; + } + + ue() { + let zeros = 0; + while (this.u(1) === 0) { + if (++zeros > 31) throw new UnsupportedSource("bad exp-Golomb code"); + } + return 2 ** zeros - 1 + this.u(zeros); + } + + se() { + const value = this.ue(); + return value % 2 === 1 ? (value + 1) / 2 : -value / 2; + } +} + +/** The fields of an H.264 SPS that decide how ffmpeg reports the stream. */ +export function readSps(nal: Uint8Array) { + // Drop emulation prevention bytes (00 00 03 -> 00 00) and the NAL header. + const raw: number[] = []; + for (let index = 1; index < nal.length; index++) { + if ( + nal[index] === 3 && + nal[index - 1] === 0 && + nal[index - 2] === 0 && + index >= 3 + ) { + continue; + } + raw.push(nal[index] as number); + } + const bits = new Bits(Uint8Array.from(raw)); + const profile = bits.u(8); + bits.u(16); + bits.ue(); + let chromaFormat = 1; + let lumaDepth = 8; + let chromaDepth = 8; + if ( + [100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134, 135].includes( + profile, + ) + ) { + chromaFormat = bits.ue(); + if (chromaFormat === 3) bits.u(1); + lumaDepth = bits.ue() + 8; + chromaDepth = bits.ue() + 8; + bits.u(1); + if (bits.u(1)) { + for (let list = 0; list < (chromaFormat === 3 ? 12 : 8); list++) { + if (!bits.u(1)) continue; + let last = 8; + let next = 8; + for (let index = 0; index < (list < 6 ? 16 : 64); index++) { + if (next !== 0) next = (last + bits.se() + 256) % 256; + last = next === 0 ? last : next; + } + } + } + } + bits.ue(); + const pocType = bits.ue(); + if (pocType === 0) bits.ue(); + else if (pocType === 1) { + bits.u(1); + bits.se(); + bits.se(); + const cycle = bits.ue(); + for (let index = 0; index < cycle; index++) bits.se(); + } + bits.ue(); + bits.u(1); + bits.ue(); + bits.ue(); + if (!bits.u(1)) bits.u(1); + bits.u(1); + if (bits.u(1)) { + bits.ue(); + bits.ue(); + bits.ue(); + bits.ue(); + } + let fullRange = false; + let reorderFrames: number | null = null; + if (bits.u(1)) { + if (bits.u(1) && bits.u(8) === 255) bits.u(32); + if (bits.u(1)) bits.u(1); + if (bits.u(1)) { + bits.u(3); + fullRange = bits.u(1) === 1; + if (bits.u(1)) bits.u(24); + } + if (bits.u(1)) { + bits.ue(); + bits.ue(); + } + if (bits.u(1)) { + bits.u(32); + bits.u(32); + bits.u(1); + } + const hrd = () => { + const count = bits.ue() + 1; + bits.u(8); + for (let index = 0; index < count; index++) { + bits.ue(); + bits.ue(); + bits.u(1); + } + bits.u(20); + }; + const nalHrd = bits.u(1); + if (nalHrd) hrd(); + const vclHrd = bits.u(1); + if (vclHrd) hrd(); + if (nalHrd || vclHrd) bits.u(1); + bits.u(1); + if (bits.u(1)) { + bits.u(1); + bits.ue(); + bits.ue(); + bits.ue(); + bits.ue(); + reorderFrames = bits.ue(); + } + } + return { + profile, + chromaFormat, + lumaDepth, + chromaDepth, + fullRange, + reorderFrames, + }; +} + +/** + * Why the transcode task would re-encode this source rather than copy it + * (see canRemux), or null when it would copy it: then the prefixed source + * decodes to the same frames as the remux, and it's used as is. + */ +export function remuxBlocker( + init: FragmentedInit, + samples: FragmentSamples, +): string | null { + if (init.codec !== "avc1" && init.codec !== "avc3") { + return `codec ${init.codec}`; + } + const avcC = init.avcC; + if (!avcC || avcC.byteLength < 8 || ((avcC[5] as number) & 0x1f) < 1) { + return "no SPS"; + } + const length = ((avcC[6] as number) << 8) | (avcC[7] as number); + const sps = readSps(avcC.subarray(8, 8 + length)); + if (sps.chromaFormat !== 1 || sps.lumaDepth !== 8 || sps.chromaDepth !== 8) { + return "not 8-bit 4:2:0"; + } + if (sps.fullRange) return "full-range video"; + if (sps.reorderFrames) return "B-frames"; + if (samples.offsets.some((offset) => offset !== 0)) { + return "presentation and decode times differ"; + } + let previous = 0; + let keyframes = 0; + for (let index = 0; index < samples.count; index++) { + if (!samples.sync[index]) continue; + const time = (samples.dts[index] as number) / init.timescale; + if (time - previous > MAX_REMUX_KEYFRAME_GAP_SECONDS) { + return "keyframes too far apart"; + } + previous = time; + keyframes++; + } + if (keyframes === 0) return "no keyframes"; + const last = (samples.dts[samples.count - 1] as number) / init.timescale; + if (last - previous > MAX_REMUX_KEYFRAME_GAP_SECONDS) { + return "keyframes too far apart"; + } + return null; +} diff --git a/apps/render-farm/src/materialize.test.ts b/apps/render-farm/src/materialize.test.ts new file mode 100644 index 00000000000..7e3aaf1f97d --- /dev/null +++ b/apps/render-farm/src/materialize.test.ts @@ -0,0 +1,56 @@ +import { expect, test } from "bun:test"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { ProjectCache } from "./materialize"; +import type { S3 } from "./s3"; + +test("a prefixed file is its prefix, then the source shifted by it", async () => { + const objects = new Map([ + ["prefix", Uint8Array.from({ length: 100 }, (_, index) => index + 1)], + [ + "source", + Uint8Array.from({ length: 10 << 20 }, (_, index) => (index % 250) + 1), + ], + ]); + const reads: string[] = []; + const s3 = { + async getRange(key: string, start: number, endInclusive: number) { + reads.push(`${key}:${start}-${endInclusive}`); + return (objects.get(key) as Uint8Array).slice(start, endInclusive + 1); + }, + } as unknown as S3; + const root = mkdtempSync(join(tmpdir(), "rf-materialize-")); + const cache = new ProjectCache(s3, root); + const size = 100 + (10 << 20); + await cache.materialize([ + { + path: "display.mp4", + key: "source", + size, + prefix: { key: "prefix", size: 100 }, + // The header, then a range crossing the source's first 4 MiB piece. + ranges: [ + [0, 100], + [100 + (4 << 20) - 10, 100 + (4 << 20) + 10], + ], + }, + ]); + cache.close(); + expect(reads.sort()).toEqual([ + "prefix:0-99", + `source:0-${(4 << 20) - 1}`, + `source:${4 << 20}-${(8 << 20) - 1}`, + ]); + const file = readFileSync(join(root, "display.mp4")); + expect(file.byteLength).toBe(size); + expect([...file.subarray(0, 100)]).toEqual([ + ...(objects.get("prefix") as Uint8Array), + ]); + const source = objects.get("source") as Uint8Array; + for (const at of [0, 1, (4 << 20) - 1, 4 << 20, (8 << 20) - 1]) { + expect(file[100 + at]).toBe(source[at] as number); + } + // Past what was asked for, the file is still a hole. + expect(file[100 + (9 << 20)]).toBe(0); +}); diff --git a/apps/render-farm/src/materialize.ts b/apps/render-farm/src/materialize.ts index 8df1d52144e..ba55050e0d2 100644 --- a/apps/render-farm/src/materialize.ts +++ b/apps/render-farm/src/materialize.ts @@ -23,6 +23,14 @@ export type FileSpec = { size: number; /** Byte ranges [start, end) to fetch, or "all". */ ranges: [number, number][] | "all"; + /** Copy this file from the machine (a built-in asset) instead of S3. */ + local?: string; + /** + * The file starts with this whole object (a built index, see + * fragment-index.ts) and `key` follows it: bytes at or past `size` come + * from `key`, `size` bytes earlier. Ranges count in the joined file. + */ + prefix?: { key: string; size: number }; }; const PIECE = 4 << 20; @@ -82,21 +90,26 @@ export class ProjectCache { return file; } + /** + * Piece `piece` of `key` (which is `size` bytes), written `at` bytes into + * the file. A prefixed file keeps its prefix as piece -1. + */ private fetchPiece( - spec: FileSpec, + source: { key: string; size: number; at: number }, file: LocalFile, piece: number, stats: FetchStats, ) { let pending = file.pieces.get(piece); if (!pending) { - const start = piece * PIECE; - const end = Math.min(spec.size, start + PIECE); + const start = Math.max(0, piece) * PIECE; + const end = + piece < 0 ? source.size : Math.min(source.size, start + PIECE); pending = this.limiter.run(async () => { - const bytes = await this.s3.getRange(spec.key, start, end - 1); + const bytes = await this.s3.getRange(source.key, start, end - 1); if (bytes.byteLength !== end - start) { throw new Error( - `short read ${spec.key} ${start}-${end}: ${bytes.byteLength}`, + `short read ${source.key} ${start}-${end}: ${bytes.byteLength}`, ); } let written = 0; @@ -106,7 +119,7 @@ export class ProjectCache { bytes, written, bytes.byteLength - written, - start + written, + source.at + start + written, ); } this.bytesFetched += bytes.byteLength; @@ -124,6 +137,15 @@ export class ProjectCache { const stats: FetchStats = { bytes: 0, requests: 0, ms: 0 }; const work: Promise[] = []; for (const spec of specs) { + if (spec.local) { + const file = this.open(spec); + const bytes = readFileSync(spec.local); + if (bytes.byteLength !== spec.size) { + throw new Error(`${spec.local} changed size`); + } + writeSync(file.fd, bytes, 0, bytes.byteLength, 0); + continue; + } if (spec.size === 0) { this.open(spec); continue; @@ -131,20 +153,34 @@ export class ProjectCache { const file = this.open(spec); const ranges: [number, number][] = spec.ranges === "all" ? [[0, spec.size]] : spec.ranges; + const at = spec.prefix?.size ?? 0; + const source = { key: spec.key, size: spec.size - at, at }; const pieces = new Set(); for (const [start, end] of ranges) { - const clampedEnd = Math.min(end, spec.size); - if (clampedEnd <= start) continue; + if (spec.prefix && start < at && end > 0) pieces.add(-1); + const from = Math.max(start, at) - at; + const to = Math.min(end, spec.size) - at; + if (to <= from) continue; for ( - let piece = Math.floor(start / PIECE); - piece <= Math.floor((clampedEnd - 1) / PIECE); + let piece = Math.floor(from / PIECE); + piece <= Math.floor((to - 1) / PIECE); piece++ ) { pieces.add(piece); } } - for (const piece of pieces) - work.push(this.fetchPiece(spec, file, piece, stats)); + for (const piece of pieces) { + work.push( + this.fetchPiece( + piece < 0 && spec.prefix + ? { key: spec.prefix.key, size: spec.prefix.size, at: 0 } + : source, + file, + piece, + stats, + ), + ); + } } await Promise.all(work); if ( diff --git a/apps/render-farm/src/mp4.ts b/apps/render-farm/src/mp4.ts index 8c4ba67d8ae..a8425d9ae61 100644 --- a/apps/render-farm/src/mp4.ts +++ b/apps/render-farm/src/mp4.ts @@ -75,7 +75,7 @@ export type TrackIndex = { // Sources are user uploads: a sample count drives several allocations, so it // is capped (about 23 h at 60 fps) and every table must fit inside its box. -const MAX_SAMPLES = 5_000_000; +export const MAX_SAMPLES = 5_000_000; /** Index the first video track of a moov box (bytes = the whole moov). */ export function indexVideoTrack(moov: Uint8Array): TrackIndex { @@ -431,11 +431,13 @@ export type TrackTable = { runs: Run[]; }; -function sampleTable( +export function sampleTable( table: TrackTable, stsd: Uint8Array, stts: [number, number][], sync: Uint32Array | null, + /** Composition offsets as [count, offset] runs; negative ones need v1. */ + ctts: [number, number][] | null = null, ) { const parts: Uint8Array[] = [stsd]; parts.push( @@ -452,6 +454,22 @@ function sampleTable( }), ), ); + if (ctts) { + parts.push( + fullBox( + "ctts", + ctts.some(([, offset]) => offset < 0) ? 1 : 0, + 0, + build((writer) => { + writer.u32(ctts.length); + for (const [count, offset] of ctts) { + writer.u32(count); + writer.u32(offset); + } + }), + ), + ); + } if (sync) { parts.push( fullBox( diff --git a/apps/render-farm/src/prepare.test.ts b/apps/render-farm/src/prepare.test.ts new file mode 100644 index 00000000000..c9f1e78e693 --- /dev/null +++ b/apps/render-farm/src/prepare.test.ts @@ -0,0 +1,234 @@ +import { describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { ProjectCache } from "./materialize"; +import { + builtinPath, + type PrepareDeps, + parsePrepareRequest, + prepareRecording, +} from "./prepare"; +import type { S3 } from "./s3"; +import { validateJobRequest } from "./validate"; + +const prefix = "owner/video/.recording/render/x/project"; +const encode = (value: unknown) => + new TextEncoder().encode(JSON.stringify(value)); +const request = { + version: 1, + sources: { title: "Take" }, + display: { key: "owner/video/raw-upload.mp4", contentType: "video/mp4" }, + inputEvents: { key: "owner/video/input-events-upload.ndjson", size: 6 }, + sessionDefaults: true, +}; + +function harness(manifest: unknown, prepare: unknown = request) { + const objects = new Map([ + [`${prefix}/manifest.json`, encode(manifest)], + [`${prefix}/prepare.json`, encode(prepare)], + [ + "owner/video/input-events-upload.ndjson", + new TextEncoder().encode("{}\n{}\n"), + ], + ]); + const engineCalls: { op: string; body: Record }[] = []; + const deps: PrepareDeps = { + async getBounded(key, limit) { + const value = objects.get(key); + if (value === undefined) throw new Error(`missing ${key}`); + const bytes = + typeof value === "string" ? new TextEncoder().encode(value) : value; + if (bytes.byteLength > limit) throw new Error("too large"); + return bytes; + }, + async put(key, body) { + objects.set(key, typeof body === "string" ? body : new Uint8Array(body)); + }, + presignGet: async (key) => `https://s3.test/${key}?signed`, + inScope: (key) => key.startsWith("owner/video/"), + async engine(op: string, body: Record) { + engineCalls.push({ op, body }); + const project = body.project as string; + mkdirSync(join(project, "content/segments/segment-0"), { + recursive: true, + }); + writeFileSync(join(project, "recording-meta.json"), '{"meta":1}'); + writeFileSync(join(project, "project-config.json"), '{"config":1}'); + writeFileSync( + join(project, "content/segments/segment-0/cursor.json"), + "[]", + ); + return { + files: [ + { path: "content/segments/segment-0/cursor.json", size: 2 }, + { path: "project-config.json", size: 12 }, + { path: "recording-meta.json", size: 10 }, + ], + } as T; + }, + }; + return { objects, engineCalls, deps }; +} + +describe("prepareRecording", () => { + test("writes the project files next to the media and lists them in the manifest", async () => { + const media = { + path: "content/segments/segment-0/display.h264.mp4", + key: "k", + transcodeFrom: "owner/video/raw-upload.mp4", + }; + const { objects, engineCalls, deps } = harness({ files: [media] }); + const work = mkdtempSync(join(tmpdir(), "rf-prepare-")); + expect(await prepareRecording(prefix, "prepare.json", work, deps)).toBe( + true, + ); + + expect(engineCalls).toHaveLength(1); + expect(engineCalls[0]?.op).toBe("prepare"); + expect(engineCalls[0]?.body).toMatchObject({ + sources: { title: "Take" }, + display_source: "https://s3.test/owner/video/raw-upload.mp4?signed", + display_content_type: "video/mp4", + session_defaults: true, + }); + expect( + readFileSync(engineCalls[0]?.body.input_events as string, "utf8"), + ).toBe("{}\n{}\n"); + expect( + new TextDecoder().decode( + objects.get(`${prefix}/recording-meta.json`) as Uint8Array, + ), + ).toBe('{"meta":1}'); + expect( + JSON.parse(objects.get(`${prefix}/manifest.json`) as string), + ).toEqual({ + files: [ + media, + { path: "content/segments/segment-0/cursor.json", size: 2 }, + { path: "project-config.json", size: 12 }, + { path: "recording-meta.json", size: 10 }, + ], + }); + }); + + test("leaves a manifest that already lists the project files alone", async () => { + const { engineCalls, deps } = harness({ + files: [{ path: "recording-meta.json", size: 1 }], + }); + expect( + await prepareRecording( + prefix, + "prepare.json", + mkdtempSync(join(tmpdir(), "rf-")), + deps, + ), + ).toBe(false); + expect(engineCalls).toHaveLength(0); + }); + + test("refuses sources outside the recording and input events that changed", async () => { + const outside = harness( + { files: [] }, + { + ...request, + display: { key: "other/video/raw.mp4", contentType: "video/mp4" }, + }, + ); + await expect( + prepareRecording( + prefix, + "prepare.json", + mkdtempSync(join(tmpdir(), "rf-")), + outside.deps, + ), + ).rejects.toThrow("outside the recording"); + const resized = harness( + { files: [] }, + { ...request, inputEvents: { ...request.inputEvents, size: 99 } }, + ); + await expect( + prepareRecording( + prefix, + "prepare.json", + mkdtempSync(join(tmpdir(), "rf-")), + resized.deps, + ), + ).rejects.toThrow("changed size"); + }); +}); + +describe("parsePrepareRequest", () => { + test("accepts only a version 1 request with a video display", () => { + expect(parsePrepareRequest(request).display.contentType).toBe("video/mp4"); + expect(() => parsePrepareRequest({ ...request, version: 2 })).toThrow(); + expect(() => + parsePrepareRequest({ + ...request, + display: { key: "k", contentType: "audio/webm" }, + }), + ).toThrow(); + expect(() => + parsePrepareRequest({ ...request, inputEvents: { key: "k", size: 0 } }), + ).toThrow(); + expect( + parsePrepareRequest({ ...request, inputEvents: null }).inputEvents, + ).toBeNull(); + expect( + parsePrepareRequest({ ...request, sessionDefaults: undefined }) + .sessionDefaults, + ).toBe(false); + }); +}); + +describe("built-in assets", () => { + test("only wallpapers and library tracks are built in", () => { + expect(builtinPath("backgrounds/macOS/sequoia-dark.jpg")).toEndWith( + "/backgrounds/macOS/sequoia-dark.jpg", + ); + expect(builtinPath("music/calm-1.mp3")).toEndWith("/music/calm-1.mp3"); + expect(builtinPath("backgrounds/../../etc/passwd")).toBeNull(); + expect(builtinPath("music/Calm.mp3")).toBeNull(); + expect(builtinPath("backgrounds/other/x.jpg")).toBeNull(); + }); + + test("a built-in file is copied from the machine, not fetched", async () => { + const dir = mkdtempSync(join(tmpdir(), "rf-builtin-")); + const local = join(dir, "wallpaper.jpg"); + writeFileSync(local, "jpeg-bytes"); + const s3 = { + getRange: () => { + throw new Error("fetched"); + }, + } as unknown as S3; + const cache = new ProjectCache(s3, join(dir, "project")); + await cache.materialize([ + { + path: "assets/backgrounds/macOS/x.jpg", + key: "", + size: 10, + ranges: "all", + local, + }, + ]); + cache.close(); + expect( + readFileSync(join(dir, "project/assets/backgrounds/macOS/x.jpg"), "utf8"), + ).toBe("jpeg-bytes"); + }); +}); + +describe("validateJobRequest", () => { + test("prepare names a JSON file in the recording", () => { + const base = { recording: prefix, sourceRoot: "owner/video/" }; + expect( + typeof validateJobRequest({ ...base, prepare: "prepare.json" }), + ).toBe("object"); + expect(validateJobRequest({ ...base, prepare: "../prepare.json" })).toBe( + "prepare must name a JSON file in the recording", + ); + expect(validateJobRequest({ ...base, prepare: 1 })).toBe( + "prepare must name a JSON file in the recording", + ); + }); +}); diff --git a/apps/render-farm/src/prepare.ts b/apps/render-farm/src/prepare.ts new file mode 100644 index 00000000000..42e8c41a241 --- /dev/null +++ b/apps/render-farm/src/prepare.ts @@ -0,0 +1,199 @@ +import { + existsSync, + mkdirSync, + readFileSync, + statSync, + writeFileSync, +} from "node:fs"; +import { join } from "node:path"; + +// A recording the web app didn't prepare on an editor worker arrives with +// `prepare.json` (what the worker would have been given) and a manifest of +// the recording's own files. The engine writes the project files the worker +// would have staged (metadata, config, pointer input) and the coordinator +// publishes them next to the media before planning. + +const BUILTIN_FILE = + /^(?:backgrounds\/(?:macOS|blue|purple|cities|dark|orange)\/[a-z0-9-]+\.jpg|music\/[a-z0-9-]+\.mp3)$/; +const MAX_PREPARE_BYTES = 8 << 20; +const MAX_INPUT_EVENTS_BYTES = 64 << 20; + +export const BUILTIN_DIR = process.env.RF_BUILTIN_DIR ?? "/opt/cap"; + +/** Where a built-in asset lives on this machine, or null when it isn't one. */ +export function builtinPath(name: string) { + return BUILTIN_FILE.test(name) ? join(BUILTIN_DIR, name) : null; +} + +export type PrepareRequest = { + version: 1; + sources: Record; + display: { key: string; contentType: string }; + inputEvents: { key: string; size: number } | null; + /** Never edited: open with the timeline and offsets a session fills in. */ + sessionDefaults: boolean; +}; + +export function parsePrepareRequest(value: unknown): PrepareRequest { + const record = (item: unknown): item is Record => + typeof item === "object" && item !== null && !Array.isArray(item); + if (!record(value) || value.version !== 1 || !record(value.sources)) { + throw new Error("prepare.json is invalid"); + } + const display = value.display; + if ( + !record(display) || + typeof display.key !== "string" || + (display.contentType !== "video/mp4" && + display.contentType !== "video/webm") + ) { + throw new Error("prepare.json names no display"); + } + const inputEvents = value.inputEvents; + if ( + inputEvents !== null && + (!record(inputEvents) || + typeof inputEvents.key !== "string" || + typeof inputEvents.size !== "number" || + !Number.isSafeInteger(inputEvents.size) || + inputEvents.size < 1 || + inputEvents.size > MAX_INPUT_EVENTS_BYTES) + ) { + throw new Error("prepare.json input events are invalid"); + } + return { + version: 1, + sources: value.sources, + display: { key: display.key, contentType: display.contentType }, + inputEvents: inputEvents as PrepareRequest["inputEvents"], + sessionDefaults: value.sessionDefaults === true, + }; +} + +type ManifestFile = { path: string; size?: number; [key: string]: unknown }; + +export type PrepareDeps = { + /** Reads a whole object, refusing ones larger than the limit. */ + getBounded(key: string, limit: number): Promise; + put( + key: string, + body: Uint8Array | string, + contentType?: string, + ): Promise; + presignGet(key: string): Promise; + /** Checks a key lies inside the recording's folder. */ + inScope(key: string): boolean; + engine(op: string, body: Record): Promise; + /** + * Sees the recording's own files before the engine runs, so work that + * needs only them (video transcodes) starts alongside. + */ + onManifest?(manifest: { files: ManifestFile[] }): void; +}; + +/** + * Writes and publishes the project files for a recording that ships + * `prepare.json`, and adds them to its manifest. Returns false when the + * manifest already lists them (a job resumed after a restart). + */ +export async function prepareRecording( + prefix: string, + name: string, + workDir: string, + deps: PrepareDeps, +) { + const manifest = JSON.parse( + new TextDecoder().decode( + await deps.getBounded(`${prefix}/manifest.json`, MAX_PREPARE_BYTES), + ), + ) as { files: ManifestFile[] }; + if (manifest.files.some((file) => file.path === "recording-meta.json")) { + return false; + } + const request = parsePrepareRequest( + JSON.parse( + new TextDecoder().decode( + await deps.getBounded(`${prefix}/${name}`, MAX_PREPARE_BYTES), + ), + ), + ); + for (const key of [request.display.key, request.inputEvents?.key]) { + if (key !== undefined && !deps.inScope(key)) { + throw new Error("prepare.json names a source outside the recording"); + } + } + deps.onManifest?.(manifest); + mkdirSync(workDir, { recursive: true }); + const project = join(workDir, "prepared"); + let inputEvents: string | undefined; + if (request.inputEvents) { + const bytes = await deps.getBounded( + request.inputEvents.key, + MAX_INPUT_EVENTS_BYTES, + ); + if (bytes.byteLength !== request.inputEvents.size) { + throw new Error("input events changed size"); + } + inputEvents = join(workDir, "input-events.ndjson"); + writeFileSync(inputEvents, bytes); + } + const written = await deps.engine<{ + files: { path: string; size: number }[]; + }>("prepare", { + project, + sources: request.sources, + input_events: inputEvents, + display_source: await deps.presignGet(request.display.key), + display_content_type: request.display.contentType, + session_defaults: request.sessionDefaults, + }); + const listed = new Set(manifest.files.map((file) => file.path)); + await Promise.all( + written.files.map((file) => + deps.put( + `${prefix}/${file.path}`, + readFileSync(join(project, file.path)), + file.path.endsWith(".json") + ? "application/json" + : "application/octet-stream", + ), + ), + ); + await deps.put( + `${prefix}/manifest.json`, + JSON.stringify({ + files: [ + ...manifest.files, + ...written.files.filter((file) => !listed.has(file.path)), + ], + }), + "application/json", + ); + return true; +} + +/** The capability /health advertises, once the engine can prepare projects. */ +export async function prepareSupport( + engine: (op: string, body: Record) => Promise, +) { + const defaultConfig = await engine>( + "default_config", + {}, + ); + const musicDir = join(BUILTIN_DIR, "music"); + const music = existsSync(musicDir) + ? (await Array.fromAsync(new Bun.Glob("*.mp3").scan(musicDir))) + .map((file) => file.slice(0, -".mp3".length)) + .filter((id) => builtinPath(`music/${id}.mp3`) !== null) + .sort() + : []; + return { version: 1 as const, defaultConfig, music }; +} + +export function builtinSize(name: string) { + const path = builtinPath(name); + if (!path || !existsSync(path)) { + throw new Error(`built-in asset ${name} is not on this machine`); + } + return { path, size: statSync(path).size }; +} diff --git a/apps/render-farm/src/probe-engine.ts b/apps/render-farm/src/probe-engine.ts index 63277149948..e06324ea9b5 100644 --- a/apps/render-farm/src/probe-engine.ts +++ b/apps/render-farm/src/probe-engine.ts @@ -11,7 +11,7 @@ export class ProbeEngine { private readonly timeoutMs = 120_000, ) {} - request(body: Record): Promise { + request(body: Record, op = "probe"): Promise { const result = this.tail.then(async () => { for (let attempt = 0; ; attempt++) { if (!this.engine?.alive) this.engine = this.create(); @@ -20,13 +20,13 @@ export class ProbeEngine { let timer: ReturnType | undefined; try { return await Promise.race([ - engine.request("probe", body), + engine.request(op, body), new Promise((_, reject) => { timer = setTimeout(() => { timedOut = true; engine.kill("SIGKILL"); this.engine = undefined; - reject(new Error("probe engine timed out")); + reject(new Error(`${op} engine timed out`)); }, this.timeoutMs); }), ]); diff --git a/apps/render-farm/src/protocol.ts b/apps/render-farm/src/protocol.ts index 9d345644f48..ed929c3fef7 100644 --- a/apps/render-farm/src/protocol.ts +++ b/apps/render-farm/src/protocol.ts @@ -80,7 +80,25 @@ export type TranscodeTask = { keyframeSeconds: number; }; -export type WorkItem = Task | TranscodeTask; +/** + * Removes the cursor burned into a browser recording: the worker writes the + * cleaned display video and the reconstructed pointer input (NDJSON, the + * format browser recordings upload) under `outputPrefix`. + */ +export type CursorTask = { + kind: "cursor"; + taskId: string; + attempt?: number; + source: string; + outputPrefix: string; +}; + +export const CURSOR_OUTPUTS = { + display: "display.mp4", + inputEvents: "input-events.ndjson", +} as const; + +export type WorkItem = Task | TranscodeTask | CursorTask; export type TaskTimings = { queuedMs: number; @@ -149,4 +167,9 @@ export type JobRequest = { callbackUrl?: string; /** Opaque, echoed in callbacks. */ reference?: string; + /** + * A file in `recording` (e.g. `prepare.json`) from which the engine writes + * the project files before planning; the manifest lists only the media. + */ + prepare?: string; }; diff --git a/apps/render-farm/src/s3.test.ts b/apps/render-farm/src/s3.test.ts index 23bb2186561..2ac674b1b4c 100644 --- a/apps/render-farm/src/s3.test.ts +++ b/apps/render-farm/src/s3.test.ts @@ -1,5 +1,7 @@ -import { describe, expect, test } from "bun:test"; -import { decodeXml } from "./s3"; +import { afterEach, describe, expect, test } from "bun:test"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { decodeXml, S3 } from "./s3"; describe("decodeXml", () => { test("decodes the predefined entities", () => { @@ -16,3 +18,90 @@ describe("decodeXml", () => { expect(decodeXml(" &")).toBe(" &"); }); }); + +describe("uploadFile", () => { + const realFetch = globalThis.fetch; + afterEach(() => { + globalThis.fetch = realFetch; + }); + + test("a cancelled upload stops sending and aborts its multipart upload", async () => { + const requests: string[] = []; + const controller = new AbortController(); + globalThis.fetch = (async (input: string | URL, init?: RequestInit) => { + const url = new URL(String(input)); + requests.push(`${init?.method} ${url.search}`); + if (url.searchParams.has("uploads")) { + return new Response( + "u1", + ); + } + if (url.searchParams.has("partNumber")) { + controller.abort(new Error("cancelled")); + init?.signal?.throwIfAborted(); + } + return new Response("", { status: 204 }); + }) as typeof fetch; + const path = join(tmpdir(), `s3-upload-${Date.now()}.bin`); + await Bun.write(path, new Uint8Array(1024)); + const s3 = new S3({ + endpoint: "https://s3.test", + region: "us-east-1", + bucket: "bucket", + accessKeyId: "a", + secretAccessKey: "b", + virtualHost: false, + }); + await expect( + s3.uploadFile("out/file.mp4", path, "video/mp4", { + signal: controller.signal, + }), + ).rejects.toThrow("cancelled"); + expect(requests).toEqual([ + "POST ?uploads=", + "PUT ?partNumber=1&uploadId=u1", + "DELETE ?uploadId=u1", + ]); + }); + + test("cleanup after a cancelled upload gives up on an unresponsive store", async () => { + const controller = new AbortController(); + let cleanupAborted = false; + globalThis.fetch = (async (input: string | URL, init?: RequestInit) => { + const url = new URL(String(input)); + if (url.searchParams.has("uploads")) { + return new Response("u1"); + } + if (init?.method === "DELETE") { + return new Promise((_, reject) => { + init.signal?.addEventListener("abort", () => { + cleanupAborted = true; + reject(init.signal?.reason); + }); + }); + } + controller.abort(new Error("cancelled")); + init?.signal?.throwIfAborted(); + return new Response("", { status: 204 }); + }) as typeof fetch; + const path = join(tmpdir(), `s3-cleanup-${Date.now()}.bin`); + await Bun.write(path, new Uint8Array(16)); + const s3 = new S3({ + endpoint: "https://s3.test", + region: "us-east-1", + bucket: "bucket", + accessKeyId: "a", + secretAccessKey: "b", + virtualHost: false, + cleanupTimeoutMs: 50, + }); + const started = performance.now(); + await expect( + s3.uploadFile("out/file.mp4", path, "video/mp4", { + signal: controller.signal, + }), + ).rejects.toThrow("cancelled"); + expect(cleanupAborted).toBe(true); + expect(performance.now() - started).toBeLessThan(5_000); + }); +}); diff --git a/apps/render-farm/src/s3.ts b/apps/render-farm/src/s3.ts index 826f3ebc1aa..4e337684380 100644 --- a/apps/render-farm/src/s3.ts +++ b/apps/render-farm/src/s3.ts @@ -15,6 +15,8 @@ export type S3Config = { virtualHost: boolean; /** Use the EC2 instance role (IMDSv2) instead of static keys. */ imds?: boolean; + /** Deadline for aborting a multipart upload after a failed one. */ + cleanupTimeoutMs?: number; /** * Canned ACL for new objects. Writing into another account's bucket needs * `bucket-owner-full-control`, or its owner cannot read what we wrote @@ -78,6 +80,14 @@ export class S3 { constructor(readonly config: S3Config) {} + /** + * Aborting a multipart upload is cleanup; an unresponsive store must not + * hold the caller, and the bucket's lifecycle rule removes what is left. + */ + private cleanupSignal() { + return AbortSignal.timeout(this.config.cleanupTimeoutMs ?? 30_000); + } + /** Refresh instance-role credentials from IMDSv2 when close to expiry. */ /** * Presigns with credentials refreshed first. A URL signed with instance-role @@ -281,6 +291,7 @@ export class S3 { headers?: Record; body?: Uint8Array | string; expect?: number[]; + signal?: AbortSignal; } = {}, ) { const payloadHash = @@ -291,6 +302,7 @@ export class S3 { : "UNSIGNED-PAYLOAD"; await this.ready(); for (let attempt = 0; ; attempt++) { + options.signal?.throwIfAborted(); const signed = this.sign( method, key, @@ -304,8 +316,10 @@ export class S3 { method, headers: signed.headers, body: options.body as BodyInit | undefined, + signal: options.signal, }); } catch (error) { + if (options.signal?.aborted) throw error; // Object stores drop connections under bursty fan-out; back off // with jitter rather than failing the chunk. if (attempt < 8) { @@ -390,10 +404,15 @@ export class S3 { }); } - async createMultipart(key: string, contentType: string) { + async createMultipart( + key: string, + contentType: string, + signal?: AbortSignal, + ) { const response = await this.send("POST", key, { query: { uploads: "" }, headers: this.writeHeaders(contentType), + signal, }); const text = await response.text(); const match = text.match(/([^<]+)<\/UploadId>/); @@ -406,10 +425,12 @@ export class S3 { uploadId: string, partNumber: number, body: Uint8Array, + signal?: AbortSignal, ) { const response = await this.send("PUT", key, { query: { partNumber: String(partNumber), uploadId }, body, + signal, }); const etag = response.headers.get("etag"); if (!etag) throw new Error(`no ETag for part ${partNumber}`); @@ -458,7 +479,7 @@ export class S3 { key: string, uploadId: string, parts: { partNumber: number; etag: string }[], - options: { ifNoneMatch?: boolean } = {}, + options: { ifNoneMatch?: boolean; signal?: AbortSignal } = {}, ) { const body = `${parts .sort((a, b) => a.partNumber - b.partNumber) @@ -475,6 +496,7 @@ export class S3 { ...(options.ifNoneMatch ? { "if-none-match": "*" } : {}), }, expect: options.ifNoneMatch ? [200, 409, 412] : undefined, + signal: options.signal, }); if (response.status === 409 || response.status === 412) return false; const text = await response.text(); @@ -519,40 +541,57 @@ export class S3 { key: string, path: string, contentType: string, - options: { ifNoneMatch?: boolean } = {}, + options: { ifNoneMatch?: boolean; signal?: AbortSignal } = {}, ) { const file = Bun.file(path); const partSize = 64 << 20; - const uploadId = await this.createMultipart(key, contentType); + const uploadId = await this.createMultipart( + key, + contentType, + options.signal, + ); try { const parts: { partNumber: number; etag: string }[] = []; for (let start = 0; start === 0 || start < file.size; start += partSize) { + options.signal?.throwIfAborted(); const body = new Uint8Array( await file.slice(start, start + partSize).arrayBuffer(), ); const partNumber = parts.length + 1; parts.push({ partNumber, - etag: await this.uploadPart(key, uploadId, partNumber, body), + etag: await this.uploadPart( + key, + uploadId, + partNumber, + body, + options.signal, + ), }); } + options.signal?.throwIfAborted(); if (await this.completeMultipart(key, uploadId, parts, options)) { return file.size; } - await this.abortMultipart(key, uploadId).catch(() => {}); + await this.abortMultipart(key, uploadId, this.cleanupSignal()).catch( + () => {}, + ); const existing = await this.head(key); if (!existing) throw new Error(`${key} was being written concurrently`); return existing.size; } catch (error) { - await this.abortMultipart(key, uploadId).catch(() => {}); + await this.abortMultipart(key, uploadId, this.cleanupSignal()).catch( + () => {}, + ); throw error; } } - async abortMultipart(key: string, uploadId: string) { + async abortMultipart(key: string, uploadId: string, signal?: AbortSignal) { await this.send("DELETE", key, { query: { uploadId }, expect: [204, 200, 404], + signal, }); } diff --git a/apps/render-farm/src/scheduler.ts b/apps/render-farm/src/scheduler.ts index 563c8f42702..14b7d3585af 100644 --- a/apps/render-farm/src/scheduler.ts +++ b/apps/render-farm/src/scheduler.ts @@ -18,11 +18,12 @@ export type SchedulableJob = { runningTasks: number; }; -export type SchedulerOptions = { +export type SchedulerOptions = { /** Unfinished chunks per job, from the front, that outrank other work. */ headChunks: number; fifo: boolean; now: number; + canTake?: (state: S) => boolean; }; /** @@ -36,18 +37,19 @@ export type SchedulerOptions = { * 2. fairness: the job currently holding the fewest running tasks; * 3. the older job, then the earlier position in the timeline. */ -export function pickQueued( - queue: readonly SchedulableState[], +export function pickQueued( + queue: readonly S[], jobs: Iterable, accepts: (kind: SchedulableTask["kind"]) => boolean, - options: SchedulerOptions, + options: SchedulerOptions, ) { const byId = new Map }>(); - const eligible = (state: SchedulableState) => + const eligible = (state: S) => state.state === "queued" && byId.has(state.task.jobId) && accepts(state.task.kind) && - !(state.heldUntil && state.heldUntil > options.now); + !(state.heldUntil && state.heldUntil > options.now) && + (options.canTake?.(state) ?? true); for (const job of jobs) { if (job.status !== "rendering") continue; const head = new Set(); diff --git a/apps/render-farm/src/transcode.test.ts b/apps/render-farm/src/transcode.test.ts index e2258eca2e4..4eeaad4fb80 100644 --- a/apps/render-farm/src/transcode.test.ts +++ b/apps/render-farm/src/transcode.test.ts @@ -1,5 +1,13 @@ import { describe, expect, test } from "bun:test"; -import { canRemux, encodedSeconds, transcodeArgs } from "./transcode"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + canRemux, + downloadSource, + encodedSeconds, + transcodeArgs, +} from "./transcode"; describe("transcodeArgs", () => { test("NVENC keeps frames on the GPU and forces one-second IDR frames", () => { @@ -89,3 +97,58 @@ describe("canRemux", () => { expect(canRemux(source.replace("dts_time=0.100000|", ""))).toBe(false); }); }); + +describe("downloadSource", () => { + test("reassembles a source from parallel ranged reads", async () => { + const data = new Uint8Array(1000).map((_, index) => index % 251); + const reads: [number, number][] = []; + const path = join(mkdtempSync(join(tmpdir(), "rf-dl-")), "source"); + const size = await downloadSource( + { + head: async () => ({ size: data.byteLength }), + getRange: async (_key, start, end) => { + reads.push([start, end]); + return data.slice(start, end + 1); + }, + }, + "k", + path, + { piece: 128, concurrency: 3 }, + ); + expect(size).toBe(1000); + expect(new Uint8Array(readFileSync(path))).toEqual(data); + expect(reads).toHaveLength(8); + }); + + test("reports progress as each range lands", async () => { + const progress: number[] = []; + await downloadSource( + { + head: async () => ({ size: 300 }), + getRange: async (_key, start, end) => new Uint8Array(end - start + 1), + }, + "k", + join(mkdtempSync(join(tmpdir(), "rf-dl-")), "source"), + { + piece: 100, + concurrency: 1, + onProgress: (bytes) => progress.push(bytes), + }, + ); + expect(progress).toEqual([100, 200, 300]); + }); + + test("refuses a short read", async () => { + const path = join(mkdtempSync(join(tmpdir(), "rf-dl-")), "source"); + await expect( + downloadSource( + { + head: async () => ({ size: 100 }), + getRange: async () => new Uint8Array(10), + }, + "k", + path, + ), + ).rejects.toThrow("short read"); + }); +}); diff --git a/apps/render-farm/src/transcode.ts b/apps/render-farm/src/transcode.ts index 0e7571ab244..fdfd5a20510 100644 --- a/apps/render-farm/src/transcode.ts +++ b/apps/render-farm/src/transcode.ts @@ -132,3 +132,68 @@ export function remuxArgs(input: string, output: string) { output, ]; } + +export type RangeSource = { + head(key: string): Promise<{ size: number } | null>; + getRange( + key: string, + start: number, + endInclusive: number, + ): Promise; +}; + +const DOWNLOAD_PIECE = 16 << 20; + +/** + * Copies a source to local disk with parallel ranged reads. ffmpeg reading a + * fragmented recording over HTTP seeks per fragment: a 2 h camera took 208 s + * that way, against about 2 s to download it and 2 s to remux it locally. + */ +export async function downloadSource( + s3: RangeSource, + key: string, + path: string, + options: { + concurrency?: number; + piece?: number; + signal?: AbortSignal; + /** Called with the bytes written so far after each range lands. */ + onProgress?: (bytes: number) => void; + } = {}, +) { + const head = await s3.head(key); + if (!head) throw new Error(`${key} is missing`); + const piece = options.piece ?? DOWNLOAD_PIECE; + const pieces = Math.ceil(head.size / piece); + const file = Bun.file(path); + const handle = await (await import("node:fs/promises")).open(path, "w"); + let written = 0; + try { + await handle.truncate(head.size); + let next = 0; + await Promise.all( + Array.from( + { length: Math.min(options.concurrency ?? 8, pieces) }, + async () => { + while (next < pieces) { + options.signal?.throwIfAborted(); + const index = next++; + const start = index * piece; + const end = Math.min(head.size, start + piece); + const bytes = await s3.getRange(key, start, end - 1); + if (bytes.byteLength !== end - start) { + throw new Error(`short read ${key} ${start}-${end}`); + } + await handle.write(bytes, 0, bytes.byteLength, start); + written += bytes.byteLength; + options.onProgress?.(written); + } + }, + ), + ); + } finally { + await handle.close(); + } + if (file.size !== head.size) throw new Error(`${key} downloaded short`); + return head.size; +} diff --git a/apps/render-farm/src/validate.ts b/apps/render-farm/src/validate.ts index ac446de1449..5d6bf5a452c 100644 --- a/apps/render-farm/src/validate.ts +++ b/apps/render-farm/src/validate.ts @@ -1,5 +1,12 @@ import { COMPRESSION_BPP, type JobRequest } from "./protocol"; +export type CursorJobRequest = { + source: string; + outputPrefix: string; + callbackUrl?: string; + reference?: string; +}; + export function validateJobRequest(body: unknown): JobRequest | string { if (!body || typeof body !== "object") return "body must be a JSON object"; const request = body as Record; @@ -112,6 +119,13 @@ export function validateJobRequest(body: unknown): JobRequest | string { ) { return "reference must be a string of at most 200 characters"; } + if ( + request.prepare !== undefined && + (typeof request.prepare !== "string" || + !/^[a-z0-9-]{1,40}\.json$/.test(request.prepare)) + ) { + return "prepare must name a JSON file in the recording"; + } return request as JobRequest; } @@ -203,6 +217,43 @@ export function checkManifestBounds( } /** A bucket key or folder with no empty or parent segments. */ +export function validateCursorJobRequest( + body: unknown, +): CursorJobRequest | string { + if (!body || typeof body !== "object") return "body must be a JSON object"; + const { sourceRoot, source, outputPrefix, callbackUrl, reference } = + body as Record; + if ( + !isKey(sourceRoot, { trailingSlash: true }) || + !sourceRoot.endsWith("/") || + !isKey(source) || + !source.startsWith(sourceRoot) || + !isKey(outputPrefix, { trailingSlash: true }) || + !outputPrefix.endsWith("/") || + !outputPrefix.startsWith(sourceRoot) || + source.startsWith(outputPrefix) + ) { + return "source and outputPrefix must be inside sourceRoot, outputPrefix a separate folder"; + } + if (callbackUrl !== undefined && !isCallbackUrl(callbackUrl)) { + return "callbackUrl must be an https URL"; + } + if ( + reference !== undefined && + (typeof reference !== "string" || reference.length > 200) + ) { + return "reference must be a string of at most 200 characters"; + } + return { + source, + outputPrefix, + ...(callbackUrl !== undefined + ? { callbackUrl: callbackUrl as string } + : {}), + ...(reference !== undefined ? { reference } : {}), + }; +} + export function isKey( value: unknown, options: { trailingSlash?: boolean } = {}, diff --git a/apps/render-farm/src/worker.test.ts b/apps/render-farm/src/worker.test.ts index a9e9e99c025..44b6badd8c5 100644 --- a/apps/render-farm/src/worker.test.ts +++ b/apps/render-farm/src/worker.test.ts @@ -1,6 +1,7 @@ import { describe, expect, test } from "bun:test"; import { randomUUID } from "node:crypto"; -import { readFileSync } from "node:fs"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import { join } from "node:path"; import { MIN_PART, type TranscodeTask, type WorkItem } from "./protocol"; import * as stitch from "./stitch"; @@ -10,6 +11,8 @@ function harness( options: { presignGate?: Promise; probeStalls?: boolean; + /** Each ranged read waits for the next gate, when given. */ + rangeGates?: Promise[]; stashFails?: boolean; } = {}, ) { @@ -17,6 +20,7 @@ function harness( const killed: string[] = []; const removed: string[] = []; const intervals: (() => void)[] = []; + const posts: { path: string; body: unknown }[] = []; let uploads = 0; const source = readFileSync( new URL("./worker.ts", import.meta.url), @@ -26,6 +30,19 @@ function harness( randomUUID, join, ...transcode, + // Sources download to a real temporary file. + downloadSource: ( + s3: transcode.RangeSource, + key: string, + _path: string, + options: { signal?: AbortSignal; onProgress?: (bytes: number) => void }, + ) => + transcode.downloadSource( + s3, + key, + join(mkdtempSync(join(tmpdir(), "rf-worker-")), "source"), + { ...options, piece: 4, concurrency: 1 }, + ), availableParallelism: () => 1, hostname: () => "worker-test", mkdirSync: () => {}, @@ -35,9 +52,13 @@ function harness( ...stitch, MIN_PART, S3: class { - async presignFresh() { + async head() { await options.presignGate; - return "https://media.test/source"; + return { size: options.rangeGates ? options.rangeGates.length * 4 : 4 }; + } + async getRange() { + await options.rangeGates?.shift(); + return new Uint8Array(4); } async uploadFile() { uploads++; @@ -57,6 +78,13 @@ function harness( on: () => {}, }, setInterval: (callback: () => void) => intervals.push(callback), + fetch: async (url: string, init?: { body?: string }) => { + posts.push({ + path: new URL(url).pathname, + body: init?.body ? JSON.parse(init.body) : null, + }); + return Response.json({ finished: [], cancel: [] }); + }, console: { log() {}, warn() {}, error() {} }, Bun: { spawn: ([command]: string[]) => { @@ -97,7 +125,7 @@ function harness( const compiled = new Bun.Transpiler({ loader: "ts" }).transformSync(source); const worker = new Function( ...Object.keys(deps), - `${compiled}\nreturn {runTranscode, cancel, busy, progress, transcoders, engineEnv, uploadLayout};`, + `${compiled}\nreturn {runTranscode, cancel, busy, progress, transcoders, engineEnv, uploadLayout, finishInBackground, finishing};`, )(...Object.values(deps)) as { runTranscode: (task: TranscodeTask, slot: number) => Promise; cancel: (taskIds: string[]) => void; @@ -112,6 +140,13 @@ function harness( segments: { bytes: Uint8Array }[], bytes: number, ) => Promise; + finishInBackground: ( + slot: number, + task: unknown, + since: number, + done: Promise, + ) => Promise; + finishing: Map; }; return { ...worker, @@ -119,6 +154,7 @@ function harness( killed, removed, intervals, + posts, uploads: () => uploads, }; } @@ -131,12 +167,17 @@ const task: TranscodeTask = { keyframeSeconds: 1, }; +/** Waits for the source download (real file I/O) to reach the probe. */ +async function until(condition: () => boolean) { + for (let i = 0; i < 200 && !condition(); i++) await Bun.sleep(5); +} + describe("transcode cancellation", () => { test("cancelling a stalled probe kills it and prevents the encoder from starting", async () => { const h = harness(); h.busy.set(0, task); const pending = h.runTranscode(task, 0); - for (let i = 0; i < 20; i++) await Promise.resolve(); + await until(() => h.spawned.length > 0); expect(h.spawned).toEqual(["ffprobe"]); h.cancel([task.taskId]); await expect(pending).rejects.toThrow("transcode cancelled"); @@ -151,7 +192,7 @@ describe("transcode cancellation", () => { test("the watchdog releases a stalled probe without waiting for cancellation", async () => { const h = harness(); const pending = h.runTranscode(task, 0); - for (let i = 0; i < 20; i++) await Promise.resolve(); + await until(() => h.spawned.length > 0); const progress = h.progress.get(0); if (!progress) throw new Error("missing progress"); progress.lastProgressAt = 0; @@ -162,6 +203,29 @@ describe("transcode cancellation", () => { expect(h.transcoders.size).toBe(0); }); + test("a download that keeps landing ranges is not taken for a stalled transcode", async () => { + const gates = [0, 1, 2].map(() => Promise.withResolvers()); + const h = harness({ rangeGates: gates.map((gate) => gate.promise) }); + const pending = h.runTranscode(task, 0); + await until(() => h.progress.has(0)); + const progress = h.progress.get(0); + if (!progress) throw new Error("missing progress"); + for (const gate of gates.slice(0, 2)) { + progress.lastProgressAt = 0; + gate.resolve(); + await until(() => progress.lastProgressAt > 0); + h.intervals[0]?.(); + expect(h.transcoders.size).toBe(1); + } + // The last range never lands: the watchdog stops it. + progress.lastProgressAt = 0; + h.intervals[0]?.(); + gates[2]?.resolve(); + await expect(pending).rejects.toThrow("transcode cancelled"); + expect(h.spawned).toEqual([]); + expect(h.transcoders.size).toBe(0); + }); + test("cancellation while obtaining credentials prevents a later probe", async () => { const gate = Promise.withResolvers(); const h = harness({ presignGate: gate.promise }); @@ -227,3 +291,89 @@ describe("chunk upload", () => { }); }); }); + +describe("chunks finishing after their render", () => { + const chunk = (index: number) => ({ + kind: "video", + taskId: `job:v${index}`, + jobId: "job", + chunk: index, + attempt: 1, + frames: [index * 100, index * 100 + 100], + }); + const settled = (promise: Promise) => + Promise.race([promise.then(() => true), Bun.sleep(20).then(() => false)]); + + test("free their slot and are reported as fully rendered", async () => { + const h = harness(); + const audio = Promise.withResolvers(); + const since = Date.now() - 5000; + expect( + await settled(h.finishInBackground(0, chunk(1), since, audio.promise)), + ).toBe(true); + // Every 3 s heartbeat lists it, frames done, with its whole task time. + for (const interval of h.intervals) await interval(); + const heartbeat = h.posts.find((post) => post.path === "/heartbeat") + ?.body as { + running: { + taskId: string; + frames: number; + total: number; + elapsedMs: number; + }[]; + }; + expect(heartbeat.running).toEqual([ + expect.objectContaining({ taskId: "job:v1", frames: 100, total: 100 }), + ]); + expect(heartbeat.running[0]?.elapsedMs).toBeGreaterThanOrEqual(5000); + audio.resolve(); + await until(() => h.finishing.size === 0); + expect(h.finishing.size).toBe(0); + }); + + test("hold their slot once it has too many waiting", async () => { + const h = harness(); + const audio = Array.from({ length: 9 }, () => + Promise.withResolvers(), + ); + for (let index = 0; index < 8; index++) { + const done = audio[index] as PromiseWithResolvers; + expect( + await settled( + h.finishInBackground(0, chunk(index), Date.now(), done.promise), + ), + ).toBe(true); + } + const ninth = h.finishInBackground( + 0, + chunk(8), + Date.now(), + (audio[8] as PromiseWithResolvers).promise, + ); + expect(await settled(ninth)).toBe(false); + // Another slot's chunks don't count against this one. + expect( + await settled( + h.finishInBackground(1, chunk(9), Date.now(), new Promise(() => {})), + ), + ).toBe(true); + (audio[3] as PromiseWithResolvers).resolve(); + expect(await settled(ninth)).toBe(true); + }); + + test("that fail are reported as failed", async () => { + const h = harness(); + await h.finishInBackground( + 0, + chunk(2), + Date.now(), + Promise.reject(new Error("audio -> 410")), + ); + await until(() => h.finishing.size === 0); + expect(h.posts).toContainEqual({ + path: "/tasks/job%3Av2/fail", + body: { worker: expect.any(String), attempt: 1, error: "audio -> 410" }, + }); + expect(h.finishing.size).toBe(0); + }); +}); diff --git a/apps/render-farm/src/worker.ts b/apps/render-farm/src/worker.ts index 4de9dfdbf24..75a961d33c5 100644 --- a/apps/render-farm/src/worker.ts +++ b/apps/render-farm/src/worker.ts @@ -1,7 +1,8 @@ -import { randomUUID } from "node:crypto"; -import { mkdirSync, rmSync } from "node:fs"; +import { createHash, randomUUID } from "node:crypto"; +import { chmodSync, mkdirSync, renameSync, rmSync } from "node:fs"; import { availableParallelism, hostname } from "node:os"; import { join } from "node:path"; +import { cursorProgress } from "./cursor"; import { Engine, processCpuSeconds, sampleThreads } from "./engine"; import { segmentHeader } from "./fmp4"; import { closesSegment, segmentKey } from "./hls"; @@ -10,6 +11,8 @@ import type { Run } from "./mp4"; import { type AudioResultMeta, type AudioTask, + CURSOR_OUTPUTS, + type CursorTask, MIN_PART, type SegmentReport, type TaskTimings, @@ -22,6 +25,7 @@ import { mediaS3ConfigFromEnv, S3, s3ConfigFromEnv } from "./s3"; import { stashBytes } from "./stitch"; import { canRemux, + downloadSource, encodedSeconds, probeArgs, remuxArgs, @@ -116,9 +120,7 @@ function dropJobs(finished: string[]) { const cache = caches.get(jobId); if (!cache) continue; if ( - [...busy.values()].some( - (task) => task.kind !== "transcode" && task.jobId === jobId, - ) + [...busy.values()].some((task) => "jobId" in task && task.jobId === jobId) ) continue; cache.close(); @@ -144,7 +146,7 @@ const progress = new Map< number, { taskId: string; - kind: "video" | "audio" | "transcode"; + kind: "video" | "audio" | "transcode" | "cursor"; frames: number; total: number; startedAt: number; @@ -219,6 +221,12 @@ class SegmentStream { private index = 0; private extradata = ""; private uploads: Promise[] = []; + /** + * Segments wait for their audio one after another: each wait is a long + * poll to the coordinator, and a chunk's 40 segments all waiting at once, + * on every chunk a worker was finishing, starved its heartbeats. + */ + private audioTurn: Promise = Promise.resolve(); private failed: unknown = null; firstUploadedMs: number | null = null; @@ -295,7 +303,11 @@ class SegmentStream { last ? task.audio.end : this.packetAt(first + b), ] : [0, 0]; - const audio = await fetchAudioRange(task.jobId, packets[0], packets[1]); + const audioReady = this.audioTurn.then(() => + fetchAudioRange(task.jobId, packets[0], packets[1]), + ); + this.audioTurn = audioReady.catch(() => {}); + const audio = await audioReady; const header = segmentHeader({ sequence: first + a + 1, firstFrame: first + a, @@ -401,21 +413,22 @@ async function uploadLayout( let pendingBytes = 0; let nextPart = upload.firstPart; const inflight = new Set>(); - let failure: unknown = null; + let failure: { error: unknown } | undefined; const track = async (promise: Promise) => { inflight.add(promise); - // Handle both outcomes here: a bare .finally() re-rejects unhandled and - // kills the whole worker (e.g. a hedge loser hitting NoSuchUpload after - // the job completed). An upload can settle and leave the set before - // anything awaits it, so its failure is kept to be thrown below. + // Record every failure here: an unhandled rejection kills the worker + // (e.g. a hedge loser hitting NoSuchUpload after the job completed), and + // an upload that fails and leaves the set before a race or the final + // wait sees it would otherwise go unnoticed. promise.then( () => inflight.delete(promise), (error) => { - failure ??= error; inflight.delete(promise); + failure ??= { error }; }, ); - if (inflight.size >= 4) await Promise.race(inflight); + if (inflight.size >= 4) await Promise.race(inflight).catch(() => {}); + if (failure) throw failure.error; }; const send = (body: Uint8Array) => { if (nextPart >= upload.firstPart + upload.partLimit) { @@ -475,8 +488,8 @@ async function uploadLayout( } if (!stashed) throw new Error("chunk ended before its stash filled"); if (pendingBytes > 0) await send(take(pendingBytes)); - await Promise.all(inflight); - if (failure) throw failure; + await Promise.allSettled(inflight); + if (failure) throw failure.error; pending = []; if (parts.reduce((sum, part) => sum + part.size, 0) + stashSize !== bytes) { throw new Error("uploaded byte count mismatch"); @@ -492,6 +505,8 @@ async function runVideo( engine: Engine, queuedMs: number, nearEnd: () => void = () => {}, + /** Called once the engine is done with the chunk, with its VRAM samples. */ + onRendered: (gpuMem: number[][] | undefined) => void = () => {}, ): Promise { const started = performance.now(); const cpuBefore = await processCpuSeconds(engine.pid); @@ -557,6 +572,7 @@ async function runVideo( progress.delete(slot); }); nearEnd(); + onRendered((result.timings as unknown as { gpu_mem?: number[][] }).gpu_mem); const segmentsDone = stream?.finish(result.sizes, result.extradata); segmentsDone?.catch(() => {}); const engineMs = performance.now() - engineStarted; @@ -720,7 +736,16 @@ async function runTranscode(task: TranscodeTask, slot: number) { transcoders.set(slot, run); try { const output = join(dir, "output.mp4"); - const input = await s3.presignFresh("GET", task.source, 6 * 3600); + const input = join(dir, "source"); + // Each range that lands counts as progress, so a long download isn't + // taken for a stalled transcode, while one that stops still is. + await downloadSource(s3, task.source, input, { + signal: run.controller.signal, + onProgress: () => { + entry.lastProgressAt = Date.now(); + }, + }); + entry.lastProgressAt = Date.now(); run.controller.signal.throwIfAborted(); const probe = Bun.spawn(["ffprobe", ...probeArgs(input)], { stdout: "pipe", @@ -789,6 +814,212 @@ async function runTranscode(task: TranscodeTask, slot: number) { } } +// The reconstruction service is private: its binary, ONNX Runtime and model +// live in the farm bucket under RF_CURSOR_BUNDLE, listed with their SHA-256 +// in the bundle's manifest.json. +const CURSOR_BUNDLE = process.env.RF_CURSOR_BUNDLE?.replace(/\/?$/, "/"); +const CURSOR_THREADS = Number( + process.env.RF_CURSOR_THREADS ?? Math.max(2, Math.floor(CPUS / 2)), +); +const CURSOR_TRANSFER_LIMIT_MS = 10 * 60_000; +let cursorBundle: Promise | null = null; + +async function fetchCursorBundle(prefix: string) { + const manifest = JSON.parse( + new TextDecoder().decode(await stashS3.get(`${prefix}manifest.json`)), + ) as { files: Record }; + const dir = join( + WORK_DIR, + "cursor-service", + createHash("sha256").update(prefix).digest("hex").slice(0, 16), + ); + mkdirSync(dir, { recursive: true }); + for (const name of [ + "cap-cursor-service", + "libonnxruntime.so", + "model.onnx", + ]) { + const expected = manifest.files[name]; + if (!expected) throw new Error(`cursor bundle manifest lacks ${name}`); + const path = join(dir, name); + const existing = Bun.file(path); + if ( + (await existing.exists()) && + createHash("sha256") + .update(new Uint8Array(await existing.arrayBuffer())) + .digest("hex") === expected + ) { + continue; + } + const partial = `${path}.partial`; + await downloadSource(stashS3, `${prefix}${name}`, partial); + const actual = createHash("sha256") + .update(new Uint8Array(await Bun.file(partial).arrayBuffer())) + .digest("hex"); + if (actual !== expected) { + rmSync(partial, { force: true }); + throw new Error(`cursor bundle ${name} does not match its manifest`); + } + chmodSync(partial, 0o755); + renameSync(partial, path); + } + return dir; +} + +function cursorBundleDir() { + if (!CURSOR_BUNDLE) { + return Promise.reject( + new Error("cursor reconstruction is not configured on this worker"), + ); + } + cursorBundle ??= fetchCursorBundle(CURSOR_BUNDLE).catch((error) => { + cursorBundle = null; + throw error; + }); + return cursorBundle; +} + +async function runCursor(task: CursorTask, slot: number) { + const dir = join(WORK_DIR, `cursor-${randomUUID()}`); + mkdirSync(dir, { recursive: true }); + const entry = { + taskId: task.taskId, + kind: "cursor" as const, + frames: 0, + total: 1000, + startedAt: Date.now(), + lastProgressAt: Date.now(), + }; + progress.set(slot, entry); + const run: TranscodeRun = { controller: new AbortController() }; + transcoders.set(slot, run); + try { + // Fetching the bundle and uploading the outputs report no progress of + // their own; neither is a stalled reconstruction unless it outlasts + // its limit. An upload past it is cancelled and settles before the + // task fails, so it cannot land after a retry wrote the same keys. + const keepAlive = () => { + const timer = setInterval(() => { + entry.lastProgressAt = Date.now(); + }, 5_000); + return () => { + clearInterval(timer); + entry.lastProgressAt = Date.now(); + }; + }; + const alive = (work: Promise, what: string) => { + const stop = keepAlive(); + let timer: ReturnType | undefined; + const limit = new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`${what} took over 10 minutes`)), + CURSOR_TRANSFER_LIMIT_MS, + ); + }); + work.catch(() => {}); + return Promise.race([work, limit]).finally(() => { + clearTimeout(timer); + stop(); + }); + }; + const upload = async (key: string, path: string, contentType: string) => { + const stop = keepAlive(); + const controller = new AbortController(); + const timer = setTimeout( + () => + controller.abort(new Error(`Uploading ${key} took over 10 minutes`)), + CURSOR_TRANSFER_LIMIT_MS, + ); + const cancel = () => controller.abort(run.controller.signal.reason); + run.controller.signal.addEventListener("abort", cancel, { once: true }); + try { + return await s3.uploadFile(key, path, contentType, { + signal: controller.signal, + }); + } finally { + clearTimeout(timer); + run.controller.signal.removeEventListener("abort", cancel); + stop(); + } + }; + const bundle = await alive(cursorBundleDir(), "Fetching the bundle"); + const input = join(dir, "source"); + await downloadSource(s3, task.source, input, { + signal: run.controller.signal, + onProgress: () => { + entry.lastProgressAt = Date.now(); + }, + }); + run.controller.signal.throwIfAborted(); + const output = join(dir, "output"); + const service = Bun.spawn( + [ + join(bundle, "cap-cursor-service"), + "process", + input, + "--model", + join(bundle, "model.onnx"), + "--output", + output, + "--inference-threads", + String(CURSOR_THREADS), + ], + { + stdout: "ignore", + stderr: "pipe", + env: { + ...process.env, + ORT_DYLIB_PATH: join(bundle, "libonnxruntime.so"), + ORT_DISABLE_TELEMETRY: "1", + }, + }, + ); + run.process = service; + const decoder = new TextDecoder(); + let buffered = ""; + let tail = ""; + for await (const bytes of service.stderr) { + buffered += decoder.decode(bytes, { stream: true }); + const lines = buffered.split("\n"); + buffered = lines.pop() ?? ""; + for (const line of lines) { + const value = cursorProgress(line.trim()); + if (value === null) { + tail = `${tail}\n${line}`.slice(-2000); + continue; + } + entry.lastProgressAt = Date.now(); + if (value > entry.frames) entry.frames = value; + } + } + const code = await service.exited; + run.controller.signal.throwIfAborted(); + if (code !== 0) { + throw new Error( + `cursor service exited ${code}: ${`${tail}${buffered}`.trim().slice(-500)}`, + ); + } + await upload( + `${task.outputPrefix}${CURSOR_OUTPUTS.inputEvents}`, + join(output, "input-events.ndjson"), + "application/x-ndjson", + ); + await upload( + `${task.outputPrefix}${CURSOR_OUTPUTS.display}`, + join(output, "reconstructed.cap/content/display.mp4"), + "video/mp4", + ); + } finally { + if (run.process?.exitCode === null) { + run.process.kill("SIGKILL"); + await run.process.exited; + } + transcoders.delete(slot); + progress.delete(slot); + rmSync(dir, { recursive: true, force: true }); + } +} + async function poll(kinds: string[] | undefined, prefetch = false) { const controller = new AbortController(); openPolls.add(controller); @@ -803,6 +1034,7 @@ async function poll(kinds: string[] | undefined, prefetch = false) { audioSlots: AUDIO_SLOTS, prefetch, draining, + features: ["prefix"], }, controller.signal, ).finally(() => openPolls.delete(controller)); @@ -824,8 +1056,68 @@ let draining = false; const reserved = new Map(); const busySince = new Map(); +/** + * Video chunks whose engine work is done, still waiting for their audio or + * uploading. Listed in heartbeats as fully rendered, so the coordinator + * neither requeues nor hedges a chunk that is only waiting on audio. + */ +const finishing = new Map< + string, + { + task: VideoTask; + slot: number; + /** When the slot took it: heartbeats report time on the whole task. */ + since: number; + done: Promise; + } +>(); +/** + * Chunks a slot may have finishing at once before it waits for one. Long + * exports render ahead of their audio (a 2 h export's ~10 min sections take + * about a minute each), so this bounds disk use rather than pacing work. + */ +const MAX_FINISHING_PER_SLOT = 8; + +async function finishInBackground( + slot: number, + task: VideoTask, + since: number, + done: Promise, +) { + const rendered = Date.now(); + finishing.set(task.taskId, { task, slot, since, done }); + done + .then( + () => + console.log( + `${task.taskId} done ${Date.now() - rendered}ms after rendering`, + ), + async (error) => { + console.error(`${task.taskId} failed: ${error}`); + await post(`/tasks/${encodeURIComponent(task.taskId)}/fail`, { + worker: WORKER_ID, + attempt: task.attempt, + error: String(error instanceof Error ? error.message : error), + }).catch(() => {}); + }, + ) + .finally(() => { + finishing.delete(task.taskId); + exitWhenIdle(); + }); + const own = [...finishing.values()].filter((entry) => entry.slot === slot); + if (own.length > MAX_FINISHING_PER_SLOT) { + await Promise.race(own.map((entry) => entry.done.catch(() => {}))); + } +} + function exitWhenIdle() { - if (draining && busy.size === 0 && reserved.size === 0) { + if ( + draining && + busy.size === 0 && + reserved.size === 0 && + finishing.size === 0 + ) { console.log("drained; exiting"); process.exit(0); } @@ -874,7 +1166,7 @@ async function slotLoop(slot: number) { .then((upcoming) => { if (upcoming) { reserved.set(slot, upcoming); - if (upcoming.kind !== "transcode") { + if ("jobId" in upcoming) { cacheFor(upcoming.jobId) .materialize(upcoming.files) .catch(() => {}); @@ -895,7 +1187,13 @@ async function slotLoop(slot: number) { ); engines[slot] = engine; } - if (task.kind === "transcode") { + if (task.kind === "cursor") { + await runCursor(task, slot); + await post(`/cursor-jobs/${task.taskId.slice(3)}/done`, { + worker: WORKER_ID, + attempt: task.attempt, + }); + } else if (task.kind === "transcode") { const size = await runTranscode(task, slot); await post(`/transcodes/${task.taskId.slice(3)}/done`, { worker: WORKER_ID, @@ -903,11 +1201,33 @@ async function slotLoop(slot: number) { size, }); } else if (task.kind === "video") { - const result = await runVideo(task, engine, 0, prefetchNext); - await post(`/tasks/${encodeURIComponent(task.taskId)}/done`, result); - const vram = ( - result.timings.engine as unknown as { gpu_mem?: number[][] } - ).gpu_mem; + // The chunk's wait for its audio and its upload don't need the + // engine: once the engine is done the slot takes its next task + // and the rest finishes alongside (see finishInBackground). + let rendered!: (gpuMem: number[][] | undefined) => void; + const engineDone = new Promise((resolve) => { + rendered = resolve; + }); + const completion = runVideo( + task, + engine, + 0, + prefetchNext, + rendered, + ).then((result) => + post(`/tasks/${encodeURIComponent(task.taskId)}/done`, result), + ); + completion.catch(() => {}); + const vram = await Promise.race([ + engineDone, + completion.then(() => undefined), + ]); + await finishInBackground( + slot, + task, + busySince.get(slot) ?? Date.now(), + completion, + ); const freeMb = vram?.[vram.length - 1]?.[2]; if (freeMb !== undefined && freeMb > 0 && freeMb < MIN_FREE_VRAM_MB) { console.warn( @@ -924,7 +1244,7 @@ async function slotLoop(slot: number) { await runAudio(task, engine, 0); } console.log( - `${task.taskId} done in ${Math.round(performance.now() - received)}ms`, + `${task.taskId} ${task.kind === "video" ? "rendered" : "done"} in ${Math.round(performance.now() - received)}ms`, ); } catch (error) { console.error(`${task.taskId} failed: ${error}`); @@ -950,9 +1270,11 @@ async function slotLoop(slot: number) { error: String(error instanceof Error ? error.message : error), }; await post( - task.kind === "transcode" - ? `/transcodes/${task.taskId.slice(3)}/fail` - : `/tasks/${encodeURIComponent(task.taskId)}/fail`, + task.kind === "cursor" + ? `/cursor-jobs/${task.taskId.slice(3)}/fail` + : task.kind === "transcode" + ? `/transcodes/${task.taskId.slice(3)}/fail` + : `/tasks/${encodeURIComponent(task.taskId)}/fail`, failure, ).catch(() => {}); } finally { @@ -1017,7 +1339,7 @@ function cancel(taskIds: string[]) { for (const [slot, task] of busy) { if (!taskIds.includes(task.taskId)) continue; console.log(`cancelling ${task.taskId}`); - if (task.kind === "transcode") { + if (task.kind === "transcode" || task.kind === "cursor") { stopTranscode(slot); continue; } @@ -1030,7 +1352,7 @@ function cancel(taskIds: string[]) { setInterval(() => { const now = Date.now(); for (const [slot, entry] of progress) { - if (entry.kind === "transcode") { + if (entry.kind === "transcode" || entry.kind === "cursor") { // Decoding a long source can start slowly; a minute without any // output means a wedged ffmpeg. if (now - entry.lastProgressAt >= Math.max(STALL_MS, 60_000)) { @@ -1066,6 +1388,17 @@ setInterval(async () => { elapsedMs: Date.now() - (busySince.get(slot) ?? Date.now()), }); } + for (const { task, since } of finishing.values()) { + const total = task.frames[1] - task.frames[0]; + running.push({ + taskId: task.taskId, + attempt: task.attempt, + phase: "running", + frames: total, + total, + elapsedMs: Date.now() - since, + }); + } for (const task of reserved.values()) { if (!task) continue; running.push({ diff --git a/apps/web/.gitignore b/apps/web/.gitignore index dec0b2d2108..1878f07aff5 100644 --- a/apps/web/.gitignore +++ b/apps/web/.gitignore @@ -39,4 +39,5 @@ next-env.d.ts # workflow devkit (generated) public/.well-known/workflow/ +/public/editor-solid/ /.swc diff --git a/apps/web/__tests__/editor-caption-access.test.ts b/apps/web/__tests__/editor-caption-access.test.ts new file mode 100644 index 00000000000..801aa1fd634 --- /dev/null +++ b/apps/web/__tests__/editor-caption-access.test.ts @@ -0,0 +1,136 @@ +import { expect, test } from "vitest"; +import { + hasEditorCaptionContent, + preserveEditorCaptionContent, + savedEditorProjectAfterFreeEdit, + shouldKeepPriorEditorCaptions, + stripEditorCaptionContent, +} from "../lib/editor-caption-access"; + +test("a Free editor opens an older Pro project without caption layers or export settings", () => { + const config = { + camera: { mirror: true }, + captions: { + segments: [{ id: "word", start: 0, end: 1, text: "Hello" }], + settings: { enabled: true, exportWithSubtitles: true, font: "Geist" }, + }, + timeline: { + segments: [{ start: 0, end: 10 }], + captionSegments: [{ id: "word", start: 0, end: 1 }], + }, + }; + const free = stripEditorCaptionContent(config); + expect(hasEditorCaptionContent(config)).toBe(true); + expect(hasEditorCaptionContent(free)).toBe(false); + expect(free).toMatchObject({ + camera: { mirror: true }, + captions: { + segments: [], + settings: { + enabled: false, + exportWithSubtitles: false, + font: "Geist", + }, + }, + timeline: { + segments: [{ start: 0, end: 10 }], + captionSegments: [], + }, + }); + expect(config.captions.segments).toHaveLength(1); +}); + +test("a compacted caption reference cannot bypass a Free entitlement", () => { + expect(hasEditorCaptionContent({ webCaptionRef: "sha", captions: {} })).toBe( + true, + ); + expect( + hasEditorCaptionContent({ + captions: { segments: [], settings: { enabled: false } }, + timeline: { captionSegments: [] }, + }), + ).toBe(false); +}); + +test("Free edits keep hidden source captions and overrides for a later Pro reopen", () => { + const prior = { + camera: { mirror: false }, + captions: { + segments: [{ id: "source-1", text: "Paid" }], + settings: { enabled: true, font: "Geist" }, + }, + timeline: { + segments: [{ recordingClip: 0, start: 0, end: 10 }], + captionSegments: [{ id: "source-1", positionOverride: "top-center" }], + }, + }; + const edited = { + ...stripEditorCaptionContent(prior), + camera: { mirror: true }, + timeline: { + segments: [{ recordingClip: 0, start: 2, end: 8 }], + captionSegments: [], + }, + }; + const stored = preserveEditorCaptionContent(edited, prior); + expect(hasEditorCaptionContent(edited)).toBe(false); + expect(stored).toMatchObject({ + camera: { mirror: true }, + captions: prior.captions, + timeline: { + segments: [{ recordingClip: 0, start: 2, end: 8 }], + captionSegments: prior.timeline.captionSegments, + }, + }); + expect(prior.camera.mirror).toBe(false); +}); + +test("a Free save receipt retains only previously saved paid caption edits", () => { + const prior = { + camera: { mirror: false }, + captions: { + segments: [{ id: "word", text: "Saved caption" }], + settings: { enabled: true, exportWithSubtitles: true }, + }, + timeline: { + segments: [{ start: 0, end: 10 }], + captionSegments: [{ id: "word", text: "Saved caption" }], + }, + }; + const local = { + camera: { mirror: true }, + captions: { + segments: [{ id: "word", text: "Unsaved caption" }], + settings: { enabled: true, exportWithSubtitles: true }, + }, + timeline: { + segments: [{ start: 2, end: 8 }], + captionSegments: [{ id: "word", text: "Unsaved caption" }], + }, + }; + const receipt = savedEditorProjectAfterFreeEdit(local, prior); + expect(receipt).toMatchObject({ + camera: { mirror: true }, + captions: prior.captions, + timeline: { + segments: [{ start: 2, end: 8 }], + captionSegments: prior.timeline.captionSegments, + }, + }); + expect(receipt).not.toEqual(local); + expect(local.captions.segments[0]?.text).toBe("Unsaved caption"); +}); + +test("a Free-origin save keeps prior captions across an upgrade while a Pro deletion removes them", () => { + const prior = { + captions: { segments: [{ id: "paid", text: "Saved" }] }, + }; + const empty = { + captions: { segments: [], settings: { enabled: false } }, + timeline: { captionSegments: [] }, + }; + expect(shouldKeepPriorEditorCaptions(empty, prior, false, false)).toBe(true); + expect(shouldKeepPriorEditorCaptions(empty, prior, true, true)).toBe(true); + expect(shouldKeepPriorEditorCaptions(empty, prior, true, false)).toBe(false); + expect(shouldKeepPriorEditorCaptions(prior, prior, true, true)).toBe(false); +}); diff --git a/apps/web/__tests__/editor-caption-assemblyai-stream-replay.ts b/apps/web/__tests__/editor-caption-assemblyai-stream-replay.ts new file mode 100644 index 00000000000..a1e0ad12b47 --- /dev/null +++ b/apps/web/__tests__/editor-caption-assemblyai-stream-replay.ts @@ -0,0 +1,106 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { isAiGenerationLanguage } from "@cap/web-domain"; +import { AssemblyAI } from "assemblyai"; +import { getAssemblyAITranscriptionOptions } from "../lib/assemblyai"; + +const audioPath = process.argv[2]; +assert.ok(audioPath); +const language = process.argv[3] ?? "auto"; +assert.ok(isAiGenerationLanguage(language)); +const expected = await readFile(audioPath); +assert.ok(expected.length > 1024); +let uploaded = Buffer.alloc(0); +let submitted: Record = {}; +let polled = false; +let serverError: Error | null = null; +let baseUrl = ""; +const server = createServer(async (request, response) => { + try { + assert.equal(request.headers.authorization, "cap-test-key"); + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + const body = Buffer.concat(chunks); + if (request.url === "/v2/upload" && request.method === "POST") { + assert.equal(request.headers["content-type"], "application/octet-stream"); + uploaded = body; + response.setHeader("Content-Type", "application/json"); + response.end(JSON.stringify({ upload_url: `${baseUrl}/uploaded-audio` })); + return; + } + if (request.url === "/v2/transcript" && request.method === "POST") { + submitted = JSON.parse(body.toString("utf8")) as Record; + response.setHeader("Content-Type", "application/json"); + response.end(JSON.stringify({ id: "caption-replay", status: "queued" })); + return; + } + if (request.url === "/v2/transcript/caption-replay") { + polled = true; + response.setHeader("Content-Type", "application/json"); + response.end( + JSON.stringify({ + id: "caption-replay", + status: "completed", + words: [], + audio_duration: 3, + }), + ); + return; + } + response.writeHead(404).end(); + } catch (error) { + serverError = error instanceof Error ? error : new Error(String(error)); + response.writeHead(500).end(); + } +}); +await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); +const address = server.address() as AddressInfo; +baseUrl = `http://127.0.0.1:${address.port}`; +try { + let nextOffset = 0; + let sourcePulls = 0; + const audio = new ReadableStream({ + pull(controller) { + if (nextOffset >= expected.length) { + controller.close(); + return; + } + const end = Math.min(nextOffset + 512, expected.length); + controller.enqueue(expected.subarray(nextOffset, end)); + nextOffset = end; + sourcePulls++; + }, + }); + const client = new AssemblyAI({ apiKey: "cap-test-key", baseUrl }); + const result = await client.transcripts.transcribe({ + audio, + ...getAssemblyAITranscriptionOptions(language), + disfluencies: true, + }); + assert.equal(result.status, "completed"); + assert.equal(serverError, null); + assert.ok(sourcePulls > 1); + assert.ok(uploaded.equals(expected)); + assert.equal(submitted.audio_url, `${baseUrl}/uploaded-audio`); + assert.deepEqual( + submitted.speech_models, + getAssemblyAITranscriptionOptions(language).speech_models, + ); + if (language === "auto") { + assert.equal(submitted.language_detection, true); + assert.equal(submitted.language_code, undefined); + } else { + assert.equal(submitted.language_detection, undefined); + assert.equal(submitted.language_code, language); + } + assert.equal(submitted.disfluencies, true); + assert.ok(polled); + process.stdout.write( + `${JSON.stringify({ uploadedBytes: uploaded.length, language, sdkStream: true })}\n`, + ); +} finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); +} diff --git a/apps/web/__tests__/editor-caption-client.test.ts b/apps/web/__tests__/editor-caption-client.test.ts new file mode 100644 index 00000000000..c02011172c0 --- /dev/null +++ b/apps/web/__tests__/editor-caption-client.test.ts @@ -0,0 +1,205 @@ +import { afterEach, expect, test, vi } from "vitest"; +import { + EditorCaptionsNeedSessionError, + generateWebEditorCaptions, + webEditorCaptionsNeedSession, +} from "../lib/editor-caption-client"; + +const captions = { + settings: null, + segments: [ + { + id: "segment-0", + start: 0.1, + end: 0.5, + text: "Hello world", + words: [ + { text: "Hello", start: 0.1, end: 0.3 }, + { text: "world", start: 0.3, end: 0.5 }, + ], + }, + ], +}; + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); + +test("web editor captions reuse an existing transcript without polling", async () => { + const fetchMock = vi.fn( + async (_input: RequestInfo | URL, _init?: RequestInit) => + Response.json({ status: "ready", captions, message: null }), + ); + vi.stubGlobal("fetch", fetchMock); + const result = await generateWebEditorCaptions( + "video", + "session", + new AbortController().signal, + ); + expect(result).toEqual(captions); + expect(fetchMock).toHaveBeenCalledOnce(); + expect(fetchMock.mock.calls[0]?.[0]).toBe( + "/api/editor/sessions/session/captions", + ); + expect(fetchMock.mock.calls[0]?.[1]?.method).toBe("POST"); + expect(JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body))).toEqual({ + videoId: "video", + language: "auto", + }); +}); + +test("web editor captions wait for the existing AssemblyAI workflow", async () => { + vi.useFakeTimers(); + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + Response.json({ status: "processing", captions: null, message: null }), + ) + .mockResolvedValueOnce( + Response.json({ status: "ready", captions, message: null }), + ); + vi.stubGlobal("fetch", fetchMock); + const generated = generateWebEditorCaptions( + "video", + "session", + new AbortController().signal, + ); + await vi.advanceTimersByTimeAsync(2000); + expect(await generated).toEqual(captions); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(fetchMock.mock.calls[1]?.[0]).toBe( + "/api/editor/sessions/session/captions?videoId=video&language=auto", + ); + expect(fetchMock.mock.calls[1]?.[1]?.method).toBe("GET"); +}); + +test("selected caption language stays with the AssemblyAI job during polling", async () => { + vi.useFakeTimers(); + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + Response.json({ status: "processing", captions: null, message: null }), + ) + .mockResolvedValueOnce( + Response.json({ status: "ready", captions, message: null }), + ); + vi.stubGlobal("fetch", fetchMock); + const generated = generateWebEditorCaptions( + "video", + "session", + new AbortController().signal, + "es", + ); + await vi.advanceTimersByTimeAsync(2000); + expect(await generated).toEqual(captions); + expect(JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body))).toEqual({ + videoId: "video", + language: "es", + }); + expect(fetchMock.mock.calls[1]?.[0]).toBe( + "/api/editor/sessions/session/captions?videoId=video&language=es", + ); +}); + +test("caption generation starts a new job when an added clip appears during processing", async () => { + vi.useFakeTimers(); + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + Response.json({ status: "processing", captions: null, message: null }), + ) + .mockResolvedValueOnce( + Response.json({ status: "missing", captions: null, message: null }), + ) + .mockResolvedValueOnce( + Response.json({ status: "processing", captions: null, message: null }), + ) + .mockResolvedValueOnce( + Response.json({ status: "ready", captions, message: null }), + ); + vi.stubGlobal("fetch", fetchMock); + const generated = generateWebEditorCaptions( + "video", + "session", + new AbortController().signal, + ); + await vi.advanceTimersByTimeAsync(4000); + expect(await generated).toEqual(captions); + expect(fetchMock.mock.calls.map((call) => call[1]?.method)).toEqual([ + "POST", + "GET", + "POST", + "GET", + ]); +}); + +test("web editor caption entitlement is enforced by the caption API", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(null, { status: 403 })), + ); + await expect( + generateWebEditorCaptions("video", "session", new AbortController().signal), + ).rejects.toThrow("Cap Pro is required"); +}); + +test("closing the editor cancels a pending caption poll", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ status: "processing", captions: null, message: null }), + ), + ); + const controller = new AbortController(); + const generated = generateWebEditorCaptions( + "video", + "session", + controller.signal, + ); + await Promise.resolve(); + await Promise.resolve(); + controller.abort(); + await expect(generated).rejects.toThrow("Caption generation cancelled"); +}); + +test("browser editor captions use the recording's route without a session", async () => { + vi.useFakeTimers(); + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + Response.json({ status: "processing", captions: null, message: null }), + ) + .mockResolvedValueOnce( + Response.json({ status: "ready", captions, message: null }), + ); + vi.stubGlobal("fetch", fetchMock); + const generated = generateWebEditorCaptions( + "video", + null, + new AbortController().signal, + ); + await vi.advanceTimersByTimeAsync(2000); + expect(await generated).toEqual(captions); + expect(fetchMock.mock.calls[0]?.[0]).toBe( + "/api/editor/videos/video/captions", + ); + expect(JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body))).toEqual({ + language: "auto", + }); + expect(fetchMock.mock.calls[1]?.[0]).toBe( + "/api/editor/videos/video/captions?language=auto", + ); +}); + +test("a recording that needs a caption job asks for an editor session", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(null, { status: 409 })), + ); + const signal = new AbortController().signal; + expect(await webEditorCaptionsNeedSession("video", "es", signal)).toBe(true); + await expect( + generateWebEditorCaptions("video", null, signal, "es"), + ).rejects.toBeInstanceOf(EditorCaptionsNeedSessionError); +}); diff --git a/apps/web/__tests__/editor-caption-job.test.ts b/apps/web/__tests__/editor-caption-job.test.ts new file mode 100644 index 00000000000..5be0508985e --- /dev/null +++ b/apps/web/__tests__/editor-caption-job.test.ts @@ -0,0 +1,212 @@ +import type { videos } from "@cap/database/schema"; +import type { VideoMetadata } from "@cap/database/types"; +import { Database, Storage } from "@cap/web-backend"; +import { Effect, Option } from "effect"; +import { afterEach, expect, test, vi } from "vitest"; +import type { EditTranscript } from "../lib/edit-transcript"; +import { inspectEditorCaptionJob } from "../lib/editor-caption-job"; +import { buildEditorCaptionSourcePlan } from "../lib/editor-caption-sources"; + +const state = vi.hoisted(() => ({ + stored: null as string | null, + instance: null as unknown, + accessOptions: [] as unknown[], + legacyEdit: null as { sourceKey: string } | null, +})); + +vi.mock("server-only", () => ({})); +vi.mock("../lib/edit-transcript-storage", () => ({ + decryptEditTranscriptObject: (value: string) => value, +})); +vi.mock("../lib/video-storage", () => ({ + decodeStorageVideo: (value: unknown) => value, +})); +vi.mock("../lib/editor-session", async () => { + const { Effect } = await import("effect"); + return { + requestMediaEditor: () => Effect.succeed(Response.json(state.instance)), + }; +}); +vi.mock("workflow/api", () => ({ start: vi.fn() })); + +function metadata(): VideoMetadata { + const path = "content/videos/00000000-0000-4000-8000-000000000001.webm"; + return { + editorSources: { + version: 1, + display: { + key: "owner/video/display.webm", + contentType: "video/webm", + size: 4000, + objectIdentity: JSON.stringify("base"), + }, + }, + webEditorVideos: { + version: 1, + items: [ + { + key: "owner/video/editor-assets/videos/00000000-0000-4000-8000-000000000001.webm", + path, + name: "Added clip", + contentType: "video/webm", + size: 3000, + objectIdentity: JSON.stringify("clip"), + }, + ], + }, + webEditorClips: { + version: 1, + items: [{ displayPath: path, duration: 5, fps: 30, hasAudio: true }], + }, + }; +} + +function transcript(): EditTranscript { + return { + version: 3, + speechModelUsed: "universal-3-5-pro", + durationMs: 15_000, + languageCode: "en", + words: [ + { + id: "first", + text: "Original", + startMs: 500, + endMs: 900, + confidence: null, + speaker: null, + channel: null, + }, + { + id: "second", + text: "Added", + startMs: 10_500, + endMs: 10_900, + confidence: null, + speaker: null, + channel: null, + }, + ], + }; +} + +function editorVideo(details: VideoMetadata) { + return { + id: "video", + ownerId: "owner", + metadata: details, + source: { type: "webMP4", outputKey: "owner/video/result.mp4" }, + } as unknown as typeof videos.$inferSelect; +} + +function read(video: typeof videos.$inferSelect) { + const bucket = { + getObject: () => Effect.succeed(Option.fromNullable(state.stored)), + headObject: () => + Effect.succeed({ ContentLength: 4000, ETag: JSON.stringify("original") }), + }; + const service = { + getAccessForVideo: (_video: unknown, options: unknown) => { + state.accessOptions.push(options); + return Effect.succeed([bucket]); + }, + } as unknown as Storage; + const database = Database.make({ + use: (callback) => + Effect.promise(() => + callback({ + select: () => ({ + from: () => ({ + where: async () => (state.legacyEdit ? [state.legacyEdit] : []), + }), + }), + } as unknown as Parameters[0]), + ), + }); + return Effect.runPromise( + inspectEditorCaptionJob(video, "/editor/sessions/session").pipe( + Effect.provideService(Storage, service), + Effect.provideService(Database, database), + ), + ); +} + +function prepare() { + state.instance = { + recordings: { + segments: [ + { display: { duration: 10 }, camera: null, mic: null }, + { display: { duration: 5 }, camera: null, mic: null }, + ], + }, + }; + const details = metadata(); + const plan = buildEditorCaptionSourcePlan( + "owner", + "video", + details, + state.instance, + ); + if (!plan) throw new Error("Missing caption plan"); + return { details, plan }; +} + +afterEach(() => { + state.stored = null; + state.instance = null; + state.accessOptions.length = 0; + state.legacyEdit = null; +}); + +test("completed clip captions are served in desktop source time", async () => { + const { details, plan } = prepare(); + state.stored = JSON.stringify(transcript()); + const result = await read(editorVideo(details)); + expect(result.plan.combinedKey).toBe(plan.combinedKey); + expect(result.snapshot.status).toBe("ready"); + expect( + result.snapshot.captions?.segments[0]?.words.map((word) => word.start), + ).toEqual([0.5, 10.5]); + expect(state.accessOptions).toEqual([{ resolvePublishedOutput: false }]); +}); + +test("a fresh clip job reports processing before its transcript exists", async () => { + const { details, plan } = prepare(); + details.webEditorCaptionJob = { + status: "processing", + requestId: "request-1", + sourceHash: plan.hash, + requestedAt: new Date().toISOString(), + }; + expect((await read(editorVideo(details))).snapshot.status).toBe("processing"); +}); + +test("a changed source invalidates an earlier clip transcript job", async () => { + const { details, plan } = prepare(); + details.webEditorCaptionJob = { + status: "processing", + requestId: "request-1", + sourceHash: plan.hash, + requestedAt: new Date().toISOString(), + }; + const asset = details.webEditorVideos?.items[0]; + if (!asset) throw new Error("Missing clip asset"); + asset.objectIdentity = JSON.stringify("new-object"); + expect((await read(editorVideo(details))).snapshot.status).toBe("missing"); +}); + +test("a migrated edit caption job measures the preserved MP4 before using its cache", async () => { + const { details } = prepare(); + state.legacyEdit = { sourceKey: "owner/video/source/original.mp4" }; + const result = await read(editorVideo(details)); + expect(result.plan.sources[0]).toMatchObject({ + key: state.legacyEdit.sourceKey, + expectedSize: 4000, + expectedIdentity: JSON.stringify("original"), + }); + expect(result.snapshot.status).toBe("missing"); + expect(state.accessOptions).toEqual([ + { resolvePublishedOutput: false }, + { resolvePublishedOutput: false }, + ]); +}); diff --git a/apps/web/__tests__/editor-caption-native-replay.ts b/apps/web/__tests__/editor-caption-native-replay.ts new file mode 100644 index 00000000000..7b6aec01766 --- /dev/null +++ b/apps/web/__tests__/editor-caption-native-replay.ts @@ -0,0 +1,31 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import type { VideoMetadata } from "@cap/database/types"; +import { buildEditorCaptionSourcePlan } from "../lib/editor-caption-sources"; + +const fixturePath = process.argv[2]; +assert.ok(fixturePath); +const fixture = JSON.parse(await readFile(fixturePath, "utf8")) as { + metadata: VideoMetadata; + instance: unknown; +}; +const plan = buildEditorCaptionSourcePlan( + "owner", + "video", + fixture.metadata, + fixture.instance, + "worker-a", +); +assert.ok(plan); +assert.equal(plan.sources.length, 4); +assert.deepEqual( + plan.sources.map((source) => source.cap?.clipCount ?? 0), + [0, 2, 0, 1], +); +const studioSegments = plan.sources[1]?.cap?.segments; +assert.ok(studioSegments); +assert.deepEqual( + studioSegments.map((segment) => segment.hasAudio), + [true, true], +); +process.stdout.write(`${JSON.stringify({ studioSegments })}\n`); diff --git a/apps/web/__tests__/editor-caption-sources.test.ts b/apps/web/__tests__/editor-caption-sources.test.ts new file mode 100644 index 00000000000..bdd17a29096 --- /dev/null +++ b/apps/web/__tests__/editor-caption-sources.test.ts @@ -0,0 +1,308 @@ +import type { VideoMetadata } from "@cap/database/types"; +import { CAP_BUNDLE_CONTENT_TYPE } from "@cap/editor-cap-bundle"; +import { expect, test } from "vitest"; +import type { EditTranscript } from "../lib/edit-transcript"; +import { + buildEditorCaptionSourcePlan, + combineEditorCaptionTranscripts, + isEditorReplacementOutput, +} from "../lib/editor-caption-sources"; + +function metadata(hasAudio = true): VideoMetadata { + return { + editorSources: { + version: 1, + display: { + key: "owner/video/display.webm", + contentType: "video/webm", + size: 4000, + objectIdentity: JSON.stringify("base"), + }, + }, + webEditorVideos: { + version: 1, + items: [ + { + key: "owner/video/editor-assets/videos/00000000-0000-4000-8000-000000000001.webm", + path: "content/videos/00000000-0000-4000-8000-000000000001.webm", + name: "Extra screen", + contentType: "video/webm", + size: 3000, + objectIdentity: JSON.stringify("clip"), + }, + ], + }, + webEditorClips: { + version: 1, + items: [ + { + displayPath: + "content/videos/00000000-0000-4000-8000-000000000001.webm", + duration: 5, + fps: 30, + hasAudio, + }, + ], + }, + }; +} + +function instance(cameraDuration = 5) { + return { + recordings: { + segments: [ + { display: { duration: 10 }, camera: null, mic: null }, + { + display: { duration: 5 }, + camera: { duration: cameraDuration }, + mic: null, + }, + ], + }, + }; +} + +function transcript(durationMs: number, text: string): EditTranscript { + return { + version: 3, + speechModelUsed: "universal-3-5-pro", + durationMs, + languageCode: "en", + words: [ + { + id: "word-0", + text, + startMs: 1000, + endMs: 1300, + confidence: 0.9, + speaker: null, + channel: null, + }, + ], + }; +} + +test("added clip speech shifts onto the same source timeline as desktop captions", () => { + const plan = buildEditorCaptionSourcePlan( + "owner", + "video", + metadata(), + instance(6), + ); + if (!plan) throw new Error("Caption source plan was not created"); + const combined = combineEditorCaptionTranscripts(plan, [ + transcript(10_000, "Original"), + transcript(5000, "Added"), + ]); + expect(combined?.words.map((word) => [word.text, word.startMs])).toEqual([ + ["Original", 1000], + ["Added", 11_000], + ]); + expect(combined?.durationMs).toBe(16_000); + expect(new Set(combined?.words.map((word) => word.id)).size).toBe(2); +}); + +test("a silent added clip preserves its gap and never requires a transcript", () => { + const plan = buildEditorCaptionSourcePlan( + "owner", + "video", + metadata(false), + instance(), + ); + if (!plan) throw new Error("Caption source plan was not created"); + const combined = combineEditorCaptionTranscripts(plan, [ + transcript(10_000, "Original"), + null, + ]); + expect(combined?.durationMs).toBe(15_000); + expect(combined?.words.map((word) => word.text)).toEqual(["Original"]); +}); + +test("Studio Cap voice tracks keep their source offsets before a later MP4", () => { + const details = metadata(); + const capId = "9a2aa734-d076-42c0-8a8a-68db1d2d2a1e"; + const capPath = `content/imports/${capId}.capbundle`; + details.webEditorVideos?.items.push({ + key: `owner/video/editor-assets/recordings/${capId}.capbundle`, + path: capPath, + name: "Studio source", + contentType: CAP_BUNDLE_CONTENT_TYPE, + size: 8000, + objectIdentity: JSON.stringify("cap"), + }); + details.webEditorImports = { + version: 1, + items: [ + { kind: "cap", path: capPath, clipCount: 2 }, + { + kind: "clip", + path: "content/videos/00000000-0000-4000-8000-000000000001.webm", + }, + ], + }; + const native = { + recordings: { + segments: [ + { display: { duration: 10 }, camera: null, mic: null }, + { + display: { duration: 5 }, + camera: { duration: 6 }, + mic: { duration: 5.5 }, + }, + { + display: { duration: 4 }, + camera: null, + mic: null, + system_audio: { duration: 4.2 }, + }, + { display: { duration: 5 }, camera: null, mic: null }, + ], + }, + }; + const plan = buildEditorCaptionSourcePlan( + "owner", + "video", + details, + native, + "worker-a", + ); + if (!plan) throw new Error("Cap caption source plan was not created"); + expect(plan.sources.map((source) => source.mediaDurationMs)).toEqual([ + 10_000, 10_200, 5000, + ]); + expect(plan.sources[1]?.cap).toEqual({ + clipCount: 2, + workerId: "worker-a", + path: capPath, + name: "Studio source", + segments: [ + { mediaDurationMs: 5000, segmentDurationMs: 6000, hasAudio: true }, + { mediaDurationMs: 4000, segmentDurationMs: 4200, hasAudio: true }, + ], + }); + const combined = combineEditorCaptionTranscripts(plan, [ + transcript(10_000, "Original"), + transcript(10_200, "Cap"), + transcript(5000, "MP4"), + ]); + expect(combined?.words.map((word) => [word.text, word.startMs])).toEqual([ + ["Original", 1000], + ["Cap", 11_000], + ["MP4", 21_200], + ]); + expect(combined?.durationMs).toBe(25_200); +}); + +test("caption cache identities change when a recorded source changes", () => { + const first = buildEditorCaptionSourcePlan( + "owner", + "video", + metadata(), + instance(), + ); + const changed = metadata(); + const asset = changed.webEditorVideos?.items[0]; + if (!asset) throw new Error("Missing clip asset"); + asset.objectIdentity = JSON.stringify("replacement"); + const second = buildEditorCaptionSourcePlan( + "owner", + "video", + changed, + instance(), + ); + expect(first?.combinedKey).not.toBe(second?.combinedKey); + expect(first?.sources[1]?.transcriptKey).not.toBe( + second?.sources[1]?.transcriptKey, + ); +}); + +test("a migrated trim captions the immutable original instead of stale raw metadata", () => { + const details = metadata(); + const original = { + key: "owner/video/source/original.mp4", + size: 4321, + objectIdentity: JSON.stringify("original"), + }; + const migrated = buildEditorCaptionSourcePlan( + "owner", + "video", + details, + instance(), + "", + "es", + original, + ); + const stale = buildEditorCaptionSourcePlan( + "owner", + "video", + details, + instance(), + "", + "es", + ); + if (!migrated || !stale) throw new Error("Missing caption plan"); + expect(migrated.sources[0]).toMatchObject({ + key: original.key, + expectedSize: original.size, + expectedIdentity: original.objectIdentity, + }); + expect(migrated.hash).not.toBe(stale.hash); + expect(migrated.sources[1]?.key).toBe(stale.sources[1]?.key); +}); + +test("different AssemblyAI languages have separate caption cache identities", () => { + const auto = buildEditorCaptionSourcePlan( + "owner", + "video", + metadata(), + instance(), + ); + const spanish = buildEditorCaptionSourcePlan( + "owner", + "video", + metadata(), + instance(), + "", + "es", + ); + expect(auto?.language).toBe("auto"); + expect(spanish?.language).toBe("es"); + expect(auto?.combinedKey).not.toBe(spanish?.combinedKey); + expect(auto?.sources[0]?.transcriptKey).not.toBe( + spanish?.sources[0]?.transcriptKey, + ); +}); + +test("stale native segment layouts are rejected before caption generation", () => { + expect( + buildEditorCaptionSourcePlan("owner", "video", metadata(), { + recordings: { segments: [{ display: { duration: 10 } }] }, + }), + ).toBeNull(); + expect( + buildEditorCaptionSourcePlan("owner", "video", metadata(), { + recordings: { + segments: [{ display: { duration: 10 } }, { display: { duration: 0 } }], + }, + }), + ).toBeNull(); +}); + +test("a reuploaded output cannot be mistaken for the editor's raw source", () => { + const outputKey = + "owner/video/.recording/outputs/reupload-00000000-0000-4000-8000-000000000001/result.mp4"; + expect( + isEditorReplacementOutput({ + id: "video", + ownerId: "owner", + source: { type: "webMP4", outputKey }, + }), + ).toBe(true); + expect( + isEditorReplacementOutput({ + id: "video", + ownerId: "owner", + source: { type: "webMP4", outputKey: "owner/video/result.mp4" }, + }), + ).toBe(false); +}); diff --git a/apps/web/__tests__/editor-caption-transport.test.ts b/apps/web/__tests__/editor-caption-transport.test.ts new file mode 100644 index 00000000000..9411770feca --- /dev/null +++ b/apps/web/__tests__/editor-caption-transport.test.ts @@ -0,0 +1,94 @@ +import { expect, test } from "vitest"; +import { + compactEditorCaptionConfig, + createEditorCaptionCache, + restoreEditorCaptionConfig, +} from "../lib/editor-caption-transport"; +import { + decodeWebEditorProject, + encodeWebEditorProject, +} from "../lib/editor-project-storage"; + +function longCaptionProject() { + const segments = Array.from({ length: 3_000 }, (_, index) => ({ + id: `segment-${index}`, + text: "A useful recording process for everyone", + start: index * 2.4, + end: index * 2.4 + 2.2, + words: Array.from({ length: 6 }, (_, wordIndex) => ({ + text: wordIndex === 0 ? "A" : "recording", + start: index * 2.4 + wordIndex * 0.4, + end: index * 2.4 + wordIndex * 0.4 + 0.3, + })), + })); + return { + captions: { + sourceTimed: true, + settings: { enabled: true, color: "#FFFFFF" }, + segments, + }, + timeline: { segments: [], captionSegments: segments }, + camera: { mirror: false }, + }; +} + +test("two-hour caption payloads are reused for small camera and style edits", async () => { + const original = longCaptionProject(); + const cache = await createEditorCaptionCache(original); + expect(cache).not.toBeNull(); + if (!cache) throw new Error("Caption cache was not created"); + const edited = { + ...original, + camera: { mirror: true }, + captions: { + ...original.captions, + settings: { enabled: true, color: "#FF0000" }, + }, + }; + const editedCache = await createEditorCaptionCache(edited); + expect(editedCache?.ref).toBe(cache.ref); + const compact = compactEditorCaptionConfig(edited, cache); + expect(JSON.stringify(compact).length).toBeLessThan( + JSON.stringify(edited).length / 20, + ); + expect(restoreEditorCaptionConfig(compact, cache)).toEqual(edited); + const saved = encodeWebEditorProject(original); + const restoredProject = decodeWebEditorProject(saved.project); + const persistedCache = await createEditorCaptionCache(restoredProject); + expect(persistedCache?.ref).toBe(cache.ref); + expect(restoreEditorCaptionConfig(compact, persistedCache)).toEqual(edited); + expect(restoreEditorCaptionConfig(compact, null)).toBeNull(); +}); + +test("caption word edits invalidate reuse and a mismatched reference is rejected", async () => { + const original = longCaptionProject(); + const cache = await createEditorCaptionCache(original); + if (!cache) throw new Error("Caption cache was not created"); + const changedSegments = original.captions.segments.map((segment, index) => + index === 0 + ? { ...segment, words: [{ ...segment.words[0], text: "Changed" }] } + : segment, + ); + const changed = { + ...original, + captions: { ...original.captions, segments: changedSegments }, + timeline: { ...original.timeline, captionSegments: changedSegments }, + }; + expect((await createEditorCaptionCache(changed))?.ref).not.toBe(cache.ref); + const compact = compactEditorCaptionConfig(original, cache); + expect( + restoreEditorCaptionConfig( + { ...compact, webCaptionRef: "0".repeat(64) }, + cache, + ), + ).toBeNull(); + expect( + restoreEditorCaptionConfig( + { + ...compact, + captions: { ...original.captions }, + }, + cache, + ), + ).toBeNull(); +}); diff --git a/apps/web/__tests__/editor-caption-workflow.test.ts b/apps/web/__tests__/editor-caption-workflow.test.ts new file mode 100644 index 00000000000..70df48fd5c7 --- /dev/null +++ b/apps/web/__tests__/editor-caption-workflow.test.ts @@ -0,0 +1,792 @@ +import type { VideoMetadata } from "@cap/database/types"; +import { CAP_BUNDLE_CONTENT_TYPE } from "@cap/editor-cap-bundle"; +import { afterEach, expect, test, vi } from "vitest"; +import { + type EditTranscript, + getEditTranscriptObjectKey, + serializeEditTranscript, +} from "../lib/edit-transcript"; +import { + buildEditorCaptionSourcePlan, + type EditorCaptionSourcePlan, +} from "../lib/editor-caption-sources"; +import { transcribeWebEditorCaptionsWorkflow } from "../lib/editor-caption-workflow"; + +const state = vi.hoisted(() => ({ + row: null as Record | null, + legacyEdit: null as { sourceKey: string } | null, + isPro: true, + baseEtag: JSON.stringify("base"), + objects: new Map(), + writes: [] as Array<{ key: string; value: string }>, + transcribe: vi.fn(), + urls: [] as string[], + workerRequests: [] as Array<{ + path: string; + body: unknown; + workerId: string | undefined; + }>, + updates: [] as unknown[], +})); + +vi.mock("server-only", () => ({})); +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ ASSEMBLY_API_KEY: "assembly-test-key" }), +})); +vi.mock("@cap/utils", () => ({ userIsPro: () => state.isPro })); +vi.mock("@cap/database/schema", () => ({ + users: { id: "users.id" }, + videoEdits: { + videoId: "video_edits.videoId", + sourceKey: "video_edits.sourceKey", + }, + videos: { + id: "videos.id", + ownerId: "videos.ownerId", + metadata: "videos.metadata", + updatedAt: "videos.updatedAt", + }, +})); +vi.mock("drizzle-orm", () => ({ + and: (...conditions: unknown[]) => conditions, + eq: (field: unknown, value: unknown) => ({ field, value }), + sql: (strings: TemplateStringsArray, ...values: unknown[]) => ({ + strings, + values, + }), +})); +vi.mock("@cap/database", () => ({ + db: () => ({ + select: () => ({ + from: (table: unknown) => + typeof table === "object" && table !== null && "videoId" in table + ? { + where: async () => (state.legacyEdit ? [state.legacyEdit] : []), + } + : { + innerJoin: () => ({ + where: async () => [ + { video: state.row, owner: { id: "owner" } }, + ], + }), + }, + }), + update: () => ({ + set: (value: unknown) => { + state.updates.push(value); + return { where: async () => [{ affectedRows: 1 }] }; + }, + }), + }), +})); +vi.mock("assemblyai", () => ({ + AssemblyAI: class { + transcripts = { transcribe: state.transcribe }; + }, +})); +vi.mock("../lib/edit-transcript-storage", () => ({ + encryptEditTranscriptObject: (value: string) => value, + decryptEditTranscriptObject: (value: string) => value, +})); +vi.mock("../lib/workflow-runtime", () => ({ + runWorkflowPromise: (value: unknown) => Promise.resolve(value), +})); +vi.mock("../lib/video-storage", () => ({ + decodeStorageVideo: (value: unknown) => value, +})); +vi.mock("../lib/editor-session", () => ({ + requestMediaEditor: ( + path: string, + init?: RequestInit, + _timeoutMs?: number, + workerId?: string, + ) => { + state.workerRequests.push({ + path, + body: JSON.parse(String(init?.body)), + workerId, + }); + const response = new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([1, 2, 3])); + controller.close(); + }, + }), + { + headers: { + "Content-Type": "audio/mp4", + "Content-Length": "3", + }, + }, + ); + return { + pipe: (runner: (value: Response) => unknown) => runner(response), + }; + }, +})); +vi.mock("@cap/web-backend/src/Storage/index", async () => { + const { Option } = await import("effect"); + const pipe = (value: unknown) => ({ + pipe: (runner: (input: unknown) => unknown) => runner(value), + }); + return { + Storage: { + getAccessForVideo: () => + pipe([ + { + getObject: (key: string) => + pipe(Option.fromNullable(state.objects.get(key))), + headObject: (key: string) => { + const details = state.row?.metadata as VideoMetadata | undefined; + const asset = details?.webEditorVideos?.items.find( + (candidate) => candidate.key === key, + ); + return pipe({ + ContentLength: asset?.size ?? 4000, + ETag: asset?.objectIdentity ?? state.baseEtag, + }); + }, + getInternalSignedObjectUrl: (key: string) => { + state.urls.push(key); + return pipe(`https://objects.example/${key}`); + }, + putObject: (key: string, value: string) => { + state.writes.push({ key, value }); + state.objects.set(key, value); + return pipe({}); + }, + }, + ]), + }, + }; +}); + +function metadata(): VideoMetadata { + const path = "content/videos/00000000-0000-4000-8000-000000000001.webm"; + return { + editorSources: { + version: 1, + display: { + key: "owner/video/display.webm", + contentType: "video/webm", + size: 4000, + objectIdentity: JSON.stringify("base"), + }, + }, + webEditorVideos: { + version: 1, + items: [ + { + key: "owner/video/editor-assets/videos/00000000-0000-4000-8000-000000000001.webm", + path, + name: "Added clip", + contentType: "video/webm", + size: 3000, + objectIdentity: JSON.stringify("clip"), + }, + ], + }, + webEditorClips: { + version: 1, + items: [{ displayPath: path, duration: 5, fps: 30, hasAudio: true }], + }, + }; +} + +function baseTranscript(): EditTranscript { + return { + version: 3, + speechModelUsed: "universal-3-5-pro", + durationMs: 10_000, + languageCode: "en", + words: [ + { + id: "base-word", + text: "Original", + startMs: 1000, + endMs: 1400, + confidence: null, + speaker: null, + channel: null, + }, + ], + }; +} + +function prepareWithAddedClips( + count: number, + language: "auto" | "es" = "auto", +): EditorCaptionSourcePlan { + const details = metadata(); + const videos = details.webEditorVideos?.items; + const clips = details.webEditorClips?.items; + const firstVideo = videos?.[0]; + const firstClip = clips?.[0]; + if (!videos || !clips || !firstVideo || !firstClip) { + throw new Error("Missing caption clip fixtures"); + } + for (let index = 2; index <= count; index++) { + const id = `00000000-0000-4000-8000-${String(index).padStart(12, "0")}`; + const path = `content/videos/${id}.webm`; + videos.push({ + ...firstVideo, + key: `owner/video/editor-assets/videos/${id}.webm`, + path, + name: `Added clip ${index}`, + }); + clips.push({ ...firstClip, displayPath: path }); + } + const plan = buildEditorCaptionSourcePlan( + "owner", + "video", + details, + { + recordings: { + segments: [ + { display: { duration: 10 }, camera: null, mic: null }, + ...clips.map(() => ({ + display: { duration: 5 }, + camera: null, + mic: null, + })), + ], + }, + }, + "", + language, + ); + if (!plan) throw new Error("Caption plan is invalid"); + details.webEditorCaptionJob = { + status: "processing", + requestId: "request-1", + sourceHash: plan.hash, + requestedAt: new Date().toISOString(), + }; + state.row = { + id: "video", + ownerId: "owner", + source: { type: "webMP4", outputKey: "owner/video/result.mp4" }, + transcriptionStatus: "COMPLETE", + metadata: details, + }; + state.objects.set( + getEditTranscriptObjectKey("owner", "video"), + serializeEditTranscript(baseTranscript()), + ); + return plan; +} + +function prepare(language: "auto" | "es" = "auto"): EditorCaptionSourcePlan { + return prepareWithAddedClips(1, language); +} + +function prepareLegacy(language: "auto" | "es" = "es") { + prepare(language); + const details = state.row?.metadata as VideoMetadata | undefined; + if (!details?.webEditorClips) + throw new Error("Missing migrated caption clips"); + const sourceKey = "owner/video/source/original.mp4"; + state.legacyEdit = { sourceKey }; + state.baseEtag = JSON.stringify("original"); + const plan = buildEditorCaptionSourcePlan( + "owner", + "video", + details, + { + recordings: { + segments: [ + { display: { duration: 10 }, camera: null, mic: null }, + { display: { duration: 5 }, camera: null, mic: null }, + ], + }, + }, + "", + language, + { key: sourceKey, size: 4000, objectIdentity: JSON.stringify("original") }, + ); + if (!plan || !details.webEditorCaptionJob) { + throw new Error("Missing migrated caption plan"); + } + details.webEditorCaptionJob.sourceHash = plan.hash; + return plan; +} + +function prepareWithCap(): EditorCaptionSourcePlan { + const details = metadata(); + const capId = "9a2aa734-d076-42c0-8a8a-68db1d2d2a1e"; + const capPath = `content/imports/${capId}.capbundle`; + const clipPath = details.webEditorClips?.items[0]?.displayPath; + if (!clipPath) throw new Error("Missing MP4 caption fixture"); + details.webEditorVideos?.items.push({ + key: `owner/video/editor-assets/recordings/${capId}.capbundle`, + path: capPath, + name: "Studio recording", + contentType: CAP_BUNDLE_CONTENT_TYPE, + size: 8000, + objectIdentity: JSON.stringify("cap"), + }); + details.webEditorImports = { + version: 1, + items: [ + { kind: "cap", path: capPath, clipCount: 2 }, + { kind: "clip", path: clipPath }, + ], + }; + const plan = buildEditorCaptionSourcePlan( + "owner", + "video", + details, + { + recordings: { + segments: [ + { display: { duration: 10 }, camera: null, mic: null }, + { + display: { duration: 3 }, + camera: null, + mic: { duration: 3 }, + }, + { + display: { duration: 2 }, + camera: null, + mic: null, + system_audio: { duration: 2 }, + }, + { display: { duration: 5 }, camera: null, mic: null }, + ], + }, + }, + "worker-a", + ); + if (!plan) throw new Error("Cap caption plan is invalid"); + details.webEditorCaptionJob = { + status: "processing", + requestId: "request-1", + sourceHash: plan.hash, + requestedAt: new Date().toISOString(), + }; + state.row = { + id: "video", + ownerId: "owner", + source: { type: "webMP4", outputKey: "owner/video/result.mp4" }, + transcriptionStatus: "COMPLETE", + metadata: details, + }; + state.objects.set( + getEditTranscriptObjectKey("owner", "video"), + serializeEditTranscript(baseTranscript()), + ); + const clipSource = plan.sources[2]; + if (!clipSource) throw new Error("Missing MP4 caption source"); + state.objects.set( + clipSource.transcriptKey, + serializeEditTranscript({ + ...baseTranscript(), + durationMs: 5000, + words: [], + }), + ); + return plan; +} + +afterEach(() => { + state.row = null; + state.legacyEdit = null; + state.isPro = true; + state.baseEtag = JSON.stringify("base"); + state.objects.clear(); + state.writes.length = 0; + state.urls.length = 0; + state.workerRequests.length = 0; + state.updates.length = 0; + state.transcribe.mockReset(); +}); + +test("added clip uses the same AssemblyAI model options and reuses share-link words", async () => { + const plan = prepare(); + state.transcribe.mockResolvedValue({ + status: "completed", + words: [{ text: "Added", start: 500, end: 900 }], + speech_model_used: "universal-3-5-pro", + language_code: "en", + }); + expect( + await transcribeWebEditorCaptionsWorkflow({ plan, requestId: "request-1" }), + ).toEqual({ success: true }); + expect(state.transcribe).toHaveBeenCalledOnce(); + expect(state.transcribe.mock.calls[0]?.[0]).toMatchObject({ + audio: `https://objects.example/${plan.sources[1]?.key}`, + speech_models: ["universal-3-5-pro", "universal-2"], + language_detection: true, + disfluencies: true, + }); + expect(state.urls).toEqual([plan.sources[1]?.key]); + const combined = state.objects.get(plan.combinedKey); + expect( + JSON.parse(combined ?? "{}").words.map( + (word: { text: string; startMs: number }) => [word.text, word.startMs], + ), + ).toEqual([ + ["Original", 1000], + ["Added", 10_500], + ]); +}); + +test("Spanish caption generation uses AssemblyAI language_code without reusing auto share words", async () => { + const plan = prepare("es"); + state.transcribe.mockResolvedValue({ + status: "completed", + words: [{ text: "Hola", start: 500, end: 900 }], + speech_model_used: "universal-3-5-pro", + language_code: "es", + }); + expect( + await transcribeWebEditorCaptionsWorkflow({ plan, requestId: "request-1" }), + ).toEqual({ success: true }); + expect(state.transcribe).toHaveBeenCalledTimes(2); + for (const [index, call] of state.transcribe.mock.calls.entries()) { + expect(call[0]).toMatchObject({ + audio: `https://objects.example/${plan.sources[index]?.key}`, + speech_models: ["universal-3-5-pro", "universal-2"], + language_code: "es", + }); + expect(call[0]).not.toHaveProperty("language_detection"); + } + expect( + JSON.parse(state.objects.get(plan.combinedKey) ?? "{}").words.map( + (word: { text: string; startMs: number }) => [word.text, word.startMs], + ), + ).toEqual([ + ["Hola", 500], + ["Hola", 10_500], + ]); +}); + +test("a migrated trim captions its preserved MP4 through AssemblyAI", async () => { + const plan = prepareLegacy(); + state.transcribe.mockResolvedValue({ + status: "completed", + words: [{ text: "Hola", start: 500, end: 900 }], + speech_model_used: "universal-3-5-pro", + language_code: "es", + }); + expect( + await transcribeWebEditorCaptionsWorkflow({ plan, requestId: "request-1" }), + ).toEqual({ success: true }); + expect(state.urls).toEqual(plan.sources.map((source) => source.key)); + expect(state.urls[0]).toBe("owner/video/source/original.mp4"); + expect(state.transcribe).toHaveBeenCalledTimes(2); + expect(state.transcribe.mock.calls[0]?.[0]).toMatchObject({ + language_code: "es", + }); +}); + +test("a changed preserved MP4 stops migrated captions before AssemblyAI", async () => { + const plan = prepareLegacy(); + state.baseEtag = JSON.stringify("replacement"); + expect( + await transcribeWebEditorCaptionsWorkflow({ plan, requestId: "request-1" }), + ).toEqual({ success: false }); + expect(state.transcribe).not.toHaveBeenCalled(); +}); + +test("Studio Cap voice audio streams through AssemblyAI without local model calls", async () => { + const plan = prepareWithCap(); + state.transcribe.mockResolvedValue({ + status: "completed", + words: [{ text: "Cap", start: 500, end: 900 }], + speech_model_used: "universal-3-5-pro", + language_code: "en", + }); + expect( + await transcribeWebEditorCaptionsWorkflow({ plan, requestId: "request-1" }), + ).toEqual({ success: true }); + expect(state.workerRequests).toHaveLength(1); + expect(state.workerRequests[0]).toMatchObject({ + path: "/editor/caption-cap-audio", + workerId: "worker-a", + body: { + asset: { + path: plan.sources[1]?.cap?.path, + name: "Studio recording", + size: 8000, + contentType: CAP_BUNDLE_CONTENT_TYPE, + objectIdentity: JSON.stringify("cap"), + }, + segments: plan.sources[1]?.cap?.segments, + }, + }); + expect(state.transcribe).toHaveBeenCalledOnce(); + expect(state.transcribe.mock.calls[0]?.[0]).toMatchObject({ + speech_models: ["universal-3-5-pro", "universal-2"], + language_detection: true, + disfluencies: true, + }); + expect(state.transcribe.mock.calls[0]?.[0].audio).toBeInstanceOf( + ReadableStream, + ); + expect( + JSON.parse(state.objects.get(plan.combinedKey) ?? "{}").words.map( + (word: { text: string; startMs: number }) => [word.text, word.startMs], + ), + ).toEqual([ + ["Original", 1000], + ["Cap", 10_500], + ]); +}); + +test("expired Pro entitlement stops a Cap caption mix before provider calls", async () => { + const plan = prepareWithCap(); + state.isPro = false; + expect( + await transcribeWebEditorCaptionsWorkflow({ plan, requestId: "request-1" }), + ).toEqual({ success: false }); + expect(state.workerRequests).toHaveLength(0); + expect(state.transcribe).not.toHaveBeenCalled(); +}); + +test("unchanged raw source and added clip keep their cached captions after a rendered reupload", async () => { + const plan = prepare(); + state.transcribe.mockResolvedValue({ + status: "completed", + words: [{ text: "Added", start: 500, end: 900 }], + }); + expect( + await transcribeWebEditorCaptionsWorkflow({ plan, requestId: "request-1" }), + ).toEqual({ success: true }); + expect( + JSON.parse(state.objects.get(plan.sources[0]?.transcriptKey ?? "") ?? "{}") + .words[0]?.text, + ).toBe("Original"); + if (!state.row) throw new Error("Missing video row"); + state.row.source = { + type: "webMP4", + outputKey: + "owner/video/.recording/outputs/reupload-00000000-0000-4000-8000-000000000001/result.mp4", + }; + state.row.transcriptionStatus = "NO_AUDIO"; + state.objects.delete(plan.combinedKey); + state.urls.length = 0; + state.transcribe.mockReset(); + expect( + await transcribeWebEditorCaptionsWorkflow({ plan, requestId: "request-1" }), + ).toEqual({ success: true }); + expect(state.transcribe).not.toHaveBeenCalled(); + expect(state.urls).toHaveLength(0); + expect( + JSON.parse(state.objects.get(plan.combinedKey) ?? "{}").words.map( + (word: { text: string }) => word.text, + ), + ).toEqual(["Original", "Added"]); +}); + +test("changed raw media metadata stops the job before any provider call", async () => { + const plan = prepare(); + if (!state.row) throw new Error("Missing video row"); + const details = state.row.metadata as VideoMetadata; + if (!details.editorSources) throw new Error("Missing source metadata"); + details.editorSources.display.objectIdentity = JSON.stringify("changed"); + expect( + await transcribeWebEditorCaptionsWorkflow({ plan, requestId: "request-1" }), + ).toEqual({ success: false }); + expect(state.transcribe).not.toHaveBeenCalled(); + expect(state.writes).toHaveLength(0); +}); + +test("changed raw object identity stops the job before added clips incur provider calls", async () => { + const plan = prepare(); + state.baseEtag = JSON.stringify("changed"); + state.transcribe.mockResolvedValue({ + status: "completed", + words: [{ text: "Added", start: 500, end: 900 }], + }); + expect( + await transcribeWebEditorCaptionsWorkflow({ plan, requestId: "request-1" }), + ).toEqual({ success: false }); + expect(state.transcribe).not.toHaveBeenCalled(); + expect(state.writes).toHaveLength(0); +}); + +test("an expired Pro entitlement prevents a paid provider call", async () => { + const plan = prepare(); + state.isPro = false; + expect( + await transcribeWebEditorCaptionsWorkflow({ plan, requestId: "request-1" }), + ).toEqual({ success: false }); + expect(state.transcribe).not.toHaveBeenCalled(); + expect(state.writes).toHaveLength(0); +}); + +test("reopened editor captions transcribe the raw source after a rendered reupload", async () => { + const plan = prepare(); + if (!state.row) throw new Error("Missing video row"); + state.row.source = { + type: "webMP4", + outputKey: + "owner/video/.recording/outputs/reupload-00000000-0000-4000-8000-000000000001/result.mp4", + }; + state.row.transcriptionStatus = "NO_AUDIO"; + state.transcribe.mockImplementation(async (options: { audio: string }) => ({ + status: "completed", + words: [ + { + text: + options.audio === `https://objects.example/${plan.sources[0]?.key}` + ? "Raw" + : "Added", + start: 500, + end: 900, + }, + ], + })); + expect( + await transcribeWebEditorCaptionsWorkflow({ plan, requestId: "request-1" }), + ).toEqual({ success: true }); + expect(state.urls).toEqual(plan.sources.map((source) => source.key)); + expect( + JSON.parse(state.objects.get(plan.combinedKey) ?? "{}").words.map( + (word: { text: string }) => word.text, + ), + ).toEqual(["Raw", "Added"]); +}); + +test("reuploaded sources transcribe two at a time and keep source order when calls finish together", async () => { + const plan = prepareWithAddedClips(4); + if (!state.row) throw new Error("Missing video row"); + state.row.source = { + type: "webMP4", + outputKey: + "owner/video/.recording/outputs/reupload-00000000-0000-4000-8000-000000000001/result.mp4", + }; + state.row.transcriptionStatus = "NO_AUDIO"; + let active = 0; + let peak = 0; + const release: Array<() => void> = []; + state.transcribe.mockImplementation(async (options: { audio: string }) => { + active++; + peak = Math.max(peak, active); + await new Promise((resolve) => release.push(resolve)); + active--; + const index = plan.sources.findIndex( + (source) => options.audio === `https://objects.example/${source.key}`, + ); + return { + status: "completed", + words: [{ text: `Clip ${index}`, start: 500, end: 900 }], + }; + }); + const running = transcribeWebEditorCaptionsWorkflow({ + plan, + requestId: "request-1", + }); + await vi.waitFor(() => expect(release).toHaveLength(1)); + expect(state.transcribe).toHaveBeenCalledTimes(1); + for (const finish of release.splice(0, 1)) finish(); + await vi.waitFor(() => expect(release).toHaveLength(2)); + expect(state.transcribe).toHaveBeenCalledTimes(3); + for (const finish of release.splice(0, 2)) finish(); + await vi.waitFor(() => expect(release).toHaveLength(2)); + expect(state.transcribe).toHaveBeenCalledTimes(5); + for (const finish of release.splice(0, 2)) finish(); + expect(await running).toEqual({ success: true }); + expect(peak).toBe(2); + expect(active).toBe(0); + expect( + JSON.parse(state.objects.get(plan.combinedKey) ?? "{}").words.map( + (word: { text: string; startMs: number }) => [word.text, word.startMs], + ), + ).toEqual([ + ["Clip 0", 500], + ["Clip 1", 10_500], + ["Clip 2", 15_500], + ["Clip 3", 20_500], + ["Clip 4", 25_500], + ]); +}); + +test("a failed clip waits for its sibling provider call before finishing the job", async () => { + const plan = prepareWithAddedClips(2); + const pending = { finish: null as ((value: unknown) => void) | null }; + state.transcribe.mockImplementation((options: { audio: string }) => { + if (options.audio === `https://objects.example/${plan.sources[1]?.key}`) { + return Promise.reject(new Error("Provider failed")); + } + return new Promise((resolve) => { + pending.finish = resolve; + }); + }); + const running = transcribeWebEditorCaptionsWorkflow({ + plan, + requestId: "request-1", + }); + await vi.waitFor(() => expect(pending.finish).not.toBeNull()); + expect(state.updates).toHaveLength(0); + if (!pending.finish) throw new Error("Second provider call did not start"); + pending.finish({ + status: "completed", + words: [{ text: "Sibling", start: 500, end: 900 }], + }); + expect(await running).toEqual({ success: false }); + expect(state.updates).toHaveLength(1); + expect(state.objects.has(plan.sources[2]?.transcriptKey ?? "")).toBe(true); + expect(state.objects.has(plan.combinedKey)).toBe(false); +}); + +test("a reuploaded single-clip recording ignores words from the rendered share page", async () => { + const details = metadata(); + delete details.webEditorClips; + delete details.webEditorVideos; + const plan = buildEditorCaptionSourcePlan("owner", "video", details, { + recordings: { + segments: [{ display: { duration: 10 }, camera: null, mic: null }], + }, + }); + if (!plan) throw new Error("Missing single source caption plan"); + details.webEditorCaptionJob = { + status: "processing", + requestId: "request-1", + sourceHash: plan.hash, + requestedAt: new Date().toISOString(), + }; + state.row = { + id: "video", + ownerId: "owner", + source: { + type: "webMP4", + outputKey: + "owner/video/.recording/outputs/reupload-00000000-0000-4000-8000-000000000001/result.mp4", + }, + transcriptionStatus: "COMPLETE", + metadata: details, + }; + state.objects.set( + getEditTranscriptObjectKey("owner", "video"), + serializeEditTranscript(baseTranscript()), + ); + state.transcribe.mockResolvedValue({ + status: "completed", + words: [{ text: "Source", start: 500, end: 900 }], + }); + expect( + await transcribeWebEditorCaptionsWorkflow({ plan, requestId: "request-1" }), + ).toEqual({ success: true }); + expect(state.transcribe).toHaveBeenCalledOnce(); + expect( + JSON.parse(state.objects.get(plan.combinedKey) ?? "{}").words[0]?.text, + ).toBe("Source"); +}); + +test("no spoken audio is cached so later clip edits cannot repay for silence", async () => { + const plan = prepare(); + state.transcribe.mockRejectedValue(new Error("no spoken audio")); + expect( + await transcribeWebEditorCaptionsWorkflow({ plan, requestId: "request-1" }), + ).toEqual({ success: true }); + expect( + JSON.parse(state.objects.get(plan.sources[1]?.transcriptKey ?? "") ?? "{}") + .words, + ).toEqual([]); + expect( + JSON.parse(state.objects.get(plan.combinedKey) ?? "{}").words, + ).toHaveLength(1); +}); diff --git a/apps/web/__tests__/editor-captions.test.ts b/apps/web/__tests__/editor-captions.test.ts new file mode 100644 index 00000000000..e4427ae2e61 --- /dev/null +++ b/apps/web/__tests__/editor-captions.test.ts @@ -0,0 +1,84 @@ +import { expect, test } from "vitest"; +import type { EditTranscript } from "../lib/edit-transcript"; +import { editTranscriptToEditorCaptions } from "../lib/editor-captions"; + +function transcript( + words: Array<{ text: string; startMs: number; endMs: number }>, +): EditTranscript { + return { + version: 3, + speechModelUsed: "universal-3-5-pro", + durationMs: 60_000, + languageCode: "en", + words: words.map((word, index) => ({ + id: `word-${index}`, + confidence: null, + speaker: null, + channel: null, + ...word, + })), + }; +} + +test("AssemblyAI source words become desktop-timed captions without filler or stretched words", () => { + const result = editTranscriptToEditorCaptions( + transcript([ + { text: "This", startMs: 100, endMs: 300 }, + { text: "is", startMs: 310, endMs: 490 }, + { text: "um", startMs: 500, endMs: 700 }, + { text: "a", startMs: 710, endMs: 800 }, + { text: "test", startMs: 810, endMs: 1000 }, + { text: ",", startMs: 1000, endMs: 1020 }, + { text: "of", startMs: 1100, endMs: 1200 }, + { text: "captions", startMs: 1300, endMs: 18_000 }, + ]), + ); + expect(result.segments).toHaveLength(1); + expect(result.segments[0]?.text).toBe("This is a test, of captions"); + expect(result.segments[0]?.start).toBe(0.1); + expect(result.segments[0]?.end).toBe(3.8); + expect(result.segments[0]?.words).toHaveLength(6); +}); + +test("caption chunking follows the desktop six-to-eight-word boundary rules", () => { + const words = [ + "one", + "two", + "three", + "four", + "five", + "of", + "seven", + "eight", + "nine", + "ten", + "eleven", + "twelve", + ]; + const result = editTranscriptToEditorCaptions( + transcript( + words.map((text, index) => ({ + text, + startMs: index * 250, + endMs: index * 250 + 200, + })), + ), + ); + expect(result.segments.map((segment) => segment.words.length)).toEqual([ + 7, 5, + ]); + expect(result.segments[0]?.words[6]?.text).toBe("seven"); + expect(result.segments[1]?.start).toBe(1.75); +}); + +test("late punctuation cannot stretch a spoken word beyond desktop caption timing", () => { + const result = editTranscriptToEditorCaptions( + transcript([ + { text: "Hello", startMs: 100, endMs: 300 }, + { text: ".", startMs: 4000, endMs: 8000 }, + ]), + ); + expect(result.segments[0]?.text).toBe("Hello."); + expect(result.segments[0]?.words[0]?.end).toBe(2.6); + expect(result.segments[0]?.end).toBe(2.6); +}); diff --git a/apps/web/__tests__/editor-image-file.test.ts b/apps/web/__tests__/editor-image-file.test.ts new file mode 100644 index 00000000000..d99dce8d82c --- /dev/null +++ b/apps/web/__tests__/editor-image-file.test.ts @@ -0,0 +1,267 @@ +import type { HttpApi } from "@effect/platform"; +import { + afterAll, + afterEach, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; + +const videoId = "123e4567-e89b-42d3-a456-426614174000"; +const sessionId = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; +const imagePath = "content/images/bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb.png"; +const imageKey = `owner/${videoId}/editor-assets/images/${imagePath.slice("content/images/".length)}`; +const imageIdentity = JSON.stringify("original-image"); + +const mocks = vi.hoisted(() => ({ + assetSize: 1024, + assetIdentity: JSON.stringify("original-image") as string | null, + storedSize: 1024, + storedIdentity: JSON.stringify("original-image"), + videoVisible: true, + sessionAllowed: true, + authenticated: true, + head: vi.fn(), + sign: vi.fn(), + dispose: async () => {}, +})); + +vi.mock("@/lib/editor-session", async () => { + const { HttpApiError } = await import("@effect/platform"); + const { Effect } = await import("effect"); + return { + loadEligibleEditorVideo: (id: string) => + Effect.suspend(() => + mocks.videoVisible + ? Effect.succeed({ + id, + ownerId: "owner", + metadata: { + webEditorAssets: { + version: 1, + items: [ + { + kind: "image", + path: imagePath, + key: imageKey, + contentType: "image/png", + size: mocks.assetSize, + objectIdentity: mocks.assetIdentity, + }, + ], + }, + }, + }) + : Effect.fail(new HttpApiError.NotFound()), + ), + verifyOwnedEditorSession: () => + Effect.suspend(() => + mocks.sessionAllowed + ? Effect.succeed(sessionId) + : Effect.fail(new HttpApiError.NotFound()), + ), + }; +}); + +vi.mock("@/lib/video-storage", () => ({ + decodeStorageVideo: (video: unknown) => video, +})); + +vi.mock("@cap/web-backend", async () => { + const { Effect } = await import("effect"); + const bucket = { + headObject: (key: string) => + Effect.sync(() => { + mocks.head(key); + return { + ContentLength: mocks.storedSize, + ETag: mocks.storedIdentity, + }; + }), + getSignedObjectUrl: (key: string, options: { expiresIn: number }) => + Effect.sync(() => { + mocks.sign(key, options); + return `https://media.example.com/${key}`; + }), + }; + return { + Storage: { + getAccessForVideo: () => Effect.succeed([bucket, false] as const), + }, + }; +}); + +vi.mock("@/lib/server", async () => { + const { HttpAuthMiddleware, Organisation, User } = await import( + "@cap/web-domain" + ); + const { HttpApiBuilder, HttpApiError, HttpServer } = await import( + "@effect/platform" + ); + const { Effect, Layer, Option } = await import("effect"); + const auth = Layer.succeed( + HttpAuthMiddleware, + Effect.suspend(() => + mocks.authenticated + ? Effect.succeed({ + id: User.UserId.make("owner"), + email: "owner@example.com", + activeOrganizationId: Organisation.OrganisationId.make("org"), + iconUrlOrKey: Option.none(), + }) + : Effect.fail(new HttpApiError.Unauthorized()), + ), + ); + return { + apiToHandler: ( + api: import("effect").Layer.Layer< + HttpApi.Api, + never, + import("@cap/web-domain").HttpAuthMiddleware + >, + ) => { + const handler = api.pipe( + Layer.provideMerge(auth), + Layer.merge(HttpServer.layerContext), + HttpApiBuilder.toWebHandler, + ); + mocks.dispose = handler.dispose; + return handler.handler; + }, + }; +}); + +import { GET } from "@/app/api/editor/sessions/[id]/file/route"; + +function request(path = imagePath, raw = false) { + const params = new URLSearchParams({ videoId, path }); + if (raw) params.set("raw", "1"); + return GET( + new Request( + `https://cap.so/api/editor/sessions/${sessionId}/file?${params}`, + ), + ); +} + +describe("web editor image file redirect", () => { + beforeEach(() => { + mocks.assetSize = 1024; + mocks.assetIdentity = imageIdentity; + mocks.storedSize = 1024; + mocks.storedIdentity = imageIdentity; + mocks.videoVisible = true; + mocks.sessionAllowed = true; + mocks.authenticated = true; + mocks.head.mockClear(); + mocks.sign.mockClear(); + }); + + afterAll(() => mocks.dispose()); + afterEach(() => vi.unstubAllGlobals()); + + it("redirects a current owned image with a private no-store response", async () => { + const response = await request(); + expect(response.status).toBe(302); + expect(response.headers.get("location")).toBe( + `https://media.example.com/${imageKey}`, + ); + expect(response.headers.get("cache-control")).toBe("private, no-store"); + expect(mocks.head).toHaveBeenCalledWith(imageKey); + expect(mocks.sign).toHaveBeenCalledWith(imageKey, { expiresIn: 300 }); + }); + + it("streams verified image bytes to the same-origin preset store", async () => { + const bytes = new Uint8Array([137, 80, 78, 71]); + mocks.assetSize = bytes.length; + mocks.storedSize = bytes.length; + const download = vi.fn( + async () => + new Response(bytes, { + headers: { + "Content-Type": "image/png", + "Content-Length": String(bytes.length), + ETag: imageIdentity, + }, + }), + ); + vi.stubGlobal("fetch", download); + const response = await request(imagePath, true); + expect(response.status).toBe(200); + expect(response.headers.get("Content-Type")).toBe("image/png"); + expect(response.headers.get("Content-Length")).toBe(String(bytes.length)); + expect(response.headers.get("Cache-Control")).toBe("private, no-store"); + expect(response.headers.get("X-Content-Type-Options")).toBe("nosniff"); + expect(new Uint8Array(await response.arrayBuffer())).toEqual(bytes); + expect(download).toHaveBeenCalledWith( + new URL(`https://media.example.com/${imageKey}`), + expect.objectContaining({ + cache: "no-store", + headers: { "If-Match": imageIdentity }, + }), + ); + }); + + it("refuses a replaced source during preset streaming", async () => { + mocks.assetSize = 4; + mocks.storedSize = 4; + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response(new Uint8Array([137, 80, 78, 71]), { + headers: { + "Content-Type": "image/png", + "Content-Length": "4", + ETag: JSON.stringify("replacement-image"), + }, + }), + ), + ); + expect((await request(imagePath, true)).status).toBe(503); + }); + + it("refuses an image replaced with different bytes of the same size", async () => { + mocks.storedIdentity = JSON.stringify("replacement-image"); + expect((await request()).status).toBe(503); + expect(mocks.head).toHaveBeenCalledWith(imageKey); + expect(mocks.sign).not.toHaveBeenCalled(); + }); + + it("refuses a truncated image before signing its URL", async () => { + mocks.storedSize = 512; + expect((await request()).status).toBe(503); + expect(mocks.sign).not.toHaveBeenCalled(); + }); + + it("refuses an asset without a saved identity", async () => { + mocks.assetIdentity = null; + expect((await request()).status).toBe(404); + expect(mocks.head).not.toHaveBeenCalled(); + expect(mocks.sign).not.toHaveBeenCalled(); + }); + + it("checks the session and path before probing storage", async () => { + mocks.sessionAllowed = false; + expect((await request()).status).toBe(404); + mocks.sessionAllowed = true; + expect((await request("content/images/../outside.png")).status).toBe(404); + expect(mocks.head).not.toHaveBeenCalled(); + expect(mocks.sign).not.toHaveBeenCalled(); + }); + + it("checks video visibility before probing storage", async () => { + mocks.videoVisible = false; + expect((await request()).status).toBe(404); + expect(mocks.head).not.toHaveBeenCalled(); + expect(mocks.sign).not.toHaveBeenCalled(); + }); + + it("requires authentication before probing storage", async () => { + mocks.authenticated = false; + expect((await request()).status).toBe(401); + expect(mocks.head).not.toHaveBeenCalled(); + expect(mocks.sign).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/__tests__/editor-legacy-sources.test.ts b/apps/web/__tests__/editor-legacy-sources.test.ts new file mode 100644 index 00000000000..e935e5e8b53 --- /dev/null +++ b/apps/web/__tests__/editor-legacy-sources.test.ts @@ -0,0 +1,255 @@ +import { Database, Storage } from "@cap/web-backend"; +import { Effect } from "effect"; +import { afterEach, expect, test, vi } from "vitest"; +import { getSignedEditorSources } from "../lib/editor-session"; + +const storage = vi.hoisted(() => ({ + head: vi.fn(), + signed: vi.fn(), + accessOptions: vi.fn(), + legacyEdit: null as null | { + sourceKey: string; + editSpec: { + version: 1; + sourceDuration: number; + keepRanges: Array<{ start: number; end: number }>; + }; + }, +})); + +vi.mock("@/lib/video-storage", () => ({ + decodeStorageVideo: () => ({ id: "video" }), +})); + +type EditorVideo = Parameters[0]; + +function video(metadata: unknown): EditorVideo { + return { + id: "video", + ownerId: "owner", + name: "Older recording", + fps: 30, + source: { type: "webMP4" }, + metadata, + } as EditorVideo; +} + +function signedSources( + editorVideo: EditorVideo, + audience: "worker" | "browser" = "worker", +) { + const access = { + headObject: (key: string) => Effect.succeed(storage.head(key)), + getInternalSignedObjectUrl: (key: string) => + Effect.succeed(storage.signed(key)), + getSignedObjectUrl: (key: string) => Effect.succeed(storage.signed(key)), + }; + const service = { + getAccessForVideo: (_video: unknown, options: unknown) => { + storage.accessOptions(options); + return Effect.succeed([access]); + }, + } as unknown as Storage; + const database = Database.make({ + use: (callback) => + Effect.promise(() => + callback({ + select: () => ({ + from: () => ({ + where: async () => + storage.legacyEdit ? [storage.legacyEdit] : [], + }), + }), + } as unknown as Parameters[0]), + ), + }); + return Effect.runPromise( + getSignedEditorSources(editorVideo, audience).pipe( + Effect.provideService(Storage, service), + Effect.provideService(Database, database), + ), + ); +} + +afterEach(() => { + vi.clearAllMocks(); + storage.legacyEdit = null; +}); + +test("a previously trimmed recording opens the immutable original with its saved cuts", async () => { + const originalKey = "owner/video/source/original.mp4"; + storage.legacyEdit = { + sourceKey: originalKey, + editSpec: { + version: 1, + sourceDuration: 12, + keepRanges: [ + { start: 1, end: 4 }, + { start: 7, end: 11 }, + ], + }, + }; + storage.head.mockReturnValue({ + ContentLength: 4321, + ETag: JSON.stringify("original"), + }); + storage.signed.mockReturnValue("https://storage.example/original"); + const result = await signedSources(video(null)); + expect(storage.head).toHaveBeenCalledExactlyOnceWith(originalKey); + expect(storage.signed).toHaveBeenCalledExactlyOnceWith(originalKey); + expect(result.display).toMatchObject({ + url: "https://storage.example/original", + size: 4321, + objectIdentity: JSON.stringify("original"), + }); + expect(result.legacyEditSpec).toEqual(storage.legacyEdit.editSpec); + expect("camera" in result).toBe(false); +}); + +test("an older recording opens from its verified published MP4", async () => { + const key = "owner/video/result.mp4"; + storage.head.mockReturnValue({ + ContentLength: 1234, + ETag: JSON.stringify("published"), + }); + storage.signed.mockReturnValue("https://storage.example/published"); + const result = await signedSources(video(null)); + expect(storage.accessOptions).toHaveBeenCalledWith({ + resolvePublishedOutput: false, + }); + expect(storage.head).toHaveBeenCalledExactlyOnceWith(key); + expect(storage.signed).toHaveBeenCalledExactlyOnceWith(key); + expect(result.display).toEqual({ + url: "https://storage.example/published", + contentType: "video/mp4", + size: 1234, + fps: 30, + objectIdentity: JSON.stringify("published"), + }); + expect("camera" in result).toBe(false); +}); + +test("an unverified legacy output cannot be opened in the editor", async () => { + storage.head.mockReturnValue({ + ContentLength: 0, + ETag: JSON.stringify("empty"), + }); + await expect(signedSources(video(null))).rejects.toThrow(); + expect(storage.signed).not.toHaveBeenCalled(); +}); + +test("a new recording still uses its separate preserved screen and camera objects", async () => { + const displayKey = "owner/video/raw-upload.webm"; + const cameraKey = "owner/video/camera-upload.webm"; + storage.head.mockImplementation((key: string) => ({ + ContentLength: key === displayKey ? 1234 : 456, + ETag: + key === displayKey ? JSON.stringify("screen") : JSON.stringify("camera"), + })); + storage.signed.mockImplementation( + (key: string) => `https://storage.example/${key}`, + ); + const result = await signedSources( + video({ + editorSources: { + version: 1, + display: { + key: displayKey, + contentType: "video/webm", + size: 1234, + objectIdentity: JSON.stringify("screen"), + }, + camera: { + key: cameraKey, + contentType: "video/webm", + size: 456, + objectIdentity: JSON.stringify("camera"), + offsetMs: 125, + }, + }, + }), + ); + expect(storage.accessOptions).toHaveBeenCalledWith({ + resolvePublishedOutput: false, + }); + expect(storage.head.mock.calls.map(([key]) => key)).toEqual([ + displayKey, + cameraKey, + ]); + expect(result.camera).toMatchObject({ + url: `https://storage.example/${cameraKey}`, + size: 456, + offsetMs: 125, + objectIdentity: JSON.stringify("camera"), + }); +}); + +test("Free preparation keeps the video edits but removes saved Pro captions", async () => { + storage.head.mockReturnValue({ + ContentLength: 1234, + ETag: JSON.stringify("published"), + }); + storage.signed.mockReturnValue("https://storage.example/published"); + const saved = { + webEditorProject: { + version: 1, + config: { + camera: { mirror: true }, + captions: { + segments: [{ text: "Paid" }], + settings: { enabled: true, exportWithSubtitles: true }, + }, + timeline: { captionSegments: [{ text: "Paid" }] }, + }, + }, + }; + const free = await signedSources(video(saved)); + expect(free.captionsEnabled).toBe(false); + expect(free.projectConfig).toMatchObject({ + camera: { mirror: true }, + captions: { + segments: [], + settings: { enabled: false, exportWithSubtitles: false }, + }, + timeline: { captionSegments: [] }, + }); + const pro = await signedSources({ ...video(saved), captionsEnabled: true }); + expect(pro.captionsEnabled).toBe(true); + expect(pro.projectConfig).toMatchObject({ + captions: { segments: [{ text: "Paid" }] }, + }); +}); + +test("the browser plays an import's own audio but not a recording screen's", async () => { + const key = "owner/video/raw-upload.mp4"; + storage.head.mockReturnValue({ + ContentLength: 1234, + ETag: JSON.stringify("upload"), + }); + storage.signed.mockReturnValue("https://storage.example/upload"); + const display = { + key, + contentType: "video/mp4", + size: 1234, + fps: 30, + objectIdentity: JSON.stringify("upload"), + }; + const imported = await signedSources( + video({ + editorSources: { + version: 1, + display: { ...display, embeddedAudio: true }, + }, + }), + "browser", + ); + expect(imported).toMatchObject({ + displayHasAudio: true, + display: { url: "https://storage.example/upload", size: 1234, fps: 30 }, + }); + const recorded = await signedSources( + video({ editorSources: { version: 1, display } }), + "browser", + ); + expect(recorded).toMatchObject({ displayHasAudio: false }); +}); diff --git a/apps/web/__tests__/editor-preparation-client.test.ts b/apps/web/__tests__/editor-preparation-client.test.ts new file mode 100644 index 00000000000..e8ccc4e648b --- /dev/null +++ b/apps/web/__tests__/editor-preparation-client.test.ts @@ -0,0 +1,183 @@ +import { afterEach, expect, test, vi } from "vitest"; +import { startWebEditorPreparation } from "../lib/editor-preparation-client"; + +afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); +}); + +test("a busy editor retries and opens a real preparation when capacity returns", async () => { + vi.useFakeTimers(); + vi.spyOn(Math, "random").mockReturnValue(0); + const onWait = vi.fn(); + let calls = 0; + const fetcher = vi.fn(async (_url: string, init: RequestInit) => { + expect(init.signal).toBeUndefined(); + expect(JSON.parse(String(init.body))).toEqual({ videoId: "recording" }); + calls++; + return calls === 1 + ? Response.json( + { _tag: "EditorCapacityBusy", retryAfterMs: 1_000 }, + { status: 503 }, + ) + : Response.json({ id: "preparation-id", status: "preparing" }); + }); + const pending = startWebEditorPreparation( + "recording", + new AbortController().signal, + onWait, + fetcher, + ); + await vi.advanceTimersByTimeAsync(0); + expect(onWait).toHaveBeenCalledWith(true); + expect(fetcher).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1_000); + expect(await pending).toEqual({ id: "preparation-id", status: "preparing" }); + expect(onWait).toHaveBeenLastCalledWith(false); + expect(fetcher).toHaveBeenCalledTimes(2); +}); + +test("leaving while capacity is busy cancels the retry before any session starts", async () => { + vi.useFakeTimers(); + const controller = new AbortController(); + const onWait = vi.fn(); + const fetcher = vi.fn(async () => + Response.json( + { _tag: "EditorCapacityBusy", retryAfterMs: 1_000 }, + { status: 503 }, + ), + ); + const pending = startWebEditorPreparation( + "recording", + controller.signal, + onWait, + fetcher, + ); + await vi.advanceTimersByTimeAsync(0); + expect(onWait).toHaveBeenCalledWith(true); + controller.abort(); + await expect(pending).rejects.toThrow("Editor preparation was canceled"); + await vi.advanceTimersByTimeAsync(30_000); + expect(fetcher).toHaveBeenCalledTimes(1); +}); + +test("an aborted successful response still returns its id for caller cleanup", async () => { + let resolveFetch: (response: Response) => void = () => undefined; + const fetcher = vi.fn((_url: string, init: RequestInit) => { + expect(init.signal).toBeUndefined(); + return new Promise((resolve) => { + resolveFetch = resolve; + }); + }); + const controller = new AbortController(); + const onWait = vi.fn(); + const pending = startWebEditorPreparation( + "recording", + controller.signal, + onWait, + fetcher, + ); + controller.abort(); + resolveFetch( + Response.json({ id: "prepared-after-close", status: "preparing" }), + ); + expect(await pending).toEqual({ + id: "prepared-after-close", + status: "preparing", + }); + expect(onWait).not.toHaveBeenCalled(); +}); + +test("an aborted busy response does not announce waiting after the editor closes", async () => { + let resolveFetch: (response: Response) => void = () => undefined; + const fetcher = vi.fn( + () => + new Promise((resolve) => { + resolveFetch = resolve; + }), + ); + const controller = new AbortController(); + const onWait = vi.fn(); + const pending = startWebEditorPreparation( + "recording", + controller.signal, + onWait, + fetcher, + ); + controller.abort(); + resolveFetch( + Response.json( + { _tag: "EditorCapacityBusy", retryAfterMs: 1_000 }, + { status: 503 }, + ), + ); + await expect(pending).rejects.toThrow("Editor preparation was canceled"); + expect(onWait).not.toHaveBeenCalled(); + expect(fetcher).toHaveBeenCalledTimes(1); +}); + +test("a missing or unhealthy editor worker fails promptly without a busy wait", async () => { + const onWait = vi.fn(); + const fetcher = vi.fn(async () => + Response.json({ _tag: "ServiceUnavailable" }, { status: 503 }), + ); + await expect( + startWebEditorPreparation( + "recording", + new AbortController().signal, + onWait, + fetcher, + ), + ).rejects.toThrow("The editor server is unavailable right now"); + expect(fetcher).toHaveBeenCalledTimes(1); + expect(onWait).not.toHaveBeenCalled(); +}); + +test("capacity waiting has a deadline rather than retrying forever", async () => { + vi.useFakeTimers(); + vi.setSystemTime(0); + vi.spyOn(Math, "random").mockReturnValue(0); + const fetcher = vi.fn(async () => + Response.json( + { _tag: "EditorCapacityBusy", retryAfterMs: 1_000 }, + { status: 503 }, + ), + ); + const pending = startWebEditorPreparation( + "recording", + new AbortController().signal, + vi.fn(), + fetcher, + ); + const rejection = expect(pending).rejects.toThrow( + "Every editor server is busy right now", + ); + await vi.advanceTimersByTimeAsync(5 * 60 * 1_000 + 20_000); + await rejection; + expect(fetcher.mock.calls.length).toBeLessThan(30); +}); + +test("a caller with somewhere else to go can wait for capacity briefly", async () => { + vi.useFakeTimers(); + vi.setSystemTime(0); + vi.spyOn(Math, "random").mockReturnValue(0); + const fetcher = vi.fn(async () => + Response.json( + { _tag: "EditorCapacityBusy", retryAfterMs: 8_000 }, + { status: 503 }, + ), + ); + const pending = startWebEditorPreparation( + "recording", + new AbortController().signal, + vi.fn(), + fetcher, + 10_000, + ); + const rejection = expect(pending).rejects.toThrow( + "Every editor server is busy right now", + ); + await vi.advanceTimersByTimeAsync(10_000); + await rejection; + expect(fetcher).toHaveBeenCalledTimes(2); +}); diff --git a/apps/web/__tests__/editor-project-storage.test.ts b/apps/web/__tests__/editor-project-storage.test.ts new file mode 100644 index 00000000000..39aea836a95 --- /dev/null +++ b/apps/web/__tests__/editor-project-storage.test.ts @@ -0,0 +1,67 @@ +import { gzipSync } from "node:zlib"; +import { expect, test } from "vitest"; +import { editTranscriptToEditorCaptions } from "../lib/editor-captions"; +import { + decodeWebEditorProject, + encodeWebEditorProject, + MAX_WEB_EDITOR_CONFIG_BYTES, +} from "../lib/editor-project-storage"; + +test("a two-hour word-level caption project survives compact save and reopen", () => { + const words = Array.from({ length: 18_000 }, (_, index) => ({ + id: `word-${index}`, + text: index % 7 === 0 ? "recording" : "process", + startMs: index * 400, + endMs: index * 400 + 240, + confidence: null, + speaker: null, + channel: null, + })); + const captions = editTranscriptToEditorCaptions({ + version: 3, + speechModelUsed: "universal-3-pro", + durationMs: 7_200_000, + languageCode: "en", + words, + }); + const config = { + captions: { ...captions, sourceTimed: true }, + timeline: { captionSegments: captions.segments }, + }; + const { project, serialized } = encodeWebEditorProject(config); + expect(Buffer.byteLength(serialized, "utf8")).toBeGreaterThan(512 * 1024); + expect(project.version).toBe(2); + if (project.version !== 2) throw new Error("Invalid stored project version"); + expect(project.configGzipBase64.length).toBeLessThan(serialized.length); + expect(decodeWebEditorProject(project)).toEqual(config); +}); + +test("compressed project decoding rejects corruption and decompression bombs", () => { + const { project } = encodeWebEditorProject({ camera: { mirror: true } }); + if (project.version !== 2) throw new Error("Invalid stored project version"); + expect( + decodeWebEditorProject({ ...project, configGzipBase64: "not-base64" }), + ).toBeNull(); + const bomb = gzipSync(Buffer.alloc(MAX_WEB_EDITOR_CONFIG_BYTES + 1, 65)); + expect( + decodeWebEditorProject({ + ...project, + configGzipBase64: bomb.toString("base64"), + uncompressedBytes: MAX_WEB_EDITOR_CONFIG_BYTES, + }), + ).toBeNull(); + expect(() => + encodeWebEditorProject({ data: "a".repeat(MAX_WEB_EDITOR_CONFIG_BYTES) }), + ).toThrow("too large"); +}); + +test("existing uncompressed editor projects remain readable", () => { + const config = { timeline: { segments: [] } }; + expect( + decodeWebEditorProject({ + version: 1, + config, + savedAt: "2026-09-17T10:00:00.000Z", + }), + ).toEqual(config); +}); diff --git a/apps/web/__tests__/editor-session-policy.test.ts b/apps/web/__tests__/editor-session-policy.test.ts new file mode 100644 index 00000000000..e03c7246d43 --- /dev/null +++ b/apps/web/__tests__/editor-session-policy.test.ts @@ -0,0 +1,80 @@ +import { expect, test } from "vitest"; +import { + isAbandonedEditorReplacementUpload, + isActiveEditorReplacementUpload, +} from "../lib/editor-replacement-upload"; + +const key = + "owner/video/.recording/outputs/reupload-123e4567-e89b-42d3-a456-426614174000/result.mp4"; + +test("only the current owned replacement upload may keep an editor session readable", () => { + expect( + isActiveEditorReplacementUpload("owner", "video", "uploading", key), + ).toBe(true); + expect( + isActiveEditorReplacementUpload("other", "video", "uploading", key), + ).toBe(false); + expect( + isActiveEditorReplacementUpload("owner", "other", "uploading", key), + ).toBe(false); + expect( + isActiveEditorReplacementUpload("owner", "video", "processing", key), + ).toBe(false); + expect( + isActiveEditorReplacementUpload( + "owner", + "video", + "uploading", + "owner/video/.recording/outputs/edit-123e4567-e89b-42d3-a456-426614174000/result.mp4", + ), + ).toBe(false); + expect( + isActiveEditorReplacementUpload( + "owner", + "video", + "uploading", + `${key}/../camera.webm`, + ), + ).toBe(false); +}); + +test("a replacement upload that stopped moving no longer blocks editing", () => { + const minutesAgo = (minutes: number) => + new Date(Date.now() - minutes * 60 * 1000); + expect( + isAbandonedEditorReplacementUpload( + "owner", + "video", + "uploading", + key, + minutesAgo(11), + ), + ).toBe(true); + expect( + isAbandonedEditorReplacementUpload( + "owner", + "video", + "uploading", + key, + minutesAgo(2), + ), + ).toBe(false); + expect( + isAbandonedEditorReplacementUpload( + "owner", + "video", + "uploading", + "owner/video/raw-upload.mp4", + minutesAgo(60), + ), + ).toBe(false); + expect( + isAbandonedEditorReplacementUpload( + "owner", + "video", + "uploading", + key, + null, + ), + ).toBe(false); +}); diff --git a/apps/web/__tests__/editor-video-upload.test.ts b/apps/web/__tests__/editor-video-upload.test.ts new file mode 100644 index 00000000000..bf18bebe2f9 --- /dev/null +++ b/apps/web/__tests__/editor-video-upload.test.ts @@ -0,0 +1,94 @@ +import type { VideoMetadata } from "@cap/database/types"; +import { describe, expect, it } from "vitest"; +import { + createEditorVideoLocation, + editorVideoExtension, + editorVideoUploadMatches, + validateEditorVideoParts, + validEditorVideoAsset, +} from "../lib/editor-video-upload"; + +const ownerId = "owner"; +const videoId = "video"; + +describe("editor video upload boundaries", () => { + it("accepts supported videos with a visible basename and matching MIME type", () => { + expect(editorVideoExtension("demo.MP4", 100, "video/mp4")).toBe("mp4"); + expect(editorVideoExtension(".mp4", 100, "video/mp4")).toBeNull(); + expect(editorVideoExtension(" .mp4", 100, "video/mp4")).toBeNull(); + expect( + editorVideoExtension("folder/demo.mp4", 100, "video/mp4"), + ).toBeNull(); + expect(editorVideoExtension("demo.mp4", 100, "video/webm")).toBeNull(); + }); + + it("binds saved assets and pending uploads to the recording and storage provider", () => { + const location = createEditorVideoLocation(ownerId, videoId, "mov"); + const asset = { + ...location, + name: "demo", + contentType: "video/quicktime", + size: 100, + objectIdentity: "etag", + }; + expect(validEditorVideoAsset(asset, ownerId, videoId)).toBe(true); + expect(validEditorVideoAsset(asset, "other", videoId)).toBe(false); + const pending: NonNullable = { + version: 1, + sessionId: "session", + ...location, + fileName: "demo.mov", + size: 100, + contentType: "video/quicktime", + uploadId: "upload", + provider: "s3", + bucketId: "bucket", + storageIntegrationId: null, + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }; + const binding = { + ownerId, + videoId, + sessionId: pending.sessionId, + uploadId: pending.uploadId, + key: pending.key, + path: pending.path, + bucketId: pending.bucketId, + storageIntegrationId: pending.storageIntegrationId, + }; + expect(editorVideoUploadMatches(pending, binding)).toBe(true); + expect( + editorVideoUploadMatches(pending, { ...binding, bucketId: "other" }), + ).toBe(false); + expect( + editorVideoUploadMatches(pending, { ...binding, sessionId: "other" }), + ).toBe(false); + expect( + editorVideoUploadMatches( + { ...pending, expiresAt: "2000-01-01" }, + binding, + ), + ).toBe(false); + }); + + it("requires complete, sequential multipart bytes with S3-sized intermediate parts", () => { + const minimum = 5 * 1024 * 1024; + const first = { partNumber: 1, etag: "first", size: minimum }; + const last = { partNumber: 2, etag: "second", size: 1 }; + const parts = [first, last]; + expect(validateEditorVideoParts(parts, minimum + 1)).toBe(true); + expect( + validateEditorVideoParts( + [{ ...first, size: minimum - 1 }, last], + minimum, + ), + ).toBe(false); + expect( + validateEditorVideoParts( + [first, { ...last, partNumber: 3 }], + minimum + 1, + ), + ).toBe(false); + expect(validateEditorVideoParts(parts, minimum + 2)).toBe(false); + }); +}); diff --git a/apps/web/__tests__/editor-worker-routing.test.ts b/apps/web/__tests__/editor-worker-routing.test.ts new file mode 100644 index 00000000000..58493771b9c --- /dev/null +++ b/apps/web/__tests__/editor-worker-routing.test.ts @@ -0,0 +1,138 @@ +import { Effect } from "effect"; +import { expect, test, vi } from "vitest"; +import { requestMediaEditor } from "../lib/editor-session"; +import { + editorWorkerForRequest, + editorWorkerIdFromSessionId, + orderedEditorWorkers, + parseEditorWorkerPool, +} from "../lib/editor-worker-routing"; + +const uuid = "00000000-0000-4000-8000-000000000001"; +const pool = JSON.stringify([ + { id: "east", url: "https://editor-east.internal:3000" }, + { id: "west", url: "https://editor-west.internal:3000" }, +]); + +const workerEnvironment = vi.hoisted(() => ({ + MEDIA_SERVER_WEBHOOK_SECRET: "private-worker-secret", + CAP_WEB_EDITOR_WORKER_URL: "https://editor.example", + CAP_WEB_EDITOR_WORKER_POOL: undefined as string | undefined, +})); + +vi.mock("@cap/env", async (importOriginal) => ({ + ...(await importOriginal()), + serverEnv: () => workerEnvironment, +})); + +test("a single legacy worker keeps UUID session routing", () => { + const workers = parseEditorWorkerPool(undefined, "http://127.0.0.1:3000"); + expect(workers).toEqual([{ id: "", origin: "http://127.0.0.1:3000" }]); + expect(parseEditorWorkerPool(undefined, "http://localhost:3000")).toEqual([ + { id: "", origin: "http://localhost:3000" }, + ]); + expect(parseEditorWorkerPool(undefined, "http://[::1]:3000")).toEqual([ + { id: "", origin: "http://[::1]:3000" }, + ]); + expect( + editorWorkerForRequest(workers, `/editor/sessions/${uuid}/instance`), + ).toEqual(workers[0]); + expect(editorWorkerIdFromSessionId(uuid)).toBe(""); +}); + +test("named sessions stay pinned to their own worker", () => { + const workers = parseEditorWorkerPool(pool, undefined); + expect( + editorWorkerForRequest( + workers, + `/editor/sessions/west.${uuid}/exports/job`, + ), + ).toEqual(workers[1]); + expect( + editorWorkerForRequest(workers, `/editor/preparations/east.${uuid}`), + ).toEqual(workers[0]); + expect( + editorWorkerForRequest(workers, `/editor/sessions/${uuid}/instance`), + ).toBeNull(); + expect(editorWorkerForRequest(workers, "/editor/preparations")).toBeNull(); + expect( + editorWorkerForRequest(workers, "/editor/preparations", "west"), + ).toEqual(workers[1]); +}); + +test("worker preference is stable while all workers remain candidates", () => { + const workers = parseEditorWorkerPool(pool, undefined); + const first = orderedEditorWorkers(workers, "video-1"); + expect(first.map((worker) => worker.id).sort()).toEqual(["east", "west"]); + expect(orderedEditorWorkers([...workers].reverse(), "video-1")).toEqual( + first, + ); +}); + +test("a pool cannot use one replicated service URL for two live workers", () => { + expect(() => + parseEditorWorkerPool( + JSON.stringify([ + { id: "east", url: "https://editor.internal" }, + { id: "west", url: "https://editor.internal" }, + ]), + undefined, + ), + ).toThrow("Duplicate editor worker pool entry"); + expect(() => + parseEditorWorkerPool( + JSON.stringify([{ id: "east", url: "https://editor.internal/path" }]), + undefined, + ), + ).toThrow("Invalid editor worker origin"); +}); + +test("remote HTTP worker origins cannot receive the worker secret", async () => { + const fetchWorker = vi.fn(); + vi.stubGlobal("fetch", fetchWorker); + try { + for (const url of [ + "http://editor.example:3000", + "http://localhost.example:3000", + "http://192.168.1.10:3000", + ]) { + workerEnvironment.CAP_WEB_EDITOR_WORKER_URL = url; + await expect( + Effect.runPromise(requestMediaEditor("/editor/preparations")), + ).rejects.toBeDefined(); + } + workerEnvironment.CAP_WEB_EDITOR_WORKER_URL = "https://editor.example"; + workerEnvironment.CAP_WEB_EDITOR_WORKER_POOL = JSON.stringify([ + { id: "east", url: "http://editor.internal:3000" }, + ]); + await expect( + Effect.runPromise(requestMediaEditor("/editor/preparations")), + ).rejects.toBeDefined(); + expect(fetchWorker).not.toHaveBeenCalled(); + } finally { + workerEnvironment.CAP_WEB_EDITOR_WORKER_POOL = undefined; + workerEnvironment.CAP_WEB_EDITOR_WORKER_URL = "https://editor.example"; + vi.unstubAllGlobals(); + } +}); + +test("HTTPS worker calls reject redirects before forwarding the secret", async () => { + const fetchWorker = vi.fn().mockResolvedValue(Response.json({ ready: true })); + vi.stubGlobal("fetch", fetchWorker); + try { + await Effect.runPromise( + requestMediaEditor("/editor/preparations", { redirect: "follow" }), + ); + expect(fetchWorker).toHaveBeenCalledOnce(); + expect(fetchWorker.mock.calls[0]?.[0]).toBe( + "https://editor.example/editor/preparations", + ); + const request = fetchWorker.mock.calls[0]?.[1] as RequestInit; + expect(request.redirect).toBe("error"); + expect(new Headers(request.headers).get("x-media-server-secret")).toBe( + "private-worker-secret", + ); + } finally { + vi.unstubAllGlobals(); + } +}); diff --git a/apps/web/__tests__/unit/browser-save-chunks.test.ts b/apps/web/__tests__/unit/browser-save-chunks.test.ts new file mode 100644 index 00000000000..4aa8b49f6f8 --- /dev/null +++ b/apps/web/__tests__/unit/browser-save-chunks.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from "vitest"; +import { + BROWSER_SAVE_CHUNK_FILE, + browserSaveChunkKey, + browserSaveMultivariantPlaylist, + browserSavePlayable, + browserSavePlaylist, +} from "@/lib/browser-save-chunks"; + +describe("browser save chunks", () => { + it("accepts only the init segment and numbered media segments", () => { + expect(BROWSER_SAVE_CHUNK_FILE.test("init.mp4")).toBe(true); + expect(BROWSER_SAVE_CHUNK_FILE.test("segment-12.m4s")).toBe(true); + expect(BROWSER_SAVE_CHUNK_FILE.test("../result.mp4")).toBe(false); + expect(BROWSER_SAVE_CHUNK_FILE.test("segment-1.m4s/x")).toBe(false); + expect(BROWSER_SAVE_CHUNK_FILE.test("segment-123456.m4s")).toBe(false); + }); + + it("keeps chunks inside the video's folder", () => { + expect(browserSaveChunkKey("owner", "video", "save", "init.mp4")).toBe( + "owner/video/.recording/browser-save/save/init.mp4", + ); + }); + + it("lists every uploaded segment after the init segment, still open", () => { + const playlist = browserSavePlaylist("abc", "save", [2, 2.5], false); + expect(playlist.split("\n")).toEqual([ + "#EXTM3U", + "#EXT-X-VERSION:7", + "#EXT-X-TARGETDURATION:4", + "#EXT-X-PLAYLIST-TYPE:EVENT", + '#EXT-X-MAP:URI="/api/videos/abc/browser-save/save/init.mp4"', + "#EXTINF:2.000,", + "/api/videos/abc/browser-save/save/segment-1.m4s", + "#EXTINF:2.500,", + "/api/videos/abc/browser-save/save/segment-2.m4s", + "", + ]); + expect(playlist).not.toContain("ENDLIST"); + }); + + it("ends the playlist once the Save is done", () => { + expect(browserSavePlaylist("abc", "save", [2], true)).toContain( + "#EXT-X-ENDLIST\n", + ); + }); + + it("raises the target duration for a long segment", () => { + expect(browserSavePlaylist("abc", "save", [6.2], false)).toContain( + "#EXT-X-TARGETDURATION:7", + ); + }); + + it("starts playback after two chunks, as with 2 s chunks", () => { + expect(browserSavePlayable([])).toBe(false); + expect(browserSavePlayable([2])).toBe(false); + expect(browserSavePlayable([2, 2])).toBe(true); + expect(browserSavePlayable([1.9853, 2])).toBe(true); + expect(browserSavePlayable([2, 0.05])).toBe(true); + }); + + it("starts playback once one longer chunk covers 4 s", () => { + expect(browserSavePlayable([3.5])).toBe(false); + expect(browserSavePlayable([4])).toBe(true); + expect(browserSavePlayable([5])).toBe(true); + expect(browserSavePlayable([10])).toBe(true); + }); + + it("wraps the media playlist in a multivariant playlist", () => { + expect(browserSaveMultivariantPlaylist("abc").split("\n")).toEqual([ + "#EXTM3U", + "#EXT-X-VERSION:7", + "#EXT-X-STREAM-INF:BANDWIDTH=8000000", + "/api/videos/abc/browser-save/playlist?media", + "", + ]); + }); +}); diff --git a/apps/web/__tests__/unit/cursor-reconstruction.test.ts b/apps/web/__tests__/unit/cursor-reconstruction.test.ts new file mode 100644 index 00000000000..3399e8d8a8b --- /dev/null +++ b/apps/web/__tests__/unit/cursor-reconstruction.test.ts @@ -0,0 +1,266 @@ +import type { VideoMetadata } from "@cap/database/types"; +import { describe, expect, it } from "vitest"; +import { + CURSOR_RECONSTRUCTION_START_TIMEOUT_MS, + CURSOR_RECONSTRUCTION_TIMEOUT_MS, + cursorReconstructionAbandoned, + cursorReconstructionBlocker, + cursorReconstructionPrefix, + cursorReconstructionView, + effectiveEditorSources, +} from "@/lib/cursor-reconstruction"; +import { parseCursorJobSummary } from "@/lib/cursor-reconstruction-jobs"; + +const owner = "owner1"; +const video = "video1"; +const runId = "0123456789abcdef"; +const prefix = cursorReconstructionPrefix(owner, video, runId); +const displayKey = `${owner}/${video}/raw-upload.webm`; + +function metadata( + reconstruction: Partial< + NonNullable + > | null = {}, + sources: Partial> = {}, +): VideoMetadata { + return { + editorSources: { + version: 1, + display: { + key: displayKey, + contentType: "video/webm", + size: 1000, + fps: 60, + objectIdentity: "etag", + }, + ...sources, + }, + ...(reconstruction === null + ? {} + : { + cursorReconstruction: { + version: 1, + runId, + jobId: "job", + status: "ready", + enabled: true, + sourceKey: displayKey, + sourceSize: 1000, + sourceIdentity: "etag", + startedAt: "2026-09-30T00:00:00.000Z", + display: { key: `${prefix}display.mp4`, size: 500 }, + inputEvents: { key: `${prefix}input-events.ndjson`, size: 20 }, + ...reconstruction, + }, + }), + } as VideoMetadata; +} + +describe("effectiveEditorSources", () => { + it("swaps in the cleaned display and reconstructed input while enabled", () => { + const sources = effectiveEditorSources(metadata(), owner, video); + expect(sources?.display).toEqual({ + key: `${prefix}display.mp4`, + contentType: "video/mp4", + size: 500, + fps: 30, + objectIdentity: null, + }); + expect(sources?.inputEvents).toEqual({ + key: `${prefix}input-events.ndjson`, + contentType: "application/x-ndjson", + size: 20, + objectIdentity: null, + }); + }); + + it("keeps the recording as uploaded otherwise", () => { + for (const input of [ + metadata(null), + metadata({ enabled: false }), + metadata({ status: "processing" }), + metadata({ status: "error" }), + metadata({ sourceKey: `${owner}/${video}/older.webm` }), + metadata({ sourceIdentity: "an-earlier-upload" }), + metadata({ sourceSize: 999 }), + metadata({ runId: "../other" }), + metadata({ display: { key: `${owner}/other/display.mp4`, size: 500 } }), + metadata({ + inputEvents: { key: `${prefix}input-events.ndjson`, size: 0 }, + }), + ]) { + expect(effectiveEditorSources(input, owner, video)).toBe( + input.editorSources, + ); + } + expect( + effectiveEditorSources(metadata(), owner, "video2")?.display.key, + ).toBe(displayKey); + }); + + it("matches runs recorded before size and identity by their key", () => { + const legacy = metadata({ + sourceSize: undefined, + sourceIdentity: undefined, + }); + expect(effectiveEditorSources(legacy, owner, video)?.display.key).toBe( + `${prefix}display.mp4`, + ); + }); + + it("never replaces pointer input a recording captured itself", () => { + const input = metadata( + {}, + { + inputEvents: { + key: `${owner}/${video}/input-events-upload.ndjson`, + contentType: "application/x-ndjson", + size: 10, + objectIdentity: null, + }, + }, + ); + expect(effectiveEditorSources(input, owner, video)).toBe( + input.editorSources, + ); + }); +}); + +describe("cursorReconstructionBlocker", () => { + it("accepts short browser recordings without pointer input", () => { + expect( + cursorReconstructionBlocker({ duration: 299, metadata: metadata(null) }), + ).toBeNull(); + }); + + it("rejects long, imported, instrumented and legacy recordings", () => { + expect( + cursorReconstructionBlocker({ duration: 301, metadata: metadata(null) }), + ).toMatch(/5 minutes/); + expect( + cursorReconstructionBlocker({ duration: null, metadata: metadata(null) }), + ).toMatch(/processing/); + expect(cursorReconstructionBlocker({ duration: 10, metadata: {} })).toMatch( + /browser recordings/, + ); + expect( + cursorReconstructionBlocker({ + duration: 10, + metadata: metadata(null, { + display: { + key: displayKey, + contentType: "video/mp4", + size: 1, + embeddedAudio: true, + }, + }), + }), + ).toMatch(/Imported/); + }); +}); + +describe("cursorReconstructionView", () => { + it("reports the current run", () => { + expect( + cursorReconstructionView({ + id: video, + ownerId: owner, + duration: 60, + metadata: metadata(), + }), + ).toEqual({ + eligible: true, + blocker: null, + status: "ready", + enabled: true, + progress: 1, + error: null, + cursorData: true, + }); + expect( + cursorReconstructionView({ + id: video, + ownerId: owner, + duration: 60, + metadata: metadata({ status: "processing", progress: 0.4 }), + }), + ).toMatchObject({ + status: "processing", + enabled: false, + progress: 0.4, + cursorData: false, + }); + }); + + it("treats a run made from an older display as idle", () => { + expect( + cursorReconstructionView({ + id: video, + ownerId: owner, + duration: 60, + metadata: metadata({ sourceKey: `${owner}/${video}/older.webm` }), + }), + ).toMatchObject({ status: "idle", enabled: false }); + }); +}); + +describe("parseCursorJobSummary", () => { + it("reads a farm summary and drops malformed ones", () => { + expect( + parseCursorJobSummary({ + id: "abc", + status: "ready", + progress: 1, + display: `${prefix}display.mp4`, + inputEvents: `${prefix}input-events.ndjson`, + bytes: { display: 5, inputEvents: 6 }, + }), + ).toMatchObject({ status: "ready", bytes: { display: 5, inputEvents: 6 } }); + expect( + parseCursorJobSummary({ + id: "abc", + status: "ready", + display: "a", + inputEvents: "b", + bytes: { display: 1.5, inputEvents: 6 }, + }), + ).not.toHaveProperty("bytes"); + expect(parseCursorJobSummary({ id: "abc", status: "done" })).toBeNull(); + expect(parseCursorJobSummary(null)).toBeNull(); + }); +}); + +describe("cursorReconstructionAbandoned", () => { + const startedAt = "2026-09-30T00:00:00.000Z"; + const at = (ms: number) => Date.parse(startedAt) + ms; + + it("gives a run the farm never accepted a short start window", () => { + const run = { jobId: "", startedAt }; + expect(cursorReconstructionAbandoned(run, at(60_000))).toBe(false); + expect( + cursorReconstructionAbandoned( + run, + at(CURSOR_RECONSTRUCTION_START_TIMEOUT_MS + 1), + ), + ).toBe(true); + }); + + it("lets an accepted run wait behind farm work until the hard limit", () => { + const run = { jobId: "job", startedAt }; + expect( + cursorReconstructionAbandoned( + run, + at(CURSOR_RECONSTRUCTION_START_TIMEOUT_MS + 1), + ), + ).toBe(false); + expect( + cursorReconstructionAbandoned( + run, + at(CURSOR_RECONSTRUCTION_TIMEOUT_MS + 1), + ), + ).toBe(true); + expect( + cursorReconstructionAbandoned({ jobId: "job", startedAt: "invalid" }), + ).toBe(true); + }); +}); diff --git a/apps/web/__tests__/unit/desktop-reupload-token.test.ts b/apps/web/__tests__/unit/desktop-reupload-token.test.ts index b486d92ce3a..3fddeca0d4c 100644 --- a/apps/web/__tests__/unit/desktop-reupload-token.test.ts +++ b/apps/web/__tests__/unit/desktop-reupload-token.test.ts @@ -247,8 +247,6 @@ describe("desktop reupload target identity", () => { audioLevelOutputKey: "audio-output", audioLevelSourceKey: "audio-source", outputKey: "output", - previewKey: "preview", - thumbnailKey: "thumbnail", type: "desktopMP4", }), ) @@ -266,7 +264,12 @@ describe("desktop reupload target identity", () => { outputKey: undefined, }), ).toBe(desktopReuploadSourceIdentity({ type: "desktopMP4" })); - for (const key of Object.keys(source)) { + for (const key of [ + "type", + "outputKey", + "audioLevelOutputKey", + "audioLevelSourceKey", + ]) { const changed = { ...source, [key]: key === "type" ? "webMP4" : "changed", @@ -275,6 +278,21 @@ describe("desktop reupload target identity", () => { } }); + it("survives preview assets remade while a replacement uploads", () => { + const published = { + type: "webMP4", + outputKey: "owner/video/.recording/render/export/result.mp4", + }; + expect( + desktopReuploadSourceIdentity({ + ...published, + thumbnailKey: + "owner/video/.recording/render/export/result/screenshot.jpg", + previewKey: "owner/video/.recording/render/export/result/preview.gif", + }), + ).toBe(desktopReuploadSourceIdentity(published)); + }); + it("rejects malformed known source fields without echoing them", () => { expect(() => desktopReuploadSourceIdentity({ type: "desktopMP4", outputKey: 123 }), diff --git a/apps/web/__tests__/unit/edit-processing.test.ts b/apps/web/__tests__/unit/edit-processing.test.ts new file mode 100644 index 00000000000..a1032370b06 --- /dev/null +++ b/apps/web/__tests__/unit/edit-processing.test.ts @@ -0,0 +1,138 @@ +// @vitest-environment jsdom + +import type { Video } from "@cap/web-domain"; +import { act, createElement } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import type { UploadProgress } from "@/app/s/[videoId]/_components/upload-progress"; + +const mocks = vi.hoisted(() => ({ + progress: null as UploadProgress | null, + router: { refresh: vi.fn() }, + retry: vi.fn(async () => ({ success: true, status: "started" })), +})); +vi.mock("next/navigation", () => ({ useRouter: () => mocks.router })); +vi.mock("@/actions/video/retry-processing", () => ({ + retryVideoProcessing: mocks.retry, +})); +vi.mock("@/app/s/[videoId]/_components/ProgressCircle", () => ({ + useUploadProgress: () => mocks.progress, +})); + +import { EditProcessing } from "@/app/s/[videoId]/edit/edit-processing"; + +let root: Root; +let container: HTMLDivElement; +const videoId = "video" as Video.VideoId; +const render = async () => { + await act(async () => + root.render(createElement(EditProcessing, { videoId })), + ); +}; + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); + mocks.progress = { status: "fetching" }; +}); + +afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); + vi.unstubAllGlobals(); +}); + +test("waits through polling and refreshes once when processing completes", async () => { + await render(); + expect(mocks.router.refresh).not.toHaveBeenCalled(); + mocks.progress = { + status: "processing", + progress: 40, + message: null, + lastUpdated: new Date(), + }; + await render(); + expect( + container + .querySelector('[role="progressbar"]') + ?.getAttribute("aria-valuenow"), + ).toBe("40"); + expect(mocks.router.refresh).not.toHaveBeenCalled(); + mocks.progress = null; + await render(); + await render(); + expect(mocks.router.refresh).toHaveBeenCalledTimes(1); +}); + +test("retries a stalled recording with retained source media", async () => { + mocks.progress = { + status: "error", + lastUpdated: new Date(), + errorMessage: "stalled", + hasRawFallback: true, + }; + await render(); + expect(container.textContent).toContain("needs attention"); + await act(async () => container.querySelector("button")?.click()); + expect(mocks.retry).toHaveBeenCalledWith({ videoId }); + expect(mocks.router.refresh).toHaveBeenCalledOnce(); +}); + +test("reports retry failures without navigating away", async () => { + mocks.progress = { + status: "error", + lastUpdated: new Date(), + errorMessage: "stalled", + hasRawFallback: true, + }; + mocks.retry.mockRejectedValueOnce(new Error("Retry unavailable")); + await render(); + await act(async () => container.querySelector("button")?.click()); + expect(container.querySelector("[role='alert']")?.textContent).toBe( + "Retry unavailable", + ); + expect(mocks.router.refresh).not.toHaveBeenCalled(); +}); + +test("does not offer processing retries when no raw source remains", async () => { + mocks.progress = { + status: "error", + lastUpdated: new Date(), + errorMessage: "source missing", + hasRawFallback: false, + }; + await render(); + expect(container.querySelector("button")).toBeNull(); + expect(container.querySelector("a")?.getAttribute("href")).toBe("/s/video"); +}); + +test("guides incomplete uploads to recovery without processing partial media", async () => { + mocks.progress = { status: "failed", lastUpdated: new Date() }; + await render(); + expect(container.textContent).toContain("did not finish uploading"); + expect(container.textContent).toContain("Recovered recordings"); + expect(container.querySelector("button")).toBeNull(); + expect(container.querySelector("a")?.getAttribute("href")).toBe( + "/dashboard/caps/record", + ); + expect(mocks.retry).not.toHaveBeenCalled(); +}); + +test("does not offer processing or recovery while an upload is active", async () => { + mocks.progress = { + status: "uploading", + lastUpdated: new Date(), + progress: 30, + }; + await render(); + expect( + container + .querySelector('[role="progressbar"]') + ?.getAttribute("aria-valuenow"), + ).toBe("30"); + expect(container.textContent).not.toContain("Retry processing"); + expect(container.textContent).not.toContain("Go to recorder"); + expect(mocks.retry).not.toHaveBeenCalled(); +}); diff --git a/apps/web/__tests__/unit/edit-transcript.test.ts b/apps/web/__tests__/unit/edit-transcript.test.ts index d82d32940d4..c64bd137eef 100644 --- a/apps/web/__tests__/unit/edit-transcript.test.ts +++ b/apps/web/__tests__/unit/edit-transcript.test.ts @@ -13,6 +13,7 @@ import { planTranscriptCut, remapEditTranscriptThroughSpec, serializeEditTranscript, + transcriptTimelineDurationMs, } from "@/lib/edit-transcript"; import { createIdentityEditSpec } from "@/lib/video-edits"; import { @@ -547,4 +548,33 @@ describe("editTranscriptWordsToCaptionVtt", () => { it("returns a header-only VTT when there are no words", () => { expect(editTranscriptWordsToCaptionVtt([])).toBe("WEBVTT\n\n"); }); + + it("keeps the stored duration when it covers the transcribed audio", () => { + expect(transcriptTimelineDurationMs(16_417, 17)).toBe(16_417); + expect(transcriptTimelineDurationMs(16_417, 18)).toBe(16_417); + expect(transcriptTimelineDurationMs(20_000, 17)).toBe(20_000); + expect(transcriptTimelineDurationMs(16_417, null)).toBe(16_417); + }); + + it("uses the audio length when the stored duration is bad metadata", () => { + expect(transcriptTimelineDurationMs(102, 17)).toBe(17_000); + expect(transcriptTimelineDurationMs(0, 17)).toBe(17_000); + expect(transcriptTimelineDurationMs(Number.NaN, 17)).toBe(17_000); + expect(transcriptTimelineDurationMs(0, null)).toBe(0); + }); + + it("keeps every word of a take whose stored duration was one fragment", () => { + const words = [ + { text: "The", start: 514, end: 594, confidence: 0.94 }, + { text: "dog.", start: 16_263, end: 16_311, confidence: 0.99 }, + ]; + const transcript = createEditTranscript( + { words, speech_model_used: "universal", language_code: "en" }, + transcriptTimelineDurationMs(102.45, 17), + ); + expect(transcript.words.map((word) => word.text)).toEqual(["The", "dog."]); + expect(editTranscriptWordsToCaptionVtt(transcript.words)).not.toBe( + "WEBVTT\n\n", + ); + }); }); diff --git a/apps/web/__tests__/unit/editor-cap-bundle.test.ts b/apps/web/__tests__/unit/editor-cap-bundle.test.ts new file mode 100644 index 00000000000..5c8ee21b1af --- /dev/null +++ b/apps/web/__tests__/unit/editor-cap-bundle.test.ts @@ -0,0 +1,107 @@ +import { + CAP_BUNDLE_HEADER_BYTES, + CAP_BUNDLE_MAGIC, + createCapBundle, + parseCapBundleManifest, + readCapBundleManifestLength, + validCapBundlePath, +} from "@cap/editor-cap-bundle"; +import { describe, expect, it } from "vitest"; + +const encoder = new TextEncoder(); + +function rawBundle(files: unknown[], dataBytes: number) { + const manifest = encoder.encode(JSON.stringify({ version: 1, files })); + return parseCapBundleManifest( + manifest, + CAP_BUNDLE_HEADER_BYTES + manifest.byteLength + dataBytes, + ); +} + +describe("Cap editor project bundle", () => { + it("keeps large media as Blob parts and records exact byte offsets", async () => { + const metadata = new Blob([JSON.stringify({ version: 1 })]); + const screen = new Blob([new Uint8Array(4 * 1024 * 1024)]); + const camera = new Blob([new Uint8Array(2 * 1024 * 1024)]); + const bundle = createCapBundle([ + { path: "content/camera.mp4", file: camera }, + { path: "recording-meta.json", file: metadata }, + { path: "content/screen.mp4", file: screen }, + ]); + const header = new Uint8Array(await bundle.slice(0, 12).arrayBuffer()); + expect(new TextDecoder().decode(header.slice(0, 8))).toBe(CAP_BUNDLE_MAGIC); + const manifestLength = readCapBundleManifestLength(header); + expect(manifestLength).not.toBeNull(); + if (manifestLength === null) throw new Error("Invalid bundle header"); + const manifest = parseCapBundleManifest( + new Uint8Array(await bundle.slice(12, 12 + manifestLength).arrayBuffer()), + bundle.size, + ); + expect(manifest?.files).toEqual([ + { path: "content/camera.mp4", size: camera.size, offset: 0 }, + { + path: "content/screen.mp4", + size: screen.size, + offset: camera.size, + }, + { + path: "recording-meta.json", + size: metadata.size, + offset: camera.size + screen.size, + }, + ]); + expect(bundle.size).toBe( + 12 + manifestLength + camera.size + screen.size + metadata.size, + ); + }); + + it("rejects traversal, absolute paths, and unknown project roots", () => { + for (const path of [ + "../recording-meta.json", + "content/../../private", + "content//screen.mp4", + "/content/screen.mp4", + "content\\screen.mp4", + "output/../screen.mp4", + "other/screen.mp4", + "content/bad\nname", + ]) { + expect(validCapBundlePath(path)).toBe(false); + expect(() => + createCapBundle([ + { path: "recording-meta.json", file: new Blob(["{}"]) }, + { path, file: new Blob(["x"]) }, + ]), + ).toThrow(); + } + }); + + it("preserves imported audio and project screenshots", () => { + for (const path of [ + "assets/audio/import-example.mp3", + "screenshots/preview.jpg", + ]) { + expect(validCapBundlePath(path)).toBe(true); + } + for (const path of ["assets/video/example.mp4", "assets/audio/../private"]) + expect(validCapBundlePath(path)).toBe(false); + }); + + it("rejects duplicate, overlapping, truncated, and oversized manifests", () => { + const meta = { path: "recording-meta.json", size: 2, offset: 0 }; + expect(rawBundle([meta, meta], 4)).toBeNull(); + expect( + rawBundle([meta, { path: "content/screen.mp4", size: 3, offset: 1 }], 5), + ).toBeNull(); + expect(rawBundle([meta], 3)).toBeNull(); + expect(rawBundle([{ ...meta, size: -1 }], 0)).toBeNull(); + expect( + rawBundle([{ ...meta, size: Number.MAX_SAFE_INTEGER }], 0), + ).toBeNull(); + expect(rawBundle([{ ...meta, size: "2" }], 2)).toBeNull(); + expect( + rawBundle([{ path: "content/screen.mp4", size: 2, offset: 0 }], 2), + ).toBeNull(); + expect(readCapBundleManifestLength(new Uint8Array(12))).toBeNull(); + }); +}); diff --git a/apps/web/__tests__/unit/editor-cap-import-client.test.ts b/apps/web/__tests__/unit/editor-cap-import-client.test.ts new file mode 100644 index 00000000000..5d37a112455 --- /dev/null +++ b/apps/web/__tests__/unit/editor-cap-import-client.test.ts @@ -0,0 +1,75 @@ +import { CAP_BUNDLE_CONTENT_TYPE } from "@cap/editor-cap-bundle"; +import { afterEach, expect, test, vi } from "vitest"; +import { importWebEditorCap } from "../../lib/editor-cap-import-client"; + +const uploader = vi.hoisted(() => ({ + finalize: vi.fn(async () => {}), + cancel: vi.fn(async () => {}), +})); + +vi.mock("@cap/recorder-core", () => ({ + InstantRecordingUploader: class { + finalize = uploader.finalize; + cancel = uploader.cancel; + }, + MultipartCompletionUncertainError: class extends Error {}, +})); + +afterEach(() => { + vi.unstubAllGlobals(); + uploader.finalize.mockClear(); + uploader.cancel.mockClear(); +}); + +test("aborting an uploaded Cap import stops its worker job", async () => { + const videoId = "video"; + const ownerId = "owner"; + const sessionId = "session"; + const fileId = "9a2aa734-d076-42c0-8a8a-68db1d2d2a1e"; + const path = `content/imports/${fileId}.capbundle`; + const key = `${ownerId}/${videoId}/editor-assets/recordings/${fileId}.capbundle`; + const jobId = "afe747f6-4c9d-47d4-9ca0-99ffad1c0050"; + const controller = new AbortController(); + const fetchMock = vi.fn( + async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (init?.method === "POST") { + return Response.json({ + key, + path, + uploadId: "upload", + provider: "s3", + }); + } + if (init?.method === "DELETE") return new Response(null, { status: 204 }); + if (url.includes(jobId)) { + controller.abort(); + return Response.json({ + id: jobId, + status: "staging", + result: null, + error: null, + }); + } + return Response.json({ id: jobId, status: "staging" }); + }, + ); + vi.stubGlobal("fetch", fetchMock); + const file = new File([new Uint8Array([1])], "recording.capbundle", { + type: CAP_BUNDLE_CONTENT_TYPE, + }); + + await expect( + importWebEditorCap(file, videoId, ownerId, sessionId, controller.signal), + ).rejects.toThrow("canceled"); + expect(uploader.finalize).toHaveBeenCalledOnce(); + expect(fetchMock.mock.calls.map((call) => call[1]?.method)).toEqual([ + "POST", + undefined, + undefined, + "DELETE", + ]); + expect(fetchMock.mock.calls[3]?.[0]).toBe( + `/api/editor/sessions/${sessionId}/video-assets/${jobId}?videoId=${videoId}&key=${encodeURIComponent(key)}&path=${encodeURIComponent(path)}`, + ); +}); diff --git a/apps/web/__tests__/unit/editor-clip-recorder-lifecycle.test.ts b/apps/web/__tests__/unit/editor-clip-recorder-lifecycle.test.ts new file mode 100644 index 00000000000..44e6286a503 --- /dev/null +++ b/apps/web/__tests__/unit/editor-clip-recorder-lifecycle.test.ts @@ -0,0 +1,175 @@ +// @vitest-environment jsdom + +import { act, createElement } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import type { + EditorClipCapture, + EditorClipCaptureOptions, + EditorClipCaptureSession, +} from "@/lib/editor-clip-recorder"; + +const capture = vi.hoisted(() => ({ start: vi.fn(), camera: vi.fn() })); + +vi.mock("@/lib/editor-clip-recorder", () => ({ + startEditorClipCapture: capture.start, +})); + +vi.mock("@cap/recorder-core/capture-streams", () => ({ + acquireCameraStream: capture.camera, + acquireDisplayStream: vi.fn(async () => { + const track = { + addEventListener: vi.fn(), + getSettings: () => ({ displaySurface: "window" }), + stop: vi.fn(), + }; + return { + getTracks: () => [track], + getVideoTracks: () => [track], + getAudioTracks: () => [], + }; + }), +})); + +import { EditorClipRecorder } from "@/app/s/[videoId]/edit/studio/editor-clip-recorder"; + +let root: Root | null; +let container: HTMLDivElement; + +function session(): EditorClipCaptureSession { + return { + cameraPreviewStream: null, + stop: vi.fn(async () => ({ + display: new File(["screen"], "screen.webm"), + camera: null, + cameraOffsetMs: 0, + })), + recover: vi.fn(async () => null), + pause: vi.fn(), + resume: vi.fn(), + discard: vi.fn(async () => undefined), + release: vi.fn(async () => undefined), + }; +} + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue(); + capture.camera.mockRejectedValue(new Error("No camera")); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); +}); + +afterEach(async () => { + if (root) await act(async () => root?.unmount()); + container.remove(); + capture.start.mockReset(); + capture.camera.mockReset(); + vi.restoreAllMocks(); + vi.unstubAllGlobals(); +}); + +async function renderRecorder(onCaptured = vi.fn(async () => undefined)) { + await act(async () => { + root?.render( + createElement(EditorClipRecorder, { + context: null, + onCaptured, + onClose: vi.fn(), + }), + ); + }); + return onCaptured; +} + +async function startRecorder() { + const button = [...container.querySelectorAll("button")].find((candidate) => + candidate.textContent?.includes("Start Recording"), + ); + expect(button).toBeDefined(); + await act(async () => button?.click()); +} + +test("closing the editor during screen permission stops a late capture", async () => { + let resolveSession: ((value: EditorClipCaptureSession) => void) | undefined; + capture.start.mockImplementation( + () => + new Promise((resolve) => { + resolveSession = resolve; + }), + ); + const onCaptured = await renderRecorder(); + await startRecorder(); + const captured = session(); + await act(async () => root?.unmount()); + root = null; + const options = capture.start.mock.calls[0]?.[0] as + | EditorClipCaptureOptions + | undefined; + expect(options?.signal?.aborted).toBe(true); + await act(async () => resolveSession?.(captured)); + expect(captured.stop).toHaveBeenCalledOnce(); + expect(captured.release).not.toHaveBeenCalled(); + expect(onCaptured).not.toHaveBeenCalled(); +}); + +test("closing the editor during recording stops the active two-track session", async () => { + const captured = session(); + capture.start.mockResolvedValue(captured); + const onCaptured = await renderRecorder(); + await startRecorder(); + expect(container.textContent).toContain("Recording"); + await act(async () => root?.unmount()); + root = null; + expect(captured.stop).toHaveBeenCalledOnce(); + expect(captured.release).not.toHaveBeenCalled(); + expect(onCaptured).not.toHaveBeenCalled(); +}); + +test("a recording that finishes after the editor closes is not imported", async () => { + let resolveClip: ((value: EditorClipCapture) => void) | undefined; + const pendingStop = new Promise((resolve) => { + resolveClip = resolve; + }); + const captured = session(); + captured.stop = vi.fn(() => pendingStop); + capture.start.mockResolvedValue(captured); + const onCaptured = await renderRecorder(); + await startRecorder(); + const stopButton = [...container.querySelectorAll("button")].find( + (candidate) => candidate.getAttribute("aria-label") === "Stop recording", + ); + expect(stopButton).toBeDefined(); + await act(async () => stopButton?.click()); + await act(async () => root?.unmount()); + root = null; + await act(async () => + resolveClip?.({ + display: new File(["screen"], "screen.webm"), + camera: null, + cameraOffsetMs: 0, + }), + ); + expect(onCaptured).not.toHaveBeenCalled(); + expect(captured.release).not.toHaveBeenCalled(); +}); + +test("starting records the camera preview instead of opening the camera again", async () => { + const cameraTrack = { stop: vi.fn(), readyState: "live" }; + const preview = { + getTracks: () => [cameraTrack], + getVideoTracks: () => [cameraTrack], + getAudioTracks: () => [], + }; + capture.camera.mockResolvedValue(preview); + capture.start.mockResolvedValue(session()); + await renderRecorder(); + await startRecorder(); + const options = capture.start.mock.calls[0]?.[0] as + | EditorClipCaptureOptions + | undefined; + expect(options?.cameraStream).toBe(preview); + expect(capture.camera).toHaveBeenCalledOnce(); + expect(cameraTrack.stop).not.toHaveBeenCalled(); +}); diff --git a/apps/web/__tests__/unit/editor-clip-recorder.test.ts b/apps/web/__tests__/unit/editor-clip-recorder.test.ts new file mode 100644 index 00000000000..574217b7181 --- /dev/null +++ b/apps/web/__tests__/unit/editor-clip-recorder.test.ts @@ -0,0 +1,286 @@ +import { afterEach, expect, test, vi } from "vitest"; +import { startEditorClipCapture } from "../../lib/editor-clip-recorder"; + +const capture = vi.hoisted(() => ({ + display: vi.fn(), + camera: vi.fn(), + mic: vi.fn(), + mix: vi.fn(), + spoolCreate: vi.fn(), +})); + +vi.mock("@cap/recorder-core/capture-streams", () => ({ + acquireDisplayStream: capture.display, + acquireCameraStream: capture.camera, + acquireMicStream: capture.mic, + createAudioMixer: capture.mix, +})); +vi.mock("@cap/recorder-core/recorder-utils", () => ({ + selectRecordingPipelineFromSupport: (hasAudio: boolean) => ({ + mimeType: hasAudio ? "video/webm;codecs=vp8,opus" : "video/webm;codecs=vp8", + }), +})); +vi.mock("@cap/recorder-core/recording-spool", () => ({ + RecordingSpool: { create: capture.spoolCreate }, +})); + +class FakeTrack extends EventTarget { + readonly stop = vi.fn(); + readyState: MediaStreamTrackState = "live"; + constructor(readonly kind: "video" | "audio") { + super(); + } +} + +class FakeStream { + constructor(readonly tracks: FakeTrack[]) {} + getTracks() { + return this.tracks; + } + getVideoTracks() { + return this.tracks.filter((track) => track.kind === "video"); + } + getAudioTracks() { + return this.tracks.filter((track) => track.kind === "audio"); + } +} + +class FakeRecorder extends EventTarget { + static instances: FakeRecorder[] = []; + static failCameraStart = false; + static isTypeSupported = vi.fn(() => true); + state: RecordingState = "inactive"; + readonly stopCalled = vi.fn(); + constructor( + readonly stream: FakeStream, + readonly options: { mimeType: string }, + ) { + super(); + FakeRecorder.instances.push(this); + } + start() { + if ( + FakeRecorder.failCameraStart && + this.stream.getVideoTracks()[0]?.kind === "video" && + this.stream.tracks.length === 1 && + FakeRecorder.instances.length === 2 + ) { + throw new Error("Camera encoder failed"); + } + this.state = "recording"; + this.dispatchEvent(new Event("start")); + } + stop() { + this.stopCalled(); + if (this.state === "inactive") throw new Error("Already stopped"); + this.state = "inactive"; + const content = + FakeRecorder.instances.indexOf(this) === 0 ? "screen" : "camera"; + const dataEvent = new Event("dataavailable") as Event & { data: Blob }; + dataEvent.data = new Blob([content], { type: this.options.mimeType }); + this.dispatchEvent(dataEvent); + this.dispatchEvent(new Event("stop")); + } + pause() { + this.state = "paused"; + } + resume() { + this.state = "recording"; + } +} + +function setup() { + FakeRecorder.instances = []; + FakeRecorder.failCameraStart = false; + vi.stubGlobal("MediaStream", FakeStream); + vi.stubGlobal("MediaRecorder", FakeRecorder); + vi.stubGlobal("navigator", { + mediaDevices: { getDisplayMedia: vi.fn(), getUserMedia: vi.fn() }, + }); + const screenTrack = new FakeTrack("video"); + const cameraTrack = new FakeTrack("video"); + const micTrack = new FakeTrack("audio"); + const audioOutputTrack = new FakeTrack("audio"); + const screen = new FakeStream([screenTrack]); + const camera = new FakeStream([cameraTrack]); + const mic = new FakeStream([micTrack]); + capture.display.mockResolvedValue(screen); + capture.camera.mockResolvedValue(camera); + capture.mic.mockResolvedValue(mic); + const close = vi.fn(async () => undefined); + capture.mix.mockResolvedValue({ + stream: new FakeStream([audioOutputTrack]), + close, + }); + const spools: Array<{ + chunks: Blob[]; + dispose: ReturnType; + }> = []; + capture.spoolCreate.mockImplementation( + async (options: { mimeType: string }) => { + const chunks: Blob[] = []; + const dispose = vi.fn(async () => undefined); + const spool = { + chunks, + dispose, + appendChunk: vi.fn(async (chunk: Blob) => { + chunks.push(chunk); + }), + flush: vi.fn(async () => undefined), + toBlob: vi.fn(async () => new Blob(chunks, { type: options.mimeType })), + recoverBlob: vi.fn( + async () => new Blob(chunks, { type: options.mimeType }), + ), + }; + spools.push(spool); + return spool; + }, + ); + return { screenTrack, cameraTrack, micTrack, close, spools }; +} + +afterEach(() => { + vi.unstubAllGlobals(); + vi.clearAllMocks(); +}); + +test("screen and webcam become distinct saved clips with screen-only audio", async () => { + const { screenTrack, cameraTrack, micTrack, close, spools } = setup(); + const session = await startEditorClipCapture({ + cameraEnabled: true, + micEnabled: true, + systemAudioEnabled: false, + }); + expect(FakeRecorder.instances).toHaveLength(2); + expect(FakeRecorder.instances[0]?.stream.getVideoTracks()).toEqual([ + screenTrack, + ]); + expect(FakeRecorder.instances[0]?.stream.getAudioTracks()).toHaveLength(1); + expect(FakeRecorder.instances[1]?.stream.getVideoTracks()).toEqual([ + cameraTrack, + ]); + expect(FakeRecorder.instances[1]?.stream.getAudioTracks()).toHaveLength(0); + session.pause(); + expect(FakeRecorder.instances.map((recorder) => recorder.state)).toEqual([ + "paused", + "paused", + ]); + session.resume(); + const captured = await session.stop(); + expect(await captured.display.text()).toBe("screen"); + expect(await captured.camera?.text()).toBe("camera"); + expect(captured.display.name).toMatch(/-screen\.webm$/); + expect(captured.camera?.name).toMatch(/-camera\.webm$/); + expect(captured.cameraOffsetMs).toBeGreaterThanOrEqual(0); + expect(screenTrack.stop).toHaveBeenCalled(); + expect(cameraTrack.stop).toHaveBeenCalled(); + expect(micTrack.stop).toHaveBeenCalled(); + expect(close).toHaveBeenCalled(); + expect(spools.every((spool) => spool.dispose.mock.calls.length === 0)).toBe( + true, + ); + await session.release(); + expect(spools.every((spool) => spool.dispose.mock.calls.length === 1)).toBe( + true, + ); +}); + +test("camera encoder startup failure stops the active screen capture", async () => { + const { screenTrack, cameraTrack, micTrack, close, spools } = setup(); + FakeRecorder.failCameraStart = true; + await expect( + startEditorClipCapture({ + cameraEnabled: true, + micEnabled: true, + systemAudioEnabled: false, + }), + ).rejects.toThrow("Camera encoder failed"); + expect(FakeRecorder.instances[0]?.stopCalled).toHaveBeenCalledOnce(); + expect(screenTrack.stop).toHaveBeenCalled(); + expect(cameraTrack.stop).toHaveBeenCalled(); + expect(micTrack.stop).toHaveBeenCalled(); + expect(close).toHaveBeenCalled(); + expect(spools.every((spool) => spool.dispose.mock.calls.length === 1)).toBe( + true, + ); +}); + +test("screen capture ending during camera permission never starts an encoder", async () => { + const { screenTrack, cameraTrack } = setup(); + capture.camera.mockImplementation(async () => { + screenTrack.readyState = "ended"; + return new FakeStream([cameraTrack]); + }); + await expect( + startEditorClipCapture({ + cameraEnabled: true, + micEnabled: false, + systemAudioEnabled: false, + }), + ).rejects.toThrow("Screen sharing ended before recording started"); + expect(FakeRecorder.instances).toHaveLength(0); + expect(screenTrack.stop).toHaveBeenCalled(); + expect(cameraTrack.stop).toHaveBeenCalled(); +}); + +test("closing the editor during camera permission stops tracks before encoders start", async () => { + const { screenTrack, cameraTrack } = setup(); + const controller = new AbortController(); + capture.camera.mockImplementation(async () => { + controller.abort(); + return new FakeStream([cameraTrack]); + }); + await expect( + startEditorClipCapture({ + cameraEnabled: true, + micEnabled: false, + systemAudioEnabled: false, + signal: controller.signal, + }), + ).rejects.toThrow("Editor clip capture was canceled"); + expect(FakeRecorder.instances).toHaveLength(0); + expect(screenTrack.stop).toHaveBeenCalled(); + expect(cameraTrack.stop).toHaveBeenCalled(); + expect(capture.spoolCreate).not.toHaveBeenCalled(); +}); + +test("a browser encoder error stops both tracks and keeps a downloadable backup", async () => { + const { screenTrack, cameraTrack, spools } = setup(); + const onError = vi.fn(); + const session = await startEditorClipCapture({ + cameraEnabled: true, + micEnabled: false, + systemAudioEnabled: false, + onError, + }); + const event = new Event("error") as Event & { error: Error }; + event.error = new Error("Screen encoder failed"); + FakeRecorder.instances[0]?.dispatchEvent(event); + await expect(session.stop()).rejects.toThrow("Screen encoder failed"); + expect(onError).toHaveBeenCalledWith(event.error); + expect(screenTrack.stop).toHaveBeenCalled(); + expect(cameraTrack.stop).toHaveBeenCalled(); + const backup = await session.recover(); + expect(await backup?.display.text()).toBe("screen"); + expect(await backup?.camera?.text()).toBe("camera"); + expect(spools.every((spool) => spool.dispose.mock.calls.length === 0)).toBe( + true, + ); + await session.release(); +}); + +test("a live camera preview is recorded instead of opening the camera again", async () => { + setup(); + const previewTrack = new FakeTrack("video"); + const preview = new FakeStream([previewTrack]); + const session = await startEditorClipCapture({ + cameraStream: preview as unknown as MediaStream, + cameraEnabled: true, + micEnabled: true, + systemAudioEnabled: false, + }); + expect(capture.camera).not.toHaveBeenCalled(); + expect(FakeRecorder.instances[1]?.stream).toBe(preview); + await session.stop(); + expect(previewTrack.stop).toHaveBeenCalled(); +}); diff --git a/apps/web/__tests__/unit/editor-clips.test.ts b/apps/web/__tests__/unit/editor-clips.test.ts new file mode 100644 index 00000000000..117f780cab5 --- /dev/null +++ b/apps/web/__tests__/unit/editor-clips.test.ts @@ -0,0 +1,133 @@ +import type { VideoMetadata } from "@cap/database/types"; +import { expect, test } from "vitest"; +import { + appendWebEditorClipToConfig, + validWebEditorClip, +} from "../../lib/editor-clips"; + +type Clip = NonNullable["items"][number]; +type Asset = NonNullable["items"][number]; + +const ownerId = "owner"; +const videoId = "video"; +const displayPath = "content/videos/00000000-0000-0000-0000-000000000001.mp4"; +const cameraPath = "content/videos/00000000-0000-0000-0000-000000000002.webm"; + +function asset(path: string, contentType: string): Asset { + return { + path, + key: `${ownerId}/${videoId}/editor-assets/videos/${path.slice("content/videos/".length)}`, + name: "Saved recording source", + contentType, + size: 32_000, + objectIdentity: null, + }; +} + +test("a saved screen and separate camera clip can only use assets from its own recording", () => { + const assets = [ + asset(displayPath, "video/mp4"), + asset(cameraPath, "video/webm"), + ]; + const clip: Clip = { + displayPath, + duration: 8.5, + fps: 30, + hasAudio: false, + cameraPath, + cameraFps: 25, + cameraOffsetMs: 125, + }; + expect(validWebEditorClip(clip, assets, ownerId, videoId)).toBe(true); + expect(validWebEditorClip(clip, assets, "another-owner", videoId)).toBe( + false, + ); + expect( + validWebEditorClip( + { ...clip, cameraOffsetMs: 30_001 }, + assets, + ownerId, + videoId, + ), + ).toBe(false); + expect( + validWebEditorClip( + { ...clip, cameraFps: undefined }, + assets, + ownerId, + videoId, + ), + ).toBe(false); + expect( + validWebEditorClip( + { ...clip, cameraPath: displayPath }, + assets, + ownerId, + videoId, + ), + ).toBe(false); +}); + +test("adding a recording clip keeps existing trims, zooms, and other project settings", () => { + const source = { + camera: { hide: false }, + clips: [{ index: 0, offsets: { camera: 0.125 } }], + timeline: { + segments: [ + { recordingSegment: 0, timescale: 1, start: 2, end: 6 }, + { recordingSegment: 0, timescale: 0.5, start: 12, end: 16 }, + ], + zoomSegments: [{ start: 1, end: 2, amount: 1.5 }], + }, + }; + const added = appendWebEditorClipToConfig(source, 1, 20, 8.5); + expect(added).not.toBeNull(); + expect(added?.camera).toBe(source.camera); + expect(added?.clips).toBe(source.clips); + expect(added?.timeline).toEqual({ + segments: [ + ...source.timeline.segments, + { recordingSegment: 1, timescale: 1, start: 0, end: 8.5 }, + ], + zoomSegments: source.timeline.zoomSegments, + }); + expect(source.timeline.segments).toHaveLength(2); +}); + +test("the first added clip creates a base timeline without mutating the saved config", () => { + const source = { camera: { hide: true } }; + expect(appendWebEditorClipToConfig(source, 1, 12, 3)).toEqual({ + camera: source.camera, + timeline: { + segments: [ + { recordingSegment: 0, timescale: 1, start: 0, end: 12 }, + { recordingSegment: 1, timescale: 1, start: 0, end: 3 }, + ], + zoomSegments: [], + }, + }); + expect(source).not.toHaveProperty("timeline"); + expect(appendWebEditorClipToConfig(source, 0, 12, 3)).toBeNull(); + expect(appendWebEditorClipToConfig(source, 1001, 12, 3)).toBeNull(); + expect(appendWebEditorClipToConfig(source, 1, 12, Number.NaN)).toBeNull(); +}); + +test("an MP4 after a multi-clip Cap import uses the next recording segment index", () => { + const source = { + timeline: { + segments: [ + { recordingSegment: 0, timescale: 1, start: 0, end: 10 }, + { recordingSegment: 1, timescale: 1.5, start: 0.2, end: 1.6 }, + { recordingSegment: 2, timescale: 1, start: 0, end: 1.2 }, + ], + zoomSegments: [], + }, + }; + expect(appendWebEditorClipToConfig(source, 3, 10, 2)?.timeline).toEqual({ + segments: [ + ...source.timeline.segments, + { recordingSegment: 3, timescale: 1, start: 0, end: 2 }, + ], + zoomSegments: [], + }); +}); diff --git a/apps/web/__tests__/unit/editor-default-style.test.ts b/apps/web/__tests__/unit/editor-default-style.test.ts new file mode 100644 index 00000000000..52cbace5883 --- /dev/null +++ b/apps/web/__tests__/unit/editor-default-style.test.ts @@ -0,0 +1,102 @@ +import { + applyDefaultStyle, + extractDefaultStyle, + parseDefaultStyle, +} from "@cap/editor-cap-bundle/default-style"; +import { describe, expect, it } from "vitest"; + +const styled = { + aspectRatio: "wide", + background: { + source: { type: "gradient", from: [1, 2, 3], to: [4, 5, 6] }, + padding: 12, + rounding: 24, + shadow: 60, + crop: { position: { x: 10, y: 10 }, size: { x: 100, y: 100 } }, + displayPosition: { x: 0.4, y: 0.5 }, + }, + camera: { hide: true, position: { x: "left", y: "top" }, size: 30 }, + cursor: { size: 150, type: "pointer" }, + timeline: { segments: [{ start: 0, end: 5 }] }, + captions: { segments: [] }, + colorCorrection: { exposure: 1 }, +}; + +describe("default style", () => { + it("keeps the look and drops what belongs to one recording", () => { + expect(extractDefaultStyle(styled)).toEqual({ + version: 1, + aspectRatio: "wide", + background: { + source: { type: "gradient", from: [1, 2, 3], to: [4, 5, 6] }, + padding: 12, + rounding: 24, + shadow: 60, + }, + camera: { position: { x: "left", y: "top" }, size: 30 }, + cursor: { size: 150, type: "pointer" }, + }); + }); + + it("leaves out project-local background images", () => { + const style = extractDefaultStyle({ + background: { + source: { type: "image", path: "content/images/a.png" }, + padding: 4, + }, + }); + expect(style?.background).toEqual({ padding: 4 }); + }); + + it("rejects styles it cannot trust", () => { + expect(parseDefaultStyle(null)).toBeNull(); + expect(parseDefaultStyle({ version: 2, background: {} })).toBeNull(); + expect(extractDefaultStyle("config")).toBeNull(); + expect( + extractDefaultStyle({ cursor: { blob: "x".repeat(70 * 1024) } }), + ).toBeNull(); + }); + + it("lays the style over a project without touching its recording", () => { + const style = extractDefaultStyle(styled); + if (!style) throw new Error("style expected"); + const fresh = { + aspectRatio: null, + background: { + source: { type: "wallpaper", path: "macOS/sequoia-dark" }, + padding: 0, + rounding: 0, + crop: { position: { x: 1, y: 2 }, size: { x: 3, y: 4 } }, + }, + camera: { + hide: false, + position: { x: "right", y: "bottom" }, + mirror: false, + }, + cursor: { size: 100, hideWhenIdle: true }, + timeline: { segments: [{ start: 0, end: 9 }] }, + }; + const next = applyDefaultStyle(fresh, style); + expect(next.aspectRatio).toBe("wide"); + expect(next.background).toEqual({ + source: { type: "gradient", from: [1, 2, 3], to: [4, 5, 6] }, + padding: 12, + rounding: 24, + shadow: 60, + crop: { position: { x: 1, y: 2 }, size: { x: 3, y: 4 } }, + }); + expect(next.camera).toEqual({ + hide: false, + position: { x: "left", y: "top" }, + size: 30, + mirror: false, + }); + expect(next.cursor).toEqual({ + size: 150, + type: "pointer", + hideWhenIdle: true, + }); + expect(next.timeline).toBe(fresh.timeline); + expect(fresh.background.padding).toBe(0); + }); +}); diff --git a/apps/web/__tests__/unit/editor-export-upload-client.test.ts b/apps/web/__tests__/unit/editor-export-upload-client.test.ts new file mode 100644 index 00000000000..b3fa996733c --- /dev/null +++ b/apps/web/__tests__/unit/editor-export-upload-client.test.ts @@ -0,0 +1,378 @@ +import { createHash } from "node:crypto"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { + uploadWebEditorExport, + uploadWebEditorFile, +} from "../../lib/editor-export-upload-client"; + +const CHUNK_BYTES = 16 * 1024 * 1024; +// Storage parts are uniform (R2 requires it), whatever size the chunks come in. +const PART_BYTES = 5 * 1024 * 1024; + +function expectedParts(total: number) { + const sizes: number[] = Array(Math.floor(total / PART_BYTES)).fill( + PART_BYTES, + ); + if (total % PART_BYTES) sizes.push(total % PART_BYTES); + return sizes.map((size, index) => ({ + partNumber: index + 1, + etag: `etag-${index + 1}`, + size, + })); +} + +function uploadedBytes() { + return new Blob( + [...MockXMLHttpRequest.parts.entries()] + .sort(([left], [right]) => left - right) + .map(([, part]) => part), + ); +} +const metadata = { duration: 95, width: 1920, height: 1080, fps: 30 }; + +class MockXMLHttpRequest { + static parts = new Map(); + static aborted = 0; + upload = { + onprogress: null as ((event: ProgressEvent) => void) | null, + }; + onload: (() => void) | null = null; + onerror: (() => void) | null = null; + onabort: (() => void) | null = null; + ontimeout: (() => void) | null = null; + status = 0; + statusText = ""; + responseType = ""; + timeout = 0; + private partNumber = 0; + private completed = false; + + open(method: string, url: string) { + expect(method).toBe("PUT"); + this.partNumber = Number(url.match(/part-(\d+)$/)?.[1]); + } + + setRequestHeader() {} + + getResponseHeader(name: string) { + return name.toLowerCase() === "etag" ? `"etag-${this.partNumber}"` : null; + } + + send(part: Blob) { + MockXMLHttpRequest.parts.set(this.partNumber, part); + queueMicrotask(() => { + if (this.completed) return; + this.upload.onprogress?.({ + lengthComputable: true, + loaded: part.size, + total: part.size, + } as ProgressEvent); + this.status = 200; + this.completed = true; + this.onload?.(); + }); + } + + abort() { + if (this.completed) return; + this.completed = true; + MockXMLHttpRequest.aborted += 1; + this.onabort?.(); + } +} + +beforeEach(() => { + MockXMLHttpRequest.parts.clear(); + MockXMLHttpRequest.aborted = 0; + vi.stubGlobal("window", globalThis as typeof globalThis & Window); + vi.stubGlobal("addEventListener", vi.fn()); + vi.stubGlobal("removeEventListener", vi.fn()); + vi.stubGlobal( + "XMLHttpRequest", + MockXMLHttpRequest as unknown as typeof XMLHttpRequest, + ); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +test("a two-chunk MP4 reaches multipart publication with identical bytes and source metadata", async () => { + const size = CHUNK_BYTES + 3 * 1024 * 1024 + 123; + const asset = new Uint8Array(size); + asset.fill(0x35, 0, CHUNK_BYTES); + asset.fill(0xa7, CHUNK_BYTES); + const chunks: Array<{ offset: number; length: number }> = []; + let completion: Record | null = null; + const progress: Array<{ stage: string; fraction: number }> = []; + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if ( + url.startsWith("/api/editor/sessions/session/exports/export/chunk?") + ) { + const query = new URL(url, "http://cap.test").searchParams; + const offset = Number(query.get("offset")); + const length = Number(query.get("length")); + chunks.push({ offset, length }); + expect(query.get("videoId")).toBe("video"); + return new Response( + new Blob([ + asset.buffer.slice(offset, offset + length) as ArrayBuffer, + ]), + { + status: 206, + headers: { + "Content-Type": "video/mp4", + "Content-Length": String(length), + "Content-Range": `bytes ${offset}-${offset + length - 1}/${size}`, + }, + }, + ); + } + const body = JSON.parse(String(init?.body)) as Record; + expect(body.replaceExisting).toBe(true); + expect(body.videoId).toBe("video"); + expect(body.subpath).toBe("result.mp4"); + if (url.endsWith("/initiate")) { + expect(body.contentType).toBe("video/mp4"); + return Response.json({ uploadId: "upload", provider: "s3" }); + } + if (url.endsWith("/presign-part")) { + return Response.json({ + presignedUrl: `https://uploads.example/part-${body.partNumber}`, + provider: "s3", + }); + } + if (url.endsWith("/complete")) { + completion = body; + return Response.json({ success: true, processingStarted: true }); + } + throw new Error(`Unexpected request: ${url}`); + }), + ); + await uploadWebEditorExport( + "video", + "session", + "export", + size, + metadata, + new AbortController().signal, + (value) => progress.push(value), + ); + expect(chunks).toEqual([ + { offset: 0, length: CHUNK_BYTES }, + { offset: CHUNK_BYTES, length: size - CHUNK_BYTES }, + ]); + expect(completion).toMatchObject({ + uploadId: "upload", + durationInSecs: metadata.duration, + width: metadata.width, + height: metadata.height, + fps: metadata.fps, + parts: expectedParts(size), + }); + const uploaded = uploadedBytes(); + expect(uploaded.size).toBe(size); + const digest = (bytes: ArrayBuffer) => + createHash("sha256").update(Buffer.from(bytes)).digest("hex"); + expect(digest(await uploaded.arrayBuffer())).toBe(digest(asset.buffer)); + expect(progress.at(-1)).toEqual({ stage: "ready", fraction: 1 }); +}); + +test("a lost multipart completion response resolves from the published recording", async () => { + const size = 6 * 1024 * 1024; + let completeAttempts = 0; + let statusChecks = 0; + let aborted = false; + const progress: Array<{ stage: string; fraction: number }> = []; + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.includes("/chunk?")) { + return new Response(new Blob([new Uint8Array(size)]), { + status: 206, + headers: { + "Content-Type": "video/mp4", + "Content-Length": String(size), + "Content-Range": `bytes 0-${size - 1}/${size}`, + }, + }); + } + if (url.endsWith("/initiate")) { + return Response.json({ + uploadId: "cap-reupload.payload.signature", + provider: "s3", + }); + } + if (url.endsWith("/presign-part")) { + return Response.json({ + presignedUrl: "https://uploads.example/part-1", + provider: "s3", + }); + } + if (url.endsWith("/complete")) { + completeAttempts += 1; + return new Response("Completion response lost", { status: 500 }); + } + if (url.endsWith("/share-status")) { + const body = JSON.parse(String(init?.body)) as { + videoId: string; + uploadId: string; + }; + expect(body).toEqual({ + videoId: "video", + uploadId: "cap-reupload.payload.signature", + }); + statusChecks += 1; + return Response.json({ + status: statusChecks === 1 ? "active" : "published", + }); + } + if (url.endsWith("/abort")) { + aborted = true; + return Response.json({ success: true }); + } + throw new Error(`Unexpected request: ${url}`); + }), + ); + await uploadWebEditorExport( + "video", + "session", + "export", + size, + metadata, + new AbortController().signal, + (value) => progress.push(value), + ); + expect(completeAttempts).toBe(4); + expect(statusChecks).toBe(2); + expect(aborted).toBe(false); + expect(progress.at(-1)).toEqual({ stage: "ready", fraction: 1 }); +}, 20_000); + +test("aborting while a chunk body is read does not publish the recording", async () => { + const controller = new AbortController(); + const requests: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + requests.push(url); + if (url.endsWith("/initiate")) + return Response.json({ uploadId: "upload", provider: "s3" }); + if (url.endsWith("/abort")) return Response.json({ success: true }); + if (url.includes("/chunk?")) { + const response = new Response(new Blob([new Uint8Array(1024)]), { + status: 206, + headers: { + "Content-Type": "video/mp4", + "Content-Length": "1024", + "Content-Range": "bytes 0-1023/1024", + }, + }); + vi.spyOn(response, "blob").mockImplementation(async () => { + controller.abort(); + return new Blob([new Uint8Array(1024)]); + }); + return response; + } + throw new Error(`Unexpected request: ${url}`); + }), + ); + await expect( + uploadWebEditorExport( + "video", + "session", + "export", + 1024, + metadata, + controller.signal, + ), + ).rejects.toThrow("Recording upload was canceled"); + expect(requests.some((url) => url.endsWith("/abort"))).toBe(true); + expect(requests.some((url) => url.endsWith("/complete"))).toBe(false); +}); + +test("a video rendered in the browser is published from its own bytes", async () => { + const size = CHUNK_BYTES + 512; + const asset = new Uint8Array(size).fill(0x42); + const requests: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + requests.push(url); + const body = JSON.parse(String(init?.body)) as Record; + expect(body.replaceExisting).toBe(true); + if (url.endsWith("/initiate")) + return Response.json({ uploadId: "upload", provider: "s3" }); + if (url.endsWith("/presign-part")) + return Response.json({ + presignedUrl: `https://uploads.example/part-${body.partNumber}`, + provider: "s3", + }); + if (url.endsWith("/complete")) + return Response.json({ success: true, processingStarted: true }); + throw new Error(`Unexpected request: ${url}`); + }), + ); + await uploadWebEditorFile( + "video", + "session", + new Blob([asset], { type: "video/mp4" }), + metadata, + new AbortController().signal, + ); + expect(requests.some((url) => url.includes("/chunk?"))).toBe(false); + const uploaded = uploadedBytes(); + const digest = (bytes: ArrayBuffer) => + createHash("sha256").update(Buffer.from(bytes)).digest("hex"); + expect(uploaded.size).toBe(size); + expect(digest(await uploaded.arrayBuffer())).toBe(digest(asset.buffer)); +}); + +test("closing the page abandons an unfinished replacement upload", async () => { + const listeners = new Map void>(); + vi.stubGlobal( + "addEventListener", + vi.fn((name: string, listener: () => void) => + listeners.set(name, listener), + ), + ); + const aborts: Array<{ body: unknown; keepalive: boolean | undefined }> = []; + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.endsWith("/initiate")) + return Response.json({ uploadId: "upload", provider: "s3" }); + if (url.endsWith("/abort")) { + aborts.push({ + body: JSON.parse(String(init?.body)), + keepalive: init?.keepalive, + }); + return Response.json({ success: true }); + } + return new Promise(() => undefined); + }), + ); + void uploadWebEditorFile( + "video", + "session", + new Blob([new Uint8Array(1024)], { type: "video/mp4" }), + metadata, + new AbortController().signal, + ).catch(() => undefined); + await vi.waitFor(() => expect(listeners.has("pagehide")).toBe(true)); + listeners.get("pagehide")?.(); + expect(aborts).toEqual([ + { + body: { videoId: "video", uploadId: "upload", subpath: "result.mp4" }, + keepalive: true, + }, + ]); +}); diff --git a/apps/web/__tests__/unit/editor-host-lifecycle.test.ts b/apps/web/__tests__/unit/editor-host-lifecycle.test.ts new file mode 100644 index 00000000000..4070d17125d --- /dev/null +++ b/apps/web/__tests__/unit/editor-host-lifecycle.test.ts @@ -0,0 +1,2738 @@ +import { afterEach, expect, test, vi } from "vitest"; +import { EditorHostBridge } from "../../app/s/[videoId]/edit/studio/editor-host"; +import { + uploadWebEditorExport, + uploadWebEditorFile, +} from "../../lib/editor-export-upload-client"; + +vi.mock("@/lib/editor-export-upload-client", () => ({ + uploadWebEditorExport: vi.fn(async () => undefined), + uploadWebEditorFile: vi.fn(async () => undefined), +})); + +const ticket = "a".repeat(43); +const credentials = { + commands: { url: "ws://127.0.0.1:1234/commands", ticket }, + events: { url: "ws://127.0.0.1:1234/events", ticket }, + audio: { url: "ws://127.0.0.1:1234/audio", ticket }, + frames: { url: "ws://127.0.0.1:1234/frames", ticket }, +}; + +function frame(status: "ready" | "error" | null = "ready") { + const postMessage = vi.fn( + (_message: unknown, _origin: string, ports?: MessagePort[]) => { + if (status) ports?.[0]?.postMessage({ kind: "mount", status }); + }, + ); + return { + postMessage, + iframe: { + contentWindow: { postMessage }, + } as unknown as HTMLIFrameElement, + }; +} + +afterEach(() => vi.unstubAllGlobals()); + +test("disposing during ticket fetch never attaches the editor frame", async () => { + let resolveFetch: (response: Response) => void = () => undefined; + const pendingFetch = new Promise((resolve) => { + resolveFetch = resolve; + }); + vi.stubGlobal( + "fetch", + vi.fn(() => pendingFetch), + ); + const { iframe, postMessage } = frame(); + const bridge = new EditorHostBridge( + "video", + "session", + "user", + vi.fn(), + vi.fn(), + ); + const connecting = bridge.connect(iframe); + bridge.dispose(); + resolveFetch(Response.json(credentials)); + await expect(connecting).rejects.toThrow("Editor bridge is closed"); + expect(postMessage).not.toHaveBeenCalled(); +}); + +test("disposing during socket negotiation closes every pending connection", async () => { + const sockets: PendingSocket[] = []; + class PendingSocket extends EventTarget { + protocol = "cap-editor-v1"; + closed = false; + constructor(_url: string, _protocols: string[]) { + super(); + sockets.push(this); + } + close() { + this.closed = true; + } + } + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json(credentials)), + ); + vi.stubGlobal("WebSocket", PendingSocket); + vi.stubGlobal("window", { setTimeout, clearTimeout }); + const { iframe, postMessage } = frame(); + const bridge = new EditorHostBridge( + "video", + "session", + "user", + vi.fn(), + vi.fn(), + ); + const connecting = bridge.connect(iframe); + await vi.waitFor(() => expect(sockets).toHaveLength(3)); + bridge.dispose(); + await expect(connecting).rejects.toThrow("Editor bridge is closed"); + expect(sockets.every((socket) => socket.closed)).toBe(true); + expect(postMessage).not.toHaveBeenCalled(); +}); + +test("a failed Solid mount rejects the web editor connection", async () => { + const sockets: Array<{ closed: boolean }> = []; + class OpenSocket extends EventTarget { + protocol = "cap-editor-v1"; + closed = false; + constructor(_url: string, _protocols: string[]) { + super(); + sockets.push(this); + queueMicrotask(() => this.dispatchEvent(new Event("open"))); + } + close() { + this.closed = true; + } + } + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json(credentials)), + ); + vi.stubGlobal("WebSocket", OpenSocket); + vi.stubGlobal("window", { + setTimeout, + clearTimeout, + location: { origin: "http://127.0.0.1:3000" }, + }); + const { iframe } = frame("error"); + const bridge = new EditorHostBridge( + "video", + "session", + "user", + vi.fn(), + vi.fn(), + ); + await expect(bridge.connect(iframe)).rejects.toThrow("Editor could not load"); + bridge.dispose(); + expect(sockets).toHaveLength(3); + expect(sockets.every((socket) => socket.closed)).toBe(true); +}); + +test("desktop autosave is acknowledged only after the persistent web API saves it", async () => { + const requests: Array<{ url: string; init?: RequestInit }> = []; + let savedAt = "first-revision"; + const save = vi.fn(async () => + Response.json({ saved: true, savedAt: "second-revision" }), + ); + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + requests.push({ url, init }); + return url.endsWith("/tickets") ? Response.json(credentials) : save(); + }), + ); + class OpenSocket extends EventTarget { + protocol = "cap-editor-v1"; + readyState = 1; + constructor(_url: string, _protocols: string[]) { + super(); + queueMicrotask(() => this.dispatchEvent(new Event("open"))); + } + close() { + this.readyState = 3; + } + send() { + throw new Error("Native command socket should not handle project saves"); + } + } + vi.stubGlobal("WebSocket", OpenSocket); + vi.stubGlobal("window", { + setTimeout, + clearTimeout, + location: { origin: "http://127.0.0.1:3000" }, + }); + const { iframe, postMessage } = frame(); + const bridge = new EditorHostBridge( + "video", + "session", + "user", + vi.fn(), + vi.fn(), + undefined, + undefined, + undefined, + false, + undefined, + (nextSavedAt) => { + savedAt = nextSavedAt; + }, + () => savedAt, + ); + await bridge.connect(iframe); + const port = postMessage.mock.calls[0]?.[2]?.[0] as MessagePort; + const reply = new Promise((resolve) => { + port.onmessage = (event) => resolve(event.data); + port.start(); + }); + port.postMessage({ + kind: "invoke", + id: 7, + name: "setProjectConfig", + args: [{ camera: { mirror: true } }], + }); + expect(await reply).toEqual({ kind: "result", id: 7, value: null }); + expect(save).toHaveBeenCalledOnce(); + expect(requests[1]?.url).toBe("/api/editor/sessions/session/config"); + expect(requests[1]?.init?.method).toBe("PUT"); + expect(JSON.parse(String(requests[1]?.init?.body))).toEqual({ + videoId: "video", + config: { camera: { mirror: true } }, + expectedSavedAt: "first-revision", + }); + expect(savedAt).toBe("second-revision"); + const nextReply = new Promise((resolve) => { + port.onmessage = (event) => resolve(event.data); + }); + port.postMessage({ + kind: "invoke", + id: 8, + name: "setProjectConfig", + args: [{ camera: { mirror: false } }, true], + }); + expect(await nextReply).toEqual({ kind: "result", id: 8, value: null }); + expect(JSON.parse(String(requests[2]?.init?.body))).toEqual({ + videoId: "video", + config: { camera: { mirror: false } }, + expectedSavedAt: "second-revision", + preserveExistingPaidCaptions: true, + }); + port.close(); + bridge.dispose(); +}); + +async function connectedExportHost( + request: (url: string, init?: RequestInit) => Promise, + captionsEnabled = false, + onUpgrade?: () => void, + parentEvents = false, +) { + vi.stubGlobal( + "fetch", + vi.fn((input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + return url.endsWith("/tickets") + ? Promise.resolve(Response.json(credentials)) + : request(url, init); + }), + ); + class OpenSocket extends EventTarget { + protocol = "cap-editor-v1"; + readyState = 1; + constructor(_url: string, _protocols: string[]) { + super(); + queueMicrotask(() => this.dispatchEvent(new Event("open"))); + } + close() { + this.readyState = 3; + } + send() { + throw new Error( + "Native command socket should not handle web export jobs", + ); + } + } + vi.stubGlobal("WebSocket", OpenSocket); + const browserWindow = { + setTimeout, + clearTimeout, + location: { origin: "http://127.0.0.1:3000" }, + }; + vi.stubGlobal( + "window", + parentEvents + ? Object.assign(new EventTarget(), browserWindow) + : browserWindow, + ); + if (parentEvents) + vi.stubGlobal( + "document", + Object.assign(new EventTarget(), { hidden: false }), + ); + const { iframe, postMessage } = frame(); + const onClose = vi.fn(); + const bridge = new EditorHostBridge( + "video", + "session", + "user", + onClose, + vi.fn(), + undefined, + undefined, + undefined, + captionsEnabled, + onUpgrade, + ); + await bridge.connect(iframe); + const port = postMessage.mock.calls[0]?.[2]?.[0] as MessagePort; + port.start(); + return { bridge, port, onClose }; +} + +test("returning to the web page sends the refreshed caption plan to Studio", async () => { + const planRequests: string[] = []; + const { bridge, port } = await connectedExportHost( + async (url) => { + if (!url.includes("/plan?")) throw new Error(`Unexpected request ${url}`); + planRequests.push(url); + return Response.json({ pro: true }); + }, + false, + undefined, + true, + ); + const plans: boolean[] = []; + port.onmessage = (event: MessageEvent) => { + const message = event.data; + if ( + typeof message === "object" && + message !== null && + "kind" in message && + message.kind === "event" && + "name" in message && + message.name === "editorCaptionPlan" && + "payload" in message && + typeof message.payload === "boolean" + ) { + plans.push(message.payload); + } + }; + try { + window.dispatchEvent(new Event("focus")); + await vi.waitFor(() => expect(plans).toEqual([true])); + document.dispatchEvent(new Event("visibilitychange")); + await vi.waitFor(() => expect(plans).toEqual([true, true])); + expect(planRequests).toHaveLength(2); + } finally { + port.close(); + bridge.dispose(); + } +}); + +test("overlapping host plan requests still deliver the latest Studio entitlement", async () => { + const replies: Array<(response: Response) => void> = []; + const { bridge, port } = await connectedExportHost( + async (url) => { + if (!url.includes("/plan?")) throw new Error(`Unexpected request ${url}`); + return new Promise((resolve) => replies.push(resolve)); + }, + false, + undefined, + true, + ); + const plans: boolean[] = []; + const commandResults: boolean[] = []; + port.onmessage = (event: MessageEvent) => { + const message = event.data; + if (typeof message !== "object" || message === null) return; + if ( + "kind" in message && + message.kind === "event" && + "name" in message && + message.name === "editorCaptionPlan" && + "payload" in message && + typeof message.payload === "boolean" + ) { + plans.push(message.payload); + } else if ( + "kind" in message && + message.kind === "result" && + "id" in message && + message.id === 73 && + "value" in message && + typeof message.value === "boolean" + ) { + commandResults.push(message.value); + } + }; + try { + window.dispatchEvent(new Event("focus")); + await vi.waitFor(() => expect(replies).toHaveLength(1)); + port.postMessage({ + kind: "invoke", + id: 73, + name: "checkUpgradedAndUpdate", + args: [], + }); + await vi.waitFor(() => expect(replies).toHaveLength(2)); + replies[1]?.(Response.json({ pro: true })); + await vi.waitFor(() => expect(plans).toEqual([true])); + await vi.waitFor(() => expect(commandResults).toEqual([true])); + replies[0]?.(Response.json({ pro: false })); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(plans).toEqual([true]); + } finally { + port.close(); + bridge.dispose(); + } +}); + +test("a failed newer plan request does not discard a successful Studio refresh", async () => { + const replies: Array<(response: Response) => void> = []; + const { bridge, port } = await connectedExportHost( + async (url) => { + if (!url.includes("/plan?")) throw new Error(`Unexpected request ${url}`); + return new Promise((resolve) => replies.push(resolve)); + }, + false, + undefined, + true, + ); + const plans: boolean[] = []; + let newerRequestFailed = false; + port.onmessage = (event: MessageEvent) => { + const message = event.data; + if (typeof message !== "object" || message === null) return; + if ( + "kind" in message && + message.kind === "event" && + "name" in message && + message.name === "editorCaptionPlan" && + "payload" in message && + typeof message.payload === "boolean" + ) { + plans.push(message.payload); + } else if ( + "kind" in message && + message.kind === "error" && + "id" in message && + message.id === 74 + ) { + newerRequestFailed = true; + } + }; + try { + window.dispatchEvent(new Event("focus")); + await vi.waitFor(() => expect(replies).toHaveLength(1)); + port.postMessage({ + kind: "invoke", + id: 74, + name: "checkUpgradedAndUpdate", + args: [], + }); + await vi.waitFor(() => expect(replies).toHaveLength(2)); + replies[1]?.(new Response("Unavailable", { status: 503 })); + await vi.waitFor(() => expect(newerRequestFailed).toBe(true)); + replies[0]?.(Response.json({ pro: true })); + await vi.waitFor(() => expect(plans).toEqual([true])); + expect(Reflect.get(bridge, "captionsEnabled")).toBe(true); + } finally { + port.close(); + bridge.dispose(); + } +}); + +test("caption cache conflicts retain the shared bridge's full-payload retry", async () => { + const { bridge, port } = await connectedExportHost( + async () => new Response("Conflict", { status: 409 }), + ); + const save = async (id: number, config: Record) => { + const reply = new Promise<{ kind: string; error?: string }>((resolve) => { + port.onmessage = (event) => resolve(event.data); + }); + port.postMessage({ + kind: "invoke", + id, + name: "setProjectConfig", + args: [config], + }); + return reply; + }; + expect( + await save(21, { webCaptionRef: "caption-cache", camera: {} }), + ).toEqual( + expect.objectContaining({ + kind: "error", + error: "Caption payload cache is unavailable", + }), + ); + expect(await save(22, { captions: { segments: [] }, camera: {} })).toEqual( + expect.objectContaining({ + kind: "error", + error: + "Editor changed in another tab or caption data is unavailable. Reload to continue.", + }), + ); + port.close(); + bridge.dispose(); +}); + +test("recording title saves through the owned web session", async () => { + const requests: Array<{ url: string; init?: RequestInit }> = []; + let conflict = false; + const { bridge, port } = await connectedExportHost(async (url, init) => { + requests.push({ url, init }); + return conflict + ? new Response("Conflict", { status: 409 }) + : Response.json({ saved: true }); + }); + const title = async (id: number, prettyName: string) => { + const reply = new Promise((resolve) => { + port.onmessage = (event) => resolve(event.data); + }); + port.postMessage({ + kind: "invoke", + id, + name: "setPrettyName", + args: [prettyName], + }); + return reply; + }; + try { + expect(await title(1, "Renamed recording")).toEqual({ + kind: "result", + id: 1, + value: null, + }); + expect(requests).toHaveLength(1); + expect(requests[0]?.url).toBe("/api/editor/sessions/session/meta"); + expect(requests[0]?.init?.method).toBe("PUT"); + expect(JSON.parse(String(requests[0]?.init?.body))).toEqual({ + videoId: "video", + prettyName: "Renamed recording", + }); + expect(await title(2, "Tiny")).toEqual({ + kind: "error", + id: 2, + error: "Recording title must be 5 to 100 characters", + }); + expect(requests).toHaveLength(1); + conflict = true; + expect(await title(3, "Another recording")).toEqual({ + kind: "error", + id: 3, + error: "Recording title changed in another editor", + }); + } finally { + port.close(); + bridge.dispose(); + } +}); + +test("both editor folder actions download an owned recording bundle", async () => { + const requests: Array<{ url: string; init?: RequestInit }> = []; + const downloadUrl = `https://media.cap.so/editor/sessions/session/project-bundle/download?ticket=${ticket}`; + const { bridge, port } = await connectedExportHost(async (url, init) => { + requests.push({ url, init }); + return Response.json({ url: downloadUrl }); + }); + const click = vi.fn(); + const remove = vi.fn(); + const append = vi.fn(); + const link = { + href: "", + rel: "", + download: "", + click, + remove, + setAttribute: vi.fn(), + }; + vi.stubGlobal("document", { + createElement: vi.fn(() => link), + body: { append }, + }); + const invoke = async (id: number, path: string) => { + const reply = new Promise((resolve) => { + port.onmessage = (event) => resolve(event.data); + }); + port.postMessage({ + kind: "invoke", + id, + name: "tauri:download_editor_bundle", + args: [{ path }], + }); + return reply; + }; + try { + for (const [id, path] of [ + [1, "cap-web-editor://session/session/"], + [2, "cap-web-editor://session/session"], + ] as const) { + expect(await invoke(id, path)).toEqual({ + kind: "result", + id, + value: null, + }); + } + expect(requests).toHaveLength(2); + for (const request of requests) { + expect(request.url).toBe( + "/api/editor/sessions/session/project-bundle/download-ticket", + ); + expect(request.init?.method).toBe("POST"); + expect(JSON.parse(String(request.init?.body))).toEqual({ + videoId: "video", + }); + } + expect(click).toHaveBeenCalledTimes(2); + expect(link.href).toBe(downloadUrl); + expect(link.download).toBe("Cap Recording.capbundle"); + // The hidden download frame is added once, then each link. + expect(append).toHaveBeenCalledTimes(3); + expect(remove).toHaveBeenCalledTimes(2); + expect(await invoke(3, "cap-web-editor://session/other")).toEqual({ + kind: "error", + id: 3, + error: "Editor bundle request was invalid", + }); + expect(requests).toHaveLength(2); + } finally { + port.close(); + bridge.dispose(); + } +}); + +test("browser Studio reprepares export workers after Cap Pro changes without restarting local playback", async () => { + let pro = true; + let now = Date.now(); + let preparations = 0; + const released: string[] = []; + const downloaded: string[] = []; + const clock = vi.spyOn(Date, "now").mockImplementation(() => now); + vi.stubGlobal("window", { + setTimeout, + clearTimeout, + location: { origin: "http://127.0.0.1:3000" }, + }); + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url === "/api/editor/videos/video/plan") + return Response.json({ pro }); + if (url === "/api/editor/preparations" && init?.method === "POST") { + preparations++; + return Response.json({ + id: `prep-${preparations}`, + status: "preparing", + }); + } + if ( + url.startsWith("/api/editor/preparations/prep-") && + init?.method !== "DELETE" + ) { + const number = url.match(/prep-(\d+)/)?.[1]; + return Response.json({ + status: "ready", + sessionId: `worker-${number}`, + }); + } + if ( + url.startsWith("/api/editor/sessions/worker-") && + init?.method === "DELETE" + ) { + released.push(url); + return Response.json({ closed: true }); + } + if ( + url.endsWith("/project-bundle/download-ticket") && + init?.method === "POST" + ) { + const number = url.match(/worker-(\d+)/)?.[1]; + return Response.json({ + url: `https://media.cap.so/editor/sessions/worker-${number}/project-bundle/download?ticket=${ticket}`, + }); + } + throw new Error(`Unexpected editor request ${url}`); + }), + ); + vi.stubGlobal("document", { + createElement: vi.fn(() => ({ + href: "", + rel: "", + download: "", + click() { + downloaded.push(this.href); + }, + remove: vi.fn(), + setAttribute: vi.fn(), + })), + body: { append: vi.fn() }, + }); + const { iframe, postMessage } = frame(); + const bridge = new EditorHostBridge( + "video", + "session", + "user", + vi.fn(), + vi.fn(), + undefined, + undefined, + undefined, + true, + undefined, + undefined, + () => "saved-revision", + true, + ); + try { + await bridge.connect(iframe); + const port = postMessage.mock.calls[0]?.[2]?.[0] as MessagePort; + port.start(); + const download = async (id: number) => { + const reply = new Promise((resolve) => { + port.onmessage = (event) => resolve(event.data); + }); + port.postMessage({ + kind: "invoke", + id, + name: "tauri:download_editor_bundle", + args: [{ path: "cap-web-editor://session/session" }], + }); + return reply; + }; + for (const [id, plan] of [ + [1, true], + [2, false], + [3, true], + ] as const) { + pro = plan; + if (id > 1) now += 31_000; + expect(await download(id)).toEqual({ kind: "result", id, value: null }); + } + expect(preparations).toBe(3); + expect(released).toEqual([ + "/api/editor/sessions/worker-1?videoId=video", + "/api/editor/sessions/worker-2?videoId=video", + ]); + expect(downloaded).toEqual( + [1, 2, 3].map( + (number) => + `https://media.cap.so/editor/sessions/worker-${number}/project-bundle/download?ticket=${ticket}`, + ), + ); + port.close(); + } finally { + bridge.dispose(); + clock.mockRestore(); + } +}); + +test("a plan change cannot release a worker during another browser Studio operation", async () => { + let planReads = 0; + let preparations = 0; + let bundleTicketRequests = 0; + let resolvePreparation: (response: Response) => void = () => undefined; + let resolveChangedPlan: (response: Response) => void = () => undefined; + let resolveBundleTicket: (response: Response) => void = () => undefined; + const preparationReady = new Promise((resolve) => { + resolvePreparation = resolve; + }); + const changedPlan = new Promise((resolve) => { + resolveChangedPlan = resolve; + }); + const bundleTicket = new Promise((resolve) => { + resolveBundleTicket = resolve; + }); + const released: string[] = []; + const downloaded: string[] = []; + vi.stubGlobal("window", { + setTimeout, + clearTimeout, + location: { origin: "http://127.0.0.1:3000" }, + }); + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url === "/api/editor/videos/video/plan") { + planReads++; + return planReads === 3 ? changedPlan : Response.json({ pro: true }); + } + if (url === "/api/editor/preparations" && init?.method === "POST") { + preparations++; + return Response.json({ id: "prep-1", status: "preparing" }); + } + if (url === "/api/editor/preparations/prep-1?videoId=video") + return preparationReady; + if ( + url === "/api/editor/sessions/worker-1?videoId=video" && + init?.method === "DELETE" + ) { + released.push(url); + return Response.json({ closed: true }); + } + if ( + url === + "/api/editor/sessions/worker-1/project-bundle/download-ticket" && + init?.method === "POST" + ) { + bundleTicketRequests++; + return bundleTicket; + } + throw new Error(`Unexpected editor request ${url}`); + }), + ); + vi.stubGlobal("document", { + createElement: vi.fn(() => ({ + href: "", + rel: "", + download: "", + click() { + downloaded.push(this.href); + }, + remove: vi.fn(), + setAttribute: vi.fn(), + })), + body: { append: vi.fn() }, + }); + const { iframe, postMessage } = frame(); + const bridge = new EditorHostBridge( + "video", + "session", + "user", + vi.fn(), + vi.fn(), + undefined, + undefined, + undefined, + true, + undefined, + undefined, + () => "saved-revision", + true, + ); + try { + await bridge.connect(iframe); + const port = postMessage.mock.calls[0]?.[2]?.[0] as MessagePort; + const replies: Array<{ kind: string; id: number; error?: string }> = []; + port.onmessage = (event) => replies.push(event.data); + port.start(); + for (const id of [1, 2]) { + port.postMessage({ + kind: "invoke", + id, + name: "tauri:download_editor_bundle", + args: [{ path: "cap-web-editor://session/session" }], + }); + } + await vi.waitFor(() => expect(planReads).toBe(1)); + await vi.waitFor(() => expect(preparations).toBe(1)); + resolvePreparation( + Response.json({ status: "ready", sessionId: "worker-1" }), + ); + await vi.waitFor(() => expect(bundleTicketRequests).toBe(1)); + await vi.waitFor(() => expect(planReads).toBe(3)); + resolveChangedPlan(Response.json({ pro: false })); + await vi.waitFor(() => + expect(replies).toContainEqual({ + kind: "error", + id: 2, + error: "Finish the current editor operation before exporting new edits", + }), + ); + expect(released).toEqual([]); + resolveBundleTicket( + Response.json({ + url: `https://media.cap.so/editor/sessions/worker-1/project-bundle/download?ticket=${ticket}`, + }), + ); + await vi.waitFor(() => + expect(replies).toContainEqual({ kind: "result", id: 1, value: null }), + ); + expect(downloaded).toHaveLength(1); + expect(released).toEqual([]); + port.close(); + } finally { + bridge.dispose(); + } +}); + +test("canceling an export estimate during worker startup never starts native rendering", async () => { + let resolvePreparation: (response: Response) => void = () => undefined; + const preparation = new Promise((resolve) => { + resolvePreparation = resolve; + }); + const requests: string[] = []; + const sends: string[] = []; + vi.stubGlobal("window", { + setTimeout, + clearTimeout, + location: { origin: "http://127.0.0.1:3000" }, + }); + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + requests.push(url); + if (url === "/api/editor/videos/video/plan") + return Response.json({ pro: false }); + if (url === "/api/editor/preparations" && init?.method === "POST") + return preparation; + if (url === "/api/editor/preparations/prep-1?videoId=video") + return Response.json({ status: "ready", sessionId: "worker-1" }); + if ( + url === "/api/editor/sessions/worker-1?videoId=video" && + init?.method === "DELETE" + ) + return Response.json({ closed: true }); + if ( + url === "/api/editor/sessions/worker-1/tickets" && + init?.method === "POST" + ) + return Response.json(credentials); + throw new Error(`Unexpected editor request ${url}`); + }), + ); + class OpenSocket extends EventTarget { + protocol = "cap-editor-v1"; + readyState = 1; + constructor(_url: string, _protocols: string[]) { + super(); + queueMicrotask(() => this.dispatchEvent(new Event("open"))); + } + close() { + this.readyState = 3; + } + send(value: string) { + sends.push(value); + } + } + vi.stubGlobal("WebSocket", OpenSocket); + const { iframe, postMessage } = frame(); + const bridge = new EditorHostBridge( + "video", + "session", + "user", + vi.fn(), + vi.fn(), + undefined, + undefined, + undefined, + false, + undefined, + undefined, + () => null, + true, + ); + try { + await bridge.connect(iframe); + const port = postMessage.mock.calls[0]?.[2]?.[0] as MessagePort; + const replies: Array<{ kind: string; id: number; error?: string }> = []; + port.onmessage = (event) => replies.push(event.data); + port.start(); + port.postMessage({ + kind: "invoke", + id: 51, + name: "getExportEstimates", + args: ["cap-web-editor://session/session", {}, 12], + }); + await vi.waitFor(() => + expect(requests).toContain("/api/editor/preparations"), + ); + port.postMessage({ + kind: "invoke", + id: 52, + name: "cancelExportEstimates", + args: [], + }); + await vi.waitFor(() => + expect(replies).toContainEqual({ kind: "result", id: 52, value: null }), + ); + resolvePreparation(Response.json({ id: "prep-1", status: "preparing" })); + await vi.waitFor(() => + expect(replies).toContainEqual({ + kind: "error", + id: 51, + error: "Editor export estimate was canceled", + }), + ); + expect(sends).toEqual([]); + expect(requests).not.toContain("/api/editor/sessions/worker-1/tickets"); + port.close(); + } finally { + bridge.dispose(); + } +}); + +test("chosen desktop wallpaper uploads as an image and returns its portable path", async () => { + const path = "content/images/22222222-2222-4222-8222-222222222222.jpg"; + const key = `user/video/editor-assets/images/${path.slice("content/images/".length)}`; + const assetUrl = "/api/editor/sessions/session/assets"; + const uploadUrl = "http://127.0.0.1:1234/upload-wallpaper"; + const requests: Array<{ url: string; init?: RequestInit }> = []; + const { bridge, port } = await connectedExportHost(async (url, init) => { + requests.push({ url, init }); + if (url === assetUrl && init?.method === "POST") { + return Response.json({ + key, + path, + upload: { type: "put", url: uploadUrl, headers: {} }, + }); + } + if (url === uploadUrl && init?.method === "PUT") + return new Response(null, { status: 200 }); + if (url === assetUrl && init?.method === "PUT") { + return Response.json({ + path, + name: "wallpaper.jpg", + width: 64, + height: 64, + }); + } + if (url === `${assetUrl}?videoId=video` && !init?.method) + return Response.json({ path }); + throw new Error(`Unexpected wallpaper request: ${url}`); + }); + try { + const reply = new Promise((resolve) => { + port.onmessage = (event) => resolve(event.data); + }); + port.postMessage({ + kind: "invoke", + id: 52, + name: "importCurrentDesktopBackground", + args: [ + new File([new Uint8Array([1, 2, 3])], "wallpaper.jpg", { + type: "image/jpeg", + }), + ], + }); + expect(await reply).toEqual({ kind: "result", id: 52, value: path }); + const storedReply = new Promise((resolve) => { + port.onmessage = (event) => resolve(event.data); + }); + port.postMessage({ + kind: "invoke", + id: 53, + name: "tauri:webEditorStoredDesktopBackground", + args: [undefined], + }); + expect(await storedReply).toEqual({ kind: "result", id: 53, value: path }); + expect( + requests.map((request) => [request.url, request.init?.method]), + ).toEqual([ + [assetUrl, "POST"], + [uploadUrl, "PUT"], + [assetUrl, "PUT"], + [`${assetUrl}?videoId=video`, undefined], + ]); + expect(JSON.parse(String(requests[0]?.init?.body)).fileName).toBe( + "current-desktop-background.jpg", + ); + expect(JSON.parse(String(requests[2]?.init?.body)).fileName).toBe( + "current-desktop-background.jpg", + ); + } finally { + port.close(); + bridge.dispose(); + } +}); + +test("deleting a recording uses the owned video API and closes to the dashboard", async () => { + const requests: Array<{ url: string; init?: RequestInit }> = []; + const { bridge, port, onClose } = await connectedExportHost( + async (url, init) => { + requests.push({ url, init }); + return Response.json({ deleted: true }); + }, + ); + try { + const reply = new Promise((resolve) => { + port.onmessage = (event) => resolve(event.data); + }); + port.postMessage({ + kind: "invoke", + id: 43, + name: "editorDeleteProject", + args: [], + }); + expect(await reply).toEqual({ kind: "result", id: 43, value: null }); + expect(requests).toHaveLength(1); + expect(requests[0]?.url).toBe("/api/video/delete?videoId=video"); + expect(requests[0]?.init?.method).toBe("DELETE"); + expect(onClose).toHaveBeenCalledWith("deleted"); + } finally { + port.close(); + bridge.dispose(); + } +}); + +test("a failed recording deletion keeps the editor open", async () => { + const { bridge, port, onClose } = await connectedExportHost( + async () => new Response("Storage unavailable", { status: 503 }), + ); + try { + const reply = new Promise((resolve) => { + port.onmessage = (event) => resolve(event.data); + }); + port.postMessage({ + kind: "invoke", + id: 44, + name: "editorDeleteProject", + args: [], + }); + expect(await reply).toEqual({ + kind: "error", + id: 44, + error: "Recording could not be deleted", + }); + expect(onClose).not.toHaveBeenCalled(); + } finally { + port.close(); + bridge.dispose(); + } +}); + +test("web editor metadata exposes the recording's existing share link", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json(credentials)), + ); + class MetaSocket extends EventTarget { + static OPEN = 1; + protocol = "cap-editor-v1"; + readyState = 1; + onmessage: ((event: MessageEvent) => void) | null = null; + constructor(_url: string, _protocols: string[]) { + super(); + queueMicrotask(() => this.dispatchEvent(new Event("open"))); + } + close() { + this.readyState = 3; + } + send(payload: string) { + const request = JSON.parse(payload) as { id: number; name: string }; + expect(["getEditorMeta", "getRecordingMetaByPath"]).toContain( + request.name, + ); + queueMicrotask(() => + this.onmessage?.({ + data: JSON.stringify({ + kind: "result", + id: request.id, + value: { pretty_name: "Shared recording", sharing: null }, + }), + } as MessageEvent), + ); + } + } + vi.stubGlobal("WebSocket", MetaSocket); + vi.stubGlobal("window", { + setTimeout, + clearTimeout, + location: { origin: "http://127.0.0.1:3000" }, + }); + const { iframe, postMessage } = frame(); + const bridge = new EditorHostBridge( + "video", + "session", + "user", + vi.fn(), + vi.fn(), + ); + await bridge.connect(iframe); + const port = postMessage.mock.calls[0]?.[2]?.[0] as MessagePort; + port.start(); + try { + for (const [id, name] of [ + [41, "getEditorMeta"], + [42, "getRecordingMetaByPath"], + ] as const) { + const reply = new Promise((resolve) => { + port.onmessage = (event) => resolve(event.data); + }); + port.postMessage({ kind: "invoke", id, name, args: [] }); + expect(await reply).toEqual({ + kind: "result", + id, + value: { + pretty_name: "Shared recording", + sharing: { + id: "video", + link: "http://127.0.0.1:3000/s/video", + }, + }, + }); + } + } finally { + port.close(); + bridge.dispose(); + } +}); + +const exportSettings = { + format: "Mp4", + fps: 30, + resolution_base: { x: 640, y: 360 }, + compression: "Social", + custom_bpp: null, +}; + +function exportState( + status: "running" | "ready", + downloadStartedAt: number | null = null, + duration = 3, +) { + return { + id: "job", + status, + format: "Mp4", + size: 160_000, + mediaMetadata: { duration, width: 640, height: 358, fps: 30 }, + progress: { + rendered_count: status === "ready" ? duration * 30 : 30, + total_frames: duration * 30, + }, + error: null, + downloadStartedAt, + }; +} + +test("a rendered MP4 is uploaded to the existing share link through bounded chunks", async () => { + const requests: Array<{ url: string; init?: RequestInit }> = []; + const { bridge, port } = await connectedExportHost(async (url, init) => { + requests.push({ url, init }); + if (url.endsWith("/exports")) + return Response.json({ id: "job", status: "running" }); + if (init?.method === "DELETE") return Response.json({ canceled: true }); + if (url.includes("/exports/job?")) + return Response.json(exportState("ready")); + throw new Error(`Unexpected editor request: ${url}`); + }); + const transfer = vi.mocked(uploadWebEditorExport); + transfer.mockClear(); + transfer.mockImplementationOnce(async (...args) => { + args[6]?.({ stage: "uploading", fraction: 0.5 }); + }); + const invoke = async (id: number, name: string, args: unknown[]) => { + const messages: unknown[] = []; + const reply = new Promise((resolve) => { + port.onmessage = (event) => { + messages.push(event.data); + if ( + typeof event.data === "object" && + event.data !== null && + "id" in event.data && + event.data.id === id && + "kind" in event.data && + event.data.kind !== "channel" + ) { + resolve(event.data); + } + }; + }); + port.postMessage({ kind: "invoke", id, name, args }); + return { reply: await reply, messages }; + }; + try { + const rendered = await invoke(11, "exportVideo", [ + "cap-web-editor://session/session", + "__CHANNEL__:8", + exportSettings, + ]); + expect(rendered.reply).toEqual({ + kind: "result", + id: 11, + value: "cap-web-editor://export/job", + }); + expect(rendered.messages).toContainEqual({ + kind: "channel", + id: 8, + value: { + type: "FramesRendered", + renderedCount: 90, + totalFrames: 90, + }, + }); + const shared = await invoke(12, "uploadExportedVideo", [ + "cap-web-editor://session/session", + "Reupload", + "__CHANNEL__:9", + null, + ]); + expect(shared.reply).toEqual({ + kind: "result", + id: 12, + value: { Success: "http://127.0.0.1:3000/s/video" }, + }); + expect(shared.messages).toContainEqual({ + kind: "channel", + id: 9, + value: { progress: 0.5 }, + }); + expect(transfer).toHaveBeenCalledOnce(); + expect(transfer.mock.calls[0]?.slice(0, 5)).toEqual([ + "video", + "session", + "job", + 160_000, + { duration: 3, width: 640, height: 358, fps: 30 }, + ]); + expect(requests.some((request) => request.init?.method === "DELETE")).toBe( + true, + ); + } finally { + port.close(); + bridge.dispose(); + } +}); + +test("long share checks the current plan before upload after a downgrade", async () => { + const requests: Array<{ url: string; init?: RequestInit }> = []; + const { bridge, port } = await connectedExportHost(async (url, init) => { + requests.push({ url, init }); + if (url.endsWith("/exports")) + return Response.json({ id: "job", status: "running" }); + if (url.includes("/exports/job?")) + return Response.json(exportState("ready", null, 420)); + if (url.endsWith("/plan?videoId=video")) + return Response.json({ pro: false }); + if (init?.method === "DELETE") return Response.json({ canceled: true }); + throw new Error(`Unexpected editor request: ${url}`); + }); + const transfer = vi.mocked(uploadWebEditorExport); + transfer.mockClear(); + const invoke = async (id: number, name: string, args: unknown[]) => { + const reply = new Promise((resolve) => { + port.onmessage = (event: MessageEvent) => { + if ( + typeof event.data === "object" && + event.data !== null && + "id" in event.data && + event.data.id === id && + "kind" in event.data && + event.data.kind !== "channel" + ) { + resolve(event.data); + } + }; + }); + port.postMessage({ kind: "invoke", id, name, args }); + return reply; + }; + try { + expect( + await invoke(20, "exportVideo", [ + "cap-web-editor://session/session", + "__CHANNEL__:8", + exportSettings, + ]), + ).toEqual({ kind: "result", id: 20, value: "cap-web-editor://export/job" }); + expect( + await invoke(21, "uploadExportedVideo", [ + "cap-web-editor://session/session", + "Reupload", + "__CHANNEL__:9", + null, + ]), + ).toEqual({ + kind: "error", + id: 21, + error: "Cap Pro is required to share recordings longer than 5 minutes", + }); + expect( + requests.some((request) => request.url.endsWith("/plan?videoId=video")), + ).toBe(true); + expect(requests.some((request) => request.init?.method === "DELETE")).toBe( + true, + ); + expect(transfer).not.toHaveBeenCalled(); + } finally { + port.close(); + bridge.dispose(); + } +}); + +test("share controls use a fresh web plan and open the existing upgrade flow", async () => { + let pro = false; + const requests: string[] = []; + const onUpgrade = vi.fn(); + const { bridge, port } = await connectedExportHost( + async (url) => { + requests.push(url); + if (url.endsWith("/plan?videoId=video")) return Response.json({ pro }); + throw new Error(`Unexpected editor request: ${url}`); + }, + false, + onUpgrade, + ); + const invoke = async (id: number, name: string, args: unknown[]) => { + const reply = new Promise((resolve) => { + port.onmessage = (event: MessageEvent) => resolve(event.data); + }); + port.postMessage({ kind: "invoke", id, name, args }); + return reply; + }; + try { + expect(await invoke(1, "checkUpgradedAndUpdate", [])).toEqual({ + kind: "result", + id: 1, + value: false, + }); + expect(await invoke(2, "showWindow", ["Upgrade"])).toEqual({ + kind: "result", + id: 2, + value: null, + }); + expect(onUpgrade).toHaveBeenCalledOnce(); + pro = true; + expect(await invoke(3, "checkUpgradedAndUpdate", [])).toEqual({ + kind: "result", + id: 3, + value: true, + }); + expect(requests).toEqual([ + "/api/editor/sessions/session/plan?videoId=video", + "/api/editor/sessions/session/plan?videoId=video", + ]); + } finally { + port.close(); + bridge.dispose(); + } +}); + +test("caption access follows a changed Cap Pro plan without reconnecting the editor", async () => { + let pro = false; + let captionRequests = 0; + const captions = { + settings: null, + segments: [ + { + id: "segment-0", + text: "Hello", + start: 0.1, + end: 0.4, + words: [{ text: "Hello", start: 0.1, end: 0.4 }], + }, + ], + }; + const { bridge, port } = await connectedExportHost(async (url, init) => { + if (url.endsWith("/plan?videoId=video")) return Response.json({ pro }); + if (url.endsWith("/captions") && init?.method === "POST") { + captionRequests++; + return Response.json({ status: "ready", captions, message: null }); + } + throw new Error(`Unexpected editor request: ${url}`); + }); + const invoke = async (id: number, name: string, args: unknown[] = []) => { + const reply = new Promise((resolve) => { + port.onmessage = (event: MessageEvent) => resolve(event.data); + }); + port.postMessage({ kind: "invoke", id, name, args }); + return reply; + }; + const transcriptionArgs = [ + "cap-web-editor://session/session", + "cap-web-editor://app-local-data/transcription_models/best.bin", + "auto", + "Parakeet", + ]; + try { + expect(await invoke(1, "transcribeAudio", transcriptionArgs)).toEqual({ + kind: "error", + id: 1, + error: "Cap Pro is required for web editor captions", + }); + pro = true; + expect(await invoke(2, "checkUpgradedAndUpdate")).toEqual({ + kind: "result", + id: 2, + value: true, + }); + expect(await invoke(3, "transcribeAudio", transcriptionArgs)).toEqual({ + kind: "result", + id: 3, + value: captions, + }); + pro = false; + expect(await invoke(4, "checkUpgradedAndUpdate")).toEqual({ + kind: "result", + id: 4, + value: false, + }); + expect(await invoke(5, "transcribeAudio", transcriptionArgs)).toEqual({ + kind: "error", + id: 5, + error: "Cap Pro is required for web editor captions", + }); + expect(captionRequests).toBe(1); + } finally { + port.close(); + bridge.dispose(); + } +}); + +/** The hidden frame the host downloads worker exports into. */ +function downloadFrame() { + return { name: "", hidden: false, setAttribute: vi.fn(), remove: vi.fn() }; +} + +test("Solid export progress is delivered before a verified direct browser download", async () => { + const requests: Array<{ url: string; init?: RequestInit }> = []; + let polled = 0; + let linkClicked = false; + const anchor = { + href: "", + rel: "", + download: "", + click: vi.fn(() => { + linkClicked = true; + }), + remove: vi.fn(), + }; + const append = vi.fn(); + vi.stubGlobal("document", { + createElement: vi.fn((tag: string) => + tag === "iframe" ? downloadFrame() : anchor, + ), + body: { append }, + }); + const { bridge, port } = await connectedExportHost(async (url, init) => { + requests.push({ url, init }); + if (url.endsWith("/exports") && init?.method === "POST") { + return Response.json({ id: "job", status: "running" }, { status: 202 }); + } + if (url.endsWith("/download-ticket")) { + return Response.json({ + url: `http://127.0.0.1:1234/editor/sessions/session/exports/job/download?ticket=${ticket}`, + }); + } + if (url.includes("/exports/job?videoId=")) { + polled++; + return Response.json( + linkClicked + ? exportState("ready", Date.now()) + : exportState(polled === 1 ? "running" : "ready"), + ); + } + throw new Error(`Unexpected request ${url}`); + }); + const messages: unknown[] = []; + const completed = new Promise((resolve) => { + port.onmessage = (event: MessageEvent) => { + messages.push(event.data); + if ( + typeof event.data === "object" && + event.data !== null && + "kind" in event.data && + event.data.kind === "result" + ) { + resolve(event.data); + } + }; + }); + port.postMessage({ + kind: "invoke", + id: 7, + name: "exportVideoToFile", + args: [ + "cap-web-editor://session/session", + "__CHANNEL__:42", + exportSettings, + "Paired test.mp4", + "mp4", + ], + }); + expect(await completed).toEqual({ + kind: "result", + id: 7, + value: "Paired test.mp4", + }); + expect(messages).toContainEqual({ + kind: "channel", + id: 42, + value: { type: "FramesRendered", renderedCount: 90, totalFrames: 90 }, + }); + expect(messages.at(-1)).toEqual({ + kind: "result", + id: 7, + value: "Paired test.mp4", + }); + expect(anchor.click).toHaveBeenCalledOnce(); + expect(anchor.href).toContain( + "/editor/sessions/session/exports/job/download", + ); + expect(anchor.download).toBe("Paired test.mp4"); + expect(append).toHaveBeenCalledWith(anchor); + expect( + requests.find((request) => request.url.endsWith("/exports"))?.init?.body, + ).toBe(JSON.stringify({ videoId: "video", settings: exportSettings })); + expect(requests.some((request) => request.init?.method === "DELETE")).toBe( + false, + ); + port.close(); + bridge.dispose(); +}); + +test("canceling while export creation is pending still deletes the native job", async () => { + let resolveStart: (response: Response) => void = () => undefined; + const pendingStart = new Promise((resolve) => { + resolveStart = resolve; + }); + const deleted = vi.fn(async () => Response.json({ canceled: true })); + const { bridge, port } = await connectedExportHost(async (url, init) => { + if (url.endsWith("/exports") && init?.method === "POST") + return pendingStart; + if (url.includes("/exports/job?videoId=") && init?.method === "DELETE") + return deleted(); + throw new Error(`Unexpected request ${url}`); + }); + const messages: unknown[] = []; + port.onmessage = (event: MessageEvent) => { + messages.push(event.data); + }; + port.postMessage({ + kind: "invoke", + id: 7, + name: "exportVideoToFile", + args: [ + "cap-web-editor://session/session", + "__CHANNEL__:42", + exportSettings, + "Paired test.mp4", + "mp4", + ], + }); + port.postMessage({ + kind: "invoke", + id: 8, + name: "cancelCurrentWindowExports", + args: [], + }); + await vi.waitFor(() => + expect(messages).toContainEqual({ kind: "result", id: 8, value: null }), + ); + resolveStart( + Response.json({ id: "job", status: "running" }, { status: 202 }), + ); + await vi.waitFor(() => + expect(messages).toContainEqual({ + kind: "error", + id: 7, + error: "Export cancelled", + }), + ); + await vi.waitFor(() => expect(deleted).toHaveBeenCalledOnce()); + port.close(); + bridge.dispose(); +}); + +test("canceling a file export acknowledges the dialog and retries after native cleanup", async () => { + let resolveStart: (response: Response) => void = () => undefined; + const pendingStart = new Promise((resolve) => { + resolveStart = resolve; + }); + let resolveDelete: (response: Response) => void = () => undefined; + const pendingDelete = new Promise((resolve) => { + resolveDelete = resolve; + }); + const requests: Array<{ url: string; init?: RequestInit }> = []; + let starts = 0; + let linkClicked = false; + const anchor = { + href: "", + rel: "", + download: "", + click: vi.fn(() => { + linkClicked = true; + }), + remove: vi.fn(), + }; + vi.stubGlobal("document", { + createElement: vi.fn((tag: string) => + tag === "iframe" ? downloadFrame() : anchor, + ), + body: { append: vi.fn() }, + }); + const { bridge, port } = await connectedExportHost(async (url, init) => { + requests.push({ url, init }); + if (url.endsWith("/exports") && init?.method === "POST") { + starts++; + return starts === 1 + ? pendingStart + : Response.json({ id: "retry", status: "running" }, { status: 202 }); + } + if (url.includes("/exports/job?videoId=") && init?.method === "DELETE") + return pendingDelete; + if (url.includes("/exports/retry?videoId=")) + return Response.json({ + ...exportState("ready", linkClicked ? Date.now() : null), + id: "retry", + }); + if (url.endsWith("/exports/retry/download-ticket")) + return Response.json({ + url: `http://127.0.0.1:1234/editor/sessions/session/exports/retry/download?ticket=${ticket}`, + }); + throw new Error(`Unexpected request ${url}`); + }); + const messages: unknown[] = []; + port.onmessage = (event: MessageEvent) => { + messages.push(event.data); + }; + const fileArgs = [ + "cap-web-editor://session/session", + "__CHANNEL__:42", + exportSettings, + "Paired test.mp4", + "mp4", + ]; + try { + port.postMessage({ + kind: "invoke", + id: 7, + name: "exportVideoToFile", + args: fileArgs, + }); + await vi.waitFor(() => expect(starts).toBe(1)); + port.postMessage({ + kind: "invoke", + id: 6, + name: "exportVideoToFile", + args: fileArgs, + }); + await vi.waitFor(() => + expect(messages).toContainEqual({ + kind: "error", + id: 6, + error: "Finish or cancel the current editor export first", + }), + ); + port.postMessage({ + kind: "invoke", + id: 8, + name: "cancelCurrentWindowExports", + args: [], + }); + await vi.waitFor(() => { + expect(messages).toContainEqual({ + kind: "error", + id: 7, + error: "Export cancelled", + }); + expect(messages).toContainEqual({ kind: "result", id: 8, value: null }); + }); + expect( + requests.find( + (request) => + request.url.endsWith("/exports") && request.init?.method === "POST", + )?.init?.signal?.aborted, + ).toBe(false); + port.postMessage({ + kind: "invoke", + id: 9, + name: "exportVideoToFile", + args: fileArgs, + }); + resolveStart( + Response.json({ id: "job", status: "running" }, { status: 202 }), + ); + await vi.waitFor(() => + expect( + requests.some( + (request) => + request.url.includes("/exports/job?videoId=") && + request.init?.method === "DELETE", + ), + ).toBe(true), + ); + expect(starts).toBe(1); + expect(messages).not.toContainEqual({ + kind: "error", + id: 9, + error: "Finish or cancel the current editor export first", + }); + resolveDelete(Response.json({ canceled: true })); + await vi.waitFor(() => + expect(messages).toContainEqual({ + kind: "result", + id: 9, + value: "Paired test.mp4", + }), + ); + expect(starts).toBe(2); + expect(anchor.click).toHaveBeenCalledOnce(); + expect( + messages.filter( + (message) => + typeof message === "object" && + message !== null && + "id" in message && + message.id === 7, + ), + ).toHaveLength(1); + } finally { + port.close(); + bridge.dispose(); + } +}); + +test("canceling a prepared share waits for job deletion before starting another export", async () => { + let resolveDelete: (response: Response) => void = () => undefined; + const pendingDelete = new Promise((resolve) => { + resolveDelete = resolve; + }); + let starts = 0; + const { bridge, port } = await connectedExportHost(async (url, init) => { + if (url.endsWith("/exports") && init?.method === "POST") { + starts++; + return Response.json( + { id: starts === 1 ? "job" : "retry", status: "running" }, + { status: 202 }, + ); + } + if (url.includes("/exports/job?videoId=") && init?.method === "DELETE") + return pendingDelete; + if (url.includes("/exports/job?videoId=")) + return Response.json(exportState("ready")); + if (url.includes("/exports/retry?videoId=")) + return Response.json({ ...exportState("ready"), id: "retry" }); + throw new Error(`Unexpected request ${url}`); + }); + const messages: unknown[] = []; + port.onmessage = (event: MessageEvent) => { + messages.push(event.data); + }; + const args = [ + "cap-web-editor://session/session", + "__CHANNEL__:42", + exportSettings, + ]; + try { + port.postMessage({ kind: "invoke", id: 7, name: "exportVideo", args }); + await vi.waitFor(() => + expect(messages).toContainEqual({ + kind: "result", + id: 7, + value: "cap-web-editor://export/job", + }), + ); + port.postMessage({ + kind: "invoke", + id: 8, + name: "cancelCurrentWindowExports", + args: [], + }); + await vi.waitFor(() => + expect(messages).toContainEqual({ kind: "result", id: 8, value: null }), + ); + port.postMessage({ kind: "invoke", id: 9, name: "exportVideo", args }); + await vi.waitFor(() => expect(starts).toBe(1)); + expect(messages).not.toContainEqual({ + kind: "error", + id: 9, + error: "Finish or cancel the current editor export first", + }); + resolveDelete(Response.json({ canceled: true })); + await vi.waitFor(() => + expect(messages).toContainEqual({ + kind: "result", + id: 9, + value: "cap-web-editor://export/retry", + }), + ); + expect(starts).toBe(2); + } finally { + port.close(); + bridge.dispose(); + } +}); + +test("a stalled export start releases a canceled retry and deletes a late native job", async () => { + let resolveStart: (response: Response) => void = () => undefined; + const pendingStart = new Promise((resolve) => { + resolveStart = resolve; + }); + let resolveFirstStarted: () => void = () => undefined; + const firstStarted = new Promise((resolve) => { + resolveFirstStarted = resolve; + }); + let resolveDeleted: () => void = () => undefined; + const nativeDeleted = new Promise((resolve) => { + resolveDeleted = resolve; + }); + let starts = 0; + const startSignals: AbortSignal[] = []; + const deleted = vi.fn(async () => Response.json({ canceled: true })); + const { bridge, port } = await connectedExportHost(async (url, init) => { + if (url.endsWith("/exports") && init?.method === "POST") { + starts++; + if (starts === 1) { + if (init.signal) startSignals.push(init.signal); + resolveFirstStarted(); + return pendingStart; + } + return Response.json({ id: "retry", status: "running" }, { status: 202 }); + } + if (url.includes("/exports/orphan?videoId=") && init?.method === "DELETE") { + resolveDeleted(); + return deleted(); + } + if (url.includes("/exports/retry?videoId=")) + return Response.json({ ...exportState("ready"), id: "retry" }); + throw new Error(`Unexpected request ${url}`); + }); + const messages: unknown[] = []; + let resolveCancel: () => void = () => undefined; + const canceled = new Promise((resolve) => { + resolveCancel = resolve; + }); + let resolveRetry: (reply: unknown) => void = () => undefined; + const retried = new Promise((resolve) => { + resolveRetry = resolve; + }); + port.onmessage = (event: MessageEvent) => { + messages.push(event.data); + const reply = event.data; + if (typeof reply !== "object" || reply === null || !("id" in reply)) return; + if (reply.id === 8 && "kind" in reply && reply.kind === "result") + resolveCancel(); + if (reply.id === 9 && "kind" in reply && reply.kind !== "channel") + resolveRetry(reply); + }; + vi.stubGlobal("window", { + ...window, + setTimeout: (handler: () => void, delay: number) => + Number(globalThis.setTimeout(handler, delay)), + clearTimeout: (id: number) => globalThis.clearTimeout(id), + }); + vi.useFakeTimers(); + const args = [ + "cap-web-editor://session/session", + "__CHANNEL__:42", + exportSettings, + ]; + try { + port.postMessage({ kind: "invoke", id: 7, name: "exportVideo", args }); + await firstStarted; + port.postMessage({ + kind: "invoke", + id: 8, + name: "cancelCurrentWindowExports", + args: [], + }); + await canceled; + expect(messages).toContainEqual({ + kind: "error", + id: 7, + error: "Export cancelled", + }); + port.postMessage({ kind: "invoke", id: 9, name: "exportVideo", args }); + await vi.advanceTimersByTimeAsync(20_001); + expect(await retried).toEqual({ + kind: "result", + id: 9, + value: "cap-web-editor://export/retry", + }); + expect(starts).toBe(2); + expect(startSignals[0]?.aborted).toBe(true); + resolveStart( + Response.json({ id: "orphan", status: "running" }, { status: 202 }), + ); + await nativeDeleted; + expect(deleted).toHaveBeenCalledOnce(); + expect( + messages.filter( + (message) => + typeof message === "object" && + message !== null && + "id" in message && + message.id === 7, + ), + ).toHaveLength(1); + } finally { + vi.useRealTimers(); + port.close(); + bridge.dispose(); + } +}); + +test("a stalled native export deletion cannot hold a prepared-share retry forever", async () => { + let resolveDeleteRequested: () => void = () => undefined; + const deleteRequested = new Promise((resolve) => { + resolveDeleteRequested = resolve; + }); + const pendingDelete = new Promise(() => undefined); + let starts = 0; + const { bridge, port } = await connectedExportHost(async (url, init) => { + if (url.endsWith("/exports") && init?.method === "POST") { + starts++; + return Response.json( + { id: starts === 1 ? "job" : "retry", status: "running" }, + { status: 202 }, + ); + } + if (url.includes("/exports/job?videoId=") && init?.method === "DELETE") { + resolveDeleteRequested(); + return pendingDelete; + } + if (url.includes("/exports/job?videoId=")) + return Response.json(exportState("ready")); + if (url.includes("/exports/retry?videoId=")) + return Response.json({ ...exportState("ready"), id: "retry" }); + throw new Error(`Unexpected request ${url}`); + }); + const messages: unknown[] = []; + const awaited = new Map void>(); + port.onmessage = (event: MessageEvent) => { + messages.push(event.data); + const reply = event.data; + if (typeof reply === "object" && reply !== null && "id" in reply) + awaited.get(Number(reply.id))?.(reply); + }; + const invoke = (id: number, name: string) => { + const reply = new Promise((resolve) => awaited.set(id, resolve)); + port.postMessage({ + kind: "invoke", + id, + name, + args: + name === "cancelCurrentWindowExports" + ? [] + : [ + "cap-web-editor://session/session", + "__CHANNEL__:42", + exportSettings, + ], + }); + return reply; + }; + vi.stubGlobal("window", { + ...window, + setTimeout: (handler: () => void, delay: number) => + Number(globalThis.setTimeout(handler, delay)), + clearTimeout: (id: number) => globalThis.clearTimeout(id), + }); + vi.useFakeTimers(); + try { + expect(await invoke(7, "exportVideo")).toEqual({ + kind: "result", + id: 7, + value: "cap-web-editor://export/job", + }); + expect(await invoke(8, "cancelCurrentWindowExports")).toEqual({ + kind: "result", + id: 8, + value: null, + }); + const retry = invoke(9, "exportVideo"); + await deleteRequested; + expect(starts).toBe(1); + await vi.advanceTimersByTimeAsync(10_001); + expect(await retry).toEqual({ + kind: "result", + id: 9, + value: "cap-web-editor://export/retry", + }); + expect(starts).toBe(2); + expect(messages).not.toContainEqual({ + kind: "error", + id: 9, + error: "Finish or cancel the current editor export first", + }); + } finally { + vi.useRealTimers(); + port.close(); + bridge.dispose(); + } +}); + +test.each(["put", "driveResumable"] as const)( + "imported audio uses a %s storage target before adding a native timeline asset", + async (targetType) => { + const path = "assets/audio/import-6516dffa-756a-4ed2-b595-5f2845d15d9a.mp3"; + const key = `user/video/editor-assets/${path.slice("assets/audio/".length)}`; + const bytes = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]); + const file = new File([bytes], "tone.mp3", { type: "audio/mpeg" }); + const requests: Array<{ url: string; init?: RequestInit }> = []; + const { bridge, port } = await connectedExportHost(async (url, init) => { + requests.push({ url, init }); + if ( + url === "/api/editor/sessions/session/assets" && + init?.method === "POST" + ) { + return Response.json({ + key, + path, + upload: { + type: targetType, + url: "https://storage.cap.so/upload/audio", + headers: { "Content-Type": "audio/mpeg" }, + }, + }); + } + if (url === "https://storage.cap.so/upload/audio") { + return new Response(null, { status: 200 }); + } + if ( + url === "/api/editor/sessions/session/assets" && + init?.method === "PUT" + ) { + return Response.json({ path, name: "tone", duration: 3.2 }); + } + throw new Error(`Unexpected request ${url}`); + }); + const completed = new Promise((resolve) => { + port.onmessage = (event: MessageEvent) => resolve(event.data); + }); + port.postMessage({ + kind: "invoke", + id: 17, + name: "importAudioTrackFile", + args: [file], + }); + expect(await completed).toEqual({ + kind: "result", + id: 17, + value: { path, name: "tone", duration: 3.2 }, + }); + expect(requests.map(({ url, init }) => [url, init?.method])).toEqual([ + ["/api/editor/sessions/session/assets", "POST"], + ["https://storage.cap.so/upload/audio", "PUT"], + ["/api/editor/sessions/session/assets", "PUT"], + ]); + expect(JSON.parse(String(requests[0]?.init?.body))).toEqual({ + kind: "audio", + videoId: "video", + fileName: "tone.mp3", + size: bytes.length, + contentType: "audio/mpeg", + }); + const upload = requests[1]?.init; + expect(upload?.body).toBeInstanceOf(File); + expect(new Uint8Array(await (upload?.body as File).arrayBuffer())).toEqual( + bytes, + ); + expect(upload?.credentials).toBe("omit"); + expect(new Headers(upload?.headers).get("Content-Type")).toBe("audio/mpeg"); + expect(new Headers(upload?.headers).get("Content-Range")).toBe( + targetType === "driveResumable" ? "bytes 0-7/8" : null, + ); + expect(JSON.parse(String(requests[2]?.init?.body))).toEqual({ + kind: "audio", + videoId: "video", + fileName: "tone.mp3", + size: bytes.length, + contentType: "audio/mpeg", + key, + path, + }); + port.close(); + bridge.dispose(); + }, +); + +test("failed audio upload never commits the timeline asset", async () => { + const path = "assets/audio/import-6516dffa-756a-4ed2-b595-5f2845d15d9a.mp3"; + const key = `user/video/editor-assets/${path.slice("assets/audio/".length)}`; + const requests: Array<{ url: string; init?: RequestInit }> = []; + const { bridge, port } = await connectedExportHost(async (url, init) => { + requests.push({ url, init }); + if (init?.method === "POST") { + return Response.json({ + key, + path, + upload: { + type: "put", + url: "https://storage.cap.so/upload/audio", + headers: { "Content-Type": "audio/mpeg" }, + }, + }); + } + return new Response(null, { status: 503 }); + }); + const reply = new Promise((resolve) => { + port.onmessage = (event: MessageEvent) => resolve(event.data); + }); + port.postMessage({ + kind: "invoke", + id: 18, + name: "importAudioTrackFile", + args: [new File([new Uint8Array([1])], "tone.mp3")], + }); + await expect(reply).resolves.toEqual({ + kind: "error", + id: 18, + error: "Audio upload failed", + }); + expect(requests).toHaveLength(2); + expect( + requests.every( + ({ init }) => + init?.method !== "PUT" || !String(init.body).includes("videoId"), + ), + ).toBe(true); + port.close(); + bridge.dispose(); +}); + +test.each([ + { + label: "JPEG", + fileExtension: "jpeg", + assetExtension: "jpg", + contentType: "image/jpeg", + bytes: new Uint8Array([255, 216, 255, 224, 1, 2, 3, 4]), + }, + { + label: "TIFF", + fileExtension: "tif", + assetExtension: "tiff", + contentType: "image/tiff", + bytes: new Uint8Array([73, 73, 42, 0, 1, 2, 3, 4]), + }, +])( + "imported $label images preserve their bytes and return desktop image dimensions", + async ({ fileExtension, assetExtension, contentType, bytes }) => { + const path = `content/images/6516dffa-756a-4ed2-b595-5f2845d15d9a.${assetExtension}`; + const key = `user/video/editor-assets/images/${path.slice("content/images/".length)}`; + const fileName = `logo.${fileExtension}`; + const file = new File([bytes], fileName, { type: contentType }); + const requests: Array<{ url: string; init?: RequestInit }> = []; + const { bridge, port } = await connectedExportHost(async (url, init) => { + requests.push({ url, init }); + if (init?.method === "POST") { + return Response.json({ + key, + path, + upload: { + type: "put", + url: "https://storage.cap.so/upload/image", + headers: { "Content-Type": contentType }, + }, + }); + } + if (url === "https://storage.cap.so/upload/image") { + return new Response(null, { status: 200 }); + } + if (init?.method === "PUT") { + return Response.json({ path, name: "logo", width: 120, height: 80 }); + } + throw new Error(`Unexpected request ${url}`); + }); + const reply = new Promise((resolve) => { + port.onmessage = (event: MessageEvent) => resolve(event.data); + }); + port.postMessage({ + kind: "invoke", + id: 19, + name: "importEditorImage", + args: [file], + }); + expect(await reply).toEqual({ + kind: "result", + id: 19, + value: { path, name: "logo", width: 120, height: 80 }, + }); + expect(requests.map(({ url, init }) => [url, init?.method])).toEqual([ + ["/api/editor/sessions/session/assets", "POST"], + ["https://storage.cap.so/upload/image", "PUT"], + ["/api/editor/sessions/session/assets", "PUT"], + ]); + expect(JSON.parse(String(requests[0]?.init?.body))).toEqual({ + kind: "image", + videoId: "video", + fileName, + size: bytes.length, + contentType, + }); + expect( + new Uint8Array(await (requests[1]?.init?.body as File).arrayBuffer()), + ).toEqual(bytes); + expect(JSON.parse(String(requests[2]?.init?.body))).toEqual({ + kind: "image", + videoId: "video", + fileName, + size: bytes.length, + contentType, + key, + path, + }); + port.close(); + bridge.dispose(); + }, +); + +test("Solid caption commands share one authenticated AssemblyAI request and bypass native models", async () => { + const requests: Array<{ url: string; init?: RequestInit }> = []; + let resolveCaption: (response: Response) => void = () => undefined; + const pendingCaption = new Promise((resolve) => { + resolveCaption = resolve; + }); + const { bridge, port } = await connectedExportHost(async (url, init) => { + requests.push({ url, init }); + if (url === "/api/editor/sessions/session/plan?videoId=video") { + return Response.json({ pro: true }); + } + if ( + url === "/api/editor/sessions/session/captions" && + init?.method === "POST" + ) { + return pendingCaption; + } + throw new Error(`Unexpected request ${url}`); + }, true); + const replies: unknown[] = []; + port.onmessage = (event: MessageEvent) => { + replies.push(event.data); + }; + for (const id of [41, 42]) { + port.postMessage({ + kind: "invoke", + id, + name: "transcribeAudio", + args: [ + "cap-web-editor://session/session", + "cap-web-editor://app-local-data/transcription_models/best.bin", + "auto", + "Parakeet", + ], + }); + } + await vi.waitFor(() => + expect( + requests.filter((request) => request.init?.method === "POST"), + ).toHaveLength(1), + ); + const captions = { + settings: null, + segments: [ + { + id: "segment-0", + text: "Hello", + start: 0.1, + end: 0.4, + words: [{ text: "Hello", start: 0.1, end: 0.4 }], + }, + ], + }; + resolveCaption(Response.json({ status: "ready", captions, message: null })); + await vi.waitFor(() => expect(replies).toHaveLength(2)); + expect(replies).toContainEqual({ kind: "result", id: 41, value: captions }); + expect(replies).toContainEqual({ kind: "result", id: 42, value: captions }); + const captionRequest = requests.find( + (request) => request.init?.method === "POST", + ); + expect(captionRequest?.url).toBe("/api/editor/sessions/session/captions"); + expect(JSON.parse(String(captionRequest?.init?.body))).toEqual({ + videoId: "video", + language: "auto", + }); + port.close(); + bridge.dispose(); +}); + +async function browserSaveHost( + request: (url: string, init?: RequestInit) => Promise, +) { + const requests: Array<{ url: string; method: string }> = []; + vi.stubGlobal( + "fetch", + vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + requests.push({ url, method: init?.method ?? "GET" }); + if (url === "/api/editor/videos/video/plan") + return Response.json({ pro: true }); + return request(url, init); + }), + ); + vi.stubGlobal("window", { + setTimeout, + clearTimeout, + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + location: { origin: "http://127.0.0.1:3000" }, + }); + const { iframe, postMessage } = frame(); + const bridge = new EditorHostBridge( + "video", + "session", + "user", + vi.fn(), + vi.fn(), + undefined, + undefined, + undefined, + true, + undefined, + undefined, + () => "saved-revision", + true, + ); + await bridge.connect(iframe); + const port = postMessage.mock.calls[0]?.[2]?.[0] as MessagePort; + port.start(); + const invoke = (name: string, args: unknown[] = [undefined]) => { + const reply = new Promise((resolve) => { + port.onmessage = (event) => { + if (event.data.kind !== "channel") resolve(event.data); + }; + }); + port.postMessage({ kind: "invoke", id: 1, name, args }); + return reply; + }; + return { bridge, port, invoke, requests }; +} + +test("Save renders in the browser without an editor worker when the farm is unavailable", async () => { + const { bridge, port, invoke, requests } = await browserSaveHost( + async (url) => { + if (url === "/api/editor/sessions/session/save?videoId=video") + return Response.json({ + renderer: "browser", + reason: "The render farm is not configured", + }); + throw new Error(`Unexpected editor request ${url}`); + }, + ); + try { + expect(await invoke("tauri:webEditorSave")).toEqual({ + kind: "result", + id: 1, + value: { + renderer: "browser", + reason: "The render farm is not configured", + }, + }); + expect(requests.some(({ url }) => url.includes("preparations"))).toBe( + false, + ); + } finally { + port.close(); + bridge.dispose(); + } +}); + +test("Save falls back to the browser when no editor worker can prepare", async () => { + const { bridge, port, invoke, requests } = await browserSaveHost( + async (url, init) => { + if (url === "/api/editor/sessions/session/save?videoId=video") + return Response.json({ renderer: "farm", reason: null }); + if (url === "/api/editor/preparations" && init?.method === "POST") + return Response.json({ _tag: "ServiceUnavailable" }, { status: 503 }); + throw new Error(`Unexpected editor request ${url}`); + }, + ); + try { + expect(await invoke("tauri:webEditorSave")).toEqual({ + kind: "result", + id: 1, + value: { + renderer: "browser", + reason: + "The editor server is unavailable right now. Try again in a moment.", + }, + }); + expect( + requests.some( + ({ url, method }) => url.endsWith("/save") && method === "POST", + ), + ).toBe(false); + } finally { + port.close(); + bridge.dispose(); + } +}); + +test("Save starts on the farm through a prepared worker and keeps plan errors", async () => { + let saveStatus = 200; + const { bridge, port, invoke } = await browserSaveHost(async (url, init) => { + if (url === "/api/editor/sessions/session/save?videoId=video") + return Response.json({ renderer: "farm", reason: null }); + if (url === "/api/editor/preparations" && init?.method === "POST") + return Response.json({ id: "prep-1", status: "preparing" }); + if (url.startsWith("/api/editor/preparations/prep-1")) + return Response.json({ status: "ready", sessionId: "worker-1" }); + if (url === "/api/editor/sessions/worker-1/save" && init?.method === "POST") + return saveStatus === 200 + ? Response.json({ + exportId: "export", + jobId: "job", + shareUrl: "http://127.0.0.1:3000/s/video", + }) + : new Response(null, { status: saveStatus }); + if (url.startsWith("/api/editor/sessions/worker-1?")) + return Response.json({ closed: true }); + throw new Error(`Unexpected editor request ${url}`); + }); + try { + expect(await invoke("tauri:webEditorSave")).toEqual({ + kind: "result", + id: 1, + value: { renderer: "farm", shareUrl: "http://127.0.0.1:3000/s/video" }, + }); + saveStatus = 403; + expect(await invoke("tauri:webEditorSave")).toEqual({ + kind: "error", + id: 1, + error: + "Saving recordings of 5 minutes or longer, or with captions, needs Cap Pro", + }); + } finally { + port.close(); + bridge.dispose(); + } +}); + +test("a browser-rendered Save withdraws any farm render before publishing the file", async () => { + const upload = vi.mocked(uploadWebEditorFile); + upload.mockClear(); + const order: string[] = []; + upload.mockImplementation(async () => { + order.push("upload"); + }); + const { bridge, port, invoke } = await browserSaveHost(async (url, init) => { + if ( + url === "/api/editor/sessions/session/save?videoId=video" && + init?.method === "DELETE" + ) { + order.push("withdraw"); + return new Response(null, { status: 204 }); + } + throw new Error(`Unexpected editor request ${url}`); + }); + const file = new Blob([new Uint8Array(16)], { type: "video/mp4" }); + const metadata = { duration: 4, width: 1920, height: 1080, fps: 30 }; + try { + expect( + await invoke("tauri:webEditorPublishRendered", [ + file, + metadata, + "__CHANNEL__:7", + ]), + ).toEqual({ + kind: "result", + id: 1, + value: { shareUrl: "http://127.0.0.1:3000/s/video" }, + }); + expect(order).toEqual(["withdraw", "upload"]); + expect(upload.mock.calls[0]?.slice(0, 4)).toEqual([ + "video", + "session", + file, + metadata, + ]); + } finally { + port.close(); + bridge.dispose(); + } +}); + +test("the default style for new recordings is loaded, saved and cleared through the account", async () => { + const style = { version: 1, background: { padding: 10 } }; + const requests: Array<{ url: string; method?: string; body?: unknown }> = []; + const { bridge, port } = await connectedExportHost(async (url, init) => { + requests.push({ + url, + method: init?.method, + body: init?.body ? JSON.parse(String(init.body)) : undefined, + }); + return Response.json({ style: init?.method === "DELETE" ? null : style }); + }); + const invoke = (id: number, name: string, args: unknown[]) => + new Promise((resolve) => { + port.onmessage = (event) => resolve(event.data); + port.postMessage({ kind: "invoke", id, name, args }); + }); + try { + expect(await invoke(1, "tauri:webEditorDefaultStyle", [])).toEqual({ + kind: "result", + id: 1, + value: style, + }); + expect( + await invoke(2, "tauri:webEditorSaveDefaultStyle", [ + { config: { background: { padding: 10 } } }, + ]), + ).toEqual({ kind: "result", id: 2, value: style }); + expect( + await invoke(3, "tauri:webEditorSaveDefaultStyle", [{ config: null }]), + ).toEqual({ kind: "result", id: 3, value: null }); + expect(requests).toEqual([ + { + url: "/api/editor/preferences/default-style", + method: "GET", + body: undefined, + }, + { + url: "/api/editor/preferences/default-style", + method: "PUT", + body: { config: { background: { padding: 10 } } }, + }, + { + url: "/api/editor/preferences/default-style", + method: "DELETE", + body: undefined, + }, + ]); + } finally { + port.close(); + bridge.dispose(); + } +}); + +async function browserCaptionHost( + request: (url: string, init?: RequestInit) => Promise, +) { + vi.stubGlobal( + "fetch", + vi.fn((input: RequestInfo | URL, init?: RequestInit) => + request(String(input), init), + ), + ); + vi.stubGlobal("window", { + setTimeout, + clearTimeout, + location: { origin: "http://127.0.0.1:3000" }, + }); + const { iframe, postMessage } = frame(); + const bridge = new EditorHostBridge( + "video", + "session", + "user", + vi.fn(), + vi.fn(), + undefined, + undefined, + undefined, + true, + undefined, + undefined, + () => "saved-revision", + true, + ); + await bridge.connect(iframe); + const port = postMessage.mock.calls[0]?.[2]?.[0] as MessagePort; + port.start(); + const transcribe = (id: number) => { + const reply = new Promise((resolve) => { + port.onmessage = (event: MessageEvent) => resolve(event.data); + }); + port.postMessage({ + kind: "invoke", + id, + name: "transcribeAudio", + args: [ + "cap-web-editor://session/session", + "cap-web-editor://app-local-data/transcription_models/best.bin", + "auto", + "Parakeet", + ], + }); + return reply; + }; + return { bridge, port, transcribe }; +} + +test("browser Studio captions read the share transcript without a worker session", async () => { + const captions = { + settings: null, + segments: [ + { + id: "segment-0", + text: "Hello", + start: 0.1, + end: 0.4, + words: [{ text: "Hello", start: 0.1, end: 0.4 }], + }, + ], + }; + const requests: string[] = []; + const { bridge, port, transcribe } = await browserCaptionHost( + async (url, init) => { + requests.push(`${init?.method ?? "GET"} ${url}`); + if (url === "/api/editor/videos/video/plan") + return Response.json({ pro: true }); + if (url.startsWith("/api/editor/videos/video/captions")) + return Response.json({ status: "ready", captions, message: null }); + throw new Error(`Unexpected editor request ${url}`); + }, + ); + try { + expect(await transcribe(1)).toEqual({ + kind: "result", + id: 1, + value: captions, + }); + expect(requests).toEqual([ + "GET /api/editor/videos/video/plan", + "GET /api/editor/videos/video/captions?language=auto", + "POST /api/editor/videos/video/captions", + ]); + } finally { + port.close(); + bridge.dispose(); + } +}); + +test("browser Studio caption jobs report an unavailable worker instead of hanging", async () => { + const { bridge, port, transcribe } = await browserCaptionHost( + async (url, init) => { + if (url === "/api/editor/videos/video/plan") + return Response.json({ pro: true }); + if (url === "/api/editor/videos/video/captions?language=auto") + return new Response(null, { status: 409 }); + if (url === "/api/editor/preparations" && init?.method === "POST") + return new Response(null, { status: 502 }); + throw new Error(`Unexpected editor request ${url}`); + }, + ); + try { + expect(await transcribe(1)).toEqual({ + kind: "error", + id: 1, + error: + "The editor server is unavailable right now. Try again in a moment.", + }); + } finally { + port.close(); + bridge.dispose(); + } +}); + +test("browser Studio captions move to a worker session when the recording comes to need one", async () => { + const requests: string[] = []; + const { bridge, port, transcribe } = await browserCaptionHost( + async (url, init) => { + requests.push(`${init?.method ?? "GET"} ${url}`); + if (url === "/api/editor/videos/video/plan") + return Response.json({ pro: true }); + if (url === "/api/editor/videos/video/captions?language=auto") + return Response.json({ + status: "missing", + captions: null, + message: null, + }); + if (url === "/api/editor/videos/video/captions") + return new Response(null, { status: 409 }); + if (url === "/api/editor/preparations" && init?.method === "POST") + return new Response(null, { status: 502 }); + throw new Error(`Unexpected editor request ${url}`); + }, + ); + try { + expect(await transcribe(1)).toEqual({ + kind: "error", + id: 1, + error: + "The editor server is unavailable right now. Try again in a moment.", + }); + expect(requests).toContain("POST /api/editor/preparations"); + } finally { + port.close(); + bridge.dispose(); + } +}); diff --git a/apps/web/__tests__/unit/editor-imports.test.ts b/apps/web/__tests__/unit/editor-imports.test.ts new file mode 100644 index 00000000000..de018339e80 --- /dev/null +++ b/apps/web/__tests__/unit/editor-imports.test.ts @@ -0,0 +1,107 @@ +import type { VideoMetadata } from "@cap/database/types"; +import { CAP_BUNDLE_CONTENT_TYPE } from "@cap/editor-cap-bundle"; +import { describe, expect, it } from "vitest"; +import { + appendWebEditorImport, + nextEditorRecordingSegmentIndex, + normalizeWebEditorImportOrder, +} from "../../lib/editor-imports"; + +const ownerId = "owner"; +const videoId = "video"; +const clipPath = "content/videos/00000000-0000-4000-8000-000000000000.mp4"; +const capPath = + "content/imports/00000000-0000-4000-8000-000000000001.capbundle"; +const clips: NonNullable["items"] = [ + { displayPath: clipPath, duration: 2, fps: 30, hasAudio: true }, +]; +const assets: NonNullable["items"] = [ + { + path: capPath, + key: `${ownerId}/${videoId}/editor-assets/recordings/${capPath.slice("content/imports/".length)}`, + name: "Studio source", + contentType: CAP_BUNDLE_CONTENT_TYPE, + size: 1024, + objectIdentity: JSON.stringify("etag"), + }, +]; + +describe("ordered web editor source replay", () => { + it("migrates existing MP4 clips and computes recording indices across Cap imports", () => { + const legacy = normalizeWebEditorImportOrder( + null, + clips, + assets, + ownerId, + videoId, + ); + expect(legacy?.items).toEqual([{ kind: "clip", path: clipPath }]); + if (!legacy) throw new Error("Invalid legacy import order"); + const capFirst = normalizeWebEditorImportOrder( + { + version: 1, + items: [ + { kind: "cap", path: capPath, clipCount: 2 }, + { kind: "clip", path: clipPath }, + ], + }, + clips, + assets, + ownerId, + videoId, + ); + expect(capFirst?.items.map((item) => item.kind)).toEqual(["cap", "clip"]); + if (!capFirst) throw new Error("Invalid Cap import order"); + expect(nextEditorRecordingSegmentIndex(capFirst)).toBe(4); + expect( + appendWebEditorImport(legacy, { + kind: "cap", + path: capPath, + clipCount: 2, + })?.items, + ).toEqual([ + { kind: "clip", path: clipPath }, + { kind: "cap", path: capPath, clipCount: 2 }, + ]); + }); + + it("rejects missing clips, duplicate or forged Cap sources, and inflated clip counts", () => { + expect( + normalizeWebEditorImportOrder( + { version: 1, items: [{ kind: "cap", path: capPath, clipCount: 2 }] }, + clips, + assets, + ownerId, + videoId, + ), + ).toBeNull(); + for (const importItems of [ + [ + { kind: "clip", path: clipPath }, + { kind: "clip", path: clipPath }, + ], + [ + { kind: "clip", path: clipPath }, + { kind: "cap", path: capPath, clipCount: 0 }, + ], + [ + { kind: "clip", path: clipPath }, + { kind: "cap", path: capPath, clipCount: 1001 }, + ], + [ + { kind: "clip", path: clipPath }, + { kind: "cap", path: "content/imports/forged.capbundle", clipCount: 2 }, + ], + ]) { + expect( + normalizeWebEditorImportOrder( + { version: 1, items: importItems }, + clips, + assets, + ownerId, + videoId, + ), + ).toBeNull(); + } + }); +}); diff --git a/apps/web/__tests__/unit/editor-local-draft.test.ts b/apps/web/__tests__/unit/editor-local-draft.test.ts new file mode 100644 index 00000000000..add07aee5e0 --- /dev/null +++ b/apps/web/__tests__/unit/editor-local-draft.test.ts @@ -0,0 +1,101 @@ +import { expect, test } from "vitest"; +import { + captureEditorLocalDraft, + clearEditorLocalDraft, + readEditorLocalDraft, +} from "../../lib/editor-local-draft"; + +function browserStorage(): Storage { + const items = new Map(); + return { + get length() { + return items.size; + }, + clear: () => items.clear(), + getItem: (key) => items.get(key) ?? null, + key: (index) => [...items.keys()][index] ?? null, + removeItem: (key) => items.delete(key), + setItem: (key, value) => items.set(key, value), + }; +} + +test("pending editor edits survive navigation only for their owner and recording", () => { + const storage = browserStorage(); + const config = { + timeline: { + audioSegments: [{ path: "assets/audio/library-lofi-beats-mirostar.mp3" }], + }, + captions: { segments: [{ id: "word", text: "Hello" }] }, + }; + expect( + captureEditorLocalDraft( + storage, + "owner", + "video", + "2026-09-18T03:00:00.000Z", + JSON.stringify(config), + ), + ).toBe(true); + expect(readEditorLocalDraft(storage, "owner", "video")?.config).toEqual( + config, + ); + expect(readEditorLocalDraft(storage, "other", "video")).toBeNull(); + expect(readEditorLocalDraft(storage, "owner", "other")).toBeNull(); + clearEditorLocalDraft(storage, "owner", "video"); + expect(readEditorLocalDraft(storage, "owner", "video")).toBeNull(); +}); + +test("a failed browser storage write cannot claim pending edits are safe", () => { + const storage = browserStorage(); + storage.setItem = () => { + throw new Error("Quota exceeded"); + }; + expect( + captureEditorLocalDraft( + storage, + "owner", + "video", + null, + JSON.stringify({ background: { padding: 12 } }), + ), + ).toBe(false); + expect(readEditorLocalDraft(storage, "owner", "video")).toBeNull(); +}); + +test("a large caption project survives the browser's local storage limit", () => { + const storage = browserStorage(); + const setItem = storage.setItem; + storage.setItem = (key, value) => { + if (new TextEncoder().encode(value).byteLength > 4 * 1024 * 1024) + throw new Error("Quota exceeded"); + setItem(key, value); + }; + const config = { + captions: { + segments: Array.from({ length: 18_000 }, (_, index) => ({ + id: `word-${index}`, + start: index / 10, + end: (index + 1) / 10, + text: "A résumé of the presentation with precise captions 🎥 ".repeat( + 7, + ), + })), + }, + }; + const serialized = JSON.stringify(config); + expect(new TextEncoder().encode(serialized).byteLength).toBeGreaterThan( + 4 * 1024 * 1024, + ); + expect(new TextEncoder().encode(serialized).byteLength).toBeLessThan( + 8 * 1024 * 1024, + ); + expect( + captureEditorLocalDraft(storage, "owner", "video", null, serialized), + ).toBe(true); + expect( + JSON.parse(storage.getItem(storage.key(0) ?? "") ?? "{}").version, + ).toBe(2); + expect(readEditorLocalDraft(storage, "owner", "video")?.config).toEqual( + config, + ); +}); diff --git a/apps/web/__tests__/unit/editor-page-processing.test.ts b/apps/web/__tests__/unit/editor-page-processing.test.ts new file mode 100644 index 00000000000..e839ea41dc1 --- /dev/null +++ b/apps/web/__tests__/unit/editor-page-processing.test.ts @@ -0,0 +1,197 @@ +import { beforeEach, describe, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + rows: [] as unknown[][], + user: { id: "owner", email: "richie@mcilroy.co" } as { + id: string; + email: string; + } | null, + studioEnabled: true, + pro: true, + queries: 0, + signedIn: Promise.resolve(), +})); + +vi.mock("@cap/database", () => ({ + db: () => ({ + select: () => { + const query = { + from: () => query, + leftJoin: () => query, + where: async () => { + mocks.queries++; + return mocks.rows.shift() ?? []; + }, + }; + return query; + }, + }), +})); +vi.mock("@cap/database/auth/session", () => ({ + getCurrentUser: async () => { + await mocks.signedIn; + return mocks.user; + }, +})); +vi.mock("@cap/database/schema", () => ({ + videos: {}, + videoUploads: {}, + videoEdits: {}, +})); +vi.mock("drizzle-orm", () => ({ eq: vi.fn() })); +vi.mock("@cap/utils", () => ({ userIsPro: () => mocks.pro })); +vi.mock("@cap/web-domain", () => ({ + Video: { VideoId: { make: (id: string) => id } }, +})); +vi.mock("next/navigation", () => ({ + notFound: () => { + throw new Error("NOT_FOUND"); + }, + redirect: (url: string) => { + throw new Error(`REDIRECT:${url}`); + }, +})); +vi.mock("@/lib/owner-custom-domain", () => ({ + ownerCustomDomain: async () => null, +})); +vi.mock("@/lib/share-link", () => ({ + shareLinkUrl: (videoId: string) => `https://cap.so/s/${videoId}`, +})); +vi.mock("@/lib/web-studio-rollout", () => ({ + isWebStudioEnabledForEmail: () => mocks.studioEnabled, +})); +vi.mock("@/lib/editor-video-duration", () => ({ + measureMissingVideoDuration: async () => null, +})); +vi.mock("@/lib/video-edit-processing", () => ({ + getEditSourceKey: (owner: string, video: string) => + `${owner}/${video}/source/original.mp4`, + isEditSourceKey: ({ rawFileKey }: { rawFileKey: string | null }) => + rawFileKey === "owner/video/source/original.mp4", +})); +vi.mock("@/lib/video-edits", () => ({ + areEditSpecsEquivalent: () => true, + createIdentityEditSpec: vi.fn(), +})); +vi.mock("@/app/s/[videoId]/edit/edit-processing", () => ({ + EditProcessing: () => null, +})); +vi.mock("@/app/s/[videoId]/edit/EditVideoClient", () => ({ + EditVideoClient: () => null, +})); +vi.mock("@/app/s/[videoId]/edit/EditUpgradeGate", () => ({ + EditUpgradeGate: () => null, +})); +vi.mock("@/app/s/[videoId]/edit/edit-recovery", () => ({ + EditRecovery: () => null, +})); +vi.mock("@/app/s/[videoId]/edit/studio/studio-editor-client", () => ({ + StudioEditorClient: () => null, +})); + +import { EditUpgradeGate } from "@/app/s/[videoId]/edit/EditUpgradeGate"; +import { EditVideoClient } from "@/app/s/[videoId]/edit/EditVideoClient"; +import { EditProcessing } from "@/app/s/[videoId]/edit/edit-processing"; +import { EditRecovery } from "@/app/s/[videoId]/edit/edit-recovery"; +import EditPage from "@/app/s/[videoId]/edit/page"; +import StudioPage from "@/app/s/[videoId]/edit/studio/page"; +import { StudioEditorClient } from "@/app/s/[videoId]/edit/studio/studio-editor-client"; + +const params = { params: Promise.resolve({ videoId: "video" }) }; +const video = { + id: "video", + ownerId: "owner", + name: "Recording", + duration: 23, + isScreenshot: false, + source: { type: "webMP4" }, + metadata: null, + uploadPhase: null, + rawFileKey: "owner/video/raw-upload.webm", +}; + +beforeEach(() => { + mocks.rows = []; + mocks.user = { id: "owner", email: "richie@mcilroy.co" }; + mocks.studioEnabled = true; + mocks.pro = true; + mocks.queries = 0; + mocks.signedIn = Promise.resolve(); +}); + +for (const [name, page] of [ + ["edit", EditPage], + ["studio", StudioPage], +] as const) { + describe(`${name} recording readiness`, () => { + test.each(["uploading", "processing", "generating_thumbnail", "error"])( + "shows progress instead of a 404 for %s, even without final duration", + async (uploadPhase) => { + mocks.rows = [[{ ...video, uploadPhase, duration: null }]]; + const result = await page(params); + expect(result.type).toBe(EditProcessing); + expect(result.props.videoId).toBe("video"); + }, + ); + test.each([ + { ...video, ownerId: "another-owner" }, + { ...video, isScreenshot: true }, + { ...video, source: { type: "desktopSegments" } }, + undefined, + ])("does not expose processing state for ineligible media", async (row) => { + mocks.rows = [row ? [{ ...row, uploadPhase: "processing" }] : []]; + await expect(page(params)).rejects.toThrow("NOT_FOUND"); + }); + test("still requires a signed-in owner", async () => { + mocks.user = null; + await expect(page(params)).rejects.toThrow("NOT_FOUND"); + }); + test("keeps legacy edit recovery separate from initial recording processing", async () => { + mocks.rows = [ + [ + { + ...video, + uploadPhase: "processing", + rawFileKey: "owner/video/source/original.mp4", + }, + ], + ]; + expect((await page(params)).type).toBe(EditRecovery); + }); + }); +} + +test("completed pilot recordings redirect to Studio", async () => { + mocks.rows = [[video], []]; + await expect(EditPage(params)).rejects.toThrow( + "REDIRECT:/s/video/edit/studio", + ); +}); + +test("completed Studio recordings mount the editor", async () => { + mocks.rows = [[video], []]; + expect((await StudioPage(params)).type).toBe(StudioEditorClient); +}); + +test("Studio looks up the recording while the sign-in check runs", async () => { + let signIn: () => void = () => {}; + mocks.signedIn = new Promise((resolve) => { + signIn = resolve; + }); + mocks.rows = [[video], []]; + const page = StudioPage(params); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(mocks.queries).toBe(2); + signIn(); + expect((await page).type).toBe(StudioEditorClient); +}); + +test("the existing editor and upgrade gate remain available outside the pilot", async () => { + mocks.studioEnabled = false; + mocks.rows = [[video], []]; + expect((await EditPage(params)).type).toBe(EditVideoClient); + mocks.pro = false; + mocks.rows = [[video], []]; + expect((await EditPage(params)).type).toBe(EditUpgradeGate); + await expect(StudioPage(params)).rejects.toThrow("NOT_FOUND"); +}); diff --git a/apps/web/__tests__/unit/editor-recorded-clips.test.ts b/apps/web/__tests__/unit/editor-recorded-clips.test.ts new file mode 100644 index 00000000000..07c9aa41154 --- /dev/null +++ b/apps/web/__tests__/unit/editor-recorded-clips.test.ts @@ -0,0 +1,69 @@ +import { afterEach, expect, test, vi } from "vitest"; +import { EditorHostBridge } from "../../app/s/[videoId]/edit/studio/editor-host"; +import { importWebEditorVideo } from "../../lib/editor-video-import-client"; + +vi.mock("@/lib/editor-video-import-client", () => ({ + importWebEditorVideo: vi.fn(), +})); + +const displayPath = "content/videos/00000000-0000-0000-0000-000000000001.webm"; +const cameraPath = "content/videos/00000000-0000-0000-0000-000000000002.webm"; +const displayJobId = "00000000-0000-0000-0000-000000000003"; +const cameraJobId = "00000000-0000-0000-0000-000000000004"; + +afterEach(() => vi.unstubAllGlobals()); + +test("a recorded screen and camera register as one synchronized editor clip", async () => { + const screen = new File(["screen"], "screen.webm", { type: "video/webm" }); + const camera = new File(["camera"], "camera.webm", { type: "video/webm" }); + const staged = vi.mocked(importWebEditorVideo); + staged.mockImplementation(async (file) => ({ + jobId: file === screen ? displayJobId : cameraJobId, + path: file === screen ? displayPath : cameraPath, + name: file.name, + duration: 3, + fps: file === screen ? 30 : 25, + width: 640, + height: 360, + hasAudio: file === screen, + })); + const requests: Array<{ url: string; body: Record }> = []; + let registrationStatus = 409; + vi.stubGlobal( + "fetch", + vi.fn(async (url: string, init: RequestInit) => { + requests.push({ + url, + body: JSON.parse(String(init.body)) as Record, + }); + return registrationStatus === 409 + ? new Response(null, { status: 409 }) + : Response.json({ count: 1 }); + }), + ); + const bridge = new EditorHostBridge( + "recording", + "session", + "owner", + vi.fn(), + vi.fn(), + ); + await expect(bridge.addRecordedClip(screen, camera, 125)).rejects.toThrow( + "Editor changed while importing the clip", + ); + registrationStatus = 200; + await bridge.addRecordedClip(screen, camera, 125); + expect(staged).toHaveBeenCalledTimes(2); + expect(staged.mock.calls.map(([file]) => file)).toEqual([screen, camera]); + expect(requests).toHaveLength(2); + expect(requests[1]).toEqual({ + url: "/api/editor/sessions/session/clips", + body: { + videoId: "recording", + path: displayPath, + jobId: displayJobId, + camera: { path: cameraPath, jobId: cameraJobId, offsetMs: 125 }, + }, + }); + bridge.dispose(); +}); diff --git a/apps/web/__tests__/unit/editor-sources-ready.test.ts b/apps/web/__tests__/unit/editor-sources-ready.test.ts new file mode 100644 index 00000000000..b6a24f0b940 --- /dev/null +++ b/apps/web/__tests__/unit/editor-sources-ready.test.ts @@ -0,0 +1,66 @@ +import type { VideoMetadata } from "@cap/database/types"; +import { describe, expect, it } from "vitest"; +import { editorSourcesUploaded } from "@/lib/editor-sources-ready"; + +const display = { + key: "user/video/raw-upload.mp4", + contentType: "video/mp4" as const, + size: 1024, +}; +const camera = { + key: "user/video/camera-upload.mp4", + contentType: "video/mp4" as const, + size: 2048, + objectIdentity: null, + offsetMs: 0, +}; + +const metadata = (sources: object): VideoMetadata => + ({ editorSources: { version: 1, ...sources } }) as VideoMetadata; + +describe("editorSourcesUploaded", () => { + it("opens a processing recording once its sources are uploaded", () => { + expect( + editorSourcesUploaded(metadata({ display, camera }), "processing"), + ).toBe(true); + expect( + editorSourcesUploaded(metadata({ display }), "generating_thumbnail"), + ).toBe(true); + }); + + it("waits while anything is still uploading or failed", () => { + expect(editorSourcesUploaded(metadata({ display }), "uploading")).toBe( + false, + ); + expect(editorSourcesUploaded(metadata({ display }), "error")).toBe(false); + expect(editorSourcesUploaded(metadata({ display }), null)).toBe(false); + }); + + it("waits for a display or camera source without a verified size", () => { + expect( + editorSourcesUploaded( + metadata({ display: { ...display, size: undefined } }), + "processing", + ), + ).toBe(false); + expect( + editorSourcesUploaded( + metadata({ display, camera: { ...camera, size: 0 } }), + "processing", + ), + ).toBe(false); + expect(editorSourcesUploaded(null, "processing")).toBe(false); + }); + + it("leaves an edit that is being processed to its recovery screen", () => { + expect( + editorSourcesUploaded( + { + ...metadata({ display }), + editProcessing: {}, + } as unknown as VideoMetadata, + "processing", + ), + ).toBe(false); + }); +}); diff --git a/apps/web/__tests__/unit/editor-studio-recovery.test.ts b/apps/web/__tests__/unit/editor-studio-recovery.test.ts new file mode 100644 index 00000000000..3afbf690a7e --- /dev/null +++ b/apps/web/__tests__/unit/editor-studio-recovery.test.ts @@ -0,0 +1,560 @@ +// @vitest-environment jsdom + +import { Video } from "@cap/web-domain"; +import { act, createElement } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { + captureEditorLocalDraft, + readEditorLocalDraft, +} from "@/lib/editor-local-draft"; + +const mocks = vi.hoisted(() => ({ + push: vi.fn(), + connect: vi.fn(async () => undefined), + dispose: vi.fn(), +})); + +vi.mock("next/navigation", () => ({ + useRouter: () => ({ push: mocks.push }), +})); +vi.mock("next/dynamic", () => ({ default: () => () => null })); +vi.mock("@/app/s/[videoId]/edit/studio/editor-clip-recorder", () => ({ + EditorClipRecorder: () => null, +})); +vi.mock("@/components/editor-shell/share-link-tab", () => ({ + EditorShareLinkTab: () => null, +})); +vi.mock("@/app/s/[videoId]/edit/studio/editor-host", () => ({ + EditorHostBridge: class { + connect = mocks.connect; + dispose = mocks.dispose; + }, +})); + +import { StudioEditorClient } from "@/app/s/[videoId]/edit/studio/studio-editor-client"; + +let root: Root; +let container: HTMLDivElement; +const requests: Array<{ url: string; method: string; body: unknown }> = []; +let currentRevision: string; +let rejectNextRestore: boolean; +let restoreUnavailable: boolean; + +function browserStorage(): Storage { + const items = new Map(); + return { + get length() { + return items.size; + }, + clear: () => items.clear(), + getItem: (key) => items.get(key) ?? null, + key: (index) => [...items.keys()][index] ?? null, + removeItem: (key) => items.delete(key), + setItem: (key, value) => items.set(key, value), + }; +} + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + Object.defineProperty(window, "localStorage", { + configurable: true, + value: browserStorage(), + }); + requests.length = 0; + currentRevision = "newer"; + rejectNextRestore = false; + restoreUnavailable = false; + mocks.connect.mockClear(); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); + vi.stubGlobal( + "fetch", + async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + const method = init?.method ?? "GET"; + const body = init?.body ? JSON.parse(String(init.body)) : null; + requests.push({ url, method, body }); + if (url === "/api/editor/videos/video/config" && method === "PUT") { + const expectedSavedAt = (body as Record) + .expectedSavedAt; + if (expectedSavedAt !== currentRevision) + return new Response("Changed in another tab", { status: 409 }); + if (restoreUnavailable) + return new Response("Unavailable", { status: 502 }); + if (rejectNextRestore) { + rejectNextRestore = false; + currentRevision = "latest"; + return new Response("Changed again", { status: 409 }); + } + currentRevision = "recovered"; + return Response.json({ saved: true, savedAt: currentRevision }); + } + if (url === "/api/editor/videos/video/config" && method === "GET") { + return Response.json({ savedAt: currentRevision }); + } + if ( + url.startsWith("/api/editor/sessions/session?") && + method === "DELETE" + ) { + return Response.json({ closed: true }); + } + throw new Error(`Unexpected editor request: ${method} ${url}`); + }, + ); +}); + +afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); + Reflect.deleteProperty(window, "localStorage"); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +async function waitFor(assertion: () => void) { + let lastError: unknown; + for (let attempt = 0; attempt < 60; attempt++) { + try { + assertion(); + return; + } catch (cause) { + lastError = cause; + } + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + }); + } + throw lastError; +} + +async function openRecoveryConflict() { + const config = { + camera: { mirror: true }, + background: { + source: { type: "image", path: "content/images/fixture.png" }, + }, + }; + expect( + captureEditorLocalDraft( + window.localStorage, + "owner", + "video", + "older", + JSON.stringify(config), + ), + ).toBe(true); + await act(async () => { + root.render( + createElement(StudioEditorClient, { + videoId: Video.VideoId.make("video"), + userId: "owner", + captionsEnabled: true, + savedAt: "newer", + isPublic: true, + shareUrl: "https://cap.so/s/video", + preparingTitle: "Paired replay", + preparingDuration: 900, + preparingTracks: ["display", "camera"], + }), + ); + }); + await waitFor(() => { + expect(container.querySelector("[role='alert']")?.textContent).toContain( + "changed", + ); + }); + return config; +} + +test("a changed recording keeps the browser draft until its owner chooses recovery", async () => { + const config = await openRecoveryConflict(); + const saves = requests.filter( + (request) => + request.url === "/api/editor/videos/video/config" && + request.method === "PUT", + ); + expect(saves).toHaveLength(1); + expect(saves[0]?.body).toEqual({ + config, + expectedSavedAt: "older", + }); + expect( + readEditorLocalDraft(window.localStorage, "owner", "video")?.config, + ).toEqual(config); + expect( + requests.filter( + (request) => + request.method === "DELETE" && + request.url.startsWith("/api/editor/sessions/session?"), + ), + ).toHaveLength(0); + const restore = Array.from(container.querySelectorAll("button")).find( + (button) => button.textContent === "Restore browser edits", + ); + if (!restore) throw new Error("Recovery choice was not shown"); + await act(async () => restore.click()); + await waitFor(() => { + expect( + container.querySelector("iframe[title='Cap editor']"), + ).not.toBeNull(); + }); + expect( + requests.filter( + (request) => + request.url === "/api/editor/videos/video/config" && + request.method === "PUT", + )[1]?.body, + ).toEqual({ + config, + expectedSavedAt: "newer", + }); + expect( + requests.filter( + (request) => + request.url === "/api/editor/videos/video/config" && + request.method === "GET", + ), + ).toHaveLength(1); + expect( + readEditorLocalDraft(window.localStorage, "owner", "video"), + ).toBeNull(); +}); + +test("a second tab save during chosen recovery keeps the draft for another owner choice", async () => { + const config = await openRecoveryConflict(); + rejectNextRestore = true; + const restore = () => + Array.from(container.querySelectorAll("button")).find( + (button) => button.textContent === "Restore browser edits", + ); + const first = restore(); + if (!first) throw new Error("Recovery choice was not shown"); + await act(async () => first.click()); + await waitFor(() => { + expect(container.querySelector("[role='alert']")?.textContent).toContain( + "changed again", + ); + }); + expect( + readEditorLocalDraft(window.localStorage, "owner", "video")?.config, + ).toEqual(config); + expect(container.querySelector("iframe[title='Cap editor']")).toBeNull(); + const second = restore(); + if (!second) throw new Error("Recovery choice disappeared"); + await act(async () => second.click()); + await waitFor(() => { + expect( + container.querySelector("iframe[title='Cap editor']"), + ).not.toBeNull(); + }); + const saves = requests.filter( + (request) => + request.url === "/api/editor/videos/video/config" && + request.method === "PUT", + ); + expect(saves.map((request) => request.body)).toEqual([ + { config, expectedSavedAt: "older" }, + { config, expectedSavedAt: "newer" }, + { config, expectedSavedAt: "latest" }, + ]); + expect( + readEditorLocalDraft(window.localStorage, "owner", "video"), + ).toBeNull(); +}); + +test("an unexpected browser-draft restore failure preserves the draft", async () => { + const config = { camera: { mirror: true } }; + expect( + captureEditorLocalDraft( + window.localStorage, + "owner", + "video", + "newer", + JSON.stringify(config), + ), + ).toBe(true); + restoreUnavailable = true; + await act(async () => { + root.render( + createElement(StudioEditorClient, { + videoId: Video.VideoId.make("video"), + userId: "owner", + captionsEnabled: true, + savedAt: "newer", + isPublic: true, + shareUrl: "https://cap.so/s/video", + preparingTitle: "Paired replay", + preparingDuration: 900, + preparingTracks: ["display", "camera"], + }), + ); + }); + await waitFor(() => { + expect(container.querySelector("[role='alert']")?.textContent).toContain( + "could not restore them", + ); + }); + expect( + requests.filter( + (request) => + request.url.startsWith("/api/editor/sessions/session?") && + request.method === "DELETE", + ), + ).toHaveLength(0); + expect( + readEditorLocalDraft(window.localStorage, "owner", "video")?.config, + ).toEqual(config); +}); + +test("browser Studio keeps the shared editor shell open and connects once when ready", async () => { + await act(async () => { + root.render( + createElement(StudioEditorClient, { + videoId: Video.VideoId.make("video"), + userId: "owner", + captionsEnabled: true, + savedAt: null, + isPublic: true, + shareUrl: "https://cap.so/s/video", + preparingTitle: "Paired replay", + preparingDuration: 900, + preparingTracks: ["display", "camera"], + }), + ); + }); + const iframe = container.querySelector( + "iframe[title='Cap editor']", + ); + if (!iframe) throw new Error("Editor shell was not shown"); + expect(requests).toHaveLength(0); + const frameDocument = + document.implementation.createHTMLDocument("Cap editor"); + Object.defineProperty(frameDocument, "URL", { + configurable: true, + value: "http://localhost/editor-solid/index.html", + }); + Object.defineProperty(frameDocument, "readyState", { + configurable: true, + value: "complete", + }); + Object.defineProperty(iframe, "contentDocument", { + configurable: true, + value: frameDocument, + }); + const childWindow = iframe.contentWindow; + if (!childWindow) throw new Error("Editor child window was unavailable"); + const postMessage = vi.spyOn(childWindow, "postMessage"); + await act(async () => iframe.dispatchEvent(new Event("load"))); + expect(postMessage).toHaveBeenCalledWith( + { + kind: "cap-editor-preparing", + version: 1, + title: "Paired replay", + durationSeconds: 900, + tracks: ["display", "camera"], + }, + window.location.origin, + ); + await waitFor(() => { + expect(mocks.connect).toHaveBeenCalledTimes(1); + }); + await act(async () => iframe.dispatchEvent(new Event("load"))); + expect(mocks.connect).toHaveBeenCalledTimes(1); +}); + +test("leaving asks first while the share link is missing this session's edits", async () => { + await act(async () => { + root.render( + createElement(StudioEditorClient, { + videoId: Video.VideoId.make("video"), + userId: "owner", + captionsEnabled: true, + savedAt: null, + isPublic: true, + shareUrl: "https://cap.so/s/video", + preparingTitle: "Paired replay", + preparingDuration: 900, + preparingTracks: ["display"], + }), + ); + }); + const iframe = container.querySelector( + "iframe[title='Cap editor']", + ); + const childWindow = iframe?.contentWindow; + if (!childWindow) throw new Error("Editor child window was unavailable"); + let unpublished = true; + Object.assign(childWindow, { + capSolidEditor: { + unsavedProject: () => null, + unpublishedEdits: () => unpublished, + }, + }); + const leave = () => { + const event = new Event("beforeunload", { cancelable: true }); + window.dispatchEvent(event); + return event.defaultPrevented; + }; + const dashboard = [...container.querySelectorAll("a")].find( + (link) => link.textContent === "Dashboard", + ); + if (!dashboard) throw new Error("Dashboard was not shown"); + const confirm = vi.spyOn(window, "confirm").mockReturnValue(false); + + expect(leave()).toBe(true); + const click = new MouseEvent("click", { bubbles: true, cancelable: true }); + await act(async () => dashboard.dispatchEvent(click)); + expect(confirm).toHaveBeenCalledTimes(1); + expect(click.defaultPrevented).toBe(true); + + unpublished = false; + expect(leave()).toBe(false); +}); + +test("a Free editor lets its owner restore non-caption edits from a Pro browser draft", async () => { + const config = { + camera: { mirror: true }, + captions: { + segments: [{ id: "word", start: 0, end: 1, text: "Hello" }], + settings: { enabled: true, exportWithSubtitles: true, font: "Geist" }, + }, + timeline: { + segments: [{ start: 0, end: 10 }], + captionSegments: [{ id: "word", start: 0, end: 1 }], + }, + }; + expect( + captureEditorLocalDraft( + window.localStorage, + "owner", + "video", + "newer", + JSON.stringify(config), + ), + ).toBe(true); + await act(async () => { + root.render( + createElement(StudioEditorClient, { + videoId: Video.VideoId.make("video"), + userId: "owner", + captionsEnabled: false, + savedAt: "newer", + isPublic: true, + shareUrl: "https://cap.so/s/video", + preparingTitle: "Paired replay", + preparingDuration: 900, + preparingTracks: ["display", "camera"], + }), + ); + }); + await waitFor(() => { + expect( + container.querySelector("[role='alert']")?.textContent ?? "", + ).toContain("Cap Pro"); + }); + expect( + requests.filter( + (request) => + request.url === "/api/editor/videos/video/config" && + request.method === "PUT", + ), + ).toHaveLength(0); + expect( + readEditorLocalDraft(window.localStorage, "owner", "video")?.config, + ).toEqual(config); + const restore = Array.from(container.querySelectorAll("button")).find( + (button) => button.textContent === "Restore browser edits", + ); + if (!restore) throw new Error("Recovery choice was not shown"); + await act(async () => restore.click()); + await waitFor(() => { + expect( + container.querySelector("iframe[title='Cap editor']"), + ).not.toBeNull(); + }); + expect( + requests.filter( + (request) => + request.url === "/api/editor/videos/video/config" && + request.method === "PUT", + )[0]?.body, + ).toEqual({ + config: { + camera: { mirror: true }, + captions: { + segments: [], + settings: { + enabled: false, + exportWithSubtitles: false, + font: "Geist", + }, + }, + timeline: { + segments: [{ start: 0, end: 10 }], + captionSegments: [], + }, + }, + expectedSavedAt: "newer", + }); + expect( + readEditorLocalDraft(window.localStorage, "owner", "video"), + ).toBeNull(); +}); + +test("a Pro editor automatically restores a caption browser draft", async () => { + const config = { + camera: { mirror: true }, + captions: { + segments: [{ id: "word", start: 0, end: 1, text: "Hello" }], + settings: { enabled: true, exportWithSubtitles: true }, + }, + }; + expect( + captureEditorLocalDraft( + window.localStorage, + "owner", + "video", + "newer", + JSON.stringify(config), + ), + ).toBe(true); + await act(async () => { + root.render( + createElement(StudioEditorClient, { + videoId: Video.VideoId.make("video"), + userId: "owner", + captionsEnabled: true, + savedAt: "newer", + isPublic: true, + shareUrl: "https://cap.so/s/video", + preparingTitle: "Paired replay", + preparingDuration: 900, + preparingTracks: ["display", "camera"], + }), + ); + }); + await waitFor(() => { + expect( + requests.filter( + (request) => + request.url === "/api/editor/videos/video/config" && + request.method === "PUT", + ), + ).toHaveLength(1); + expect( + readEditorLocalDraft(window.localStorage, "owner", "video"), + ).toBeNull(); + }); + expect( + requests.filter( + (request) => + request.url === "/api/editor/videos/video/config" && + request.method === "PUT", + )[0]?.body, + ).toEqual({ config, expectedSavedAt: "newer" }); + expect( + readEditorLocalDraft(window.localStorage, "owner", "video"), + ).toBeNull(); +}); diff --git a/apps/web/__tests__/unit/editor-video-import-client.test.ts b/apps/web/__tests__/unit/editor-video-import-client.test.ts new file mode 100644 index 00000000000..33004774a51 --- /dev/null +++ b/apps/web/__tests__/unit/editor-video-import-client.test.ts @@ -0,0 +1,64 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@cap/recorder-core", () => ({ + InstantRecordingUploader: class {}, + MultipartCompletionUncertainError: class extends Error {}, +})); +vi.mock("@cap/web-domain", () => ({ + Video: { VideoId: { make: (id: string) => id } }, +})); + +import { importWebEditorVideo } from "@/lib/editor-video-import-client"; + +const start = (file: File) => + importWebEditorVideo( + file, + "video", + "owner", + "session", + new AbortController().signal, + ); + +describe("importWebEditorVideo", () => { + afterEach(() => vi.unstubAllGlobals()); + + it("shortens a long file name to what the upload accepts", async () => { + const fetch = vi.fn(async () => new Response(null, { status: 500 })); + vi.stubGlobal("fetch", fetch); + const name = `${"Quarterly review 🎬 ".repeat(8)}final.MP4`; + + await expect(start(new File(["x"], name))).rejects.toThrow( + "Video upload could not start in the editor", + ); + + const body = JSON.parse( + (fetch.mock.calls[0] as unknown as [string, RequestInit])[1] + .body as string, + ); + expect(body.fileName.length).toBeLessThanOrEqual(100); + expect(body.fileName).toMatch(/^Quarterly review 🎬 .*\.mp4$/); + expect(body.fileName).not.toMatch(/�|[\uD800-\uDBFF]\.mp4$/); + }); + + it("keeps short names as they are", async () => { + const fetch = vi.fn(async () => new Response(null, { status: 500 })); + vi.stubGlobal("fetch", fetch); + + await expect(start(new File(["x"], "Démo 视频.mov"))).rejects.toThrow(); + + const body = JSON.parse( + (fetch.mock.calls[0] as unknown as [string, RequestInit])[1] + .body as string, + ); + expect(body.fileName).toBe("Démo 视频.mov"); + }); + + it("says why a file can't be imported", async () => { + await expect(start(new File(["x"], "notes.txt"))).rejects.toThrow( + "This video format isn't supported", + ); + await expect(start(new File([], "empty.mp4"))).rejects.toThrow( + "This video file is empty", + ); + }); +}); diff --git a/apps/web/__tests__/unit/editor-worker-session-client.test.ts b/apps/web/__tests__/unit/editor-worker-session-client.test.ts new file mode 100644 index 00000000000..37c06d5b751 --- /dev/null +++ b/apps/web/__tests__/unit/editor-worker-session-client.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it, vi } from "vitest"; +import { openWorkerEditorSession } from "@/lib/editor-worker-session-client"; + +function responses( + ...queue: (Response | ((url: string, init: RequestInit) => Response))[] +) { + const calls: { url: string; init: RequestInit }[] = []; + const fetcher = vi.fn(async (url: string, init: RequestInit) => { + calls.push({ url, init }); + if (init.method === "DELETE") return new Response(null, { status: 204 }); + const next = queue.shift(); + if (!next) throw new Error(`unexpected request ${url}`); + return typeof next === "function" ? next(url, init) : next; + }); + return { fetcher, calls }; +} + +describe("worker-rendered editor preview", () => { + it("returns the session once the worker has prepared it", async () => { + const { fetcher, calls } = responses( + Response.json({ id: "prep-1", status: "preparing" }), + Response.json({ status: "preparing" }), + Response.json({ status: "ready", sessionId: "worker.session-1" }), + ); + await expect( + openWorkerEditorSession( + "video-1", + new AbortController().signal, + () => undefined, + fetcher, + 1, + ), + ).resolves.toBe("worker.session-1"); + expect(calls.map((call) => call.init.method ?? "GET")).toEqual([ + "POST", + "GET", + "GET", + ]); + expect(calls[1]?.url).toBe( + "/api/editor/preparations/prep-1?videoId=video-1", + ); + }); + + it("cancels the preparation when it fails", async () => { + const { fetcher, calls } = responses( + Response.json({ id: "prep-2", status: "preparing" }), + Response.json({ status: "error" }), + ); + await expect( + openWorkerEditorSession( + "video-1", + new AbortController().signal, + () => undefined, + fetcher, + 1, + ), + ).rejects.toThrow("Editor preparation failed"); + expect(calls.at(-1)?.init.method).toBe("DELETE"); + expect(calls.at(-1)?.url).toBe( + "/api/editor/preparations/prep-2?videoId=video-1", + ); + }); + + it("stops and cancels the preparation when the page leaves", async () => { + const controller = new AbortController(); + const { fetcher, calls } = responses( + Response.json({ id: "prep-3", status: "preparing" }), + () => { + controller.abort(); + return Response.json({ status: "preparing" }); + }, + ); + await expect( + openWorkerEditorSession( + "video-1", + controller.signal, + () => undefined, + fetcher, + 1, + ), + ).rejects.toThrow("canceled"); + expect(calls.at(-1)?.init.method).toBe("DELETE"); + }); +}); diff --git a/apps/web/__tests__/unit/embed-video-playback-chrome.test.ts b/apps/web/__tests__/unit/embed-video-playback-chrome.test.ts index fac9bc0aa5c..98675546f93 100644 --- a/apps/web/__tests__/unit/embed-video-playback-chrome.test.ts +++ b/apps/web/__tests__/unit/embed-video-playback-chrome.test.ts @@ -26,6 +26,27 @@ import { vi.mock("@cap/env", () => ({ NODE_ENV: "test" })); +// next/dynamic's loader relies on Next's own React context; resolve the +// (mocked) modules through React.lazy instead. +vi.mock("next/dynamic", async () => { + const { createElement, lazy, Suspense } = await import("react"); + return { + default: ( + loader: () => Promise< + | ((props: object) => ReactNode) + | { default: (props: object) => ReactNode } + >, + ) => { + const Lazy = lazy(async () => { + const loaded = await loader(); + return { default: "default" in loaded ? loaded.default : loaded }; + }); + return (props: object) => + createElement(Suspense, { fallback: null }, createElement(Lazy, props)); + }, + }; +}); + vi.mock("@cap/ui", async () => { const { createElement } = await import("react"); return { @@ -223,6 +244,38 @@ describe("EmbedVideo playback chrome", () => { }, ); + it("starts a lazily loaded HLS embed at its start time", async () => { + const container = document.createElement("div"); + document.body.append(container); + const root = createRoot(container); + + await act(async () => { + root.render( + createElement(EmbedVideo, { + ...createProps({ type: "MediaConvert" }), + startTime: 30, + }), + ); + }); + const video = container.querySelector("video"); + if (!video) throw new Error("Expected the HLS player to mount"); + let currentTime = 0; + Object.defineProperty(video, "duration", { value: 60 }); + Object.defineProperty(video, "currentTime", { + get: () => currentTime, + set: (value: number) => { + currentTime = value; + }, + }); + + await act(async () => { + video.dispatchEvent(new Event("loadedmetadata")); + }); + expect(currentTime).toBe(30); + + await act(async () => root.unmount()); + }); + it.each([ ["an asynchronously mounted MP4", { type: "desktopMP4" } as const], ["an HLS video", { type: "MediaConvert" } as const], diff --git a/apps/web/__tests__/unit/hover-prefetch-link.test.ts b/apps/web/__tests__/unit/hover-prefetch-link.test.ts new file mode 100644 index 00000000000..a88a53db181 --- /dev/null +++ b/apps/web/__tests__/unit/hover-prefetch-link.test.ts @@ -0,0 +1,79 @@ +// @vitest-environment jsdom + +import { act, createElement } from "react"; +import { createRoot } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { HoverPrefetchLink } from "@/components/hover-prefetch-link"; + +const mocks = vi.hoisted(() => ({ + prefetch: vi.fn(), + linkProps: [] as Record[], +})); + +vi.mock("next/navigation", () => ({ + useRouter: () => ({ prefetch: mocks.prefetch }), +})); + +vi.mock("next/link", async () => { + const { createElement, forwardRef } = await import("react"); + return { + default: forwardRef>( + function MockLink({ prefetch, ...props }, ref) { + mocks.linkProps.push({ prefetch }); + return createElement("a", { ...props, ref }); + }, + ), + }; +}); + +const env = globalThis as typeof globalThis & { + IS_REACT_ACT_ENVIRONMENT?: boolean; +}; + +describe("HoverPrefetchLink", () => { + let container: HTMLDivElement; + beforeEach(() => { + env.IS_REACT_ACT_ENVIRONMENT = true; + container = document.createElement("div"); + document.body.append(container); + mocks.prefetch.mockReset(); + mocks.linkProps.length = 0; + }); + afterEach(() => container.remove()); + + it("doesn't prefetch on render, then prefetches once on hover or focus", async () => { + const onPointerEnter = vi.fn(); + const root = createRoot(container); + await act(async () => { + root.render( + createElement( + HoverPrefetchLink, + { href: "/dashboard/caps", onPointerEnter }, + "Dashboard", + ), + ); + }); + expect(mocks.linkProps.at(-1)?.prefetch).toBe(false); + expect(mocks.prefetch).not.toHaveBeenCalled(); + + const link = container.querySelector("a"); + if (!link) throw new Error("link missing"); + await act(async () => { + // jsdom has no PointerEvent; React derives pointerenter from pointerover. + link.dispatchEvent( + new MouseEvent("pointerover", { + bubbles: true, + relatedTarget: document.body, + }), + ); + }); + await act(async () => { + link.focus(); + }); + + expect(mocks.prefetch).toHaveBeenCalledTimes(1); + expect(mocks.prefetch).toHaveBeenCalledWith("/dashboard/caps"); + expect(onPointerEnter).toHaveBeenCalled(); + await act(async () => root.unmount()); + }); +}); diff --git a/apps/web/__tests__/unit/import-editor-source.test.ts b/apps/web/__tests__/unit/import-editor-source.test.ts new file mode 100644 index 00000000000..40d23501f6f --- /dev/null +++ b/apps/web/__tests__/unit/import-editor-source.test.ts @@ -0,0 +1,276 @@ +import { describe, expect, it } from "vitest"; +import { + type ImportMediaFacts, + importEditorSourcePlan, + importEditorSourcesPatch, + isobmffLayout, +} from "@/lib/import-editor-source"; + +const video = { + codec: "avc", + codecString: "avc1.640028", + width: 1920, + height: 1080, + rotation: 0, + squarePixels: true, + firstTimestamp: 0, + packetRate: 29.97, + decodable: true, +}; +const audio = { codec: "aac", firstTimestamp: 0, decodable: true }; +const facts = ( + overrides: Partial = {}, +): ImportMediaFacts => ({ + container: "mp4", + layout: "faststart", + duration: 30.5, + videoTracks: [video], + audioTracks: [audio], + ...overrides, +}); + +describe("importEditorSourcePlan", () => { + it("opens an H.264 and AAC MP4 or MOV from its upload", () => { + expect(importEditorSourcePlan(facts())).toEqual({ + duration: 30.5, + fps: 30, + }); + expect( + importEditorSourcePlan( + facts({ container: "quicktime", audioTracks: [] }), + ), + ).toEqual({ duration: 30.5, fps: 30 }); + expect( + importEditorSourcePlan( + facts({ videoTracks: [{ ...video, codecString: "avc1.42E01E" }] }), + ), + ).not.toBeNull(); + }); + + it.each([ + ["another container", facts({ container: "other" })], + ["a fragmented file", facts({ layout: "fragmented" })], + ["an index at the end", facts({ layout: "index-at-end" })], + ["an unreadable layout", facts({ layout: "unknown" })], + ["no video", facts({ videoTracks: [] })], + ["two video tracks", facts({ videoTracks: [video, video] })], + ["two audio tracks", facts({ audioTracks: [audio, audio] })], + [ + "HEVC", + facts({ + videoTracks: [ + { ...video, codec: "hevc", codecString: "hvc1.1.6.L93.B0" }, + ], + }), + ], + [ + "10-bit H.264", + facts({ videoTracks: [{ ...video, codecString: "avc1.6E0028" }] }), + ], + [ + "4:4:4 H.264", + facts({ videoTracks: [{ ...video, codecString: "avc1.F40028" }] }), + ], + [ + "undecodable video", + facts({ videoTracks: [{ ...video, decodable: false }] }), + ], + ["rotated video", facts({ videoTracks: [{ ...video, rotation: 90 }] })], + [ + "non-square pixels", + facts({ videoTracks: [{ ...video, squarePixels: false }] }), + ], + [ + "8K video", + facts({ videoTracks: [{ ...video, width: 7680, height: 4320 }] }), + ], + [ + "late video start", + facts({ videoTracks: [{ ...video, firstTimestamp: 0.5 }] }), + ], + [ + "negative video start", + facts({ videoTracks: [{ ...video, firstTimestamp: -0.02 }] }), + ], + ["Opus audio", facts({ audioTracks: [{ ...audio, codec: "opus" }] })], + [ + "undecodable audio", + facts({ audioTracks: [{ ...audio, decodable: false }] }), + ], + ["offset audio", facts({ audioTracks: [{ ...audio, firstTimestamp: 1 }] })], + [ + "too high a frame rate", + facts({ videoTracks: [{ ...video, packetRate: 240 }] }), + ], + [ + "no frame rate", + facts({ videoTracks: [{ ...video, packetRate: Number.NaN }] }), + ], + ["no duration", facts({ duration: 0 })], + ])("waits for processing on %s", (_, input) => { + expect(importEditorSourcePlan(input)).toBeNull(); + }); +}); + +describe("importEditorSourcesPatch", () => { + const input = { + ownerId: "owner", + videoId: "video", + isPro: true, + source: { duration: 42.25, fps: 30 }, + upload: { phase: "uploading", rawFileKey: "owner/video/raw-upload.mp4" }, + existingSources: undefined, + head: { size: 1234, identity: '"etag"' }, + }; + + it("registers the verified upload as the editor's display source", () => { + expect(importEditorSourcesPatch(input)).toEqual({ + duration: 42.25, + editorSources: { + version: 1, + display: { + key: "owner/video/raw-upload.mp4", + contentType: "video/mp4", + size: 1234, + fps: 30, + objectIdentity: '"etag"', + embeddedAudio: true, + }, + }, + }); + }); + + it.each([ + ["an invalid source", { source: { duration: 10, fps: 30.5 } }], + [ + "a Free import over the plan length", + { isPro: false, source: { duration: 301, fps: 30 } }, + ], + [ + "another upload key", + { upload: { phase: "uploading", rawFileKey: "owner/video/result.mp4" } }, + ], + [ + "an upload already processing", + { + upload: { + phase: "processing", + rawFileKey: "owner/video/raw-upload.mp4", + }, + }, + ], + ["no upload row", { upload: null }], + ["existing sources", { existingSources: { version: 1 } }], + ["an empty object", { head: { size: 0, identity: '"etag"' } }], + ["an unidentified object", { head: { size: 1234, identity: undefined } }], + ])("leaves %s to processing", (_, overrides) => { + expect(importEditorSourcesPatch({ ...input, ...overrides })).toBeNull(); + }); + + it("keeps a short Free import eligible", () => { + expect( + importEditorSourcesPatch({ + ...input, + isPro: false, + source: { duration: 120, fps: 60 }, + }), + ).not.toBeNull(); + }); +}); + +function box(type: string, body: Uint8Array | number = 0) { + const content = typeof body === "number" ? new Uint8Array(body) : body; + const bytes = new Uint8Array(8 + content.byteLength); + new DataView(bytes.buffer).setUint32(0, bytes.byteLength); + bytes.set( + [...type].map((c) => c.charCodeAt(0)), + 4, + ); + bytes.set(content, 8); + return bytes; +} + +function largeBox(type: string, size: number) { + const bytes = new Uint8Array(16); + const view = new DataView(bytes.buffer); + view.setUint32(0, 1); + bytes.set( + [...type].map((c) => c.charCodeAt(0)), + 4, + ); + view.setBigUint64(8, BigInt(size)); + return bytes; +} + +function concat(...parts: Uint8Array[]) { + const bytes = new Uint8Array(parts.reduce((n, p) => n + p.byteLength, 0)); + let offset = 0; + for (const part of parts) { + bytes.set(part, offset); + offset += part.byteLength; + } + return bytes; +} + +function layout(bytes: Uint8Array, size = bytes.byteLength) { + const reads: Array<[number, number]> = []; + return { + reads, + result: isobmffLayout(async (start, end) => { + reads.push([start, end]); + return bytes.subarray(start, Math.min(end, bytes.byteLength)); + }, size), + }; +} + +describe("isobmffLayout", () => { + const moov = box("moov", concat(box("mvhd", 100), box("trak", 64))); + + it("finds a faststart index without reading the media", async () => { + const { result, reads } = layout( + concat(box("ftyp", 16), moov, box("mdat", 4096)), + ); + await expect(result).resolves.toBe("faststart"); + expect(reads.every(([start, end]) => end - start <= 16)).toBe(true); + }); + + it("finds an index after the media, skipping it by size", async () => { + const { result, reads } = layout( + concat(box("ftyp", 16), box("mdat", 4096), moov), + ); + await expect(result).resolves.toBe("index-at-end"); + expect(reads.every(([start, end]) => end - start <= 16)).toBe(true); + const header = largeBox("mdat", 16 + 4096); + await expect( + layout(concat(box("ftyp", 16), header, new Uint8Array(4096), moov)) + .result, + ).resolves.toBe("index-at-end"); + }); + + it("flags fragmented and indexed layouts", async () => { + const fragmentedMoov = box( + "moov", + concat(box("mvhd", 100), box("trak", 64), box("mvex", 32)), + ); + for (const file of [ + concat(box("ftyp", 16), fragmentedMoov, box("moof", 32), box("mdat", 64)), + concat(box("ftyp", 16), box("moof", 32), box("mdat", 64)), + concat(box("ftyp", 16), box("sidx", 32), moov), + ]) { + await expect(layout(file).result).resolves.toBe("fragmented"); + } + }); + + it("gives up on a truncated or index-less file", async () => { + const noIndex = concat(box("ftyp", 16), box("mdat", 64)); + await expect(layout(noIndex).result).resolves.toBe("unknown"); + await expect(layout(noIndex, noIndex.byteLength + 4).result).resolves.toBe( + "unknown", + ); + const corrupt = concat( + box("ftyp", 16), + new Uint8Array([0, 0, 0, 4, 1, 2, 3, 4]), + ); + await expect(layout(corrupt).result).resolves.toBe("unknown"); + }); +}); diff --git a/apps/web/__tests__/unit/media-processing-workflows.test.ts b/apps/web/__tests__/unit/media-processing-workflows.test.ts index 57deff5af8c..e3629eb8974 100644 --- a/apps/web/__tests__/unit/media-processing-workflows.test.ts +++ b/apps/web/__tests__/unit/media-processing-workflows.test.ts @@ -1,8 +1,11 @@ +import type { SQL } from "drizzle-orm"; +import { MySqlDialect } from "drizzle-orm/mysql-core"; import { Effect } from "effect"; import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ rows: [] as unknown[][], + updates: [] as Record[], where: vi.fn(), write: vi.fn(), getAccess: vi.fn(), @@ -22,7 +25,12 @@ vi.mock("@cap/database", () => ({ innerJoin: () => ({ where: mocks.where }), }), }), - update: () => ({ set: () => ({ where: mocks.write }) }), + update: () => ({ + set: (values: Record) => { + mocks.updates.push(values); + return { where: mocks.write }; + }, + }), delete: () => ({ where: mocks.write }), }), })); @@ -86,6 +94,7 @@ const pending = { phase: "processing", processingProgress: 25 }; describe("media processing workflows", () => { beforeEach(() => { mocks.rows = []; + mocks.updates = []; mocks.where .mockReset() .mockImplementation(async () => mocks.rows.shift() ?? []); @@ -133,6 +142,8 @@ describe("media processing workflows", () => { [], [metadata], [video], + [video], + [video], [], ]; await expect(processVideoWorkflow(payload)).resolves.toMatchObject({ @@ -144,6 +155,90 @@ describe("media processing workflows", () => { expect(mocks.remove).toHaveBeenCalledWith(payload.rawFileKey); }); + it("retains a screen-only raw source after processing completes", async () => { + const screenVideo = { + ...video, + ownerId: "owner", + metadata: { + editorSources: { + version: 1, + display: { + key: payload.rawFileKey, + contentType: "video/mp4", + size: 2048, + objectIdentity: "source-etag", + }, + }, + }, + }; + mocks.rows = [ + [screenVideo], + [{ ...pending, rawFileKey: payload.rawFileKey }], + [screenVideo], + [pending], + [pending], + [], + [metadata], + [screenVideo], + [screenVideo], + [], + ]; + await expect(processVideoWorkflow(payload)).resolves.toMatchObject({ + success: true, + }); + expect(mocks.remove).not.toHaveBeenCalled(); + }); + + it("verifies the finished direct-MP4 source before enabling Studio", async () => { + const directKey = "owner/video/result.mp4"; + const directVideo = { + ...video, + ownerId: "owner", + metadata: { + editorSources: { + version: 1, + display: { key: directKey, contentType: "video/mp4" }, + }, + }, + }; + mocks.head.mockReturnValue( + Effect.succeed({ ContentLength: 4096, ETag: "final-etag" }), + ); + mocks.rows = [ + [directVideo], + [{ ...pending, rawFileKey: directKey }], + [directVideo], + [pending], + [pending], + [], + [metadata], + [directVideo], + [directVideo], + [], + ]; + await expect( + processVideoWorkflow({ ...payload, rawFileKey: directKey }), + ).resolves.toMatchObject({ success: true }); + expect(mocks.head).toHaveBeenCalledWith(directKey); + const dialect = new MySqlDialect(); + const sourceSql = mocks.updates.find((values) => "metadata" in values) + ?.metadata as SQL | undefined; + if (!sourceSql) throw new Error("Finished editor source was not written"); + const patch = dialect + .sqlToQuery(sourceSql) + .params.find( + (value): value is string => + typeof value === "string" && + value.includes(JSON.stringify("editorSources")), + ); + expect(JSON.parse(patch ?? "null")).toEqual({ + editorSources: { + display: { size: 4096, objectIdentity: "final-etag" }, + }, + }); + expect(mocks.remove).not.toHaveBeenCalled(); + }); + it("keeps uploaded source data when processing fails", async () => { mocks.rows = [ [video], @@ -163,6 +258,20 @@ describe("media processing workflows", () => { expect(mocks.sleep.mock.calls).toEqual([[15_000], [30_000]]); }); + it("fails an unreadable upload without retrying it", async () => { + mocks.rows = [ + [video], + [{ ...pending, rawFileKey: payload.rawFileKey }], + [video], + [{ ...pending, processingError: "Media input could not be read" }], + ]; + await expect(processVideoWorkflow(payload)).rejects.toThrow( + "Media input could not be read", + ); + expect(mocks.fetch).toHaveBeenCalledTimes(1); + expect(mocks.sleep).not.toHaveBeenCalled(); + }); + it("recovers from a confirmed worker failure with a bounded durable retry", async () => { mocks.rows = [ [video], @@ -173,6 +282,8 @@ describe("media processing workflows", () => { [], [metadata], [video], + [video], + [video], [], ]; await expect(processVideoWorkflow(payload)).resolves.toMatchObject({ diff --git a/apps/web/__tests__/unit/media-server-webhook-url.test.ts b/apps/web/__tests__/unit/media-server-webhook-url.test.ts new file mode 100644 index 00000000000..80cab08a651 --- /dev/null +++ b/apps/web/__tests__/unit/media-server-webhook-url.test.ts @@ -0,0 +1,79 @@ +import { describe, expect, test } from "vitest"; +import { createMediaServerWebhookUrl } from "@/lib/media-server-webhook-url"; + +const input = { + webUrl: "https://staging.cap.so", + deploymentEnvironment: "preview", + deploymentHost: "cap-preview.vercel.app", + automationBypassSecret: "test-bypass&value", + searchParams: { retryable: "true" }, +}; + +describe("media processing webhook destination", () => { + test("returns to the originating preview with both callback parameters", () => { + const url = new URL(createMediaServerWebhookUrl(input)); + expect(url.origin).toBe("https://cap-preview.vercel.app"); + expect(url.pathname).toBe("/api/webhooks/media-server/progress"); + expect(url.searchParams.get("retryable")).toBe("true"); + expect(url.searchParams.get("x-vercel-protection-bypass")).toBe( + input.automationBypassSecret, + ); + }); + + test.each(["production", "development", undefined])( + "preserves the normal callback in %s without forwarding the bypass", + (deploymentEnvironment) => { + expect( + createMediaServerWebhookUrl({ ...input, deploymentEnvironment }), + ).toBe( + "https://staging.cap.so/api/webhooks/media-server/progress?retryable=true", + ); + }, + ); + + test("honors a custom callback destination without leaking preview credentials", () => { + expect( + createMediaServerWebhookUrl({ + ...input, + webhookBaseUrl: "https://callbacks.example.com", + }), + ).toBe( + "https://callbacks.example.com/api/webhooks/media-server/progress?retryable=true", + ); + }); + + test("authenticates an explicit destination only when it matches this deployment", () => { + const url = new URL( + createMediaServerWebhookUrl({ + ...input, + webhookBaseUrl: "https://cap-preview.vercel.app", + }), + ); + expect(url.searchParams.get("x-vercel-protection-bypass")).toBe( + input.automationBypassSecret, + ); + }); + + test.each([ + "cap-preview.vercel.app.attacker.example", + "user@cap-preview.vercel.app", + "cap-preview.vercel.app/path", + "cap-preview.vercel.app:443", + undefined, + ])("rejects an invalid preview hostname: %s", (deploymentHost) => { + expect(createMediaServerWebhookUrl({ ...input, deploymentHost })).toBe( + "https://staging.cap.so/api/webhooks/media-server/progress?retryable=true", + ); + }); + + test("does not invent a bypass when the project has none", () => { + const url = new URL( + createMediaServerWebhookUrl({ + ...input, + automationBypassSecret: undefined, + }), + ); + expect(url.origin).toBe("https://cap-preview.vercel.app"); + expect(url.searchParams.has("x-vercel-protection-bypass")).toBe(false); + }); +}); diff --git a/apps/web/__tests__/unit/multipart-presign.test.ts b/apps/web/__tests__/unit/multipart-presign.test.ts index 0e4b0625680..96c4e3c4273 100644 --- a/apps/web/__tests__/unit/multipart-presign.test.ts +++ b/apps/web/__tests__/unit/multipart-presign.test.ts @@ -11,12 +11,14 @@ const mocks = vi.hoisted(() => ({ database: vi.fn(), prepareReplacement: vi.fn(), invalidate: vi.fn(), + refreshPreviews: vi.fn(), head: vi.fn(), queueTranscription: vi.fn(), shouldQueueTranscription: vi.fn(), complete: vi.fn(), create: vi.fn(), abort: vi.fn(), + startProcessing: vi.fn(async () => "started"), })); vi.mock("@cap/env", () => ({ serverEnv: () => ({ NEXTAUTH_SECRET: "test-reupload-secret" }), @@ -26,6 +28,9 @@ vi.mock("@/lib/desktop-reupload", () => ({ prepareDesktopReupload: mocks.prepareReplacement, invalidateReuploadedVideo: mocks.invalidate, })); +vi.mock("@/lib/refresh-preview-assets", () => ({ + enqueuePreviewAssetsRefresh: mocks.refreshPreviews, +})); vi.mock("@cap/web-backend", async () => { const { Context, Layer } = await import("effect"); return { @@ -75,7 +80,7 @@ vi.mock("@/lib/queue-video-transcription", () => ({ mocks.shouldQueueTranscription, })); vi.mock("@/lib/video-processing", () => ({ - startVideoProcessingWorkflow: vi.fn(async () => "started"), + startVideoProcessingWorkflow: mocks.startProcessing, })); import { app } from "@/app/api/upload/[...route]/multipart"; @@ -139,6 +144,287 @@ describe("multipart presign ownership failures", () => { }); }); +describe("recorder source upload", () => { + const cameraKey = "owner/video/camera-upload.webm"; + const displayKey = "owner/video/raw-upload.webm"; + const dialect = new MySqlDialect(); + let updates: Record[]; + let uploadMutations: string[]; + beforeEach(() => { + vi.clearAllMocks(); + updates = []; + uploadMutations = []; + mocks.getOwnedById.mockReturnValue( + Effect.succeed( + Option.some([ + { + id: Video.VideoId.make("video"), + ownerId: User.UserId.make("owner"), + orgId: "org", + source: { type: "webMP4" }, + metadata: Option.none(), + bucketId: Option.none(), + storageIntegrationId: Option.none(), + }, + ]), + ), + ); + mocks.database.mockReturnValue({ + insert: () => ({ + values: () => ({ + onDuplicateKeyUpdate: async () => { + uploadMutations.push("insert"); + }, + }), + }), + update: () => ({ + set: (values: Record) => ({ + where: async () => { + updates.push(values); + }, + }), + }), + delete: () => ({ + where: async () => { + uploadMutations.push("delete"); + }, + }), + }); + mocks.create.mockReturnValue( + Effect.succeed({ UploadId: "camera-session" }), + ); + mocks.complete.mockReturnValue( + Effect.succeed({ ETag: "camera-etag", Location: cameraKey }), + ); + mocks.head.mockReturnValue( + Effect.succeed({ ETag: "camera-etag", ContentLength: 100 }), + ); + mocks.abort.mockReturnValue(Effect.succeed({})); + mocks.storage.mockReturnValue( + Effect.succeed([ + { + provider: "s3", + bucketName: "bucket", + multipart: { + create: mocks.create, + complete: mocks.complete, + abort: mocks.abort, + }, + headObject: mocks.head, + }, + ]), + ); + }); + const completeCamera = (body: Record = {}) => + app.request("/complete", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + videoId: "video", + uploadId: "camera-session", + subpath: "camera-upload.webm", + screenSubpath: "raw-upload.webm", + cameraOffsetMs: 8, + durationInSecs: 5, + parts: [{ partNumber: 1, etag: "part-etag", size: 100 }], + ...body, + }), + }); + const sourcePatch = () => { + const metadata = updates[0]?.metadata; + if (!metadata) throw new Error("Missing editor source metadata update"); + const params = dialect.sqlToQuery(metadata as SQL).params; + const json = params.find( + (value): value is string => + typeof value === "string" && + value.includes(JSON.stringify("editorSources")), + ); + if (!json) throw new Error("Missing editor source JSON patch"); + return JSON.parse(json) as Record; + }; + + it("retains a verified screen source when no webcam was recorded", async () => { + mocks.complete.mockReturnValue( + Effect.succeed({ ETag: "display-etag", Location: displayKey }), + ); + mocks.head.mockReturnValue( + Effect.succeed({ ETag: "display-etag", ContentLength: 100 }), + ); + mocks.startProcessing.mockResolvedValue("started"); + const response = await app.request("/complete", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + videoId: "video", + uploadId: "display-session", + subpath: "raw-upload.webm", + durationInSecs: 5, + parts: [{ partNumber: 1, etag: "part-etag", size: 100 }], + }), + }); + expect(response.status).toBe(200); + expect(sourcePatch()).toEqual({ + editorSources: { + version: 1, + display: { + key: displayKey, + contentType: "video/webm", + size: 100, + objectIdentity: "display-etag", + }, + }, + }); + expect(mocks.startProcessing).toHaveBeenCalledWith( + expect.objectContaining({ rawFileKey: displayKey }), + ); + }); + + it("completes the private camera asset without publishing or clearing the screen upload", async () => { + const initiated = await app.request("/initiate", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + videoId: "video", + subpath: "camera-upload.webm", + contentType: "video/webm", + }), + }); + expect(initiated.status).toBe(200); + expect(uploadMutations).toEqual([]); + const response = await completeCamera(); + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ + success: true, + fileKey: cameraKey, + processingStarted: false, + }); + expect(mocks.complete).toHaveBeenCalledWith( + cameraKey, + "camera-session", + expect.anything(), + ); + expect(updates).toHaveLength(1); + expect(sourcePatch()).toEqual({ + editorSources: { + version: 1, + display: { key: displayKey, contentType: "video/webm" }, + camera: { + key: cameraKey, + contentType: "video/webm", + size: 100, + objectIdentity: "camera-etag", + offsetMs: 8, + }, + }, + }); + expect(uploadMutations).toEqual([]); + expect(mocks.startProcessing).not.toHaveBeenCalled(); + }); + + it("rejects an invalid display path before completing camera bytes", async () => { + const response = await completeCamera({ + screenSubpath: "../result.mp4", + }); + expect(response.status).toBe(400); + expect(mocks.complete).not.toHaveBeenCalled(); + expect(updates).toEqual([]); + }); + + it("recovers a lost completion response from the verified camera object", async () => { + mocks.complete.mockReturnValueOnce(Effect.fail(new Error("NoSuchUpload"))); + const response = await completeCamera(); + expect(response.status).toBe(200); + expect(updates).toHaveLength(1); + expect(uploadMutations).toEqual([]); + }); + + it("requires verified display bytes before starting the composited output", async () => { + mocks.getOwnedById.mockReturnValue( + Effect.succeed( + Option.some([ + { + id: Video.VideoId.make("video"), + ownerId: User.UserId.make("owner"), + orgId: "org", + source: { type: "webMP4" }, + metadata: Option.some({ + editorSources: { + version: 1, + display: { key: displayKey, contentType: "video/webm" }, + camera: { + key: cameraKey, + contentType: "video/webm", + size: 100, + objectIdentity: "camera-etag", + offsetMs: 8, + }, + }, + }), + bucketId: Option.none(), + storageIntegrationId: Option.none(), + }, + ]), + ), + ); + mocks.startProcessing.mockResolvedValue("started"); + mocks.head.mockReturnValue( + Effect.succeed({ ETag: "display-etag", ContentLength: 99 }), + ); + const response = await app.request("/complete", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + videoId: "video", + uploadId: "display-session", + subpath: "raw-upload.webm", + durationInSecs: 5, + parts: [{ partNumber: 1, etag: "part-etag", size: 100 }], + }), + }); + expect(response.status).toBe(500); + expect(mocks.startProcessing).not.toHaveBeenCalled(); + }); + + it("aborts only the camera multipart session", async () => { + const response = await app.request("/abort", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + videoId: "video", + subpath: "camera-upload.webm", + uploadId: "camera-session", + }), + }); + expect(response.status).toBe(200); + expect(mocks.abort).toHaveBeenCalledWith(cameraKey, "camera-session"); + expect(uploadMutations).toEqual([]); + }); + + it("keeps an uncertain Drive camera mapping for retry", async () => { + mocks.storage.mockReturnValue( + Effect.succeed([ + { + provider: "googleDrive", + bucketName: "drive", + multipart: { abort: mocks.abort }, + }, + ]), + ); + const response = await app.request("/abort", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + videoId: "video", + subpath: "camera-upload.webm", + uploadId: "camera-session", + }), + }); + expect(response.status).toBe(200); + expect(mocks.abort).not.toHaveBeenCalled(); + expect(uploadMutations).toEqual([]); + }); +}); + describe("desktop reupload completion", () => { const oldKey = "owner/video/.recording/outputs/original/result.mp4"; let events: string[]; @@ -316,6 +602,9 @@ describe("desktop reupload completion", () => { mocks.invalidate.mockImplementation(async () => { events.push("invalidate"); }); + mocks.refreshPreviews.mockImplementation(async () => { + events.push("refresh-previews"); + }); mocks.shouldQueueTranscription.mockReturnValue(false); mocks.queueTranscription.mockResolvedValue({ success: true }); }); @@ -393,7 +682,9 @@ describe("desktop reupload completion", () => { "publish", "delete-upload", "invalidate", + "refresh-previews", ]); + expect(mocks.refreshPreviews).toHaveBeenCalledWith("video"); expect(mocks.complete).toHaveBeenCalledWith( outputKey, "backend-upload", @@ -420,6 +711,7 @@ describe("desktop reupload completion", () => { expect(response.status).toBe(200); expect(mocks.prepareReplacement).not.toHaveBeenCalled(); expect(mocks.invalidate).not.toHaveBeenCalled(); + expect(mocks.refreshPreviews).not.toHaveBeenCalled(); expect(updates[0]).not.toHaveProperty("source"); }, ); @@ -657,6 +949,7 @@ describe("raw recording completion integrity", () => { source: { type: "webMP4" }, bucketId: Option.none(), storageIntegrationId: Option.none(), + metadata: Option.none(), }, ]), ), diff --git a/apps/web/__tests__/unit/multipart-upload-utils.test.ts b/apps/web/__tests__/unit/multipart-upload-utils.test.ts index 5a1f8068155..62243ca4c58 100644 --- a/apps/web/__tests__/unit/multipart-upload-utils.test.ts +++ b/apps/web/__tests__/unit/multipart-upload-utils.test.ts @@ -1,7 +1,10 @@ import { describe, expect, it } from "vitest"; import { + getAudioRecorderUploadKind, getMultipartFileKey, getSubpath, + isCameraRecorderUpload, + isDisplayRecorderUpload, isRawRecorderUpload, } from "@/app/api/upload/[...route]/multipart-utils"; @@ -45,6 +48,42 @@ describe("multipart upload utils", () => { expect(isRawRecorderUpload("result.mp4")).toBe(false); }); + it("accepts only canonical paired recording media paths", () => { + for (const extension of ["webm", "mp4"]) { + expect(isDisplayRecorderUpload(`raw-upload.${extension}`)).toBe(true); + expect(isCameraRecorderUpload(`camera-upload.${extension}`)).toBe(true); + } + for (const subpath of [ + "camera-upload.webm/extra", + "camera-upload.webm.bak", + "raw-upload.webm/extra", + "../camera-upload.webm", + "result.mp4", + ]) { + expect(isDisplayRecorderUpload(subpath)).toBe(false); + expect(isCameraRecorderUpload(subpath)).toBe(false); + } + }); + + it("accepts only canonical microphone and system-audio sidecar paths", () => { + for (const extension of ["webm", "mp4"]) { + expect(getAudioRecorderUploadKind(`mic-upload.${extension}`)).toBe("mic"); + expect( + getAudioRecorderUploadKind(`system-audio-upload.${extension}`), + ).toBe("systemAudio"); + } + for (const subpath of [ + "../mic-upload.webm", + "mic-upload.webm/extra", + "mic-upload.webm.bak", + "system-audio-upload.wav", + "system-audio-upload.webm/extra", + "audio-upload.webm", + ]) { + expect(getAudioRecorderUploadKind(subpath)).toBeNull(); + } + }); + it("rejects missing video ids", () => { expect(() => getMultipartFileKey("user-123", { diff --git a/apps/web/__tests__/unit/playback-source.test.ts b/apps/web/__tests__/unit/playback-source.test.ts index a3c9ce976e0..e4d76371c69 100644 --- a/apps/web/__tests__/unit/playback-source.test.ts +++ b/apps/web/__tests__/unit/playback-source.test.ts @@ -514,3 +514,62 @@ describe("shouldFallbackToRawPlaybackSource", () => { ); }); }); + +describe("resolvePlaybackSource with a trusted initial URL", () => { + const initialUrl = + "https://bucket.s3.us-east-1.amazonaws.com/o/v/result.mp4?sig"; + + it("starts on it without a probe request", async () => { + const fetchImpl = vi.fn(); + const source = await resolvePlaybackSource({ + videoSrc: "/api/playlist?videoType=mp4", + initialUrl, + enableCrossOrigin: true, + trustInitialUrl: true, + fetchImpl, + }); + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(source).toEqual({ + url: initialUrl, + type: "mp4", + supportsCrossOrigin: true, + unprobed: true, + }); + }); + + it("still probes when the initial URL isn't trusted", async () => { + const fetchImpl = vi.fn(async () => + createResponse(initialUrl, { status: 206 }), + ); + const source = await resolvePlaybackSource({ + videoSrc: "/api/playlist?videoType=mp4", + initialUrl, + enableCrossOrigin: true, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(source?.unprobed).toBeUndefined(); + }); + + it("ignores trust once the player has moved to the raw upload", async () => { + const fetchImpl = vi.fn(async () => + createResponse("/api/playlist?videoType=raw-preview", { + status: 206, + headers: { "content-type": "video/mp4" }, + }), + ); + const source = await resolvePlaybackSource({ + videoSrc: "/api/playlist?videoType=mp4", + initialUrl, + rawFallbackSrc: "/api/playlist?videoType=raw-preview", + preferredSource: "raw", + trustInitialUrl: true, + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + expect(source?.type).toBe("raw"); + expect(source?.unprobed).toBeUndefined(); + }); +}); diff --git a/apps/web/__tests__/unit/published-preview-assets.test.ts b/apps/web/__tests__/unit/published-preview-assets.test.ts new file mode 100644 index 00000000000..48bb36bc8de --- /dev/null +++ b/apps/web/__tests__/unit/published-preview-assets.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@cap/web-backend/src/Storage/index", () => ({ Storage: {} })); +vi.mock("@/lib/workflow-runtime", () => ({ runWorkflowPromise: vi.fn() })); + +const { + getOutputPreviewAssetKeys, + getReplacementAwaitingPreviewAssets, + getReplacementOutputKey, +} = await import("@/lib/published-preview-assets"); +const { awaitsReplacementPreviewGif } = await import("@/lib/published-output"); + +const render = + "owner/video/.recording/render/11111111-1111-4111-8111-111111111111/result.mp4"; +const reupload = + "owner/video/.recording/outputs/reupload-22222222-2222-4222-8222-222222222222/result.mp4"; +const video = (source: Record) => ({ + id: "video", + ownerId: "owner", + source: { type: "webMP4", ...source }, +}); + +describe("published output preview assets", () => { + it("keeps each output's previews beside it", () => { + expect(getOutputPreviewAssetKeys(render)).toEqual({ + thumbnailKey: + "owner/video/.recording/render/11111111-1111-4111-8111-111111111111/result/screenshot.jpg", + previewKey: + "owner/video/.recording/render/11111111-1111-4111-8111-111111111111/result/preview.gif", + }); + }); + + it.each([render, reupload])("recognizes the replacement %s", (outputKey) => { + expect(getReplacementOutputKey(video({ outputKey }))).toBe(outputKey); + expect( + getReplacementOutputKey(video({ type: "desktopMP4", outputKey })), + ).toBe(outputKey); + }); + + it.each([ + undefined, + "owner/video/result.mp4", + "owner/video/.recording/outputs/generation/attempt.mp4", + "owner/video/.recording/sources/generation/snapshot/mp4/0.mp4", + "owner/video/.recording/outputs/edit-token/result.mp4", + "owner/video/.recording/render/11111111-1111-4111-8111-111111111111/export.mp4", + "owner/other/.recording/render/11111111-1111-4111-8111-111111111111/result.mp4", + ])("leaves outputs made with their own previews alone: %s", (outputKey) => { + expect( + getReplacementOutputKey(video(outputKey ? { outputKey } : {})), + ).toBeNull(); + }); + + it("ignores sources that are not single recordings", () => { + expect( + getReplacementOutputKey( + video({ type: "desktopSegments", outputKey: render }), + ), + ).toBeNull(); + }); + + it("awaits previews until both belong to the replacement", () => { + const keys = getOutputPreviewAssetKeys(render); + expect( + getReplacementAwaitingPreviewAssets(video({ outputKey: render })), + ).toBe(render); + expect( + getReplacementAwaitingPreviewAssets( + video({ outputKey: render, thumbnailKey: keys.thumbnailKey }), + ), + ).toBe(render); + expect( + getReplacementAwaitingPreviewAssets( + video({ outputKey: render, ...keys }), + ), + ).toBeNull(); + }); + + it("knows a replacement has no preview GIF until its own is made", () => { + expect(awaitsReplacementPreviewGif(video({ outputKey: render }))).toBe( + true, + ); + expect(awaitsReplacementPreviewGif(video({ outputKey: reupload }))).toBe( + true, + ); + expect( + awaitsReplacementPreviewGif( + video({ + outputKey: render, + previewKey: + "owner/video/.recording/render/11111111-1111-4111-8111-111111111111/result/preview.gif", + }), + ), + ).toBe(false); + expect(awaitsReplacementPreviewGif(video({}))).toBe(false); + expect( + awaitsReplacementPreviewGif( + video({ outputKey: "owner/video/.recording/outputs/gen/attempt.mp4" }), + ), + ).toBe(false); + }); +}); diff --git a/apps/web/__tests__/unit/recorder-camera-style.test.ts b/apps/web/__tests__/unit/recorder-camera-style.test.ts new file mode 100644 index 00000000000..7f373f16538 --- /dev/null +++ b/apps/web/__tests__/unit/recorder-camera-style.test.ts @@ -0,0 +1,85 @@ +import { + applyDefaultStyle, + parseRecorderCamera, + recordingDefaultStyle, + withRecorderCamera, +} from "@cap/editor-cap-bundle/default-style"; +import { describe, expect, it } from "vitest"; + +const layout = { + version: 1, + position: { x: "left", y: "top" }, + size: 40, + mirror: true, + shape: "round", +} as const; + +describe("recorder camera layout", () => { + it("rejects layouts the editor can't place", () => { + expect(parseRecorderCamera({ ...layout, size: 90 })).toBeNull(); + expect( + parseRecorderCamera({ ...layout, position: { x: "middle", y: "top" } }), + ).toBeNull(); + expect(parseRecorderCamera({ ...layout, shape: "hexagon" })).toBeNull(); + expect(parseRecorderCamera(layout)).toEqual(layout); + }); + + it("places the camera over a saved style without dropping its look", () => { + const style = withRecorderCamera( + { + version: 1, + camera: { shadow: 10, position: { x: "right", y: "bottom" } }, + }, + layout, + ); + expect(style?.camera).toMatchObject({ + shadow: 10, + position: { x: "left", y: "top" }, + manualPosition: null, + size: 40, + mirror: true, + shape: "square", + rounding: 100, + }); + }); + + it("maps full frame to the camera's own aspect", () => { + const style = withRecorderCamera(null, { ...layout, shape: "full" }); + expect(style?.camera).toMatchObject({ shape: "source", rounding: 25 }); + }); + + it("leaves the style alone without a layout", () => { + const saved = { version: 1 as const, cursor: { size: 2 } }; + expect(withRecorderCamera(saved, null)).toBe(saved); + }); + + it("lands in the project the editor opens with", () => { + const project = applyDefaultStyle( + { + camera: { + hide: false, + size: 30, + position: { x: "right", y: "bottom" }, + }, + }, + withRecorderCamera(null, layout) ?? { version: 1 }, + ); + expect(project.camera).toMatchObject({ + hide: false, + size: 40, + mirror: true, + position: { x: "left", y: "top" }, + }); + }); + + it("opens a new recording edge to edge until its owner saves a style", () => { + expect(recordingDefaultStyle(null, layout)).toMatchObject({ + background: { padding: 0, rounding: 0, shadow: 0 }, + camera: { position: { x: "left", y: "top" }, size: 40 }, + }); + expect( + recordingDefaultStyle({ version: 1, background: { padding: 12 } }, null) + ?.background, + ).toEqual({ padding: 12 }); + }); +}); diff --git a/apps/web/__tests__/unit/recording-output.test.ts b/apps/web/__tests__/unit/recording-output.test.ts index 0620e360274..703acd4dddb 100644 --- a/apps/web/__tests__/unit/recording-output.test.ts +++ b/apps/web/__tests__/unit/recording-output.test.ts @@ -107,8 +107,34 @@ describe("published recording storage keys", () => { } }); + it("resolves previews made beside a render that replaced the upload", () => { + const render = `${prefix}.recording/render/export/result`; + const rendered = { + ...video, + source: { + type: "webMP4", + outputKey: `${render}.mp4`, + thumbnailKey: `${render}/screenshot.jpg`, + previewKey: `${render}/preview.gif`, + }, + }; + expect( + resolveRecordingObjectKey( + rendered, + `${prefix}screenshot/screen-capture.jpg`, + ), + ).toBe(`${render}/screenshot.jpg`); + expect( + resolveRecordingObjectKey( + rendered, + `${prefix}preview/animated-preview.gif`, + ), + ).toBe(`${render}/preview.gif`); + }); + it.each([ "other/video/.recording/outputs/attempt/screenshot.jpg", + "other/video/.recording/render/export/result/screenshot.jpg", `${prefix}.recording/sources/snapshot/video/0.mp4`, `${prefix}.recording/outputs/../screenshot.jpg`, ])("does not alias an invalid thumbnail pointer %s", (key) => { diff --git a/apps/web/__tests__/unit/recording-tail-duration.test.ts b/apps/web/__tests__/unit/recording-tail-duration.test.ts new file mode 100644 index 00000000000..58fe9a04705 --- /dev/null +++ b/apps/web/__tests__/unit/recording-tail-duration.test.ts @@ -0,0 +1,169 @@ +import { describe, expect, it } from "vitest"; +import { + fragmentedMp4End, + fragmentedMp4Tracks, +} from "../../lib/fragmented-mp4-duration"; +import { webmEnd, webmTimecodeScale } from "../../lib/webm-audio-duration"; + +const u32 = (value: number) => { + const bytes = new Uint8Array(4); + new DataView(bytes.buffer).setUint32(0, value); + return [...bytes]; +}; + +const box = (type: string, ...body: number[][]) => { + const content = body.flat(); + return [ + ...u32(8 + content.length), + ...[...type].map((c) => c.charCodeAt(0)), + ...content, + ]; +}; + +const fullBox = ( + type: string, + version: number, + flags: number, + ...body: number[][] +) => + box( + type, + [version, (flags >> 16) & 0xff, (flags >> 8) & 0xff, flags & 0xff], + ...body, + ); + +const trak = (trackId: number, timescale: number) => + box( + "trak", + fullBox("tkhd", 0, 3, u32(0), u32(0), u32(trackId), new Array(68).fill(0)), + box("mdia", fullBox("mdhd", 0, 0, u32(0), u32(0), u32(timescale), u32(0))), + ); + +const moov = (...traks: number[][]) => + box( + "moov", + fullBox("mvhd", 0, 0, new Array(96).fill(0)), + ...traks, + box( + "mvex", + fullBox("trex", 0, 0, u32(1), u32(1), u32(0), u32(0), u32(0)), + fullBox("trex", 0, 0, u32(2), u32(1), u32(1024), u32(0), u32(0)), + ), + ); + +// A fragment whose track 1 samples carry explicit durations and whose +// track 2 samples use the trex default. +const fragment = (videoStart: number, audioStart: number) => [ + ...box( + "moof", + fullBox("mfhd", 0, 0, u32(7)), + box( + "traf", + fullBox("tfhd", 0, 0x20000, u32(1)), + fullBox("tfdt", 0, 0, u32(videoStart)), + fullBox("trun", 0, 0x100, u32(3), u32(1000), u32(1000), u32(1000)), + ), + box( + "traf", + fullBox("tfhd", 0, 0x20000, u32(2)), + fullBox("tfdt", 0, 0, u32(audioStart)), + fullBox("trun", 0, 0, u32(4)), + ), + ), + ...box("mdat", new Array(64).fill(0x6d)), +]; + +describe("fragmented MP4 tail duration", () => { + const head = new Uint8Array([ + ...box("ftyp", [0x69, 0x73, 0x6f, 0x6d]), + ...moov(trak(1, 30000), trak(2, 48000)), + ]); + + it("reads the end of the last fragment across tracks", () => { + const tracks = fragmentedMp4Tracks(head); + expect(tracks).not.toBeNull(); + const tail = new Uint8Array([ + ...new Array(100).fill(0), + ...fragment(0, 0), + ...fragment(597000, 954_000), + ]); + // video: (597000 + 3 * 1000) / 30000 = 20 s; audio: (954000 + 4 * 1024) / 48000 + expect(fragmentedMp4End(tail, tracks ?? new Map())).toBeCloseTo(20, 6); + }); + + it("gives up when the tail ends inside a fragment", () => { + const tracks = fragmentedMp4Tracks(head) ?? new Map(); + const tail = new Uint8Array(fragment(0, 0).slice(0, -8)); + expect(fragmentedMp4End(tail, tracks)).toBeNull(); + }); + + it("ignores files that are not fragmented", () => { + const plain = new Uint8Array( + box("moov", fullBox("mvhd", 0, 0, new Array(96).fill(0)), trak(1, 600)), + ); + expect(fragmentedMp4Tracks(plain)).toBeNull(); + }); +}); + +const ebml = (id: number[], body: number[]) => { + if (body.length > 0x7f) throw new Error("fixture element too large"); + return [...id, 0x80 | body.length, ...body]; +}; +const unknownSize = [0x01, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff]; +const block = (relative: number) => + ebml( + [0xa3], + [0x81, (relative >> 8) & 0xff, relative & 0xff, 0x80, 0xfc, 0xff], + ); + +describe("WebM audio tail duration", () => { + const head = new Uint8Array([ + ...ebml( + [0x1a, 0x45, 0xdf, 0xa3], + ebml([0x42, 0x82], [0x77, 0x65, 0x62, 0x6d]), + ), + 0x18, + 0x53, + 0x80, + 0x67, + ...unknownSize, + ...ebml( + [0x15, 0x49, 0xa9, 0x66], + ebml([0x2a, 0xd7, 0xb1], [0x0f, 0x42, 0x40]), + ), + ]); + + it("reads the timecode scale", () => { + expect(webmTimecodeScale(head)).toBe(1_000_000); + }); + + it("returns the last block time of the final cluster", () => { + const cluster = [ + 0x1f, + 0x43, + 0xb6, + 0x75, + ...unknownSize, + ...ebml([0xe7], [0x1c, 0x20]), + ...block(0), + ...block(20), + ...block(980), + ]; + const tail = new Uint8Array([...new Array(40).fill(0), ...cluster]); + // 7200 ms cluster + 980 ms + expect(webmEnd(tail, 1_000_000)).toBeCloseTo(8.18, 6); + }); + + it("gives up on a truncated cluster", () => { + const tail = new Uint8Array([ + 0x1f, + 0x43, + 0xb6, + 0x75, + 0x90, + ...ebml([0xe7], [0x10]), + ...block(5), + ]); + expect(webmEnd(tail, 1_000_000)).toBeNull(); + }); +}); diff --git a/apps/web/__tests__/unit/refresh-preview-assets.test.ts b/apps/web/__tests__/unit/refresh-preview-assets.test.ts new file mode 100644 index 00000000000..39029fbc98d --- /dev/null +++ b/apps/web/__tests__/unit/refresh-preview-assets.test.ts @@ -0,0 +1,259 @@ +import type { Video } from "@cap/web-domain"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { FatalError, RetryableError } from "workflow"; + +const mocks = vi.hoisted(() => ({ + video: null as null | { + id: string; + ownerId: string; + source: Record; + }, + objects: new Map(), + updates: [] as unknown[], + start: vi.fn(), + fetchPreviewAssets: vi.fn(), + mediaServerConfigured: vi.fn(() => true), +})); + +const effect = (value: T) => ({ + pipe: (run: (value: T) => unknown) => run(value), +}); + +vi.mock("@cap/database", () => { + const select = () => ({ + from: () => ({ + where: () => + Object.assign(Promise.resolve([mocks.video]), { + for: async () => [mocks.video], + }), + }), + }); + return { + db: () => ({ + select, + transaction: async (run: (tx: unknown) => Promise) => + run({ + select, + update: () => ({ + set: (values: { source: Record }) => ({ + where: async () => { + mocks.updates.push(values); + if (mocks.video) mocks.video.source = values.source; + }, + }), + }), + }), + }), + }; +}); +vi.mock("@cap/database/schema", () => ({ + videos: { id: "id", ownerId: "ownerId", source: "source" }, +})); +vi.mock("drizzle-orm", () => ({ eq: (...args: unknown[]) => args })); +vi.mock("@cap/web-backend/src/Storage/index", () => ({ + Storage: { + getAccessForVideo: () => + effect([ + { + getInternalSignedObjectUrl: (key: string) => + effect(`https://storage.test/get/${key}`), + getInternalPresignedPutUrl: (key: string) => + effect(`https://storage.test/put/${key}`), + headObject: async (key: string) => { + const head = mocks.objects.get(key); + if (!head) throw new Error("Object missing"); + return head; + }, + }, + ]), + }, +})); +vi.mock("@/lib/video-storage", () => ({ + decodeStorageVideo: (video: unknown) => video, +})); +vi.mock("@/lib/workflow-runtime", () => ({ + runWorkflowPromise: async (value: unknown) => + value && typeof value === "object" && "pipe" in value + ? (value as ReturnType).pipe((inner) => inner) + : value, +})); +vi.mock("@/lib/media-client", () => ({ + fetchPreviewAssetsViaMediaServer: mocks.fetchPreviewAssets, + isMediaServerConfigured: mocks.mediaServerConfigured, +})); +vi.mock("workflow/api", () => ({ start: mocks.start })); + +const { enqueuePreviewAssetsRefresh } = await import( + "@/lib/refresh-preview-assets" +); +const { refreshPreviewAssetsWorkflow } = await import( + "@/workflows/refresh-preview-assets" +); + +const outputKey = + "owner/video/.recording/render/11111111-1111-4111-8111-111111111111/result.mp4"; +const thumbnailKey = + "owner/video/.recording/render/11111111-1111-4111-8111-111111111111/result/screenshot.jpg"; +const previewKey = + "owner/video/.recording/render/11111111-1111-4111-8111-111111111111/result/preview.gif"; +const videoId = "video" as Video.VideoId; + +function storeGeneratedAssets() { + mocks.fetchPreviewAssets.mockImplementation(async () => { + mocks.objects.set(thumbnailKey, { ContentLength: 10 }); + mocks.objects.set(previewKey, { ContentLength: 20 }); + return new Response(JSON.stringify({ success: true })); + }); +} + +beforeEach(() => { + vi.clearAllMocks(); + mocks.video = { + id: "video", + ownerId: "owner", + source: { type: "webMP4", outputKey }, + }; + mocks.objects.clear(); + mocks.updates = []; + mocks.mediaServerConfigured.mockReturnValue(true); +}); + +describe("preview asset refresh queue", () => { + it("starts a refresh for a render shown with the upload's previews", async () => { + await enqueuePreviewAssetsRefresh(videoId); + + expect(mocks.start).toHaveBeenCalledWith(refreshPreviewAssetsWorkflow, [ + { videoId: "video", outputKey }, + ]); + }); + + it("skips videos that already have previews of their output", async () => { + mocks.video = { + id: "video", + ownerId: "owner", + source: { type: "webMP4", outputKey, thumbnailKey, previewKey }, + }; + await enqueuePreviewAssetsRefresh(videoId); + + mocks.video = { + id: "video", + ownerId: "owner", + source: { + type: "desktopMP4", + outputKey: "owner/video/.recording/outputs/generation/attempt.mp4", + }, + }; + await enqueuePreviewAssetsRefresh(videoId); + + expect(mocks.start).not.toHaveBeenCalled(); + }); + + it("does nothing without a media server and never fails the publish", async () => { + mocks.mediaServerConfigured.mockReturnValue(false); + await enqueuePreviewAssetsRefresh(videoId); + expect(mocks.start).not.toHaveBeenCalled(); + + mocks.mediaServerConfigured.mockReturnValue(true); + mocks.start.mockRejectedValueOnce(new Error("workflow unavailable")); + const log = vi.spyOn(console, "error").mockImplementation(() => {}); + await expect(enqueuePreviewAssetsRefresh(videoId)).resolves.toBeUndefined(); + expect(log).toHaveBeenCalled(); + log.mockRestore(); + }); +}); + +describe("preview asset refresh workflow", () => { + it("makes both assets from the published output and points the video at them", async () => { + storeGeneratedAssets(); + + await refreshPreviewAssetsWorkflow({ videoId: "video", outputKey }); + + expect(mocks.fetchPreviewAssets).toHaveBeenCalledWith({ + videoUrl: `https://storage.test/get/${outputKey}`, + thumbnailPresignedUrl: `https://storage.test/put/${thumbnailKey}`, + previewGifPresignedUrl: `https://storage.test/put/${previewKey}`, + }); + expect(mocks.video?.source).toEqual({ + type: "webMP4", + outputKey, + thumbnailKey, + previewKey, + }); + }); + + it("skips an output that was replaced before its previews were made", async () => { + mocks.video = { + id: "video", + ownerId: "owner", + source: { + type: "webMP4", + outputKey: + "owner/video/.recording/outputs/reupload-22222222-2222-4222-8222-222222222222/result.mp4", + }, + }; + + await refreshPreviewAssetsWorkflow({ videoId: "video", outputKey }); + + expect(mocks.fetchPreviewAssets).not.toHaveBeenCalled(); + expect(mocks.updates).toEqual([]); + }); + + it("does not attach previews to an output replaced while they were made", async () => { + const next = + "owner/video/.recording/outputs/reupload-22222222-2222-4222-8222-222222222222/result.mp4"; + mocks.fetchPreviewAssets.mockImplementation(async () => { + mocks.objects.set(thumbnailKey, { ContentLength: 10 }); + mocks.objects.set(previewKey, { ContentLength: 20 }); + if (mocks.video) mocks.video.source = { type: "webMP4", outputKey: next }; + return new Response("{}"); + }); + + await refreshPreviewAssetsWorkflow({ videoId: "video", outputKey }); + + expect(mocks.updates).toEqual([]); + expect(mocks.video?.source).toEqual({ type: "webMP4", outputKey: next }); + }); + + it("retries a busy media server and one that predates the route", async () => { + mocks.fetchPreviewAssets.mockResolvedValueOnce( + new Response("busy", { status: 503 }), + ); + await expect( + refreshPreviewAssetsWorkflow({ videoId: "video", outputKey }), + ).rejects.toBeInstanceOf(RetryableError); + + mocks.fetchPreviewAssets.mockResolvedValueOnce( + new Response("Not Found", { status: 404 }), + ); + await expect( + refreshPreviewAssetsWorkflow({ videoId: "video", outputKey }), + ).rejects.toBeInstanceOf(RetryableError); + + mocks.fetchPreviewAssets.mockResolvedValueOnce( + new Response("bad request", { status: 400 }), + ); + await expect( + refreshPreviewAssetsWorkflow({ videoId: "video", outputKey }), + ).rejects.toBeInstanceOf(FatalError); + + mocks.fetchPreviewAssets.mockResolvedValueOnce( + new Response("failed", { status: 500 }), + ); + const failure = refreshPreviewAssetsWorkflow({ + videoId: "video", + outputKey, + }); + await expect(failure).rejects.toThrow("Preview assets failed (500)"); + await expect(failure).rejects.toBeInstanceOf(FatalError); + expect(mocks.updates).toEqual([]); + }); + + it("does not point the video at assets that were never stored", async () => { + mocks.fetchPreviewAssets.mockResolvedValue(new Response("{}")); + + await expect( + refreshPreviewAssetsWorkflow({ videoId: "video", outputKey }), + ).rejects.toThrow("Preview assets were not stored"); + expect(mocks.updates).toEqual([]); + }); +}); diff --git a/apps/web/__tests__/unit/render-farm-browser-save.test.ts b/apps/web/__tests__/unit/render-farm-browser-save.test.ts new file mode 100644 index 00000000000..7c44cf3970e --- /dev/null +++ b/apps/web/__tests__/unit/render-farm-browser-save.test.ts @@ -0,0 +1,25 @@ +import { expect, test } from "vitest"; +import { recentBrowserSave } from "@/lib/render-farm-status"; + +const browserSave = (updatedAt: string) => ({ updatedAt, progress: 0.4 }); + +test("a save the tab is still reporting, or just finished, is recent", () => { + const now = Date.parse("2026-09-28T12:00:20.000Z"); + expect( + recentBrowserSave( + { browserSave: browserSave("2026-09-28T12:00:10.000Z") }, + now, + )?.progress, + ).toBe(0.4); +}); + +test("a report that has gone quiet is not", () => { + const now = Date.parse("2026-09-28T12:01:00.000Z"); + expect( + recentBrowserSave( + { browserSave: browserSave("2026-09-28T12:00:10.000Z") }, + now, + ), + ).toBeNull(); + expect(recentBrowserSave(null, now)).toBeNull(); +}); diff --git a/apps/web/__tests__/unit/render-farm-direct-plan.test.ts b/apps/web/__tests__/unit/render-farm-direct-plan.test.ts new file mode 100644 index 00000000000..085a60e92c3 --- /dev/null +++ b/apps/web/__tests__/unit/render-farm-direct-plan.test.ts @@ -0,0 +1,441 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@cap/env", () => ({ serverEnv: () => ({}) })); + +import { + DirectRenderForbidden, + type DirectRenderInput, + directRenderLikely, + parseRenderFarmPrepareSupport, + planDirectRenderProject, +} from "@/lib/render-farm-direct-plan"; +import { buildRenderProject } from "@/lib/render-farm-project"; + +const OWNER = "owner123456789a"; +const VIDEO = "video1234567890"; +const root = `${OWNER}/${VIDEO}/`; +const target = { root, recording: `${root}.recording/render/x/project` }; +const defaultConfig = { + audio: { mute: false, improve: false, isolation: "balanced" }, + background: { source: { type: "color", value: [255, 255, 255] } }, + timeline: null, + clips: [], +}; +const support = { version: 1 as const, defaultConfig, music: ["calm-1"] }; +const key = (name: string) => `${root}${name}`; + +function input(overrides: Partial = {}): DirectRenderInput { + return { + sources: { + title: "Take", + display: { contentType: "video/mp4", fps: 30 }, + camera: { contentType: "video/webm", fps: 20, offsetMs: 4 }, + mic: { contentType: "audio/webm", offsetMs: 2 }, + }, + editorSources: { + version: 1, + display: { + key: key("raw-upload.mp4"), + contentType: "video/mp4", + size: 10, + fps: 30, + }, + camera: { + key: key("camera-upload.webm"), + contentType: "video/webm", + size: 20, + fps: 20, + objectIdentity: null, + offsetMs: 4, + }, + mic: { + key: key("mic-upload.webm"), + contentType: "audio/webm", + size: 30, + objectIdentity: null, + offsetMs: 2, + }, + }, + heads: new Map([ + [key("raw-upload.mp4"), { size: 10, etag: '"d"' }], + [key("camera-upload.webm"), { size: 20, etag: '"c"' }], + [key("mic-upload.webm"), { size: 30, etag: '"m"' }], + ]), + savedAssets: [], + ownerId: OWNER, + videoId: VIDEO, + hasSavedProject: false, + defaultStyle: null, + captionsEnabled: true, + target, + support, + mixedAudioInDisplay: false, + ...overrides, + }; +} + +function plan(overrides: Partial = {}) { + const result = planDirectRenderProject(input(overrides)); + if ("unsupported" in result) throw new Error(result.unsupported); + return result; +} + +describe("planDirectRenderProject", () => { + it("names every file where a worker stages it and the farm transcodes it", () => { + const { prepare, entries } = plan(); + expect(prepare.display).toEqual({ + key: key("raw-upload.mp4"), + contentType: "video/mp4", + }); + expect(prepare.sessionDefaults).toBe(true); + expect(prepare.sources).toMatchObject({ + title: "Take", + display: { + path: "content/segments/segment-0/display.h264.mp4", + fps: 30, + offsetMs: 0, + }, + camera: { + path: "content/segments/segment-0/camera.h264.mp4", + fps: 20, + offsetMs: 4, + }, + mic: { path: "content/segments/segment-0/mic.mic.opus", offsetMs: 2 }, + systemAudio: null, + }); + expect(entries.map((entry) => entry.path).sort()).toEqual([ + "content/segments/segment-0/camera.h264.mp4", + "content/segments/segment-0/display.h264.mp4", + "content/segments/segment-0/mic.mic.opus", + ]); + expect( + entries.find((entry) => entry.path.endsWith("display.h264.mp4")), + ).toMatchObject({ transcodeFrom: key("raw-upload.mp4") }); + }); + + it("builds the same manifest as a worker-prepared project", () => { + const direct = plan(); + const worker = buildRenderProject({ + ...target, + files: [ + { + path: "content/segments/segment-0/display.mp4", + size: 10, + inode: "1", + }, + { + path: "content/segments/segment-0/camera.webm", + size: 20, + inode: "2", + }, + { path: "content/segments/segment-0/mic.webm", size: 30, inode: "3" }, + { path: "recording-meta.json", size: 1, inode: "4" }, + { path: "project-config.json", size: 1, inode: "5" }, + ], + recordingMeta: { + segments: [ + { + display: { + path: "content/segments/segment-0/display.mp4", + fps: 30, + }, + camera: { path: "content/segments/segment-0/camera.webm", fps: 20 }, + mic: { path: "content/segments/segment-0/mic.webm" }, + }, + ], + }, + config: defaultConfig, + sources: new Map([ + [ + "content/segments/segment-0/display.mp4", + { key: key("raw-upload.mp4"), size: 10, identity: '"d"' }, + ], + [ + "content/segments/segment-0/camera.webm", + { key: key("camera-upload.webm"), size: 20, identity: '"c"' }, + ], + [ + "content/segments/segment-0/mic.webm", + { key: key("mic-upload.webm"), size: 30, identity: '"m"' }, + ], + ]), + }); + expect(worker.uploads).toEqual([]); + const byPath = (a: { path: string }, b: { path: string }) => + a.path.localeCompare(b.path); + expect([...direct.entries].sort(byPath)).toEqual( + [...worker.entries].sort(byPath), + ); + }); + + it("opens a never-edited recording with the Studio Sound preference and owner style", () => { + const { config } = plan({ + sources: { + ...input().sources, + audioDefault: { enabledByDefault: true, isolation: "strong" }, + }, + defaultStyle: { + version: 1, + background: { padding: 0, rounding: 0 }, + }, + }); + expect(config.audio).toEqual({ + mute: false, + improve: true, + isolation: "strong", + }); + expect(config.background).toMatchObject({ padding: 0, rounding: 0 }); + expect(defaultConfig.audio.improve).toBe(false); + }); + + it("keeps a saved project as saved, without the owner style", () => { + const saved = { ...defaultConfig, aspectRatio: "wide" }; + const { config } = plan({ + sources: { ...input().sources, projectConfig: saved }, + hasSavedProject: true, + defaultStyle: { version: 1, aspectRatio: "square" }, + }); + expect(config.aspectRatio).toBe("wide"); + expect( + planDirectRenderProject( + input({ + sources: { ...input().sources, projectConfig: saved }, + hasSavedProject: true, + }), + ), + ).toMatchObject({ prepare: { sessionDefaults: false } }); + }); + + it("plays mixed display audio from the display, as a worker does", () => { + const { prepare, entries } = plan({ + sources: { + title: "Take", + display: { contentType: "video/webm", fps: 30 }, + }, + editorSources: { + version: 1, + display: { + key: key("raw-upload.webm"), + contentType: "video/webm", + size: 10, + }, + }, + heads: new Map([[key("raw-upload.webm"), { size: 10, etag: '"d"' }]]), + mixedAudioInDisplay: true, + }); + expect(prepare.sources.systemAudio).toEqual({ + path: "content/segments/segment-0/display.system.opus", + offsetMs: 0, + }); + expect( + entries.find((entry) => entry.path.endsWith("display.system.opus")), + ).toEqual({ + path: "content/segments/segment-0/display.system.opus", + key: key("raw-upload.webm"), + size: 10, + }); + }); + + it("reads MP4 system audio in place as AAC", () => { + const { prepare } = plan({ + sources: { + title: "Take", + display: { contentType: "video/mp4", fps: 30 }, + systemAudio: { contentType: "audio/mp4", offsetMs: -3 }, + }, + editorSources: { + version: 1, + display: { + key: key("raw-upload.mp4"), + contentType: "video/mp4", + size: 10, + }, + systemAudio: { + key: key("system-audio-upload.mp4"), + contentType: "audio/mp4", + size: 5, + objectIdentity: null, + offsetMs: -3, + }, + }, + heads: new Map([ + [key("raw-upload.mp4"), { size: 10, etag: '"d"' }], + [key("system-audio-upload.mp4"), { size: 5, etag: '"s"' }], + ]), + }); + expect(prepare.sources.systemAudio).toEqual({ + path: "content/segments/segment-0/system-audio.system.m4a", + offsetMs: -3, + }); + }); + + it("hands the farm the recording's pointer input", () => { + const { prepare } = plan({ + sources: { ...input().sources, inputEvents: { url: "https://x" } }, + editorSources: { + ...input().editorSources, + inputEvents: { + key: key("input-events-upload.ndjson"), + contentType: "application/x-ndjson", + size: 99, + objectIdentity: null, + }, + }, + }); + expect(prepare.inputEvents).toEqual({ + key: key("input-events-upload.ndjson"), + size: 99, + }); + }); + + it("reads saved assets in place and ships wallpapers and library music from the farm", () => { + const image = { + path: "content/images/1b0e7a6a-5f52-4b2a-9d3e-2f0a1c0d7e11.png", + key: key("editor-assets/images/1b0e7a6a-5f52-4b2a-9d3e-2f0a1c0d7e11.png"), + size: 7, + objectIdentity: '"i"', + }; + const { entries, config } = plan({ + sources: { + ...input().sources, + projectConfig: { + ...defaultConfig, + background: { + source: { + type: "wallpaper", + path: "cap-web-wallpaper://assets/backgrounds/macOS/sequoia-dark.jpg", + }, + }, + timeline: { + audioSegments: [{ path: "assets/audio/library-calm-1.mp3" }], + }, + }, + imageAssets: [{ path: image.path, size: image.size }], + }, + hasSavedProject: true, + savedAssets: [image], + }); + expect(entries).toContainEqual({ + path: image.path, + key: image.key, + size: image.size, + }); + expect(entries).toContainEqual({ + path: "assets/backgrounds/macOS/sequoia-dark.jpg", + builtin: "backgrounds/macOS/sequoia-dark.jpg", + }); + expect(entries).toContainEqual({ + path: "assets/audio/library-calm-1.mp3", + builtin: "music/calm-1.mp3", + }); + + expect(config.background).toEqual({ + source: { + type: "wallpaper", + path: "$RF_PROJECT/assets/backgrounds/macOS/sequoia-dark.jpg", + }, + }); + }); + + it("leaves a project using a library track the farm lacks to the worker", () => { + expect( + planDirectRenderProject( + input({ + sources: { + ...input().sources, + projectConfig: { + ...defaultConfig, + timeline: { + audioSegments: [ + { path: "assets/audio/library-calm-1.mp3" }, + { path: "assets/audio/library-gone.mp3" }, + ], + }, + }, + }, + hasSavedProject: true, + }), + ), + ).toEqual({ unsupported: "A library track isn't on the render farm" }); + }); + + it("refuses captions the plan doesn't include", () => { + expect(() => + planDirectRenderProject( + input({ + sources: { + ...input().sources, + projectConfig: { + ...defaultConfig, + captions: { segments: [{ text: "hi" }] }, + }, + }, + hasSavedProject: true, + captionsEnabled: false, + }), + ), + ).toThrow(DirectRenderForbidden); + }); + + it("leaves recordings only a worker can prepare to the worker", () => { + const unsupported = (overrides: Partial) => + planDirectRenderProject( + input({ sources: { ...input().sources, ...overrides } }), + ); + expect(unsupported({ legacyEditSpec: { version: 1 } })).toEqual({ + unsupported: "The recording has a legacy edit", + }); + expect(unsupported({ clips: [] })).toEqual({ + unsupported: "The project has imported clips", + }); + expect(unsupported({ imports: [] })).toEqual({ + unsupported: "The project has imported clips", + }); + expect(unsupported({ videoAssets: [] })).toEqual({ + unsupported: "The project has imported clips", + }); + expect( + unsupported({ display: { contentType: "video/mp4", fps: 29.97 } }), + ).toEqual({ unsupported: "A source frame rate needs probing" }); + expect( + unsupported({ + camera: { contentType: "video/mp4", fps: undefined, offsetMs: 0 }, + }), + ).toEqual({ unsupported: "A source frame rate needs probing" }); + }); +}); + +describe("directRenderLikely", () => { + const editorSources = input().editorSources; + it("expects a farm-prepared render from stored metadata alone", () => { + expect(directRenderLikely({ editorSources } as never)).toBe(true); + expect(directRenderLikely(null)).toBe(false); + expect( + directRenderLikely({ + editorSources: { + ...editorSources, + display: { ...editorSources.display, fps: undefined }, + }, + } as never), + ).toBe(false); + expect( + directRenderLikely({ + editorSources, + webEditorClips: { version: 1, items: [{}] }, + } as never), + ).toBe(false); + }); +}); + +describe("parseRenderFarmPrepareSupport", () => { + it("accepts only a complete version 1 capability", () => { + expect(parseRenderFarmPrepareSupport(support)).toEqual(support); + expect(parseRenderFarmPrepareSupport(undefined)).toBeNull(); + expect( + parseRenderFarmPrepareSupport({ ...support, version: 2 }), + ).toBeNull(); + expect( + parseRenderFarmPrepareSupport({ ...support, music: [1] }), + ).toBeNull(); + }); +}); diff --git a/apps/web/__tests__/unit/render-farm-exports.test.ts b/apps/web/__tests__/unit/render-farm-exports.test.ts new file mode 100644 index 00000000000..a2f06c65c0c --- /dev/null +++ b/apps/web/__tests__/unit/render-farm-exports.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, it } from "vitest"; +import { + parseRenderFarmReference, + renderFarmKeys, + renderFarmReference, +} from "@/lib/render-farm"; +import { + attachmentDisposition, + awaitingPendingRenderJob, + pickRenderExport, + RENDER_EXPORT_TTL_MS, + renderExportFileName, + renderExportView, + upsertRenderFarmExport, +} from "@/lib/render-farm-status"; + +const item = { + exportId: "export-1", + jobId: "job-1", + status: "ready" as const, + startedAt: "2026-09-26T00:00:00.000Z", + completedAt: "2026-09-26T00:05:00.000Z", + outputKey: "owner/video/.recording/render/export-1/export.mp4", + fileName: "Demo.mp4", + resolution: [1920, 1080] as [number, number], + fps: 30, + bytes: 1234, +}; + +describe("render farm references", () => { + it("keeps saves on the bare video id and tags other jobs", () => { + expect(renderFarmReference("save", "abc123")).toBe("abc123"); + expect(renderFarmReference("export", "abc123")).toBe("export:abc123"); + expect(parseRenderFarmReference("abc123")).toEqual({ + kind: "save", + videoId: "abc123", + }); + expect(parseRenderFarmReference("recording:abc123")).toEqual({ + kind: "recording", + videoId: "abc123", + }); + expect(parseRenderFarmReference("export:abc123")).toEqual({ + kind: "export", + videoId: "abc123", + }); + }); + + it("rejects unknown kinds and malformed ids", () => { + expect(parseRenderFarmReference("thumbnail:abc123")).toBeNull(); + expect(parseRenderFarmReference("export:")).toBeNull(); + expect(parseRenderFarmReference("export:../other")).toBeNull(); + expect(parseRenderFarmReference("")).toBeNull(); + }); + + it("writes exports beside, not over, the share render", () => { + expect(renderFarmKeys("owner", "video", "e1").outputKey).toBe( + "owner/video/.recording/render/e1/result.mp4", + ); + expect(renderFarmKeys("owner", "video", "e1", "export").outputKey).toBe( + "owner/video/.recording/render/e1/export.mp4", + ); + }); +}); + +describe("background export status", () => { + const completed = Date.parse(item.completedAt); + + it("maps stored exports to what the download page shows", () => { + expect(renderExportView(item, completed + 1000)).toMatchObject({ + state: "ready", + bytes: 1234, + error: null, + }); + expect( + renderExportView( + { ...item, status: "rendering", completedAt: undefined }, + completed, + ).state, + ).toBe("rendering"); + expect( + renderExportView({ ...item, status: "error" }, completed), + ).toMatchObject({ state: "error", error: "Export failed" }); + }); + + it("expires ready exports after the retention window", () => { + expect( + renderExportView(item, completed + RENDER_EXPORT_TTL_MS + 1).state, + ).toBe("expired"); + expect( + renderExportView({ ...item, completedAt: undefined }, completed).state, + ).toBe("expired"); + }); + + it("keeps the newest ten exports and replaces by id", () => { + const many = Array.from({ length: 12 }, (_, index) => ({ + ...item, + exportId: `export-${index}`, + })); + const next = upsertRenderFarmExport(many, { + ...item, + exportId: "export-5", + status: "error", + }); + expect(next).toHaveLength(10); + expect(next[0]).toMatchObject({ exportId: "export-5", status: "error" }); + expect( + next.filter((candidate) => candidate.exportId === "export-5"), + ).toHaveLength(1); + }); + + it("picks the named export, or the newest", () => { + const views = [ + renderExportView({ ...item, exportId: "b" }, completed), + renderExportView({ ...item, exportId: "a" }, completed), + ]; + expect(pickRenderExport(views, "a")?.exportId).toBe("a"); + expect(pickRenderExport(views, null)?.exportId).toBe("b"); + expect(pickRenderExport(views, "missing")).toBeNull(); + expect(pickRenderExport([], null)).toBeNull(); + }); + + it("waits for a recording render's job only for a while", () => { + const startedAt = "2026-09-26T00:00:00.000Z"; + expect( + awaitingPendingRenderJob({ startedAt }, Date.parse(startedAt) + 60_000), + ).toBe(true); + expect( + awaitingPendingRenderJob( + { startedAt }, + Date.parse(startedAt) + 31 * 60_000, + ), + ).toBe(false); + }); +}); + +describe("export downloads", () => { + it("names the file after the video, safely", () => { + expect(renderExportFileName("Q3 Roadmap: Review")).toBe( + "Q3 Roadmap Review.mp4", + ); + expect(renderExportFileName("Café déjà vu")).toBe("Cafe deja vu.mp4"); + expect(renderExportFileName("../../etc/passwd")).toBe("etc passwd.mp4"); + expect(renderExportFileName("Screen Recording.MP4")).toBe( + "Screen Recording.mp4", + ); + expect(renderExportFileName("")).toBe("Cap Export.mp4"); + expect(renderExportFileName(null)).toBe("Cap Export.mp4"); + expect(renderExportFileName("录屏")).toBe("Cap Export.mp4"); + expect(renderExportFileName("x".repeat(300))).toBe( + `${"x".repeat(120)}.mp4`, + ); + }); + + it("builds an attachment disposition with a UTF-8 fallback", () => { + expect(attachmentDisposition("Demo.mp4")).toBe( + `attachment; filename="Demo.mp4"; filename*=UTF-8''Demo.mp4`, + ); + expect(attachmentDisposition('a"b\\c é.mp4')).toBe( + `attachment; filename="a_b_c _.mp4"; filename*=UTF-8''a%22b%5Cc%20%C3%A9.mp4`, + ); + }); +}); diff --git a/apps/web/__tests__/unit/render-farm-save-current.test.ts b/apps/web/__tests__/unit/render-farm-save-current.test.ts new file mode 100644 index 00000000000..0e6e9647001 --- /dev/null +++ b/apps/web/__tests__/unit/render-farm-save-current.test.ts @@ -0,0 +1,63 @@ +import type { VideoMetadata } from "@cap/database/types"; +import { expect, test } from "vitest"; +import { renderFarmSaveIsCurrent } from "@/lib/render-farm-status"; + +const save = ( + status: "rendering" | "published" | "error", + projectSavedAt?: string | null, +) => ({ + version: 1 as const, + exportId: "export", + jobId: "job", + status, + projectSavedAt, + startedAt: "2026-09-28T00:00:00.000Z", + outputKey: "owner/video/output.mp4", + hlsPrefix: "owner/video/hls/", +}); + +const project = (savedAt: string) => + ({ version: 1, config: {}, savedAt }) as VideoMetadata["webEditorProject"]; + +test("a render of the project as saved now is current", () => { + expect( + renderFarmSaveIsCurrent({ + renderFarmSave: save("published", "a"), + webEditorProject: project("a"), + }), + ).toBe(true); + expect( + renderFarmSaveIsCurrent({ + renderFarmSave: save("rendering", "a"), + webEditorProject: project("a"), + }), + ).toBe(true); + expect( + renderFarmSaveIsCurrent({ renderFarmSave: save("published", null) }), + ).toBe(true); +}); + +test("edits, failures and older renders can be saved again", () => { + expect( + renderFarmSaveIsCurrent({ + renderFarmSave: save("published", "a"), + webEditorProject: project("b"), + }), + ).toBe(false); + expect( + renderFarmSaveIsCurrent({ + renderFarmSave: save("published", null), + webEditorProject: project("a"), + }), + ).toBe(false); + expect( + renderFarmSaveIsCurrent({ + renderFarmSave: save("error", "a"), + webEditorProject: project("a"), + }), + ).toBe(false); + expect(renderFarmSaveIsCurrent({ renderFarmSave: save("published") })).toBe( + false, + ); + expect(renderFarmSaveIsCurrent(null)).toBe(false); +}); diff --git a/apps/web/__tests__/unit/render-farm-save-publish.test.ts b/apps/web/__tests__/unit/render-farm-save-publish.test.ts new file mode 100644 index 00000000000..27cc1bf08a9 --- /dev/null +++ b/apps/web/__tests__/unit/render-farm-save-publish.test.ts @@ -0,0 +1,119 @@ +import type { Video } from "@cap/web-domain"; +import { beforeEach, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + video: null as null | Record, + updates: [] as unknown[], + refreshPreviews: vi.fn(), + invalidate: vi.fn(async () => {}), +})); + +vi.mock("@cap/database", () => { + const select = () => ({ + from: () => ({ + where: () => + Object.assign(Promise.resolve([mocks.video]), { + for: async () => [mocks.video], + }), + }), + }); + return { + db: () => ({ + select, + transaction: async (run: (tx: unknown) => Promise) => + run({ + select, + update: () => ({ + set: (values: Record) => ({ + where: async () => { + mocks.updates.push(values); + }, + }), + }), + }), + }), + }; +}); +vi.mock("@cap/database/emails/config", () => ({ sendEmail: vi.fn() })); +vi.mock("@cap/database/emails/export-ready", () => ({ ExportReady: vi.fn() })); +vi.mock("@cap/env", () => ({ serverEnv: () => ({}) })); +vi.mock("@cap/web-backend", () => ({ Storage: {} })); +vi.mock("@/lib/server", () => ({ + runPromise: async () => ({ ContentLength: 100 }), +})); +vi.mock("@/lib/video-storage", () => ({ + decodeStorageVideo: (video: unknown) => video, +})); +vi.mock("@/lib/desktop-recording-jobs", () => ({ + retireDesktopRecordingJobForOutputReplacement: vi.fn(), +})); +vi.mock("@/lib/desktop-reupload", () => ({ + invalidateReuploadedVideo: mocks.invalidate, +})); +vi.mock("@/lib/refresh-preview-assets", () => ({ + enqueuePreviewAssetsRefresh: mocks.refreshPreviews, +})); +vi.mock("@/lib/queue-video-transcription", () => ({ + queueVideoTranscription: vi.fn(), + shouldQueueTranscriptionAfterMultipartComplete: () => false, +})); +vi.mock("@/lib/render-farm-records", () => ({ + changeRenderFarmExports: vi.fn(), + clearRecordingRender: vi.fn(), + failRenderFarmSave: vi.fn(), +})); +vi.mock("@/lib/render-farm", () => ({ + renderFarmConfig: () => null, + renderFarmFetch: vi.fn(), + mapRenderFarmJob: vi.fn(), +})); + +const { finalizeRenderFarmSave } = await import("@/lib/render-farm-save"); + +const videoId = "video" as Video.VideoId; +const output = { + width: 1920, + height: 1080, + fps: 30, + durationSeconds: 10, + bytes: 100, +}; + +beforeEach(() => { + vi.clearAllMocks(); + mocks.updates = []; + mocks.video = { + id: "video", + ownerId: "owner", + source: { type: "webMP4" }, + metadata: { + renderFarmSave: { + version: 1, + exportId: "export", + jobId: "job", + status: "rendering", + trigger: "recording", + startedAt: new Date().toISOString(), + outputKey: "owner/video/.recording/render/export/result.mp4", + hlsPrefix: "owner/video/.recording/render/export/hls", + }, + }, + }; +}); + +test("a published render remakes the share video's previews", async () => { + await expect(finalizeRenderFarmSave(videoId, "job", output)).resolves.toBe( + "published", + ); + + expect(mocks.updates).toHaveLength(1); + expect(mocks.refreshPreviews).toHaveBeenCalledWith("video"); +}); + +test("a stale render leaves the previews alone", async () => { + await expect( + finalizeRenderFarmSave(videoId, "other-job", output), + ).resolves.toBe("stale"); + + expect(mocks.refreshPreviews).not.toHaveBeenCalled(); +}); diff --git a/apps/web/__tests__/unit/render-farm-save-refresh.test.ts b/apps/web/__tests__/unit/render-farm-save-refresh.test.ts new file mode 100644 index 00000000000..6ac9cfdfb30 --- /dev/null +++ b/apps/web/__tests__/unit/render-farm-save-refresh.test.ts @@ -0,0 +1,84 @@ +import { beforeEach, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + rows: [] as unknown[], + jobs: new Map< + string, + { state: string; output?: unknown; progress?: number } + >(), +})); + +vi.mock("@cap/database", () => ({ + db: () => ({ + select: () => ({ + from: () => ({ where: async () => [mocks.rows.shift()] }), + }), + }), +})); +vi.mock("@cap/database/emails/config", () => ({ sendEmail: vi.fn() })); +vi.mock("@cap/database/emails/export-ready", () => ({ ExportReady: vi.fn() })); +vi.mock("@cap/env", () => ({ serverEnv: () => ({}) })); +vi.mock("@cap/web-backend", () => ({ Storage: {} })); +vi.mock("@/lib/server", () => ({ runPromise: vi.fn() })); +vi.mock("@/lib/desktop-recording-jobs", () => ({ + retireDesktopRecordingJobForOutputReplacement: vi.fn(), +})); +vi.mock("@/lib/desktop-reupload", () => ({ + invalidateReuploadedVideo: vi.fn(), +})); +vi.mock("@/lib/queue-video-transcription", () => ({ + queueVideoTranscription: vi.fn(), + shouldQueueTranscriptionAfterMultipartComplete: () => false, +})); +vi.mock("@/lib/render-farm-records", () => ({ + changeRenderFarmExports: vi.fn(), + clearRecordingRender: vi.fn(), + failRenderFarmSave: vi.fn(), +})); +vi.mock("@/lib/render-farm", () => ({ + renderFarmConfig: () => ({}), + renderFarmFetch: async (_config: unknown, path: string) => + Response.json({ path }), + mapRenderFarmJob: ({ body }: { body: { path: string } }) => + mocks.jobs.get(body.path.replace("/jobs/", "")) ?? null, +})); + +import { refreshRenderFarmSave } from "@/lib/render-farm-save"; + +const save = (jobId: string, exportId: string) => ({ + version: 1, + exportId, + jobId, + status: "rendering", + startedAt: new Date().toISOString(), + outputKey: `owner/video/.recording/outputs/${exportId}/result.mp4`, + hlsPrefix: `owner/video/.recording/outputs/${exportId}/hls/`, +}); + +const video = (renderFarmSave: ReturnType) => ({ + id: "video", + fps: 30, + metadata: { renderFarmSave }, +}); + +beforeEach(() => { + mocks.rows = []; + mocks.jobs.clear(); +}); + +test("a render replaced while it was polled reports the new save, not ready", async () => { + const first = save("first", "export-1"); + const second = save("second", "export-2"); + mocks.jobs.set("first", { + state: "ready", + output: { width: 1920, height: 1080, fps: 30, frames: 300, bytes: 10 }, + }); + mocks.jobs.set("second", { state: "rendering", progress: 0.4 }); + mocks.rows = [video(second), video(second)]; + + const status = await refreshRenderFarmSave(video(first) as never); + + expect(status.state).toBe("rendering"); + expect(status.exportId).toBe("export-2"); + expect(status.progress).toBe(0.4); +}); diff --git a/apps/web/__tests__/unit/render-farm-save-target.test.ts b/apps/web/__tests__/unit/render-farm-save-target.test.ts new file mode 100644 index 00000000000..522dd7ccbd7 --- /dev/null +++ b/apps/web/__tests__/unit/render-farm-save-target.test.ts @@ -0,0 +1,85 @@ +import { afterEach, beforeEach, expect, test, vi } from "vitest"; + +const env = vi.hoisted(() => ({ + value: {} as Record, +})); + +vi.mock("@cap/env", () => ({ serverEnv: () => env.value })); + +const farm = { + RENDER_FARM_URL: "https://render.example", + RENDER_FARM_TOKEN: "token", + RENDER_FARM_CALLBACK_SECRET: "secret", +}; +const worker = { + CAP_WEB_EDITOR_WORKER_URL: "https://editor.example", + MEDIA_SERVER_WEBHOOK_SECRET: "media-secret", +}; + +async function saveTarget() { + vi.resetModules(); + const { renderFarmSaveUnavailable } = await import("@/lib/render-farm-start"); + return renderFarmSaveUnavailable(); +} + +beforeEach(() => { + env.value = {}; +}); + +afterEach(() => vi.unstubAllGlobals()); + +test("Save renders in the browser when the farm or an editor worker is not configured", async () => { + const health = vi.fn(async () => Response.json({ ok: true, workers: 3 })); + vi.stubGlobal("fetch", health); + expect(await saveTarget()).toBe("The render farm is not configured"); + expect(health).not.toHaveBeenCalled(); + env.value = { ...farm }; + expect(await saveTarget()).toBe("No editor worker is configured"); + env.value = { ...farm, CAP_WEB_EDITOR_WORKER_POOL: "not json" }; + expect(await saveTarget()).toBe("No editor worker is configured"); +}); + +test("Save needs no editor worker when the farm prepares projects itself", async () => { + env.value = { ...farm }; + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ + ok: true, + workers: 3, + prepare: { version: 1, defaultConfig: { audio: {} }, music: [] }, + }), + ), + ); + expect(await saveTarget()).toBeNull(); +}); + +test("Save uses the farm only while its health check reports workers", async () => { + env.value = { ...farm, ...worker }; + const health = vi.fn(async (_url: string, init?: RequestInit) => { + expect(new Headers(init?.headers).get("authorization")).toBe( + "Bearer token", + ); + return Response.json({ ok: true, workers: 3 }); + }); + vi.stubGlobal("fetch", health); + expect(await saveTarget()).toBeNull(); + expect(health).toHaveBeenCalledWith( + "https://render.example/health", + expect.anything(), + ); + + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json({ ok: true, workers: 0 })), + ); + expect(await saveTarget()).toBe("The render farm is not responding"); + + vi.stubGlobal( + "fetch", + vi.fn(async () => { + throw new TypeError("fetch failed"); + }), + ); + expect(await saveTarget()).toBe("The render farm is not responding"); +}); diff --git a/apps/web/__tests__/unit/render-farm-save.test.ts b/apps/web/__tests__/unit/render-farm-save.test.ts new file mode 100644 index 00000000000..a8575b5ff68 --- /dev/null +++ b/apps/web/__tests__/unit/render-farm-save.test.ts @@ -0,0 +1,290 @@ +import { createHmac } from "node:crypto"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const env = vi.hoisted(() => ({ + value: {} as Record, +})); +const saves = vi.hoisted(() => ({ + finalize: vi.fn(async () => "published" as const), + fail: vi.fn(async () => undefined), + clearRecording: vi.fn(async () => undefined), + finalizeExport: vi.fn(async () => "ready" as const), + failExport: vi.fn(async () => null), +})); + +vi.mock("@cap/env", () => ({ serverEnv: () => env.value })); +vi.mock("@/lib/render-farm-save", () => ({ + finalizeRenderFarmSave: saves.finalize, + finalizeRenderFarmExport: saves.finalizeExport, + failRenderFarmExport: saves.failExport, +})); +vi.mock("@/lib/render-farm-records", () => ({ + failRenderFarmSave: saves.fail, + clearRecordingRender: saves.clearRecording, +})); + +import { POST } from "@/app/api/render-farm/callback/route"; +import { + publishedRenderFarmUpdate, + renderSaveStatusFromMetadata, +} from "@/lib/render-farm-status"; + +const save = { + version: 1 as const, + exportId: "export-1", + jobId: "job-1", + status: "rendering" as const, + startedAt: "2026-09-26T00:00:00.000Z", + outputKey: "owner/video/.recording/render/export-1/result.mp4", + hlsPrefix: "owner/video/.recording/render/export-1/hls", +}; +const output = { + width: 1920, + height: 1080, + fps: 30, + durationSeconds: 20, + bytes: 1234, +}; + +describe("publishedRenderFarmUpdate", () => { + const video = { + source: { type: "webMP4" as const, outputKey: "old.mp4" }, + metadata: { + renderFarmSave: save, + summary: "old summary", + chapters: [{ title: "Intro", start: 0 }], + webEditorAudioDefault: { + enabledByDefault: true, + isolation: "light" as const, + }, + }, + }; + + it("switches the source to the render and clears stale AI output", () => { + const update = publishedRenderFarmUpdate( + video, + "job-1", + output, + new Date("2026-09-26T01:00:00.000Z"), + ); + expect(update?.source).toEqual({ + type: "webMP4", + outputKey: save.outputKey, + }); + expect(update).toMatchObject({ + duration: 20, + width: 1920, + height: 1080, + fps: 30, + transcriptionStatus: null, + }); + expect(update?.metadata.summary).toBeUndefined(); + expect(update?.metadata.chapters).toBeUndefined(); + expect(update?.metadata.webEditorAudioDefault).toEqual( + video.metadata.webEditorAudioDefault, + ); + expect(update?.metadata.renderFarmSave).toMatchObject({ + status: "published", + publishedAt: "2026-09-26T01:00:00.000Z", + }); + }); + + it("keeps transcript and AI output for a render of the untouched recording", () => { + const update = publishedRenderFarmUpdate( + { + ...video, + metadata: { + ...video.metadata, + renderFarmSave: { ...save, trigger: "recording" as const }, + }, + }, + "job-1", + output, + new Date("2026-09-26T01:00:00.000Z"), + ); + expect(update?.source.outputKey).toBe(save.outputKey); + expect(update?.metadata.summary).toBe("old summary"); + expect(update?.metadata.chapters).toEqual(video.metadata.chapters); + expect(update).not.toHaveProperty("transcriptionStatus"); + }); + + it("ignores superseded, repeated or invalid results", () => { + const now = new Date(); + expect(publishedRenderFarmUpdate(video, "job-0", output, now)).toBeNull(); + expect( + publishedRenderFarmUpdate( + { + ...video, + metadata: { renderFarmSave: { ...save, status: "published" } }, + }, + "job-1", + output, + now, + ), + ).toBeNull(); + expect( + publishedRenderFarmUpdate(video, "job-1", { ...output, bytes: 0 }, now), + ).toBeNull(); + expect( + publishedRenderFarmUpdate( + { ...video, source: { type: "MediaConvert" } } as never, + "job-1", + output, + now, + ), + ).toBeNull(); + }); +}); + +describe("renderSaveStatusFromMetadata", () => { + it("settles published and failed saves without asking the farm", () => { + expect(renderSaveStatusFromMetadata(undefined)?.state).toBe("idle"); + expect( + renderSaveStatusFromMetadata({ ...save, status: "published" }), + ).toMatchObject({ state: "ready", progress: 1 }); + expect( + renderSaveStatusFromMetadata({ ...save, status: "error", error: "x" }), + ).toMatchObject({ state: "error", error: "x" }); + expect(renderSaveStatusFromMetadata(save)).toBeNull(); + }); +}); + +describe("render farm callback route", () => { + const post = (body: string, signature?: string) => + POST( + new Request("https://cap.test/api/render-farm/callback", { + method: "POST", + headers: signature ? { "x-render-farm-signature": signature } : {}, + body, + }), + ); + const sign = (body: string) => + `sha256=${createHmac("sha256", "callback-secret").update(body).digest("hex")}`; + + beforeEach(() => { + env.value = { + RENDER_FARM_URL: "https://farm.test", + RENDER_FARM_TOKEN: "token", + RENDER_FARM_CALLBACK_SECRET: "callback-secret", + }; + for (const mock of Object.values(saves)) mock.mockClear(); + }); + + it("is unavailable until the farm is configured", async () => { + env.value = {}; + expect((await post("{}", sign("{}"))).status).toBe(503); + }); + + it("rejects unsigned or tampered callbacks", async () => { + const body = JSON.stringify({ + id: "job-1", + reference: "video", + status: "ready", + }); + expect((await post(body)).status).toBe(401); + expect((await post(body, sign(`${body}x`))).status).toBe(401); + expect(saves.finalize).not.toHaveBeenCalled(); + }); + + it("publishes ready exports and records failures", async () => { + const ready = JSON.stringify({ + id: "job-1", + reference: "video", + status: "ready", + width: 1920, + height: 1080, + fps: 30, + durationSeconds: 20, + bytes: 1234, + }); + expect((await post(ready, sign(ready))).status).toBe(200); + expect(saves.finalize).toHaveBeenCalledWith("video", "job-1", output); + const failed = JSON.stringify({ + id: "job-2", + reference: "video", + status: "error", + error: "gpu fell over", + }); + expect((await post(failed, sign(failed))).status).toBe(200); + expect(saves.fail).toHaveBeenCalledWith( + "video", + { jobId: "job-2" }, + "gpu fell over", + ); + }); + + it("routes background exports and recording renders by reference kind", async () => { + const ready = JSON.stringify({ + id: "job-3", + reference: "export:video", + status: "ready", + width: 1920, + height: 1080, + fps: 30, + durationSeconds: 20, + bytes: 1234, + }); + expect((await post(ready, sign(ready))).status).toBe(200); + expect(saves.finalizeExport).toHaveBeenCalledWith("video", "job-3", output); + expect(saves.finalize).not.toHaveBeenCalled(); + + const exportFailed = JSON.stringify({ + id: "job-4", + reference: "export:video", + status: "error", + error: "out of disk", + }); + expect((await post(exportFailed, sign(exportFailed))).status).toBe(200); + expect(saves.failExport).toHaveBeenCalledWith( + "video", + "job-4", + "out of disk", + ); + + const recordingReady = JSON.stringify({ + id: "job-5", + reference: "recording:video", + status: "ready", + width: 1920, + height: 1080, + fps: 30, + durationSeconds: 20, + bytes: 1234, + }); + expect((await post(recordingReady, sign(recordingReady))).status).toBe(200); + expect(saves.finalize).toHaveBeenCalledWith("video", "job-5", output); + + const recordingFailed = JSON.stringify({ + id: "job-6", + reference: "recording:video", + status: "error", + }); + expect((await post(recordingFailed, sign(recordingFailed))).status).toBe( + 200, + ); + expect(saves.clearRecording).toHaveBeenCalledWith("video", { + jobId: "job-6", + }); + expect(saves.fail).not.toHaveBeenCalled(); + }); + + it("rejects references it cannot route", async () => { + const body = JSON.stringify({ + id: "job-7", + reference: "thumbnail:video", + status: "ready", + }); + expect((await post(body, sign(body))).status).toBe(400); + expect(saves.finalize).not.toHaveBeenCalled(); + }); + + it("asks the farm to retry when publishing fails", async () => { + saves.finalize.mockRejectedValueOnce(new Error("db down")); + const ready = JSON.stringify({ + id: "job-1", + reference: "video", + status: "ready", + }); + expect((await post(ready, sign(ready))).status).toBe(503); + }); +}); diff --git a/apps/web/__tests__/unit/render-farm.test.ts b/apps/web/__tests__/unit/render-farm.test.ts new file mode 100644 index 00000000000..9ca8b4b32b2 --- /dev/null +++ b/apps/web/__tests__/unit/render-farm.test.ts @@ -0,0 +1,387 @@ +import { createHmac } from "node:crypto"; +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@cap/env", () => ({ serverEnv: () => ({}) })); + +import { + mapRenderFarmJob, + renderFarmCallbackUrl, + renderFarmKeys, + renderFarmTranscodeKey, + verifyRenderFarmSignature, +} from "@/lib/render-farm"; +import { + buildRenderProject, + finishRenderManifest, + RenderProjectError, +} from "@/lib/render-farm-project"; +import { awaitingUnknownRenderJob } from "@/lib/render-farm-status"; + +describe("render farm callbacks", () => { + const body = JSON.stringify({ id: "job", status: "ready" }); + const sign = (secret: string) => + `sha256=${createHmac("sha256", secret).update(body).digest("hex")}`; + + it("accepts only the farm's HMAC of the exact body", () => { + expect(verifyRenderFarmSignature(body, sign("secret"), "secret")).toBe( + true, + ); + expect(verifyRenderFarmSignature(body, sign("other"), "secret")).toBe( + false, + ); + expect( + verifyRenderFarmSignature(`${body} `, sign("secret"), "secret"), + ).toBe(false); + expect(verifyRenderFarmSignature(body, null, "secret")).toBe(false); + expect(verifyRenderFarmSignature(body, "sha256=zz", "secret")).toBe(false); + }); +}); + +describe("render farm keys", () => { + it("keep every export inside the video's render folder", () => { + expect(renderFarmKeys("owner", "video", "export")).toEqual({ + root: "owner/video/", + recording: "owner/video/.recording/render/export/project", + outputKey: "owner/video/.recording/render/export/result.mp4", + hlsPrefix: "owner/video/.recording/render/export/hls", + }); + }); + + it("reuse a transcode for the same source object and not for a changed one", () => { + const first = renderFarmTranscodeKey("o/v/", "o/v/raw-upload.webm", "etag"); + expect(first).toMatch( + /^o\/v\/\.recording\/render\/sources\/[0-9a-f]{32}\.mp4$/, + ); + expect(renderFarmTranscodeKey("o/v/", "o/v/raw-upload.webm", "etag")).toBe( + first, + ); + expect( + renderFarmTranscodeKey("o/v/", "o/v/raw-upload.webm", "etag-2"), + ).not.toBe(first); + }); +}); + +describe("mapRenderFarmJob", () => { + it("reports progress and playability while rendering", () => { + expect( + mapRenderFarmJob( + { + status: 200, + body: { + status: "rendering", + progress: 0.4, + hlsSegments: 3, + hlsUrl: "https://bucket/hls/index.m3u8", + }, + }, + 30, + ), + ).toMatchObject({ + state: "rendering", + progress: 0.4, + playable: true, + hlsUrl: "https://bucket/hls/index.m3u8", + }); + expect( + mapRenderFarmJob( + { status: 200, body: { status: "planning", hlsSegments: 0 } }, + 30, + ), + ).toMatchObject({ state: "rendering", playable: false }); + }); + + it("maps a finished job's output and failures", () => { + expect( + mapRenderFarmJob( + { + status: 200, + body: { + status: "ready", + fps: 30, + output: { width: 1920, height: 1080, frames: 900, bytes: 1234 }, + }, + }, + 60, + ).output, + ).toEqual({ width: 1920, height: 1080, frames: 900, fps: 30, bytes: 1234 }); + expect( + mapRenderFarmJob( + { status: 200, body: { status: "error", error: "x" } }, + 30, + ), + ).toMatchObject({ state: "error", error: "x" }); + expect(mapRenderFarmJob({ status: 404, body: null }, 30).state).toBe( + "gone", + ); + expect(() => mapRenderFarmJob({ status: 500, body: null }, 30)).toThrow(); + }); +}); + +describe("buildRenderProject", () => { + const root = "owner/video/"; + const recording = "owner/video/.recording/render/e/project"; + const segment = "content/segments/segment-0"; + const baseFiles = [ + { path: `${segment}/display.webm`, size: 1000, inode: "1:1" }, + { path: `${segment}/camera.webm`, size: 500, inode: "1:2" }, + { path: `${segment}/mic.webm`, size: 200, inode: "1:3" }, + { path: `${segment}/cursor.json`, size: 50, inode: "1:4" }, + { path: "content/cursors/web-default.png", size: 9, inode: "1:5" }, + { path: "recording-meta.json", size: 90, inode: "1:6" }, + { path: "project-config.json", size: 80, inode: "1:7" }, + { path: "assets/audio/import-1.ogg", size: 70, inode: "1:8" }, + ]; + const meta = { + segments: [ + { + display: { path: `${segment}/display.webm`, fps: 30 }, + camera: { path: `${segment}/camera.webm`, fps: 30 }, + mic: { path: `${segment}/mic.webm`, start_time: 0 }, + cursor: `${segment}/cursor.json`, + }, + ], + }; + const sources = new Map([ + [ + `${segment}/display.webm`, + { key: "owner/video/raw-upload.webm", size: 1000, identity: "d" }, + ], + [ + `${segment}/camera.webm`, + { key: "owner/video/camera-upload.webm", size: 500, identity: "c" }, + ], + [ + `${segment}/mic.webm`, + { key: "owner/video/mic-upload.webm", size: 200, identity: "m" }, + ], + [ + "assets/audio/import-1.ogg", + { key: "owner/video/editor-assets/a.ogg", size: 70, identity: "a" }, + ], + ]); + + it("transcodes recorded video from its source, reads audio in place and uploads the rest", () => { + const plan = buildRenderProject({ + root, + recording, + files: baseFiles, + recordingMeta: meta, + config: { + background: { + source: { + type: "wallpaper", + path: "cap-web-wallpaper://assets/backgrounds/blue/sky.jpg", + }, + }, + }, + sources, + }); + const byPath = new Map(plan.entries.map((entry) => [entry.path, entry])); + expect(byPath.get(`${segment}/display.h264.mp4`)).toEqual({ + path: `${segment}/display.h264.mp4`, + key: renderFarmTranscodeKey(root, "owner/video/raw-upload.webm", "d"), + transcodeFrom: "owner/video/raw-upload.webm", + }); + expect(byPath.get(`${segment}/mic.mic.opus`)).toEqual({ + path: `${segment}/mic.mic.opus`, + key: "owner/video/mic-upload.webm", + size: 200, + }); + expect(byPath.get("assets/audio/import-1.ogg")?.key).toBe( + "owner/video/editor-assets/a.ogg", + ); + expect(byPath.has(`${segment}/display.webm`)).toBe(false); + expect(byPath.has("recording-meta.json")).toBe(false); + expect(plan.uploads.sort()).toEqual([ + "content/cursors/web-default.png", + `${segment}/cursor.json`, + ]); + expect(plan.wallpapers).toEqual(["blue/sky.jpg"]); + expect(plan.config.background).toEqual({ + source: { + type: "wallpaper", + path: "$RF_PROJECT/assets/backgrounds/blue/sky.jpg", + }, + }); + const segment0 = ( + plan.recordingMeta.segments as Record[] + )[0]; + expect(segment0?.display).toEqual({ + path: `${segment}/display.h264.mp4`, + fps: 30, + }); + expect(segment0?.mic).toEqual({ + path: `${segment}/mic.mic.opus`, + start_time: 0, + }); + + const manifest = finishRenderManifest( + plan, + new Map([ + [`${segment}/cursor.json`, 55], + ["content/cursors/web-default.png", 9], + ["assets/backgrounds/blue/sky.jpg", 77], + ]), + [{ path: "recording-meta.json", size: 10 }], + ); + const manifestByPath = new Map( + manifest.files.map((entry) => [entry.path, entry]), + ); + expect(manifestByPath.get(`${segment}/cursor.json`)).toEqual({ + path: `${segment}/cursor.json`, + size: 55, + }); + expect(manifestByPath.get("assets/backgrounds/blue/sky.jpg")?.size).toBe( + 77, + ); + expect(manifestByPath.get("recording-meta.json")?.size).toBe(10); + }); + + it("maps clip segments to their imported asset through hard links", () => { + const plan = buildRenderProject({ + root, + recording, + files: [ + ...baseFiles, + { path: "content/videos/clip.webm", size: 300, inode: "9:9" }, + { + path: "content/segments/segment-1/display.webm", + size: 300, + inode: "9:9", + }, + ], + recordingMeta: { + segments: [ + ...meta.segments, + { display: { path: "content/segments/segment-1/display.webm" } }, + ], + }, + config: {}, + sources: new Map([ + ...sources, + [ + "content/videos/clip.webm", + { + key: "owner/video/editor-videos/clip.webm", + size: 300, + identity: "x", + }, + ], + ]), + }); + const clip = plan.entries.find( + (entry) => entry.path === "content/segments/segment-1/display.h264.mp4", + ); + expect(clip?.transcodeFrom).toBe("owner/video/editor-videos/clip.webm"); + expect(plan.uploads).not.toContain("content/videos/clip.webm"); + expect(plan.uploads).not.toContain( + "content/segments/segment-1/display.webm", + ); + }); + + it("uploads and transcodes media with no stored source, and splits audio mixed into the display", () => { + const plan = buildRenderProject({ + root, + recording, + files: [ + { path: `${segment}/display.mp4`, size: 1000, inode: "1:1" }, + { path: "recording-meta.json", size: 1, inode: "1:6" }, + ], + recordingMeta: { + segments: [ + { + display: { path: `${segment}/display.mp4` }, + system_audio: { path: `${segment}/display.mp4` }, + }, + ], + }, + config: {}, + sources: new Map(), + }); + expect(plan.uploads).toEqual([`${segment}/display.mp4`]); + const byPath = new Map(plan.entries.map((entry) => [entry.path, entry])); + expect(byPath.get(`${segment}/display.h264.mp4`)).toEqual({ + path: `${segment}/display.h264.mp4`, + key: `${recording}/transcoded/${segment}/display.h264.mp4`, + transcodeFrom: `${recording}/${segment}/display.mp4`, + }); + expect(byPath.get(`${segment}/display.system.m4a`)).toEqual({ + path: `${segment}/display.system.m4a`, + key: `${recording}/${segment}/display.mp4`, + size: 1000, + }); + }); + + it("refuses projects the farm cannot render faithfully", () => { + const build = (config: unknown, recordingMeta: unknown = meta) => + buildRenderProject({ + root, + recording, + files: baseFiles, + recordingMeta, + config, + sources, + }); + expect(() => + build({ + timeline: { videoSegments: [{ path: "content/videos/x.mp4" }] }, + }), + ).toThrow(RenderProjectError); + expect(() => + build({ + background: { + source: { type: "wallpaper", path: "/etc/passwd" }, + }, + }), + ).toThrow(RenderProjectError); + expect(() => build({}, { display: { path: "x" } })).toThrow( + RenderProjectError, + ); + expect(() => + buildRenderProject({ + root, + recording, + files: baseFiles.filter((file) => !file.path.endsWith("cursor.json")), + recordingMeta: meta, + config: {}, + sources, + }), + ).not.toThrow(); + expect(() => finishRenderManifest(build({}), new Map(), [])).toThrow( + RenderProjectError, + ); + }); +}); + +describe("awaitingUnknownRenderJob", () => { + const startedAt = "2026-09-26T12:00:00.000Z"; + const at = (minutes: number) => Date.parse(startedAt) + minutes * 60_000; + + it("waits while a restarted coordinator may still be reloading the job", () => { + expect(awaitingUnknownRenderJob({ startedAt }, at(1))).toBe(true); + expect(awaitingUnknownRenderJob({ startedAt }, at(14))).toBe(true); + }); + + it("gives up on a job the farm still does not know later", () => { + expect(awaitingUnknownRenderJob({ startedAt }, at(16))).toBe(false); + expect(awaitingUnknownRenderJob({ startedAt: "not a date" }, at(1))).toBe( + false, + ); + }); +}); + +describe("renderFarmCallbackUrl", () => { + it("adds the protection bypass only for Vercel preview hosts", () => { + expect( + renderFarmCallbackUrl("https://cap-web-git-x.vercel.app", "secret"), + ).toBe( + "https://cap-web-git-x.vercel.app/api/render-farm/callback?x-vercel-protection-bypass=secret", + ); + expect(renderFarmCallbackUrl("https://cap.so", "secret")).toBe( + "https://cap.so/api/render-farm/callback", + ); + expect(renderFarmCallbackUrl("https://cap-web.vercel.app", undefined)).toBe( + "https://cap-web.vercel.app/api/render-farm/callback", + ); + }); +}); diff --git a/apps/web/__tests__/unit/render-recording-eligibility.test.ts b/apps/web/__tests__/unit/render-recording-eligibility.test.ts new file mode 100644 index 00000000000..688215844ab --- /dev/null +++ b/apps/web/__tests__/unit/render-recording-eligibility.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from "vitest"; +import { + PRO_DURATION_SECONDS, + recordingRenderEligible, + recordingRenderSourcesReady, +} from "@/lib/render-recording-eligibility"; + +describe("render on recording finish", () => { + const candidate = { + isScreenshot: false, + sourceType: "webMP4", + duration: 60, + ownerIsPro: false, + metadata: { + editorSources: { + version: 1 as const, + display: { + key: "owner/video/raw-upload.webm", + contentType: "video/webm" as const, + size: 1000, + }, + }, + }, + }; + + it("renders untouched browser recordings with editor sources", () => { + expect(recordingRenderEligible(candidate)).toBe(true); + }); + + it("skips screenshots, imports, desktop and edited or saved videos", () => { + expect(recordingRenderEligible({ ...candidate, isScreenshot: true })).toBe( + false, + ); + expect(recordingRenderEligible({ ...candidate, metadata: {} })).toBe(false); + expect( + recordingRenderEligible({ ...candidate, sourceType: "desktopMP4" }), + ).toBe(false); + expect( + recordingRenderEligible({ + ...candidate, + metadata: { + ...candidate.metadata, + webEditorProject: { version: 1, config: {}, savedAt: "now" }, + }, + }), + ).toBe(false); + expect( + recordingRenderEligible({ + ...candidate, + metadata: { + ...candidate.metadata, + renderFarmSave: { + version: 1, + exportId: "e", + jobId: "j", + status: "published", + startedAt: "now", + outputKey: "k", + hlsPrefix: "h", + }, + }, + }), + ).toBe(false); + }); + + it("keeps the Pro limit Save has", () => { + const long = { ...candidate, duration: PRO_DURATION_SECONDS }; + expect(recordingRenderEligible(long)).toBe(false); + expect(recordingRenderEligible({ ...long, ownerIsPro: true })).toBe(true); + }); + + it("waits until the screen source is verified and uploaded", () => { + expect( + recordingRenderSourcesReady(candidate.metadata, 60, "processing"), + ).toBe(true); + expect(recordingRenderSourcesReady(candidate.metadata, 60, null)).toBe( + true, + ); + expect( + recordingRenderSourcesReady(candidate.metadata, 60, "uploading"), + ).toBe(false); + expect(recordingRenderSourcesReady(candidate.metadata, null, null)).toBe( + false, + ); + expect( + recordingRenderSourcesReady( + { + editorSources: { + version: 1, + display: { + key: "owner/video/result.mp4", + contentType: "video/mp4", + }, + }, + }, + 60, + null, + ), + ).toBe(false); + }); +}); diff --git a/apps/web/__tests__/unit/render-recording-start.test.ts b/apps/web/__tests__/unit/render-recording-start.test.ts new file mode 100644 index 00000000000..b4d709e3fd9 --- /dev/null +++ b/apps/web/__tests__/unit/render-recording-start.test.ts @@ -0,0 +1,145 @@ +import { afterEach, beforeEach, expect, test, vi } from "vitest"; + +const state = vi.hoisted(() => ({ + afters: [] as Promise[], + sources: [] as string[], + direct: "started" as string | Error, + workflowStarts: 0, + cleared: 0, +})); + +vi.mock("@cap/database", () => ({ + db: () => ({ + select: () => ({ + from: () => ({ + innerJoin: () => ({ + leftJoin: () => ({ + where: async () => [ + { + video: { + id: "video", + ownerId: "owner", + isScreenshot: false, + source: { type: "webMP4" }, + duration: 10, + metadata: { + editorSources: { + version: 1, + display: { key: "k", size: 1 }, + }, + }, + }, + owner: { email: "owner@cap.so" }, + uploadPhase: null, + }, + ], + }), + }), + }), + }), + }), +})); +vi.mock("@cap/database/schema", () => ({ + users: {}, + videos: {}, + videoUploads: {}, +})); +vi.mock("@cap/utils", () => ({ userIsPro: () => true })); +vi.mock("drizzle-orm", () => ({ eq: () => ({}) })); +vi.mock("next/server", () => ({ + after: (task: () => Promise) => { + state.afters.push(task()); + }, +})); +vi.mock("workflow/api", () => ({ + start: async () => { + state.workflowStarts++; + }, +})); +vi.mock("@/lib/render-farm", () => ({ + renderFarmConfig: () => ({ callbackSecret: "secret" }), + renderFarmKeys: () => ({ outputKey: "out", hlsPrefix: "hls" }), +})); +vi.mock("@/lib/render-farm-records", () => ({ + recordPendingRecordingRender: async () => true, + clearRecordingRender: async () => { + state.cleared++; + }, +})); +vi.mock("@/lib/web-studio-rollout", () => ({ + isWebStudioEnabledForEmail: () => true, +})); +vi.mock("@/lib/render-recording-workflow", () => ({ + renderRecordingWorkflow: () => undefined, +})); +vi.mock("@/lib/render-recording-start", () => ({ + recordingSourcesState: async () => state.sources.shift() ?? "ready", + startRecordingRenderDirectly: async () => { + if (state.direct instanceof Error) throw state.direct; + return state.direct; + }, +})); + +const { startRecordingRender } = await import("@/lib/render-recording"); + +async function run() { + expect(await startRecordingRender("video" as never, "https://cap.test")).toBe( + true, + ); + await Promise.all(state.afters); +} + +beforeEach(() => { + state.afters = []; + state.sources = []; + state.direct = "started"; + state.workflowStarts = 0; + state.cleared = 0; + vi.spyOn(console, "error").mockImplementation(() => undefined); +}); + +afterEach(() => { + vi.restoreAllMocks(); + vi.useRealTimers(); +}); + +test("a recording whose sources are in starts from the upload, without the workflow", async () => { + await run(); + expect(state.workflowStarts).toBe(0); +}); + +test("a source landing a moment later is waited for", async () => { + state.sources = ["waiting", "waiting", "ready"]; + await run(); + expect(state.workflowStarts).toBe(0); +}); + +test("a recording only an editor worker can prepare goes to the workflow", async () => { + state.direct = "unsupported"; + await run(); + expect(state.workflowStarts).toBe(1); +}); + +test("a failed inline start hands over to the workflow", async () => { + state.direct = new Error("farm unreachable"); + await run(); + expect(state.workflowStarts).toBe(1); + expect(state.cleared).toBe(0); +}); + +test("sources still uploading after the wait go to the workflow", async () => { + vi.useFakeTimers(); + state.sources = Array.from({ length: 1000 }, () => "waiting"); + const started = startRecordingRender("video" as never, "https://cap.test"); + await vi.advanceTimersByTimeAsync(6_000); + expect(await started).toBe(true); + await vi.advanceTimersByTimeAsync(6_000); + await Promise.all(state.afters); + expect(state.workflowStarts).toBe(1); +}); + +test("a replaced render starts nothing", async () => { + state.sources = ["superseded"]; + await run(); + expect(state.workflowStarts).toBe(0); +}); diff --git a/apps/web/__tests__/unit/render-save-status.test.ts b/apps/web/__tests__/unit/render-save-status.test.ts new file mode 100644 index 00000000000..e6b41057f70 --- /dev/null +++ b/apps/web/__tests__/unit/render-save-status.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; +import { + renderStatusPollDelay, + unansweredPollDelay, +} from "@/app/s/[videoId]/_components/render-save-status"; + +describe("render status polling", () => { + it("polls quickly while a render normally reports", () => { + expect(renderStatusPollDelay(0, false)).toBe(3000); + expect(renderStatusPollDelay(119_000, true)).toBe(3000); + }); + + it("polls often for a page showing the render, most often near the end", () => { + const watching = { progress: 0, playable: false }; + expect(renderStatusPollDelay(0, false, watching)).toBe(1000); + expect(renderStatusPollDelay(5_000, true, watching)).toBe(1000); + expect( + renderStatusPollDelay(5_000, true, { progress: 0.99, playable: true }), + ).toBe(500); + expect( + renderStatusPollDelay(59_000, true, { progress: 0.6, playable: false }), + ).toBe(500); + expect(renderStatusPollDelay(60_000, true, watching)).toBe(3000); + }); + + it("backs off for a render that runs on", () => { + expect(renderStatusPollDelay(2 * 60_000, true)).toBe(6000); + expect(renderStatusPollDelay(10 * 60_000, true)).toBe(15_000); + }); + + it("stops asking about a render that never started", () => { + expect(renderStatusPollDelay(60 * 60_000, false)).toBeNull(); + }); + + it("keeps watching a render that is still running after an hour", () => { + expect(renderStatusPollDelay(60 * 60_000, true)).toBe(15_000); + expect(renderStatusPollDelay(3 * 60 * 60_000, true)).toBe(15_000); + }); + + it("stops within a bounded number of requests when no render starts", () => { + let elapsed = 0; + let requests = 0; + for (;;) { + requests++; + const delay = renderStatusPollDelay(elapsed, false); + if (delay === null) break; + elapsed += delay; + } + expect(requests).toBeLessThan(400); + }); + + it("checks rarely while the status route keeps failing, then stops", () => { + expect(unansweredPollDelay(90 * 60_000)).toBe(60_000); + expect(unansweredPollDelay(4 * 60 * 60_000)).toBeNull(); + }); +}); diff --git a/apps/web/__tests__/unit/share-page-loading.test.ts b/apps/web/__tests__/unit/share-page-loading.test.ts index 30dd23513fe..151097319a1 100644 --- a/apps/web/__tests__/unit/share-page-loading.test.ts +++ b/apps/web/__tests__/unit/share-page-loading.test.ts @@ -106,6 +106,7 @@ vi.mock("@/app/s/[videoId]/_components/ShareHeader", () => ({ vi.mock("@/app/s/[videoId]/Share", () => ({ Share: () => null })); import ShareVideoPage from "@/app/s/[videoId]/page"; +import { Share } from "@/app/s/[videoId]/Share"; const createVideo = () => ({ id: "video", @@ -159,18 +160,21 @@ async function renderAuthorizedContent() { if (!isValidElement(content) || typeof content.type !== "function") { throw new Error("Expected authorized share content"); } - return (await (content.type as (props: unknown) => Promise)( - content.props, - )) as ReactElement<{ - children: ReactElement<{ - initialPlaybackUrl?: Promise; - screenshotImageUrl?: string | null; - data: { - sharedOrganizations: unknown[]; - ownerIsOverShareLimit: boolean; - callToAction: { label: string; url: string } | null; - }; - }>; + const rendered = (await ( + content.type as (props: unknown) => Promise + )(content.props)) as ReactElement<{ children: unknown }>; + const share = [rendered.props.children] + .flat() + .find((child) => isValidElement(child) && child.type === Share); + if (!share) throw new Error("Expected the share page"); + return share as ReactElement<{ + initialPlaybackUrl?: Promise; + screenshotImageUrl?: string | null; + data: { + sharedOrganizations: unknown[]; + ownerIsOverShareLimit: boolean; + callToAction: { label: string; url: string } | null; + }; }>; } @@ -194,7 +198,7 @@ describe("share page loading", () => { async (ownerIsPro) => { mocks.ownerIsPro = ownerIsPro; const content = await renderAuthorizedContent(); - const callToAction = content.props.children.props.data.callToAction; + const callToAction = content.props.data.callToAction; if (ownerIsPro) { expect(callToAction).toMatchObject( createVideo().videoSettings.callToAction, @@ -210,7 +214,7 @@ describe("share page loading", () => { { id: "shared", organizationId: "shared", name: "Shared team" }, ]; const content = await renderAuthorizedContent(); - expect(content.props.children.props.data.sharedOrganizations).toEqual([ + expect(content.props.data.sharedOrganizations).toEqual([ { id: "shared", name: "Shared team" }, ]); expect(mocks.select).toHaveBeenCalledTimes(3); @@ -284,17 +288,15 @@ describe("share page loading", () => { it("streams the authorized MP4 URL without waiting for signing to finish", async () => { mocks.playbackUrl.mockReturnValue(new Promise(() => {})); const content = await renderAuthorizedContent(); - expect(content.props.children.props.initialPlaybackUrl).toBeInstanceOf( - Promise, - ); + expect(content.props.initialPlaybackUrl).toBeInstanceOf(Promise); expect(mocks.playbackUrl).toHaveBeenCalledOnce(); }); it("does not sign media hidden by the shareable link quota", async () => { mocks.quota.mockResolvedValue(true); const content = await renderAuthorizedContent(); - expect(await content.props.children.props.initialPlaybackUrl).toBeNull(); - expect(content.props.children.props.data.ownerIsOverShareLimit).toBe(true); + expect(await content.props.initialPlaybackUrl).toBeNull(); + expect(content.props.data.ownerIsOverShareLimit).toBe(true); expect(mocks.playbackUrl).not.toHaveBeenCalled(); }); @@ -307,7 +309,7 @@ describe("share page loading", () => { ])("retains the existing player resolution for %j", async (changes) => { arrangeRows([{ ...createVideo(), ...changes }]); const content = await renderAuthorizedContent(); - expect(content.props.children.props.initialPlaybackUrl).toBeUndefined(); + expect(content.props.initialPlaybackUrl).toBeUndefined(); expect(mocks.playbackUrl).not.toHaveBeenCalled(); }); @@ -325,7 +327,7 @@ describe("share page loading", () => { ); const content = await renderAuthorizedContent(); - expect(content.props.children.props.screenshotImageUrl).toBe( + expect(content.props.screenshotImageUrl).toBe( "https://media.example.com/image.jpg", ); }); diff --git a/apps/web/__tests__/unit/share-video-updates.test.ts b/apps/web/__tests__/unit/share-video-updates.test.ts new file mode 100644 index 00000000000..a026dd03c7b --- /dev/null +++ b/apps/web/__tests__/unit/share-video-updates.test.ts @@ -0,0 +1,284 @@ +// @vitest-environment jsdom + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { shareVideoRevision } from "@/lib/share-video-revision"; +import { + announceShareVideoUpdate, + watchShareVideoUpdates, +} from "@/lib/share-video-updates"; + +function bus() { + const channels = new Set(); + const open = () => { + const channel = Object.assign(new EventTarget(), { closed: false }); + channels.add(channel); + return { + postMessage: (data: unknown) => { + for (const other of channels) + if (other !== channel && !other.closed) + other.dispatchEvent(new MessageEvent("message", { data })); + }, + close: () => { + channel.closed = true; + channels.delete(channel); + }, + addEventListener: channel.addEventListener.bind(channel), + removeEventListener: channel.removeEventListener.bind(channel), + } as unknown as BroadcastChannel; + }; + return { open, count: () => channels.size }; +} + +function statusFetch(revision: () => string | null) { + return vi.fn(async () => + Response.json({ state: "idle", revision: revision() }), + ) as unknown as typeof fetch & ReturnType; +} + +const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); + +function setVisibility(state: "visible" | "hidden") { + Object.defineProperty(document, "visibilityState", { + value: state, + configurable: true, + }); + document.dispatchEvent(new Event("visibilitychange")); +} + +describe("share video revision", () => { + it("tells published files apart without exposing their storage key", () => { + const first = shareVideoRevision({ + type: "webMP4", + outputKey: "owner/video/.recording/outputs/reupload-a/result.mp4", + }); + const second = shareVideoRevision({ + type: "webMP4", + outputKey: "owner/video/.recording/outputs/reupload-b/result.mp4", + }); + expect(first).toBeTypeOf("string"); + expect(first).not.toBe(second); + expect(first).not.toContain("reupload"); + expect( + shareVideoRevision({ + type: "webMP4", + outputKey: "owner/video/.recording/outputs/reupload-a/result.mp4", + }), + ).toBe(first); + expect(shareVideoRevision({ type: "webMP4" })).not.toBe(first); + expect(shareVideoRevision({ type: "desktopMP4" })).not.toBe( + shareVideoRevision({ type: "webMP4" }), + ); + }); + + it("leaves sources a Save never replaces unwatched", () => { + expect(shareVideoRevision({ type: "MediaConvert" })).toBeTruthy(); + expect(shareVideoRevision({ type: "MediaConvert" })).not.toBe( + shareVideoRevision({ type: "webMP4" }), + ); + expect(shareVideoRevision(null)).toBeNull(); + }); +}); + +describe("an open share page after a Save", () => { + const stops: (() => void)[] = []; + const clock = { now: 0 }; + const later = () => { + clock.now += 60_000; + }; + afterEach(() => { + for (const stop of stops.splice(0)) stop(); + setVisibility("visible"); + }); + + it("costs nothing while nobody looks, then checks once on return", async () => { + let now = 0; + const fetchImpl = statusFetch(() => "new"); + const onNewer = vi.fn(); + stops.push( + watchShareVideoUpdates({ + videoId: "video", + revision: "old", + onNewer, + fetchImpl, + now: () => now, + open: bus().open, + }), + ); + await flush(); + expect(fetchImpl).not.toHaveBeenCalled(); + + setVisibility("hidden"); + now = 60_000; + setVisibility("visible"); + window.dispatchEvent(new Event("focus")); + await flush(); + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(fetchImpl.mock.calls[0]?.[0]).toBe( + "/api/videos/video/render-status", + ); + expect(onNewer).toHaveBeenCalledWith("new"); + + now = 70_000; + window.dispatchEvent(new Event("focus")); + await flush(); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it("checks a page left and returned to soon after it loaded", async () => { + let now = 0; + const fetchImpl = statusFetch(() => "new"); + const onNewer = vi.fn(); + stops.push( + watchShareVideoUpdates({ + videoId: "video", + revision: "old", + onNewer, + fetchImpl, + now: () => now, + open: bus().open, + }), + ); + window.dispatchEvent(new Event("focus")); + await flush(); + expect(fetchImpl).not.toHaveBeenCalled(); + + now = 5_000; + window.dispatchEvent(new Event("blur")); + now = 10_000; + window.dispatchEvent(new Event("focus")); + await flush(); + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(onNewer).toHaveBeenCalledWith("new"); + + now = 15_000; + window.dispatchEvent(new Event("blur")); + window.dispatchEvent(new Event("focus")); + await flush(); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it("stays put while the server still publishes the version shown", async () => { + const fetchImpl = statusFetch(() => "same"); + const onNewer = vi.fn(); + stops.push( + watchShareVideoUpdates({ + videoId: "video", + revision: "same", + onNewer, + fetchImpl, + now: () => clock.now, + open: bus().open, + }), + ); + later(); + window.dispatchEvent(new Event("focus")); + await flush(); + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(onNewer).not.toHaveBeenCalled(); + }); + + it("checks straight away when another tab announces a Save of this video", async () => { + const channels = bus(); + const fetchImpl = statusFetch(() => "new"); + const onNewer = vi.fn(); + stops.push( + watchShareVideoUpdates({ + videoId: "video", + revision: "old", + onNewer, + fetchImpl, + now: () => 0, + open: channels.open, + }), + ); + announceShareVideoUpdate("another-video", channels.open); + await flush(); + expect(fetchImpl).not.toHaveBeenCalled(); + + announceShareVideoUpdate("video", channels.open); + await flush(); + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(onNewer).toHaveBeenCalledWith("new"); + expect(channels.count()).toBe(1); + }); + + it("looks again when a Save lands during a check that began before it", async () => { + const channels = bus(); + let published = "old"; + let release: () => void = () => undefined; + const fetchImpl = vi.fn(async () => { + const revision = published; + await new Promise((resolve) => { + release = resolve; + }); + return Response.json({ revision }); + }) as unknown as typeof fetch & ReturnType; + const onNewer = vi.fn(); + stops.push( + watchShareVideoUpdates({ + videoId: "video", + revision: "old", + onNewer, + fetchImpl, + now: () => clock.now, + open: channels.open, + }), + ); + later(); + window.dispatchEvent(new Event("focus")); + await flush(); + published = "new"; + announceShareVideoUpdate("video", channels.open); + release(); + await flush(); + await flush(); + release(); + await flush(); + await flush(); + expect(fetchImpl).toHaveBeenCalledTimes(2); + expect(onNewer).toHaveBeenCalledTimes(1); + expect(onNewer).toHaveBeenCalledWith("new"); + }); + + it("stops listening once the page moves on", async () => { + const channels = bus(); + const fetchImpl = statusFetch(() => "new"); + const onNewer = vi.fn(); + const stop = watchShareVideoUpdates({ + videoId: "video", + revision: "old", + onNewer, + fetchImpl, + now: () => clock.now, + open: channels.open, + }); + stop(); + later(); + window.dispatchEvent(new Event("focus")); + announceShareVideoUpdate("video", channels.open); + await flush(); + expect(fetchImpl).not.toHaveBeenCalled(); + expect(channels.count()).toBe(0); + }); + + it("ignores a status it can't read", async () => { + const fetchImpl = vi.fn( + async () => new Response("nope", { status: 500 }), + ) as unknown as typeof fetch; + const onNewer = vi.fn(); + stops.push( + watchShareVideoUpdates({ + videoId: "video", + revision: "old", + onNewer, + fetchImpl, + now: () => clock.now, + open: bus().open, + }), + ); + later(); + window.dispatchEvent(new Event("focus")); + await flush(); + expect(onNewer).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/__tests__/unit/video-preview-route.test.ts b/apps/web/__tests__/unit/video-preview-route.test.ts new file mode 100644 index 00000000000..e77c96ea19d --- /dev/null +++ b/apps/web/__tests__/unit/video-preview-route.test.ts @@ -0,0 +1,101 @@ +import { Effect } from "effect"; +import { NextRequest } from "next/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + source: {} as Record, + headObject: vi.fn(), +})); + +vi.mock("@cap/web-backend", async () => { + const { Effect, Option } = await import("effect"); + return { + provideOptionalAuth: (effect: T) => effect, + Videos: Effect.succeed({ + getByIdForViewing: () => + Effect.succeed( + Option.some([ + { id: "video", ownerId: "owner", source: mocks.source }, + ]), + ), + }), + Storage: { + getAccessForVideo: (video: { source: { previewKey?: string } }) => + Effect.succeed([ + { + headObject: (key: string) => { + const resolved = + key === "owner/video/preview/animated-preview.gif" + ? (video.source.previewKey ?? key) + : key; + mocks.headObject(resolved); + return Effect.succeed({ ContentLength: 1 }); + }, + getSignedObjectUrl: (key: string) => + Effect.succeed( + `https://storage.test/${video.source.previewKey ?? key}`, + ), + }, + ]), + }, + }; +}); +vi.mock("@cap/web-backend/src/Storage/index", () => ({ Storage: {} })); +vi.mock("@/lib/workflow-runtime", () => ({ runWorkflowPromise: vi.fn() })); +vi.mock("@/lib/server", () => ({ runPromise: Effect.runPromise })); + +const { GET } = await import("@/app/api/video/preview/route"); + +const render = + "owner/video/.recording/render/11111111-1111-4111-8111-111111111111/result"; +const request = (fallback: string) => + new NextRequest( + `https://cap.test/api/video/preview?videoId=video&fallback=${fallback}`, + ); + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("video preview GIF route", () => { + it("serves the upload's GIF while it is still the share video", async () => { + mocks.source = { type: "webMP4" }; + + const response = await GET(request("none")); + + expect(response.status).toBe(302); + expect(response.headers.get("location")).toBe( + "https://storage.test/owner/video/preview/animated-preview.gif", + ); + expect(response.headers.get("cache-control")).toBe("private, max-age=60"); + }); + + it("hides the upload's GIF once a render replaced the share video", async () => { + mocks.source = { type: "webMP4", outputKey: `${render}.mp4` }; + + const none = await GET(request("none")); + expect(none.status).toBe(404); + + const og = await GET(request("og")); + expect(og.status).toBe(302); + expect(og.headers.get("location")).toBe( + "https://cap.test/api/video/og?videoId=video", + ); + expect(mocks.headObject).not.toHaveBeenCalled(); + }); + + it("serves the GIF made from the render", async () => { + mocks.source = { + type: "webMP4", + outputKey: `${render}.mp4`, + previewKey: `${render}/preview.gif`, + }; + + const response = await GET(request("none")); + + expect(response.status).toBe(302); + expect(response.headers.get("location")).toBe( + `https://storage.test/${render}/preview.gif`, + ); + }); +}); diff --git a/apps/web/__tests__/unit/web-recorder-camera-memory.test.ts b/apps/web/__tests__/unit/web-recorder-camera-memory.test.ts new file mode 100644 index 00000000000..05370fbc2a9 --- /dev/null +++ b/apps/web/__tests__/unit/web-recorder-camera-memory.test.ts @@ -0,0 +1,658 @@ +// @vitest-environment jsdom + +import { Exit } from "effect"; +import { act, createElement } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + displayStream: vi.fn(), + cameraStream: vi.fn(), + micStream: vi.fn(), + createVideo: vi.fn(), + deleteVideo: vi.fn(), + initiateMultipart: vi.fn(), + refresh: vi.fn(), + push: vi.fn(), + refetchQueries: vi.fn(), + warning: vi.fn(), + uploaders: [] as Array<{ + subpath: string; + handleChunk: ReturnType; + finalize: ReturnType; + cancel: ReturnType; + }>, + failedScreenUpload: false, + sidecars: [] as Array<{ + kind: "mic" | "systemAudio"; + start: ReturnType; + pause: ReturnType; + resume: ReturnType; + stop: ReturnType; + finalize: ReturnType; + markUploadedBackup: ReturnType; + disposeBackup: ReturnType; + recoverBlob: ReturnType; + abortUploadRetainSpool: ReturnType; + }>, +})); + +vi.mock("@cap/recorder-core", () => ({ + AudioRecordingSidecar: { + async create(options: { kind: "mic" | "systemAudio" }) { + let backupAvailable = true; + const sidecar = { + kind: options.kind, + start: vi.fn(), + pause: vi.fn(), + resume: vi.fn(), + stop: vi.fn(async () => 8), + finalize: vi.fn(async () => undefined), + markUploadedBackup: vi.fn(), + disposeBackup: vi.fn(async () => { + backupAvailable = false; + }), + recoverBlob: vi.fn(async () => + backupAvailable ? new Blob(["mic body"]) : null, + ), + abortUploadRetainSpool: vi.fn(async () => undefined), + cancel: vi.fn(async () => undefined), + metadata: { kind: options.kind }, + }; + mocks.sidecars.push(sidecar); + return sidecar; + }, + }, +})); + +vi.mock("@cap/recorder-core/capture-streams", () => ({ + acquireDisplayStream: mocks.displayStream, + acquireCameraStream: mocks.cameraStream, + acquireMicStream: mocks.micStream, + createAudioMixer: vi.fn(), + getCaptureErrorMessage: (error: unknown) => String(error), +})); +vi.mock("@cap/recorder-core/recorder-utils", () => ({ + detectCapabilities: () => ({ + assessed: true, + hasMediaRecorder: true, + hasUserMedia: true, + hasDisplayMedia: true, + }), + selectRecordingPipeline: () => ({ + mode: "streaming", + mimeType: "video/webm;codecs=vp8", + fileExtension: "webm", + }), + openShareUrlInNewTab: () => false, +})); +vi.mock("@cap/recorder-core/recording-spool", () => ({ + canUseRecordingSpool: () => false, + createRecordingSessionId: () => "paired-recording-session", + deleteRecoveredRecordingSpool: vi.fn(), + RECORDING_SPOOL_HEARTBEAT_INTERVAL_MS: 10000, + RecordingSpool: { create: vi.fn() }, +})); +vi.mock("@cap/recorder-core/instant-mp4-uploader", () => ({ + initiateMultipartUpload: mocks.initiateMultipart, + MultipartCompletionUncertainError: class extends Error {}, + InstantRecordingUploader: class { + handleChunk = vi.fn(); + finalize: ReturnType; + uploadRemaining = vi.fn(async () => undefined); + cancel = vi.fn(async () => undefined); + constructor(options: { subpath: string }) { + this.finalize = vi.fn(async () => { + if (mocks.failedScreenUpload && options.subpath === "raw-upload.webm") { + throw new Error("Screen upload failed"); + } + }); + mocks.uploaders.push({ + subpath: options.subpath, + handleChunk: this.handleChunk, + finalize: this.finalize, + cancel: this.cancel, + }); + } + getProcessingStarted() { + return true; + } + }, +})); +vi.mock("@/lib/EffectRuntime", () => ({ + useRpcClient: () => ({ + VideoInstantCreate: mocks.createVideo, + VideoDelete: mocks.deleteVideo, + }), + useEffectMutation: ({ + mutationFn, + }: { + mutationFn: (variables: unknown) => Promise; + }) => ({ mutateAsync: mutationFn }), +})); +vi.mock("@tanstack/react-query", () => ({ + useQueryClient: () => ({ refetchQueries: mocks.refetchQueries }), +})); +vi.mock("next/navigation", () => ({ + useRouter: () => ({ refresh: mocks.refresh, push: mocks.push }), +})); +vi.mock("@/actions/video/trigger-instant-recording-processing", () => ({ + triggerInstantRecordingProcessing: vi.fn(), +})); +vi.mock("@/actions/video/upload", () => ({ + createVideoAndGetUploadUrl: vi.fn(), +})); +vi.mock("@/app/(org)/dashboard/caps/UploadingContext", () => ({ + useUploadingContext: () => ({ setUploadStatus: vi.fn() }), +})); +vi.mock("sonner", () => ({ + toast: { + dismiss: vi.fn(), + error: vi.fn(), + info: vi.fn(), + success: vi.fn(), + warning: mocks.warning, + }, +})); +vi.mock( + "@/app/(org)/dashboard/caps/components/web-recorder-dialog/useSurfaceDetection", + () => ({ + useSurfaceDetection: () => ({ scheduleSurfaceDetection: vi.fn() }), + }), +); +vi.mock( + "@/app/(org)/dashboard/caps/components/web-recorder-dialog/recovered-recording-cache", + () => ({ + loadRecoveredRecordingSpools: vi.fn(async () => []), + removeRecoveredRecordingSpoolFromCache: vi.fn(), + }), +); + +import { useMediaRecorderSetup } from "@/app/(org)/dashboard/caps/components/web-recorder-dialog/useMediaRecorderSetup"; +import { useWebRecorder } from "@/app/(org)/dashboard/caps/components/web-recorder-dialog/useWebRecorder"; + +class FakeTrack extends EventTarget { + constructor(readonly kind: "video" | "audio" = "video") { + super(); + } + readonly readyState = "live"; + readonly stop = vi.fn(); + readonly clones: FakeTrack[] = []; + clone() { + const copy = new FakeTrack(this.kind); + this.clones.push(copy); + return copy; + } + getSettings() { + return { width: 1280, height: 720, frameRate: 30 }; + } +} + +class FakeStream { + constructor(readonly tracks: FakeTrack[]) {} + getTracks() { + return this.tracks; + } + getVideoTracks() { + return this.tracks.filter((track) => track.kind === "video"); + } + getAudioTracks() { + return this.tracks.filter((track) => track.kind === "audio"); + } +} + +class FakeRecorder extends EventTarget { + static instances: FakeRecorder[] = []; + state: RecordingState = "inactive"; + readonly stopCalled = vi.fn(); + ondataavailable: ((event: BlobEvent) => void) | null = null; + onstop: (() => void) | null = null; + onerror: ((event: Event) => void) | null = null; + constructor( + readonly stream: FakeStream, + readonly options: { mimeType: string }, + ) { + super(); + FakeRecorder.instances.push(this); + } + start() { + this.state = "recording"; + } + pause() { + if (this.state !== "recording") throw new Error("Recorder is not active"); + this.state = "paused"; + } + resume() { + if (this.state !== "paused") throw new Error("Recorder is not paused"); + this.state = "recording"; + } + stop() { + if (this.state === "inactive") throw new Error("Already stopped"); + this.stopCalled(); + this.state = "inactive"; + this.emitData(new Blob(["final"], { type: this.options.mimeType })); + this.dispatchEvent(new Event("stop")); + this.onstop?.(); + } + emitData(data: Blob) { + const event = new Event("dataavailable") as BlobEvent; + Object.defineProperty(event, "data", { value: data }); + this.dispatchEvent(event); + this.ondataavailable?.(event); + } +} + +let root: Root; +let container: HTMLDivElement; +let latest: ReturnType; +let displayTrack: FakeTrack; +let cameraTrack: FakeTrack; +let micTrack: FakeTrack; +let enableMic = false; + +let beforeRecordingStarts: (() => Promise) | undefined; +let takeSharedDisplayStream: (() => MediaStream | null) | undefined; + +function Harness() { + latest = useWebRecorder({ + beforeRecordingStarts, + takeSharedDisplayStream, + organisationId: "organisation", + selectedMicId: enableMic ? "mic" : null, + micEnabled: enableMic, + systemAudioEnabled: false, + recordingMode: "fullscreen", + selectedCameraId: "camera", + getCameraPreviewStream: () => null, + isProUser: true, + }); + return null; +} + +async function waitFor(assertion: () => void) { + let lastError: unknown; + for (let attempt = 0; attempt < 100; attempt++) { + try { + assertion(); + return; + } catch (cause) { + lastError = cause; + } + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + }); + } + throw lastError; +} + +beforeEach(async () => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + vi.stubGlobal("MediaStream", FakeStream); + vi.stubGlobal("MediaRecorder", FakeRecorder); + vi.stubGlobal("URL", { + createObjectURL: vi.fn(() => "blob:recording"), + revokeObjectURL: vi.fn(), + }); + FakeRecorder.instances = []; + mocks.uploaders.length = 0; + mocks.sidecars.length = 0; + mocks.failedScreenUpload = false; + mocks.warning.mockClear(); + enableMic = false; + displayTrack = new FakeTrack(); + cameraTrack = new FakeTrack(); + micTrack = new FakeTrack("audio"); + mocks.displayStream.mockResolvedValue(new FakeStream([displayTrack])); + mocks.cameraStream.mockResolvedValue(new FakeStream([cameraTrack])); + mocks.micStream.mockResolvedValue(new FakeStream([micTrack])); + mocks.createVideo.mockResolvedValue( + Exit.succeed({ + id: "video", + shareUrl: "https://cap.example/s/video", + upload: {}, + }), + ); + mocks.deleteVideo.mockResolvedValue(Exit.succeed(undefined)); + mocks.initiateMultipart.mockResolvedValue({ + uploadId: "upload", + provider: "s3", + }); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); + await act(async () => root.render(createElement(Harness))); + await act(async () => latest.startRecording()); + await waitFor(() => expect(latest.phase).toBe("recording")); +}); + +afterEach(async () => { + beforeRecordingStarts = undefined; + takeSharedDisplayStream = undefined; + await act(async () => root.unmount()); + container.remove(); + vi.unstubAllGlobals(); +}); + +async function restartWithMicAudio() { + await act(async () => latest.stopRecording()); + await act(async () => root.unmount()); + mocks.uploaders.length = 0; + mocks.sidecars.length = 0; + FakeRecorder.instances = []; + enableMic = true; + root = createRoot(container); + await act(async () => root.render(createElement(Harness))); + await act(async () => latest.startRecording()); + await waitFor(() => expect(latest.phase).toBe("recording")); + const sidecar = mocks.sidecars[0]; + if (!sidecar) throw new Error("Microphone sidecar did not start"); + return sidecar; +} + +test("microphone capture stays paired with screen and camera through stop", async () => { + const sidecar = await restartWithMicAudio(); + await act(async () => latest.pauseRecording()); + await waitFor(() => expect(latest.phase).toBe("paused")); + expect(sidecar.pause).toHaveBeenCalledOnce(); + await act(async () => latest.resumeRecording()); + await waitFor(() => expect(latest.phase).toBe("recording")); + expect(sidecar.resume).toHaveBeenCalledOnce(); + await act(async () => latest.stopRecording()); + await waitFor(() => expect(latest.phase).toBe("completed")); + expect(mocks.push).toHaveBeenCalledWith("/s/video"); + expect(sidecar.kind).toBe("mic"); + expect(sidecar.start).toHaveBeenCalledOnce(); + expect(sidecar.stop).toHaveBeenCalledOnce(); + expect(sidecar.finalize).toHaveBeenCalledOnce(); + expect(sidecar.disposeBackup).toHaveBeenCalledOnce(); + expect( + FakeRecorder.instances.map( + (recorder) => recorder.stopCalled.mock.calls.length, + ), + ).toEqual([1, 1]); + expect(micTrack.stop).toHaveBeenCalled(); +}); + +test("failed screen upload offers the separate microphone recording for recovery", async () => { + const sidecar = await restartWithMicAudio(); + mocks.failedScreenUpload = true; + await act(async () => latest.stopRecording()); + await waitFor(() => expect(latest.phase).toBe("error")); + expect(latest.audioErrorDownloads.map((source) => source.kind)).toEqual([ + "mic", + ]); + expect(latest.audioErrorDownloads[0]?.download).toBeDefined(); + expect(sidecar.recoverBlob).toHaveBeenCalled(); + expect(sidecar.disposeBackup).not.toHaveBeenCalled(); + expect(sidecar.abortUploadRetainSpool).toHaveBeenCalledOnce(); +}); + +async function overflowCameraBackup() { + const camera = FakeRecorder.instances[1]; + if (!camera) throw new Error("Camera recorder did not start"); + const chunk = new Blob([new Uint8Array(1024 * 1024)], { + type: "video/webm;codecs=vp8", + }); + await act(async () => { + for (let second = 0; second < 257; second++) { + camera.emitData(chunk); + } + }); + expect(latest.phase).toBe("recording"); + expect(FakeRecorder.instances[0]?.stopCalled).not.toHaveBeenCalled(); + await act(async () => latest.stopRecording()); + await waitFor(() => { + expect(latest.phase).toMatch(/completed|error/); + }); + return camera; +} + +async function overflowScreenBackup() { + const screen = FakeRecorder.instances[0]; + if (!screen) throw new Error("Screen recorder did not start"); + const chunk = new Blob([new Uint8Array(1024 * 1024)], { + type: "video/webm;codecs=vp8", + }); + await act(async () => { + for (let second = 0; second < 257; second++) { + screen.emitData(chunk); + } + }); + await waitFor(() => expect(latest.phase).toBe("completed")); + return screen; +} + +test("a long camera capture streams after its backup limit and finalizes the paired upload", async () => { + const camera = await overflowCameraBackup(); + const screen = FakeRecorder.instances[0]; + expect(screen?.stopCalled).toHaveBeenCalledOnce(); + expect(camera.stopCalled).toHaveBeenCalledOnce(); + expect(displayTrack.stop).toHaveBeenCalled(); + expect(cameraTrack.stop).toHaveBeenCalled(); + expect(latest.phase).toBe("completed"); + expect(mocks.warning).toHaveBeenCalledWith( + "Camera backup is unavailable. Recording continues, but camera video cannot be recovered if upload fails.", + ); + expect(mocks.uploaders.map((uploader) => uploader.subpath)).toEqual([ + "raw-upload.webm", + "camera-upload.webm", + ]); + expect( + mocks.uploaders.every( + (uploader) => uploader.finalize.mock.calls.length === 1, + ), + ).toBe(true); + expect( + mocks.uploaders.find( + (uploader) => uploader.subpath === "camera-upload.webm", + )?.handleChunk, + ).toHaveBeenCalledTimes(258); + expect(latest.cameraErrorDownload).toBeNull(); +}); + +test("a long screen capture stops both clips at its memory limit", async () => { + const screen = await overflowScreenBackup(); + const camera = FakeRecorder.instances[1]; + expect(screen.stopCalled).toHaveBeenCalledOnce(); + expect(camera?.stopCalled).toHaveBeenCalledOnce(); + expect(displayTrack.stop).toHaveBeenCalled(); + expect(cameraTrack.stop).toHaveBeenCalled(); + expect(mocks.warning).toHaveBeenCalledWith( + "Recording memory backup reached its limit. Finishing now.", + ); + expect( + mocks.uploaders.find((uploader) => uploader.subpath === "raw-upload.webm") + ?.handleChunk, + ).toHaveBeenCalledTimes(258); + expect( + mocks.uploaders.every( + (uploader) => uploader.finalize.mock.calls.length === 1, + ), + ).toBe(true); +}); + +test("an upload failure before the memory limit keeps both complete recovery clips", async () => { + mocks.failedScreenUpload = true; + const camera = FakeRecorder.instances[1]; + if (!camera) throw new Error("Camera recorder did not start"); + const cameraChunk = new Blob(["camera body"], { + type: "video/webm;codecs=vp8", + }); + await act(async () => { + camera.emitData(cameraChunk); + await latest.stopRecording(); + }); + await waitFor(() => expect(latest.phase).toBe("error")); + expect(latest.errorDownload).not.toBeNull(); + expect(latest.cameraErrorDownload).not.toBeNull(); + const recoveryBlobs = vi + .mocked(URL.createObjectURL) + .mock.calls.map(([blob]) => blob as Blob); + expect(recoveryBlobs.map((blob) => blob.size)).toEqual([ + 5, + cameraChunk.size + 5, + ]); +}); + +test("an upload failure after overflow never offers a truncated camera download", async () => { + mocks.failedScreenUpload = true; + const camera = await overflowCameraBackup(); + expect(FakeRecorder.instances[0]?.stopCalled).toHaveBeenCalledOnce(); + expect(camera.stopCalled).toHaveBeenCalledOnce(); + expect(latest.phase).toBe("error"); + expect(latest.errorDownload).not.toBeNull(); + expect(latest.cameraErrorDownload).toBeNull(); + expect( + mocks.uploaders.find((uploader) => uploader.subpath === "raw-upload.webm") + ?.cancel, + ).toHaveBeenCalledOnce(); +}); + +test("screen fallback overflow remains visible after the recorder stops", async () => { + const setupRef = { + current: null as ReturnType | null, + }; + function SetupHarness() { + setupRef.current = useMediaRecorderSetup(); + return null; + } + const setupContainer = document.createElement("div"); + document.body.append(setupContainer); + const setupRoot = createRoot(setupContainer); + try { + await act(async () => setupRoot.render(createElement(SetupHarness))); + const setup = setupRef.current; + if (!setup) throw new Error("Recorder setup did not mount"); + const strategy = { mode: "capped" as const, maxBytes: 5 }; + setup.setLocalRecordingStrategy(strategy); + const data = new Event("dataavailable") as BlobEvent; + Object.defineProperty(data, "data", { value: new Blob(["overflow"]) }); + expect(setup.onRecorderDataAvailable(data)).toBe(true); + setup.onRecorderStop(); + expect(setup.localRecordingOverflowedRef.current).toBe(true); + expect( + setup.replaceLocalRecording( + [], + strategy, + setup.localRecordingOverflowedRef.current, + ), + ).toBe(true); + } finally { + await act(async () => setupRoot.unmount()); + setupContainer.remove(); + } +}); + +test("recording waits for the countdown while setup finishes underneath it", async () => { + await act(async () => latest.stopRecording()); + await act(async () => root.unmount()); + FakeRecorder.instances = []; + mocks.createVideo.mockClear(); + let finishCountdown = () => {}; + beforeRecordingStarts = () => + new Promise((resolve) => { + finishCountdown = () => resolve(true); + }); + root = createRoot(container); + await act(async () => root.render(createElement(Harness))); + let started: Promise | undefined; + await act(async () => { + started = latest.startRecording(); + }); + await waitFor(() => expect(mocks.createVideo).toHaveBeenCalledOnce()); + expect(latest.phase).toBe("idle"); + expect( + FakeRecorder.instances.some((recorder) => recorder.state !== "inactive"), + ).toBe(false); + await act(async () => { + finishCountdown(); + await started; + }); + await waitFor(() => expect(latest.phase).toBe("recording")); + expect( + FakeRecorder.instances.every((recorder) => recorder.state === "recording"), + ).toBe(true); +}); + +test("cancelling the countdown starts nothing and removes the pending video", async () => { + await act(async () => latest.stopRecording()); + await act(async () => root.unmount()); + FakeRecorder.instances = []; + mocks.uploaders.length = 0; + mocks.deleteVideo.mockClear(); + beforeRecordingStarts = async () => false; + root = createRoot(container); + await act(async () => root.render(createElement(Harness))); + await act(async () => latest.startRecording()); + expect(latest.phase).toBe("idle"); + expect( + FakeRecorder.instances.some((recorder) => recorder.state !== "inactive"), + ).toBe(false); + expect(mocks.deleteVideo).toHaveBeenCalledOnce(); + expect(mocks.uploaders).not.toHaveLength(0); + expect( + mocks.uploaders.every((uploader) => uploader.cancel.mock.calls.length > 0), + ).toBe(true); +}); + +test("a screen shared before starting records without opening the picker again", async () => { + await act(async () => latest.stopRecording()); + await act(async () => root.unmount()); + FakeRecorder.instances = []; + mocks.displayStream.mockClear(); + const sharedTrack = new FakeTrack(); + const shared = new FakeStream([sharedTrack]); + takeSharedDisplayStream = vi.fn(() => shared as unknown as MediaStream); + root = createRoot(container); + await act(async () => root.render(createElement(Harness))); + await act(async () => latest.startRecording()); + await waitFor(() => expect(latest.phase).toBe("recording")); + expect(takeSharedDisplayStream).toHaveBeenCalledOnce(); + expect(mocks.displayStream).not.toHaveBeenCalled(); + expect(latest.getActiveDisplayStream()).toBe(shared); + expect( + FakeRecorder.instances.some((recorder) => + recorder.stream.getVideoTracks().includes(sharedTrack), + ), + ).toBe(true); +}); + +test("restarting records the shared screen again without reopening the picker", async () => { + const firstVideo = latest.videoId; + mocks.displayStream.mockClear(); + mocks.deleteVideo.mockClear(); + mocks.createVideo.mockResolvedValue( + Exit.succeed({ + id: "second-video", + shareUrl: "https://cap.example/s/second-video", + upload: {}, + }), + ); + await act(async () => latest.restartRecording()); + await waitFor(() => expect(latest.phase).toBe("recording")); + expect(mocks.displayStream).not.toHaveBeenCalled(); + expect(mocks.deleteVideo).toHaveBeenCalledOnce(); + expect(latest.videoId).toBe("second-video"); + expect(latest.videoId).not.toBe(firstVideo); + const [kept] = displayTrack.clones; + expect(kept).toBeDefined(); + expect(displayTrack.stop).toHaveBeenCalled(); + expect(kept?.stop).not.toHaveBeenCalled(); + const live = FakeRecorder.instances.filter( + (recorder) => recorder.state === "recording", + ); + expect(live).toHaveLength(2); + expect(live[0]?.stream.getVideoTracks()).toContain(kept); +}); + +test("a restart whose countdown is cancelled releases the kept screen", async () => { + beforeRecordingStarts = async () => false; + await act(async () => root.render(createElement(Harness))); + await act(async () => latest.restartRecording()); + expect(latest.phase).toBe("idle"); + expect(latest.isRestarting).toBe(false); + expect( + FakeRecorder.instances.some((recorder) => recorder.state !== "inactive"), + ).toBe(false); + expect(displayTrack.clones[0]?.stop).toHaveBeenCalled(); +}); diff --git a/apps/web/__tests__/unit/web-recorder-reliability.test.ts b/apps/web/__tests__/unit/web-recorder-reliability.test.ts index a040236fa1b..d4f875ff70c 100644 --- a/apps/web/__tests__/unit/web-recorder-reliability.test.ts +++ b/apps/web/__tests__/unit/web-recorder-reliability.test.ts @@ -15,11 +15,17 @@ const mocks = vi.hoisted(() => ({ finalize: vi.fn(), open: vi.fn(), refresh: vi.fn(), + push: vi.fn(), status: vi.fn(), spoolCreate: vi.fn(), recovered: vi.fn(), + recoveredAudio: vi.fn(async () => {}), + sidecars: [] as Array<{ + abortUploadRetainSpool: ReturnType; + recoverBlob: ReturnType; + }>, pipeline: { - mode: "streaming-webm", + mode: "streaming", mimeType: "video/webm", fileExtension: "webm", }, @@ -32,6 +38,41 @@ const mocks = vi.hoisted(() => ({ }>, })); +// The branch records the microphone and system audio as their own tracks; +// these tests cover the screen and camera recording, so audio tracks are stubs. +vi.mock("@cap/recorder-core", async (original) => ({ + ...(await original()), + uploadRecoveredAudioSidecar: mocks.recoveredAudio, + AudioRecordingSidecar: { + create: vi.fn(async ({ kind }: { kind: "mic" | "systemAudio" }) => { + const sidecar = { + metadata: { + kind, + sessionId: `audio-${kind}`, + mimeType: "audio/webm", + subpath: `${kind}.webm`, + offsetMs: 0, + recordedBytes: 1, + }, + isUploadCompleted: false, + start: vi.fn(), + pause: vi.fn(), + resume: vi.fn(), + stop: vi.fn(async () => 1), + finalize: vi.fn(async () => {}), + cancel: vi.fn(async () => {}), + abortUploadRetainSpool: vi.fn(async () => {}), + recoverBlob: vi.fn( + async () => new Blob([kind], { type: "audio/webm" }), + ), + markUploadedBackup: vi.fn(), + disposeBackup: vi.fn(async () => {}), + }; + mocks.sidecars.push(sidecar); + return sidecar; + }), + }, +})); vi.mock("@cap/recorder-core/capture-streams", () => ({ acquireCameraStream: mocks.acquire, acquireDisplayStream: mocks.acquire, @@ -60,6 +101,7 @@ vi.mock("@cap/recorder-core/instant-mp4-uploader", async (original) => ({ initiateMultipartUpload: mocks.initiate, InstantRecordingUploader: class { handleChunk = vi.fn(); + uploadRemaining = vi.fn(async () => {}); finalize = mocks.finalize; cancel = vi.fn(async () => {}); suspend = vi.fn(); @@ -81,7 +123,7 @@ vi.mock("@/lib/EffectRuntime", () => ({ }) => ({ mutateAsync: mutationFn }), })); vi.mock("next/navigation", () => ({ - useRouter: () => ({ refresh: mocks.refresh }), + useRouter: () => ({ refresh: mocks.refresh, push: mocks.push }), })); vi.mock("sonner", () => ({ toast: { @@ -109,7 +151,7 @@ vi.mock( import { MultipartCompletionUncertainError } from "@cap/recorder-core/instant-mp4-uploader"; import { MicrophoneUnavailablePrompt } from "@/app/(org)/dashboard/caps/components/web-recorder-dialog/MicrophoneUnavailablePrompt"; -import type { RecordingMode } from "@/app/(org)/dashboard/caps/components/web-recorder-dialog/RecordingModeSelector"; +import type { RecordingMode } from "@/app/(org)/dashboard/caps/components/web-recorder-dialog/recording-mode"; import { useWebRecorder } from "@/app/(org)/dashboard/caps/components/web-recorder-dialog/useWebRecorder"; class Track extends EventTarget { @@ -146,6 +188,9 @@ class Recorder extends EventTarget { }); Object.defineProperty(event, "target", { value: this }); this.ondataavailable?.(event as BlobEvent); + this.dispatchEvent( + Object.assign(new Event("dataavailable"), { data: event.data }), + ); } stop() { this.state = "inactive"; @@ -154,6 +199,7 @@ class Recorder extends EventTarget { const event = new Event("stop"); Object.defineProperty(event, "target", { value: this }); this.onstop?.(event); + this.dispatchEvent(new Event("stop")); }); } } @@ -181,6 +227,16 @@ const makeSpool = () => { ), touch: vi.fn(async () => {}), dispose: vi.fn(async () => {}), + markUploaded: vi.fn(), + getUnwrittenChunks: vi.fn(() => []), + recoverPersistedBlob: vi.fn(async () => + chunks.length + ? new Blob(chunks, { type: mocks.pipeline.mimeType }) + : null, + ), + get totalBytes() { + return chunks.reduce((total, chunk) => total + chunk.size, 0); + }, }; }; @@ -192,6 +248,7 @@ describe("web recording upload recovery", () => { let track: Track; let micEnabled: boolean; let recordingMode: RecordingMode; + let selectedCameraId: string | null; let systemAudioEnabled: boolean; let rerender: () => Promise; @@ -200,12 +257,14 @@ describe("web recording upload recovery", () => { vi.useFakeTimers(); micEnabled = false; recordingMode = "camera"; + selectedCameraId = "camera"; systemAudioEnabled = false; mocks.mic.mockReset(); mocks.uploaders.length = 0; + mocks.sidecars.length = 0; Recorder.instances = []; mocks.pipeline = { - mode: "streaming-webm", + mode: "streaming", mimeType: "video/webm", fileExtension: "webm", }; @@ -241,7 +300,8 @@ describe("web recording upload recovery", () => { micEnabled, systemAudioEnabled, recordingMode, - selectedCameraId: "camera", + selectedCameraId, + getCameraPreviewStream: () => null, isProUser: true, }); return recorder.isMicrophoneUnavailable @@ -301,7 +361,7 @@ describe("web recording upload recovery", () => { }); expect(recorder.phase).toBe("completed"); expect(spool.dispose).toHaveBeenCalledOnce(); - expect(mocks.open).toHaveBeenCalledWith("https://cap.so/s/video"); + expect(mocks.push).toHaveBeenCalledWith("/s/video"); }, ); @@ -571,6 +631,8 @@ describe("web recording upload recovery", () => { "continues the same %s capture only after choosing to record without a microphone", async (mode) => { recordingMode = mode; + // A screen recording would also record the selected camera. + if (mode === "tab") selectedCameraId = null; const { starting } = await startWithUnavailableMicrophone(); await act(async () => recorder.startRecording()); expect(mocks.acquire).toHaveBeenCalledOnce(); @@ -585,12 +647,112 @@ describe("web recording upload recovery", () => { expect(Recorder.instances[0]?.stream.getVideoTracks()).toEqual([track]); await act(async () => recorder.stopRecording()); expect(recorder.phase).toBe("completed"); - expect(recorder.completedShareUrl).toBe("https://cap.so/s/video"); + expect(mocks.push).toHaveBeenCalledWith("/s/video"); + expect(recorder.completedShareUrl).toBe( + `${window.location.origin}/s/video`, + ); }, ); + it("retries a paired take from its camera and microphone backups after the upload fails", async () => { + recordingMode = "tab"; + micEnabled = true; + const mic = new Track(); + mic.kind = "audio"; + mocks.mic.mockResolvedValue(new Stream([mic])); + mocks.acquire + .mockResolvedValueOnce(new Stream([track])) + .mockResolvedValueOnce(new Stream([new Track()])); + const cameraSpool = makeSpool(); + mocks.spoolCreate + .mockResolvedValueOnce(spool) + .mockResolvedValueOnce(cameraSpool); + await rerender(); + await act(async () => recorder.startRecording()); + expect(recorder.phase).toBe("recording"); + const [screenRecorder, cameraRecorder] = Recorder.instances; + screenRecorder?.chunk("first"); + cameraRecorder?.chunk("face"); + expect(mocks.uploaders).toHaveLength(2); + expect(mocks.sidecars).toHaveLength(1); + + mocks.finalize.mockRejectedValueOnce(new Error("Network failed")); + await act(async () => recorder.stopRecording()); + expect(recorder.phase).toBe("error"); + expect(recorder.canRetryUpload).toBe(true); + expect(mocks.delete).not.toHaveBeenCalled(); + expect(mocks.uploaders[0]?.cancel).toHaveBeenCalled(); + expect(mocks.uploaders[1]?.cancel).toHaveBeenCalled(); + expect(mocks.sidecars[0]?.abortUploadRetainSpool).toHaveBeenCalled(); + expect(cameraSpool.dispose).not.toHaveBeenCalled(); + + mocks.finalize.mockResolvedValue(undefined); + await act(async () => recorder.retryUpload()); + expect(recorder.phase).toBe("completed"); + expect(mocks.uploaders).toHaveLength(4); + const retried = mocks.finalize.mock.calls + .slice(-2) + .map(([options]) => options as { finalBlob: Blob; subpath: string }); + expect( + await Promise.all( + retried.map(async ({ finalBlob, subpath }) => [ + subpath, + await finalBlob.text(), + ]), + ), + ).toEqual([ + ["camera-upload.webm", "facetail"], + ["raw-upload.webm", "firsttail"], + ]); + expect(mocks.recoveredAudio).toHaveBeenCalledWith( + expect.objectContaining({ + videoId: "video", + screenSubpath: "raw-upload.webm", + }), + ); + expect(cameraSpool.dispose).toHaveBeenCalledOnce(); + }); + + it("retries only the screen when the paired camera already uploaded", async () => { + recordingMode = "tab"; + micEnabled = true; + const mic = new Track(); + mic.kind = "audio"; + mocks.mic.mockResolvedValue(new Stream([mic])); + mocks.acquire + .mockResolvedValueOnce(new Stream([track])) + .mockResolvedValueOnce(new Stream([new Track()])); + const cameraSpool = makeSpool(); + mocks.spoolCreate + .mockResolvedValueOnce(spool) + .mockResolvedValueOnce(cameraSpool); + await rerender(); + await act(async () => recorder.startRecording()); + const [screenRecorder, cameraRecorder] = Recorder.instances; + screenRecorder?.chunk("first"); + cameraRecorder?.chunk("face"); + + mocks.finalize + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new Error("Network failed")); + await act(async () => recorder.stopRecording()); + expect(recorder.phase).toBe("error"); + expect(recorder.canRetryUpload).toBe(true); + const attempted = mocks.finalize.mock.calls.length; + + mocks.finalize.mockResolvedValue(undefined); + await act(async () => recorder.retryUpload()); + expect(recorder.phase).toBe("completed"); + expect( + mocks.finalize.mock.calls + .slice(attempted) + .map(([options]) => (options as { subpath: string }).subpath), + ).toEqual(["raw-upload.webm"]); + }); + it("retains captured system audio when continuing without the microphone", async () => { recordingMode = "tab"; + selectedCameraId = null; systemAudioEnabled = true; const systemAudio = new Track(); systemAudio.kind = "audio"; @@ -714,7 +876,9 @@ describe("web recording upload recovery", () => { it("clears the previous share link when starting the next recording", async () => { await start(); await act(async () => recorder.stopRecording()); - expect(recorder.completedShareUrl).toBe("https://cap.so/s/video"); + expect(recorder.completedShareUrl).toBe( + `${window.location.origin}/s/video`, + ); mocks.acquire.mockResolvedValue(new Stream([new Track()])); mocks.spoolCreate.mockResolvedValue(makeSpool()); await act(async () => recorder.startRecording()); diff --git a/apps/web/__tests__/unit/web-recorder-sources.test.ts b/apps/web/__tests__/unit/web-recorder-sources.test.ts new file mode 100644 index 00000000000..d2d5e38912f --- /dev/null +++ b/apps/web/__tests__/unit/web-recorder-sources.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it } from "vitest"; +import { + canRecordMicOnly, + micOnlyFileExtension, + micOnlyLanding, + micOnlyMimeType, + parseCameraOnlyChoice, + startRecordingChoice, +} from "@/app/(org)/dashboard/caps/components/web-recorder-dialog/recording-sources"; + +const base = { + screenShared: false, + cameraEnabled: false, + screenSupported: true, + cameraOnlyChoice: null, +}; + +describe("startRecordingChoice", () => { + it("records straight away once a screen is shared", () => { + expect( + startRecordingChoice({ + ...base, + screenShared: true, + cameraEnabled: true, + }), + ).toBe("record"); + expect(startRecordingChoice({ ...base, screenShared: true })).toBe( + "record", + ); + }); + + it("opens the screen picker when nothing visual is on", () => { + expect(startRecordingChoice(base)).toBe("share-then-record"); + }); + + it("asks before recording just the camera", () => { + expect(startRecordingChoice({ ...base, cameraEnabled: true })).toBe( + "confirm-camera-only", + ); + }); + + it("repeats the answer given when the question was turned off", () => { + expect( + startRecordingChoice({ + ...base, + cameraEnabled: true, + cameraOnlyChoice: "camera", + }), + ).toBe("record"); + expect( + startRecordingChoice({ + ...base, + cameraEnabled: true, + cameraOnlyChoice: "screen", + }), + ).toBe("share-then-record"); + }); + + it("only accepts a stored choice it knows", () => { + expect(parseCameraOnlyChoice("camera")).toBe("camera"); + expect(parseCameraOnlyChoice("screen")).toBe("screen"); + expect(parseCameraOnlyChoice("true")).toBeNull(); + expect(parseCameraOnlyChoice(null)).toBeNull(); + }); + + it("never asks when the browser can't record a screen", () => { + expect( + startRecordingChoice({ + ...base, + cameraEnabled: true, + screenSupported: false, + }), + ).toBe("record"); + }); +}); + +describe("canRecordMicOnly", () => { + const idle = { + screenShared: false, + cameraEnabled: false, + micEnabled: true, + idle: true, + }; + + it("is offered with only a microphone on", () => { + expect(canRecordMicOnly(idle)).toBe(true); + }); + + it("isn't offered with a screen, a camera, no mic, or mid-recording", () => { + expect(canRecordMicOnly({ ...idle, screenShared: true })).toBe(false); + expect(canRecordMicOnly({ ...idle, cameraEnabled: true })).toBe(false); + expect(canRecordMicOnly({ ...idle, micEnabled: false })).toBe(false); + expect(canRecordMicOnly({ ...idle, idle: false })).toBe(false); + }); +}); + +describe("microphone recording format", () => { + it("prefers Opus in WebM and falls back to AAC in MP4", () => { + expect(micOnlyMimeType(() => true)).toBe("audio/webm;codecs=opus"); + expect(micOnlyMimeType((type) => type.startsWith("audio/mp4"))).toBe( + "audio/mp4;codecs=mp4a.40.2", + ); + expect(micOnlyMimeType(() => false)).toBeUndefined(); + expect( + micOnlyMimeType(() => { + throw new Error("unsupported"); + }), + ).toBeUndefined(); + }); + + it("names the file for its container", () => { + expect(micOnlyFileExtension("audio/webm;codecs=opus")).toBe("webm"); + expect(micOnlyFileExtension("audio/mp4")).toBe("m4a"); + expect(micOnlyFileExtension("audio/ogg;codecs=opus")).toBe("ogg"); + }); +}); + +describe("micOnlyLanding", () => { + it("opens the editor for web editor users and the share page otherwise", () => { + expect(micOnlyLanding("abc123", true)).toBe("/s/abc123/edit?from=import"); + expect(micOnlyLanding("abc123", false)).toBe("/s/abc123"); + }); +}); diff --git a/apps/web/__tests__/web-recorder-browser-harness.tsx b/apps/web/__tests__/web-recorder-browser-harness.tsx new file mode 100644 index 00000000000..445a73197c0 --- /dev/null +++ b/apps/web/__tests__/web-recorder-browser-harness.tsx @@ -0,0 +1,41 @@ +import { + RecordingSpool, + recoverOrphanedRecordingSpools, +} from "@cap/recorder-core/recording-spool"; +import { useEffect } from "react"; +import { createRoot } from "react-dom/client"; +import { useWebRecorder } from "../app/(org)/dashboard/caps/components/web-recorder-dialog/useWebRecorder"; + +declare global { + interface Window { + capRecorderHarness?: ReturnType; + capRecorderSpool?: typeof RecordingSpool; + capRecorderRecoverOrphans?: typeof recoverOrphanedRecordingSpools; + } +} + +window.capRecorderSpool = RecordingSpool; +window.capRecorderRecoverOrphans = recoverOrphanedRecordingSpools; + +function RecorderHarness() { + const params = new URLSearchParams(location.search); + const captureMic = params.get("mic") === "1"; + const recorder = useWebRecorder({ + organisationId: "test-org", + selectedMicId: captureMic ? "test-mic" : null, + micEnabled: captureMic, + systemAudioEnabled: false, + recordingMode: "fullscreen", + selectedCameraId: params.get("camera") === "0" ? null : "test-camera", + getCameraPreviewStream: () => null, + isProUser: true, + }); + useEffect(() => { + window.capRecorderHarness = recorder; + }); + return
{recorder.phase}
; +} + +const element = document.getElementById("root"); +if (!element) throw new Error("Recorder replay root is unavailable"); +createRoot(element).render(); diff --git a/apps/web/__tests__/web-recorder-paired-browser-replay.mjs b/apps/web/__tests__/web-recorder-paired-browser-replay.mjs new file mode 100644 index 00000000000..594dedd4def --- /dev/null +++ b/apps/web/__tests__/web-recorder-paired-browser-replay.mjs @@ -0,0 +1,902 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { STREAMING_MP4_MIME_TYPES } from "@cap/recorder-core/recorder-constants"; +import { chromium, webkit } from "@playwright/test"; +import { build } from "esbuild"; + +const root = resolve(import.meta.dirname, "../../.."); +const webRoot = resolve(root, "apps/web"); +const artifactDirectory = await mkdtemp( + join(tmpdir(), "cap-web-recorder-replay-"), +); +const artifact = join(artifactDirectory, "harness.js"); +const mockSources = new Map([ + [ + "@tanstack/react-query", + "export function useQueryClient(){return {refetchQueries(){}}}", + ], + [ + "next/navigation", + "export function useRouter(){return {push(){},refresh(){}}}", + ], + ["sonner", "export const toast={error(){},warning(){},success(){},info(){}}"], + [ + "@cap/web-domain", + "export const Organisation={OrganisationId:{make(value){return value}}}", + ], + [ + "@/actions/video/trigger-instant-recording-processing", + "export async function triggerInstantRecordingProcessing(){}", + ], + [ + "@/actions/video/upload", + "export async function createVideoAndGetUploadUrl(){throw new Error('Unused upload path')}", + ], + [ + "@/lib/EffectRuntime", + "import {Exit} from 'effect';export function useRpcClient(){return {VideoInstantCreate:async()=>Exit.succeed({id:'test-video',shareUrl:'https://capture.test/share',upload:{}}),VideoDelete:async()=>Exit.succeed(null)}}export function useEffectMutation({mutationFn}){return {mutateAsync:mutationFn}}", + ], + [ + "@/lib/Requests/ThumbnailRequest", + "export const ThumbnailRequest={queryKey:()=>[]}", + ], + ["@/utils/upload-target", "export async function uploadWithTarget(){}"], + [ + "../../UploadingContext", + "export function useUploadingContext(){return {setUploadStatus(){}}}", + ], + ["../sendProgressUpdate", "export async function sendProgressUpdate(){}"], + [ + "./recording-conversion", + "export async function canConvertToMp4InBrowser(){return false}export async function captureThumbnail(){return null}export async function convertToMp4(){throw new Error('Unused conversion path')}", + ], + [ + "./recovered-recording-cache", + "export async function loadRecoveredRecordingSpools(){return []}export function removeRecoveredRecordingSpoolFromCache(){}export function resetRecoveredRecordingSpoolsCache(){}", + ], +]); + +await build({ + entryPoints: [resolve(webRoot, "__tests__/web-recorder-browser-harness.tsx")], + outfile: artifact, + bundle: true, + format: "iife", + platform: "browser", + jsx: "automatic", + target: "es2022", + nodePaths: [resolve(webRoot, "node_modules"), resolve(root, "node_modules")], + plugins: [ + { + name: "pr-recorder-dependencies", + setup(plugin) { + plugin.onResolve({ filter: /^react(\/jsx-runtime)?$/ }, (args) => ({ + path: resolve( + root, + "node_modules/react", + `${args.path === "react" ? "index" : "jsx-runtime"}.js`, + ), + })); + plugin.onResolve({ filter: /^react-dom\/client$/ }, () => ({ + path: resolve(root, "node_modules/react-dom/client.js"), + })); + plugin.onResolve({ filter: /^@cap\/recorder-core(\/.*)?$/ }, (args) => { + const subpath = args.path.replace(/^@cap\/recorder-core\/?/, ""); + const basename = subpath || "index"; + return { + path: resolve(root, "packages/recorder-core/src", `${basename}.ts`), + }; + }); + plugin.onResolve({ filter: /.*/ }, (args) => { + if (!args.importer.endsWith("useWebRecorder.ts")) return; + if (!mockSources.has(args.path)) return; + return { path: args.path, namespace: "recorder-replay-mock" }; + }); + plugin.onLoad( + { filter: /.*/, namespace: "recorder-replay-mock" }, + (args) => ({ + contents: mockSources.get(args.path), + loader: "js", + resolveDir: webRoot, + }), + ); + }, + }, + ], +}); + +async function replayPairedCapture( + browser, + bundle, + pauseResume, + engine, + failCameraSpool = false, + failCameraCompletion = false, + failAudioResume = false, + failBackupDeletion = false, + stallAudioSpool = false, + cameraEnabled = true, + unwritableBackupStore = false, +) { + const requests = []; + const parts = []; + const browserErrors = []; + const browserWarnings = []; + const cameraExtension = engine.name === "WebKit" ? "webm" : engine.extension; + const cameraSubpath = `camera-upload.${cameraExtension}`; + const screenSubpath = `raw-upload.${engine.extension}`; + // A profile on disk gives IndexedDB the durable storage a normal browser + // window has. WebKit's ephemeral contexts refuse to store Blobs, which is + // how a backup store fails in Safari private browsing. + const profileDirectory = unwritableBackupStore + ? null + : await mkdtemp(join(tmpdir(), "cap-web-recorder-profile-")); + const context = profileDirectory + ? await engine.browserType.launchPersistentContext(profileDirectory, { + headless: process.env.CAP_REPLAY_HEADED !== "true", + }) + : await browser.newContext(); + try { + await context.addInitScript( + (options) => { + if (options.failCameraSpool) { + const put = IDBObjectStore.prototype.put; + IDBObjectStore.prototype.put = function (value, ...rest) { + if ( + this.name === "chunks" && + value?.sessionId?.endsWith("-camera") + ) { + window.capRecorderCameraSpoolFailures = + (window.capRecorderCameraSpoolFailures ?? 0) + 1; + throw new DOMException( + "Simulated camera Blob store failure", + "UnknownError", + ); + } + return put.call(this, value, ...rest); + }; + } + window.capRecorderMediaRecorders = []; + window.capRecorderCanvasSources = []; + window.capRecorderAudioContexts = []; + const recorderStart = MediaRecorder.prototype.start; + MediaRecorder.prototype.start = function (...args) { + const stats = { + dataEvents: 0, + bytes: 0, + startEvents: 0, + stopEvents: 0, + pauseEvents: 0, + resumeEvents: 0, + errorEvents: 0, + }; + this.addEventListener("dataavailable", (event) => { + stats.dataEvents++; + stats.bytes += event.data.size; + }); + for (const name of ["start", "stop", "pause", "resume", "error"]) { + this.addEventListener(name, () => { + stats[`${name}Events`]++; + }); + } + window.capRecorderMediaRecorders.push({ recorder: this, stats }); + return recorderStart.apply(this, args); + }; + const createCanvasStream = (width, height, color) => { + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + const canvasContext = canvas.getContext("2d"); + if (!canvasContext) throw new Error("Canvas capture is unavailable"); + let frame = 0; + let track; + const paint = () => { + canvasContext.fillStyle = color; + canvasContext.fillRect(0, 0, width, height); + canvasContext.fillStyle = "white"; + canvasContext.font = "24px sans-serif"; + canvasContext.fillText(String(frame++), 20, 40); + track?.requestFrame?.(); + }; + paint(); + const stream = canvas.captureStream(30); + track = stream.getVideoTracks()[0]; + window.capRecorderCanvasSources.push({ + width, + height, + framesPainted: () => frame, + }); + track?.requestFrame?.(); + setInterval(paint, 33); + return stream; + }; + let display; + let camera; + const mediaDevices = navigator.mediaDevices; + window.capRecorderMediaDevices = mediaDevices; + const getDisplayMedia = async () => { + display ??= createCanvasStream(640, 360, "#203060"); + return display; + }; + const getUserMedia = async (constraints) => { + if (constraints?.audio) { + const context = new AudioContext(); + window.capRecorderAudioContexts.push(context); + const oscillator = context.createOscillator(); + const output = context.createMediaStreamDestination(); + oscillator.connect(output); + oscillator.start(); + return output.stream; + } + camera ??= createCanvasStream(320, 180, "#b02040"); + return camera; + }; + Object.defineProperty(mediaDevices, "getDisplayMedia", { + configurable: true, + value: getDisplayMedia, + }); + Object.defineProperty(mediaDevices, "getUserMedia", { + configurable: true, + value: getUserMedia, + }); + window.capRecorderPutBodies = []; + const open = XMLHttpRequest.prototype.open; + XMLHttpRequest.prototype.open = function (method, url, ...options) { + this.capRecorderUploadUrl = method === "PUT" ? String(url) : null; + return open.call(this, method, url, ...options); + }; + const send = XMLHttpRequest.prototype.send; + XMLHttpRequest.prototype.send = function (body) { + if ( + this.capRecorderUploadUrl?.includes("/part/") && + body instanceof Blob + ) { + const url = this.capRecorderUploadUrl; + window.capRecorderPutBodies.push( + body + .arrayBuffer() + .then(async (buffer) => ({ + header: Array.from( + new Uint8Array(buffer).subarray(0, 4), + (byte) => byte.toString(16).padStart(2, "0"), + ).join(""), + digest: await crypto.subtle.digest("SHA-256", buffer), + })) + .then(({ digest, header }) => ({ + url, + bytes: body.size, + header, + sha256: Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, "0"), + ).join(""), + })), + ); + } + return send.call(this, body); + }; + }, + { failCameraSpool }, + ); + const page = await context.newPage(); + page.on("console", (message) => { + if (message.type() === "error") browserErrors.push(message.text()); + if (message.type() === "warning") browserWarnings.push(message.text()); + }); + page.on("pageerror", (error) => browserErrors.push(error.message)); + await page.route("https://capture.test/**", async (route) => { + const request = route.request(); + const path = new URL(request.url()).pathname; + if (path === "/harness") { + await route.fulfill({ + status: 200, + contentType: "text/html", + body: "
", + }); + return; + } + if (path === "/harness.js") { + await route.fulfill({ + status: 200, + contentType: "text/javascript", + body: bundle, + }); + return; + } + if (path.startsWith("/part/")) { + const data = request.postDataBuffer(); + parts.push({ + path, + bytes: data?.length ?? 0, + sha256: data ? createHash("sha256").update(data).digest("hex") : null, + }); + await route.fulfill({ + status: 200, + headers: { etag: JSON.stringify("replay-etag") }, + body: "", + }); + return; + } + if (path.startsWith("/api/upload/multipart/")) { + const body = request.postDataJSON(); + requests.push({ path, body }); + if (path.endsWith("/initiate")) { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ uploadId: body.subpath, provider: "s3" }), + }); + return; + } + if (path.endsWith("/presign-part")) { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + presignedUrl: `https://capture.test/part/${encodeURIComponent(body.subpath)}/${body.partNumber}`, + provider: "s3", + }), + }); + return; + } + if (path.endsWith("/complete")) { + await route.fulfill({ + status: + failCameraCompletion && body.subpath === cameraSubpath + ? 400 + : 200, + contentType: "application/json", + body: JSON.stringify( + failCameraCompletion && body.subpath === cameraSubpath + ? { error: "Simulated camera completion rejection" } + : { success: true, processingStarted: true }, + ), + }); + return; + } + if (path.endsWith("/abort")) { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ success: true }), + }); + return; + } + } + await route.fulfill({ status: 404, body: `Unexpected request: ${path}` }); + }); + + const params = new URLSearchParams(); + if (failAudioResume) params.set("mic", "1"); + if (!cameraEnabled) params.set("camera", "0"); + await page.goto( + `https://capture.test/harness${params.size ? `?${params}` : ""}`, + ); + try { + await page.waitForFunction( + () => window.capRecorderHarness?.canStartRecording === true, + ); + } catch (error) { + const diagnostics = await page.evaluate(() => ({ + readyState: document.readyState, + mediaDevices: Boolean(navigator.mediaDevices), + mediaRecorder: typeof MediaRecorder, + harnessMounted: Boolean(window.capRecorderHarness), + })); + throw new Error( + "Capture harness failed to initialize: " + + JSON.stringify({ + engine: engine.name, + pauseResume, + browserErrors, + diagnostics, + }), + { cause: error }, + ); + } + await page.evaluate(() => window.capRecorderHarness.startRecording()); + await page.waitForFunction( + () => window.capRecorderHarness?.phase === "recording", + ); + if (stallAudioSpool) { + await page.evaluate(() => { + const append = window.capRecorderSpool.prototype.appendChunk; + const flush = window.capRecorderSpool.prototype.flush; + window.capRecorderSpool.prototype.appendChunk = function (chunk) { + if (this.session.mimeType.startsWith("audio/")) { + this.simulatedAudioSpoolFailure = true; + return Promise.reject( + new Error("Simulated IndexedDB audio backup write failure"), + ); + } + return append.call(this, chunk); + }; + window.capRecorderSpool.prototype.flush = function () { + if (this.simulatedAudioSpoolFailure) { + return new Promise(() => undefined); + } + return flush.call(this); + }; + }); + } + if (pauseResume) { + if (failAudioResume) { + await page.evaluate(() => { + window.capRecorderPauseInstances = []; + const pause = MediaRecorder.prototype.pause; + MediaRecorder.prototype.pause = function () { + window.capRecorderPauseInstances.push(this); + return pause.call(this); + }; + const resume = MediaRecorder.prototype.resume; + MediaRecorder.prototype.resume = function () { + if ( + window.capRecorderFailAudioResume && + this.stream.getVideoTracks().length === 0 && + this.stream.getAudioTracks().length > 0 + ) { + window.capRecorderFailAudioResume = false; + throw new DOMException( + "Simulated audio sidecar resume failure", + "InvalidStateError", + ); + } + return resume.call(this); + }; + }); + } + await page.waitForTimeout(600); + await page.evaluate(() => window.capRecorderHarness.pauseRecording()); + await page.waitForFunction( + () => window.capRecorderHarness?.phase === "paused", + ); + await page.waitForTimeout(350); + if (failAudioResume) { + await page.evaluate(() => { + window.capRecorderFailAudioResume = true; + }); + } + await page.evaluate(() => window.capRecorderHarness.resumeRecording()); + if (failAudioResume) { + const pausedStates = await page.evaluate(() => + Array.from(new Set(window.capRecorderPauseInstances)).map( + (recorder) => ({ + state: recorder.state, + videoTracks: recorder.stream.getVideoTracks().length, + audioTracks: recorder.stream.getAudioTracks().length, + }), + ), + ); + assert.equal(pausedStates.length, 3); + assert.deepEqual( + pausedStates.map((recorder) => recorder.state), + ["paused", "paused", "paused"], + ); + assert.deepEqual( + pausedStates.map((recorder) => [ + recorder.videoTracks, + recorder.audioTracks, + ]), + [ + [1, 0], + [0, 1], + [1, 1], + ], + ); + assert.equal( + await page.evaluate(() => window.capRecorderHarness.phase), + "paused", + ); + await page.evaluate(() => window.capRecorderHarness.resumeRecording()); + } + await page.waitForFunction( + () => window.capRecorderHarness?.phase === "recording", + ); + } + await page.waitForTimeout(engine.name === "WebKit" ? 5000 : 1700); + if (failBackupDeletion) { + await page.evaluate(() => { + window.capRecorderSpool.prototype.dispose = async () => { + throw new Error("Simulated backup deletion failure"); + }; + }); + } + await page.evaluate(() => window.capRecorderHarness.stopRecording()); + try { + await page.waitForFunction( + () => ["completed", "error"].includes(window.capRecorderHarness?.phase), + null, + { timeout: 30000 }, + ); + const actualPhase = await page.evaluate( + () => window.capRecorderHarness?.phase, + ); + assert.equal(actualPhase, failCameraCompletion ? "error" : "completed"); + } catch (error) { + const state = await page.evaluate(() => ({ + phase: window.capRecorderHarness?.phase, + videoId: window.capRecorderHarness?.videoId, + cameraDownload: Boolean(window.capRecorderHarness?.cameraErrorDownload), + displayDownload: Boolean(window.capRecorderHarness?.errorDownload), + recorders: window.capRecorderMediaRecorders?.map( + ({ recorder, stats }) => ({ + state: recorder.state, + mimeType: recorder.mimeType, + tracks: recorder.stream.getTracks().map((track) => ({ + kind: track.kind, + readyState: track.readyState, + muted: track.muted, + settings: track.getSettings(), + })), + stats, + }), + ), + canvasSources: window.capRecorderCanvasSources?.map((source) => ({ + width: source.width, + height: source.height, + framesPainted: source.framesPainted(), + })), + audioContexts: window.capRecorderAudioContexts?.map((context) => ({ + state: context.state, + currentTime: context.currentTime, + })), + })); + throw new Error( + "Paired capture did not reach its terminal phase: " + + JSON.stringify({ + engine: engine.name, + pauseResume, + failCameraCompletion, + browserErrors, + browserWarnings, + state, + requests: requests.map((request) => request.path), + partBytes: parts.map((part) => part.bytes), + }), + { cause: error }, + ); + } + if (failCameraCompletion) { + const evidence = await page.evaluate(async () => { + const readBytes = async (download) => + download ? (await (await fetch(download.url)).blob()).size : 0; + return { + cameraBackupBytes: await readBytes( + window.capRecorderHarness?.cameraErrorDownload, + ), + displayBackupBytes: await readBytes( + window.capRecorderHarness?.errorDownload, + ), + }; + }); + const sentParts = await page.evaluate(() => + Promise.all(window.capRecorderPutBodies), + ); + const cameraBytes = sentParts + .filter((part) => part.url.includes(cameraSubpath)) + .reduce((total, part) => total + part.bytes, 0); + assert.ok(cameraBytes > 0); + assert.equal(evidence.cameraBackupBytes, cameraBytes); + assert.ok(evidence.displayBackupBytes > 0); + assert.ok( + browserErrors.some((error) => + error.includes("Failed to upload camera recording"), + ), + ); + return { + engine: engine.name, + pauseResume, + failCameraCompletion, + ...evidence, + }; + } + if (failAudioResume) { + assert.equal(browserErrors.length, 1); + assert.ok(browserErrors[0].includes("Failed to resume recording")); + } else if (failBackupDeletion) { + assert.ok( + browserErrors.some((error) => + error.includes("Failed to dispose camera recording spool"), + ), + ); + } else if (unwritableBackupStore) { + assert.ok( + browserErrors.length > 0, + "The ephemeral context stored the backup, so the failing store was not exercised", + ); + assert.ok( + browserErrors.every((error) => + error.startsWith("Failed to persist recording chunk locally"), + ), + JSON.stringify(browserErrors), + ); + } else { + assert.deepEqual(browserErrors, []); + } + if (failCameraSpool) { + assert.ok( + (await page.evaluate(() => window.capRecorderCameraSpoolFailures)) > 0, + ); + } + if (stallAudioSpool) { + assert.ok( + browserWarnings.some((warning) => + warning.includes("Simulated IndexedDB audio backup write failure"), + ), + ); + } + const completions = requests.filter((request) => + request.path.endsWith("/complete"), + ); + assert.equal( + completions.length, + failAudioResume ? 3 : cameraEnabled ? 2 : 1, + ); + const micComplete = completions.find((request) => + request.body.subpath.startsWith("mic-upload."), + ); + if (failAudioResume) assert.ok(micComplete); + assert.deepEqual( + completions.map((request) => request.body.subpath).sort(), + failAudioResume + ? [cameraSubpath, micComplete?.body.subpath, screenSubpath].sort() + : cameraEnabled + ? [cameraSubpath, screenSubpath] + : [screenSubpath], + ); + const sentParts = await page.evaluate(() => + Promise.all(window.capRecorderPutBodies), + ); + const cameraParts = sentParts.filter((part) => + part.url.includes(cameraSubpath), + ); + const screenParts = sentParts.filter((part) => + part.url.includes(screenSubpath), + ); + const cameraBytes = cameraParts.reduce( + (total, part) => total + part.bytes, + 0, + ); + const screenBytes = screenParts.reduce( + (total, part) => total + part.bytes, + 0, + ); + if (cameraEnabled) assert.ok(cameraBytes > 0); + else assert.equal(cameraBytes, 0); + assert.ok(screenBytes > 0); + if (engine.name === "WebKit" && cameraEnabled) { + const cameraMime = await page.evaluate( + () => window.capRecorderMediaRecorders?.[1]?.recorder.mimeType, + ); + assert.ok(cameraMime?.startsWith("video/webm")); + assert.equal(cameraParts[0].header, "1a45dfa3"); + } + if (failAudioResume) { + const micParts = sentParts.filter((part) => + part.url.includes(micComplete.body.subpath), + ); + assert.ok(micParts.reduce((total, part) => total + part.bytes, 0) > 0); + assert.ok(Number.isInteger(micComplete.body.audioOffsetMs)); + } + if (cameraEnabled) + assert.notEqual(cameraParts[0].sha256, screenParts[0].sha256); + if (engine.streamingDisplay && cameraEnabled) { + const cameraPart = parts.find((part) => + part.path.includes(cameraSubpath), + ); + const screenPart = parts.find((part) => + part.path.includes(screenSubpath), + ); + assert.ok(cameraPart?.bytes > 0); + assert.ok(screenPart?.bytes > 0); + assert.notEqual(cameraPart.sha256, screenPart.sha256); + } + const cameraComplete = completions.find( + (request) => request.body.subpath === cameraSubpath, + ); + const screenComplete = completions.find( + (request) => request.body.subpath === screenSubpath, + ); + if (cameraEnabled) { + assert.equal(cameraComplete.body.screenSubpath, screenSubpath); + assert.equal( + cameraComplete.body.parts.reduce((total, part) => total + part.size, 0), + cameraBytes, + ); + } else { + assert.equal(cameraComplete, undefined); + } + assert.equal( + screenComplete.body.parts.reduce((total, part) => total + part.size, 0), + screenBytes, + ); + if (cameraEnabled) { + assert.ok(Number.isInteger(cameraComplete.body.cameraOffsetMs)); + assert.ok(Math.abs(cameraComplete.body.cameraOffsetMs) < 500); + assert.deepEqual( + [cameraComplete.body.width, cameraComplete.body.height], + [320, 180], + ); + } + assert.deepEqual( + [screenComplete.body.width, screenComplete.body.height], + [640, 360], + ); + if (failBackupDeletion) { + const markers = await page.evaluate(() => + Object.keys(localStorage).filter((key) => + key.startsWith("cap-recording-spool-uploaded:"), + ), + ); + assert.equal(markers.length, 2); + assert.deepEqual( + await page.evaluate(() => window.capRecorderRecoverOrphans()), + [], + ); + await page.waitForFunction( + () => + Object.keys(localStorage).every( + (key) => !key.startsWith("cap-recording-spool-uploaded:"), + ), + null, + { timeout: 5000 }, + ); + } + return { + engine: engine.name, + cameraEnabled, + pauseResume, + failAudioResume, + failBackupDeletion, + failCameraSpool, + stallAudioSpool, + unwritableBackupStore, + cameraBytes, + screenBytes, + cameraOffsetMs: cameraComplete?.body.cameraOffsetMs ?? null, + }; + } finally { + await context.close(); + if (profileDirectory) await rm(profileDirectory, { recursive: true }); + } +} + +const bundle = await readFile(artifact, "utf8"); + +async function supportsStreamingMp4(browser) { + const page = await browser.newPage(); + try { + return await page.evaluate( + (candidates) => + candidates.some((candidate) => + MediaRecorder.isTypeSupported(candidate), + ), + [ + ...STREAMING_MP4_MIME_TYPES.videoOnly, + ...STREAMING_MP4_MIME_TYPES.withAudio, + ], + ); + } finally { + await page.close(); + } +} +const requestedEngine = process.argv[2]?.toLowerCase(); +const engines = [ + { name: "Chromium", browserType: chromium, streamingDisplay: true }, + { name: "WebKit", browserType: webkit, streamingDisplay: false }, +].filter( + (engine) => !requestedEngine || engine.name.toLowerCase() === requestedEngine, +); +assert.ok(engines.length > 0, `Unknown browser engine: ${requestedEngine}`); +try { + const results = []; + for (const engine of engines) { + const browser = await engine.browserType.launch({ + headless: process.env.CAP_REPLAY_HEADED !== "true", + }); + engine.extension = engine.streamingDisplay + ? (await supportsStreamingMp4(browser)) + ? "mp4" + : "webm" + : "mp4"; + try { + for (const pauseResume of [false, true]) { + results.push( + await replayPairedCapture(browser, bundle, pauseResume, engine), + ); + results.push( + await replayPairedCapture( + browser, + bundle, + pauseResume, + engine, + false, + false, + false, + false, + false, + false, + ), + ); + } + results.push( + await replayPairedCapture( + browser, + bundle, + true, + engine, + false, + false, + true, + ), + ); + if (engine.name === "WebKit") { + results.push( + await replayPairedCapture( + browser, + bundle, + true, + engine, + false, + false, + true, + false, + true, + ), + ); + } + if (engine.name === "Chromium") { + results.push( + await replayPairedCapture(browser, bundle, false, engine, true), + ); + } + if (engine.name === "WebKit") { + results.push( + await replayPairedCapture( + browser, + bundle, + false, + engine, + false, + false, + false, + false, + false, + true, + true, + ), + ); + } + if (engine.name === "WebKit") { + results.push( + await replayPairedCapture( + browser, + bundle, + false, + engine, + false, + true, + ), + ); + } + results.push( + await replayPairedCapture( + browser, + bundle, + false, + engine, + false, + false, + false, + true, + ), + ); + } finally { + await browser.close(); + } + } + process.stdout.write(`${JSON.stringify(results)}\n`); +} finally { + await rm(artifactDirectory, { recursive: true }); +} diff --git a/apps/web/actions/notifications/update-preferences.ts b/apps/web/actions/notifications/update-preferences.ts index 9fa59e7bb8e..3061fbeac80 100644 --- a/apps/web/actions/notifications/update-preferences.ts +++ b/apps/web/actions/notifications/update-preferences.ts @@ -3,7 +3,7 @@ import { db } from "@cap/database"; import { getCurrentUser } from "@cap/database/auth/session"; import { users } from "@cap/database/schema"; -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import { revalidatePath } from "next/cache"; export const updatePreferences = async ({ @@ -26,10 +26,7 @@ export const updatePreferences = async ({ await db() .update(users) .set({ - preferences: { - ...(currentUser.preferences ?? {}), - notifications, - }, + preferences: sql`JSON_SET(COALESCE(${users.preferences}, JSON_OBJECT()), '$.notifications', CAST(${JSON.stringify(notifications)} AS JSON))`, }) .where(eq(users.id, currentUser.id)); revalidatePath("/dashboard"); diff --git a/apps/web/actions/video/create-for-processing.ts b/apps/web/actions/video/create-for-processing.ts index 7a5660bbf49..c4c8dd6e5f8 100644 --- a/apps/web/actions/video/create-for-processing.ts +++ b/apps/web/actions/video/create-for-processing.ts @@ -35,11 +35,15 @@ export async function createVideoForServerProcessing({ resolution, folderId, orgId, + name, + audioOnly = false, }: { duration?: number; resolution?: string; folderId?: Folder.FolderId; orgId: Organisation.OrganisationId; + name?: string; + audioOnly?: boolean; }): Promise { const user = await getCurrentUser(); @@ -70,7 +74,8 @@ export async function createVideoForServerProcessing({ })} ${date.getFullYear()}`; const rawFileKey = `${user.id}/${videoId}/raw-upload.mp4`; - const videoTitle = `Cap Upload - ${formattedDate}`; + const videoTitle = + name?.trim().slice(0, 100) || `Cap Upload - ${formattedDate}`; const uploadResult = await StorageService.createUploadTargetForUser( user.id, @@ -99,6 +104,7 @@ export async function createVideoForServerProcessing({ bucket: Option.getOrNull(uploadResult.bucketId), storageIntegrationId: Option.getOrNull(uploadResult.storageIntegrationId), public: await getNewVideoPublic(orgId), + ...(audioOnly === true ? { metadata: { audioOnly: true } } : {}), ...(folderId ? { folderId } : {}), }); diff --git a/apps/web/actions/video/set-recorder-camera.ts b/apps/web/actions/video/set-recorder-camera.ts new file mode 100644 index 00000000000..85aeceb9180 --- /dev/null +++ b/apps/web/actions/video/set-recorder-camera.ts @@ -0,0 +1,37 @@ +"use server"; + +import { db } from "@cap/database"; +import { getCurrentUser } from "@cap/database/auth/session"; +import { videos } from "@cap/database/schema"; +import { + parseRecorderCamera, + type RecorderCameraLayout, +} from "@cap/editor-cap-bundle/default-style"; +import type { Video } from "@cap/web-domain"; +import { and, eq, sql } from "drizzle-orm"; + +/** + * Keeps where the camera sat in the browser recorder, so the editor and the + * first render open with the same layout. Only the owner can set it. + */ +export async function setRecorderCamera({ + videoId, + layout, +}: { + videoId: Video.VideoId; + layout: RecorderCameraLayout; +}): Promise<{ success: boolean }> { + const user = await getCurrentUser(); + if (!user) throw new Error("Unauthorized"); + const recorderCamera = parseRecorderCamera(layout); + if (!recorderCamera) throw new Error("Invalid camera layout"); + + const patch = JSON.stringify({ recorderCamera }); + const [result] = await db() + .update(videos) + .set({ + metadata: sql`JSON_MERGE_PATCH(COALESCE(${videos.metadata}, JSON_OBJECT()), ${patch})`, + }) + .where(and(eq(videos.id, videoId), eq(videos.ownerId, user.id))); + return { success: result.affectedRows > 0 }; +} diff --git a/apps/web/actions/video/trigger-instant-recording-processing.ts b/apps/web/actions/video/trigger-instant-recording-processing.ts index ce117d11a40..56e59be4e50 100644 --- a/apps/web/actions/video/trigger-instant-recording-processing.ts +++ b/apps/web/actions/video/trigger-instant-recording-processing.ts @@ -4,7 +4,9 @@ import { db } from "@cap/database"; import { getCurrentUser } from "@cap/database/auth/session"; import { videos } from "@cap/database/schema"; import type { Video } from "@cap/web-domain"; -import { eq } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; +import { headers } from "next/headers"; +import { startRecordingRender } from "@/lib/render-recording"; import { startVideoProcessingWorkflow } from "@/lib/video-processing"; export async function triggerInstantRecordingProcessing({ @@ -24,6 +26,29 @@ export async function triggerInstantRecordingProcessing({ if (video.ownerId !== user.id) throw new Error("Unauthorized"); const rawFileKey = `${user.id}/${videoId}/result.mp4`; + if (!video.metadata?.editorSources) { + const sourcePatch = JSON.stringify({ + editorSources: { + version: 1, + display: { + key: rawFileKey, + contentType: "video/mp4", + }, + }, + }); + await db() + .update(videos) + .set({ + metadata: sql`JSON_MERGE_PATCH(COALESCE(${videos.metadata}, JSON_OBJECT()), ${sourcePatch})`, + }) + .where( + and( + eq(videos.id, videoId), + eq(videos.ownerId, user.id), + sql`JSON_EXTRACT(${videos.metadata}, '$.editorSources') IS NULL`, + ), + ); + } await startVideoProcessingWorkflow({ videoId, @@ -35,5 +60,22 @@ export async function triggerInstantRecordingProcessing({ mode: "singlepart", }); + const requestHeaders = await headers(); + const host = ( + requestHeaders.get("x-forwarded-host") ?? requestHeaders.get("host") + ) + ?.split(",")[0] + ?.trim(); + if (host) { + const protocol = requestHeaders.get("x-forwarded-proto") ?? "https"; + await startRecordingRender(videoId, `${protocol}://${host}`).catch( + (error) => + console.error( + "Failed to start the render of a finished recording", + error, + ), + ); + } + return { success: true }; } diff --git a/apps/web/actions/video/trigger-processing.ts b/apps/web/actions/video/trigger-processing.ts index ed6ed5df90c..9382a587046 100644 --- a/apps/web/actions/video/trigger-processing.ts +++ b/apps/web/actions/video/trigger-processing.ts @@ -2,14 +2,21 @@ import { db } from "@cap/database"; import { getCurrentUser } from "@cap/database/auth/session"; -import { videos } from "@cap/database/schema"; +import { videos, videoUploads } from "@cap/database/schema"; +import { userIsPro } from "@cap/utils"; import { Storage } from "@cap/web-backend"; +import { getRecordingObjectIdentity } from "@cap/web-backend/src/Storage/recording-object-identity"; import type { Video } from "@cap/web-domain"; -import { eq } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import { Effect, Schedule } from "effect"; +import { + type ImportEditorSource, + importEditorSourcesPatch, +} from "@/lib/import-editor-source"; import { runPromise } from "@/lib/server"; import { startVideoProcessingWorkflow } from "@/lib/video-processing"; import { decodeStorageVideo } from "@/lib/video-storage"; +import { isWebStudioEnabledForEmail } from "@/lib/web-studio-rollout"; async function verifyRawFileUploaded( video: typeof videos.$inferSelect, @@ -28,16 +35,59 @@ async function verifyRawFileUploaded( if ((head.ContentLength ?? 0) <= 0) { throw new Error("Uploaded video file is empty"); } + return head; +} + +async function registerImportEditorSource( + user: NonNullable>>, + video: typeof videos.$inferSelect, + source: ImportEditorSource, + head: Awaited>, +) { + const [upload] = await db() + .select({ phase: videoUploads.phase, rawFileKey: videoUploads.rawFileKey }) + .from(videoUploads) + .where(eq(videoUploads.videoId, video.id)); + const patch = importEditorSourcesPatch({ + ownerId: video.ownerId, + videoId: video.id, + isPro: userIsPro(user), + source, + upload: upload ?? null, + existingSources: video.metadata?.editorSources, + head: { + size: head.ContentLength, + identity: getRecordingObjectIdentity(head), + }, + }); + if (!patch) return; + const sourcePatch = JSON.stringify({ editorSources: patch.editorSources }); + await db() + .update(videos) + .set({ + duration: sql`COALESCE(${videos.duration}, ${patch.duration})`, + metadata: sql`JSON_MERGE_PATCH(COALESCE(${videos.metadata}, JSON_OBJECT()), ${sourcePatch})`, + }) + .where( + and( + eq(videos.id, video.id), + eq(videos.ownerId, user.id), + sql`JSON_EXTRACT(${videos.metadata}, '$.editorSources') IS NULL`, + ), + ); } export async function triggerVideoProcessing({ videoId, rawFileKey, bucketId, + editorSource, }: { videoId: Video.VideoId; rawFileKey: string; bucketId: string | null; + /** Opens the import in the editor from its upload; see import-editor-source. */ + editorSource?: ImportEditorSource | null; }): Promise<{ success: boolean }> { const user = await getCurrentUser(); if (!user) throw new Error("Unauthorized"); @@ -50,7 +100,14 @@ export async function triggerVideoProcessing({ if (!video) throw new Error("Video not found"); if (video.ownerId !== user.id) throw new Error("Unauthorized"); - await verifyRawFileUploaded(video, rawFileKey); + const head = await verifyRawFileUploaded(video, rawFileKey); + + if (editorSource && isWebStudioEnabledForEmail(user.email)) { + await registerImportEditorSource(user, video, editorSource, head).catch( + (error) => + console.error("Import will open in the editor after processing", error), + ); + } await startVideoProcessingWorkflow({ videoId, diff --git a/apps/web/actions/videos/get-editor-sharing.ts b/apps/web/actions/videos/get-editor-sharing.ts new file mode 100644 index 00000000000..1d36921e228 --- /dev/null +++ b/apps/web/actions/videos/get-editor-sharing.ts @@ -0,0 +1,34 @@ +"use server"; + +import { db } from "@cap/database"; +import { getCurrentUser } from "@cap/database/auth/session"; +import { videos } from "@cap/database/schema"; +import type { Video } from "@cap/web-domain"; +import { eq, sql } from "drizzle-orm"; +import { getDashboardSpacesData } from "@/app/(org)/dashboard/dashboard-data"; +import { getSharedSpacesForVideo } from "@/lib/video-shared-spaces"; + +/** What the editor's sharing controls need for the owner's own recording. */ +export async function getEditorSharing(videoId: Video.VideoId) { + const user = await getCurrentUser(); + if (!user) throw new Error("Unauthorized"); + const [video] = await db() + .select({ + ownerId: videos.ownerId, + isPublic: videos.public, + hasPassword: sql`${videos.password} IS NOT NULL`.mapWith(Boolean), + }) + .from(videos) + .where(eq(videos.id, videoId)); + if (!video || video.ownerId !== user.id) throw new Error("Video not found"); + const [{ sharedSpaces }, spacesData] = await Promise.all([ + getSharedSpacesForVideo(videoId), + getDashboardSpacesData(user), + ]); + return { + isPublic: video.isPublic, + hasPassword: video.hasPassword, + sharedSpaces, + spacesData, + }; +} diff --git a/apps/web/app/(org)/dashboard/Contexts.tsx b/apps/web/app/(org)/dashboard/Contexts.tsx index fe9cfc25f47..657bd10d6e1 100644 --- a/apps/web/app/(org)/dashboard/Contexts.tsx +++ b/apps/web/app/(org)/dashboard/Contexts.tsx @@ -35,6 +35,7 @@ export function DashboardContexts({ initialSidebarCollapsed, referClicked, shareableLinkUsage, + webStudioEnabled = false, }: { children: React.ReactNode; organizationData: SharedContext["organizationData"]; @@ -48,6 +49,7 @@ export function DashboardContexts({ initialSidebarCollapsed: boolean; referClicked: boolean; shareableLinkUsage: SharedContext["shareableLinkUsage"]; + webStudioEnabled?: boolean; }) { const user = useCurrentUser(); if (!user) redirect("/login"); @@ -167,6 +169,7 @@ export function DashboardContexts({ developerApps, setDeveloperApps, shareableLinkUsage, + webStudioEnabled, }} > {children} diff --git a/apps/web/app/(org)/dashboard/DashboardContext.ts b/apps/web/app/(org)/dashboard/DashboardContext.ts index 489f5938c0b..17d1a837edb 100644 --- a/apps/web/app/(org)/dashboard/DashboardContext.ts +++ b/apps/web/app/(org)/dashboard/DashboardContext.ts @@ -41,6 +41,8 @@ export type SharedContext = { developerApps: DeveloperApp[] | null; setDeveloperApps: (apps: DeveloperApp[] | null) => void; shareableLinkUsage: { used: number; limit: number } | null; + /** The web studio editor (and its Editor page) is on for this user. */ + webStudioEnabled: boolean; }; export type ITheme = "light" | "dark"; diff --git a/apps/web/app/(org)/dashboard/_components/AnimatedIcons/Editor.tsx b/apps/web/app/(org)/dashboard/_components/AnimatedIcons/Editor.tsx new file mode 100644 index 00000000000..3b80fb9d6b6 --- /dev/null +++ b/apps/web/app/(org)/dashboard/_components/AnimatedIcons/Editor.tsx @@ -0,0 +1,92 @@ +"use client"; + +import { motion, useAnimation } from "motion/react"; +import type { HTMLAttributes } from "react"; +import { forwardRef, useCallback, useImperativeHandle, useRef } from "react"; +import { cn } from "@/lib/utils"; + +export interface EditorIconHandle { + startAnimation: () => void; + stopAnimation: () => void; +} + +interface EditorIconProps extends HTMLAttributes { + size?: number; +} + +const EditorIcon = forwardRef( + ({ onMouseEnter, onMouseLeave, className, size = 28, ...props }, ref) => { + const controls = useAnimation(); + const isControlledRef = useRef(false); + + useImperativeHandle(ref, () => { + isControlledRef.current = true; + + return { + startAnimation: () => controls.start("animate"), + stopAnimation: () => controls.start("normal"), + }; + }); + + const handleMouseEnter = useCallback( + (e: React.MouseEvent) => { + if (!isControlledRef.current) { + controls.start("animate"); + } else { + onMouseEnter?.(e); + } + }, + [controls, onMouseEnter], + ); + + const handleMouseLeave = useCallback( + (e: React.MouseEvent) => { + if (!isControlledRef.current) { + controls.start("normal"); + } else { + onMouseLeave?.(e); + } + }, + [controls, onMouseLeave], + ); + + return ( +
+ + + + + + +
+ ); + }, +); + +EditorIcon.displayName = "EditorIcon"; + +export default EditorIcon; diff --git a/apps/web/app/(org)/dashboard/_components/AnimatedIcons/index.ts b/apps/web/app/(org)/dashboard/_components/AnimatedIcons/index.ts index f3570be3a5f..80eddb7506f 100644 --- a/apps/web/app/(org)/dashboard/_components/AnimatedIcons/index.ts +++ b/apps/web/app/(org)/dashboard/_components/AnimatedIcons/index.ts @@ -7,6 +7,7 @@ import ClapIcon from "./Clap"; import CodeIcon from "./Code"; import CogIcon from "./Cog"; import DownloadIcon from "./Download"; +import EditorIcon from "./Editor"; import HomeIcon from "./Home"; import ImportIcon from "./Import"; import LayersIcon from "./Layers"; @@ -22,6 +23,7 @@ export { CodeIcon, CogIcon, DownloadIcon, + EditorIcon, HomeIcon, ImportIcon, LayersIcon, diff --git a/apps/web/app/(org)/dashboard/_components/Navbar/Items.tsx b/apps/web/app/(org)/dashboard/_components/Navbar/Items.tsx index e91138d70e5..7b2eaac1235 100644 --- a/apps/web/app/(org)/dashboard/_components/Navbar/Items.tsx +++ b/apps/web/app/(org)/dashboard/_components/Navbar/Items.tsx @@ -56,6 +56,7 @@ import { ChartLineIcon, CodeIcon, CogIcon, + EditorIcon, ImportIcon, RecordIcon, } from "../AnimatedIcons"; @@ -71,7 +72,8 @@ interface Props { const AdminNavItems = ({ toggleMobileNav }: Props) => { const pathname = usePathname(); const [open, setOpen] = useState(false); - const { user, sidebarCollapsed, userCapsCount } = useDashboardContext(); + const { user, sidebarCollapsed, userCapsCount, webStudioEnabled } = + useDashboardContext(); const DEVELOPER_DASHBOARD_ALLOWED_EMAILS = ["richie@cap.so"]; @@ -100,6 +102,17 @@ const AdminNavItems = ({ toggleMobileNav }: Props) => { icon: , subNav: [], }, + ...(webStudioEnabled + ? [ + { + name: "Editor", + href: `/dashboard/editor`, + matchChildren: true, + icon: , + subNav: [] as { name: string; href: string }[], + }, + ] + : []), { name: "Import Media", href: loomImportPageHref(loomImportDestinationFromPathname(pathname)), diff --git a/apps/web/app/(org)/dashboard/_components/Navbar/Top.tsx b/apps/web/app/(org)/dashboard/_components/Navbar/Top.tsx index dab863e629a..33926ea05ec 100644 --- a/apps/web/app/(org)/dashboard/_components/Navbar/Top.tsx +++ b/apps/web/app/(org)/dashboard/_components/Navbar/Top.tsx @@ -63,6 +63,7 @@ const Top = () => { "/dashboard/folder": "Caps", "/dashboard/shared-caps": "Shared Caps", "/dashboard/caps/record": "Record a Cap", + "/dashboard/editor": "Editor", "/dashboard/settings/organization": "Organization Settings", "/dashboard/settings/organization/preferences": "Organization Settings", "/dashboard/settings/organization/content": "Organization Settings", diff --git a/apps/web/app/(org)/dashboard/caps/components/CapCard/CapCard.tsx b/apps/web/app/(org)/dashboard/caps/components/CapCard/CapCard.tsx index 1a2318821a1..ab5feac8dcf 100644 --- a/apps/web/app/(org)/dashboard/caps/components/CapCard/CapCard.tsx +++ b/apps/web/app/(org)/dashboard/caps/components/CapCard/CapCard.tsx @@ -41,6 +41,11 @@ import { toast } from "sonner"; import { ConfirmationDialog } from "@/app/(org)/dashboard/_components/ConfirmationDialog"; import { useDashboardContext } from "@/app/(org)/dashboard/Contexts"; import { useUploadProgress } from "@/app/s/[videoId]/_components/ProgressCircle"; +import { + renderProgressLabel, + useRenderSaveStatus, +} from "@/app/s/[videoId]/_components/render-save-status"; +import { RenderFog } from "@/components/render-fog"; import { UpgradeModal } from "@/components/UpgradeModal"; import { type ImageLoadingStatus, @@ -249,7 +254,17 @@ export const CapCard = ({ ); const hasRawFallback = uploadProgress?.status === "error" && uploadProgress.hasRawFallback; + // A render in progress takes the card over, as it does the share page. + const farmRendering = cap.metadata?.renderFarmSave?.status === "rendering"; + const renderStatus = useRenderSaveStatus( + cap.id, + farmRendering || !!cap.metadata?.browserSave, + ); + const isRendering = renderStatus + ? renderStatus.state === "rendering" + : farmRendering; const hasVisibleUploadProgress = + !isRendering && uploadProgress !== null && uploadProgress.status !== "fetching" && !hasRawFallback; @@ -704,6 +719,15 @@ export const CapCard = ({ }} href={`/s/${cap.id}`} > + {isRendering && ( + + )} {hasVisibleUploadProgress && ( <>
diff --git a/apps/web/app/(org)/dashboard/caps/components/CapCard/CapCardAnalytics.tsx b/apps/web/app/(org)/dashboard/caps/components/CapCard/CapCardAnalytics.tsx index 6139a4a2117..fbb6c0a86a1 100644 --- a/apps/web/app/(org)/dashboard/caps/components/CapCard/CapCardAnalytics.tsx +++ b/apps/web/app/(org)/dashboard/caps/components/CapCard/CapCardAnalytics.tsx @@ -10,6 +10,7 @@ import { forwardRef, type PropsWithChildren, } from "react"; +import { HoverPrefetchLink } from "@/components/hover-prefetch-link"; import { Tooltip } from "@/components/Tooltip"; interface CapCardAnalyticsProps { @@ -19,6 +20,7 @@ interface CapCardAnalyticsProps { isLoadingAnalytics: boolean; totalReactions: number; isOwner?: boolean; + prefetchOnHover?: boolean; } export const CapCardAnalytics = Object.assign( @@ -29,8 +31,10 @@ export const CapCardAnalytics = Object.assign( totalReactions, isLoadingAnalytics, isOwner = true, - }: CapCardAnalyticsProps) => - isLoadingAnalytics ? ( + prefetchOnHover = false, + }: CapCardAnalyticsProps) => { + const AnalyticsLink = prefetchOnHover ? HoverPrefetchLink : Link; + return isLoadingAnalytics ? ( ) : ( @@ -40,56 +44,57 @@ export const CapCardAnalytics = Object.assign( className="bg-gray-12 text-gray-1 border-gray-11 shadow-lg" delayDuration={100} > - {displayCount} - + - {totalComments} - + - {totalReactions} - +
{isOwner && ( - View analytics - + )} - ), + ); + }, { Skeleton: () => ( diff --git a/apps/web/app/(org)/dashboard/caps/components/CapCard/CapCardContent.tsx b/apps/web/app/(org)/dashboard/caps/components/CapCard/CapCardContent.tsx index 34e7c2ca4a1..42c2761c5b8 100644 --- a/apps/web/app/(org)/dashboard/caps/components/CapCard/CapCardContent.tsx +++ b/apps/web/app/(org)/dashboard/caps/components/CapCard/CapCardContent.tsx @@ -217,6 +217,7 @@ export const CapCardContent: React.FC = ({

{showFullDate ? moment(effectiveDate).format("YYYY-MM-DD HH:mm:ss") diff --git a/apps/web/app/(org)/dashboard/caps/components/web-recorder-dialog/CameraPreviewWindow.tsx b/apps/web/app/(org)/dashboard/caps/components/web-recorder-dialog/CameraPreviewWindow.tsx index 9f252ead87f..d0b3cd7d5da 100644 --- a/apps/web/app/(org)/dashboard/caps/components/web-recorder-dialog/CameraPreviewWindow.tsx +++ b/apps/web/app/(org)/dashboard/caps/components/web-recorder-dialog/CameraPreviewWindow.tsx @@ -1,5 +1,7 @@ "use client"; +import { cameraVideoConstraints } from "@cap/recorder-core/capture-streams"; + import { LoadingSpinner } from "@cap/ui"; import clsx from "clsx"; import { @@ -84,17 +86,22 @@ const getPreviewMetrics = ( export interface CameraPreviewWindowHandle { stopStream: () => void; + getVideoStream: () => MediaStream | null; + closePictureInPicture: () => Promise; } interface CameraPreviewWindowProps { cameraId: string; + hidden?: boolean; + /** 16:9 capture height. Screen recordings record a copy of this stream. */ + captureHeight?: number; onClose: () => void; } export const CameraPreviewWindow = forwardRef< CameraPreviewWindowHandle, CameraPreviewWindowProps ->(({ cameraId, onClose }, ref) => { +>(({ cameraId, hidden = false, captureHeight = 1080, onClose }, ref) => { const [size, setSize] = useState("sm"); const [shape, setShape] = useState("round"); const [mirrored, setMirrored] = useState(false); @@ -150,15 +157,34 @@ export const CameraPreviewWindow = forwardRef< videoRef.current.srcObject = null; } }, []); + const getVideoStream = useCallback(() => streamRef.current, []); + const closePictureInPicture = useCallback(async () => { + const video = videoRef.current as AutoPictureInPictureVideo | null; + if (video && "autoPictureInPicture" in video) { + video.autoPictureInPicture = false; + } + autoPictureInPictureRef.current = false; + if (video && document.pictureInPictureElement === video) { + await document.exitPictureInPicture(); + } + }, []); useImperativeHandle( ref, () => ({ stopStream, + getVideoStream, + closePictureInPicture, }), - [stopStream], + [stopStream, getVideoStream, closePictureInPicture], ); + useEffect(() => { + if (hidden && document.pictureInPictureElement === videoRef.current) { + void document.exitPictureInPicture().catch(() => {}); + } + }, [hidden]); + useEffect(() => { if (!canUseAutoPiPAttribute) { return; @@ -178,7 +204,7 @@ export const CameraPreviewWindow = forwardRef< } pipVideo = maybeVideo; - pipVideo.autoPictureInPicture = true; + pipVideo.autoPictureInPicture = !hidden; }; attachAttribute(); @@ -192,7 +218,7 @@ export const CameraPreviewWindow = forwardRef< pipVideo.autoPictureInPicture = false; } }; - }, [canUseAutoPiPAttribute]); + }, [canUseAutoPiPAttribute, hidden]); useEffect(() => { setMounted(true); @@ -206,9 +232,7 @@ export const CameraPreviewWindow = forwardRef< const startCamera = async () => { try { const stream = await navigator.mediaDevices.getUserMedia({ - video: { - deviceId: { exact: cameraId }, - }, + video: cameraVideoConstraints(cameraId, { height: captureHeight }), }); if (cancelled) { for (const track of stream.getTracks()) track.stop(); @@ -231,7 +255,7 @@ export const CameraPreviewWindow = forwardRef< cancelled = true; stopStream(); }; - }, [cameraId, stopStream]); + }, [cameraId, captureHeight, stopStream]); useEffect(() => { const metrics = getPreviewMetrics(size, shape, videoDimensions); @@ -372,7 +396,7 @@ export const CameraPreviewWindow = forwardRef< return; } - if (!isPictureInPictureSupported || canUseAutoPiPAttribute) { + if (!isPictureInPictureSupported || canUseAutoPiPAttribute || hidden) { return; } @@ -448,7 +472,12 @@ export const CameraPreviewWindow = forwardRef< return () => { document.removeEventListener("visibilitychange", handleVisibilityChange); }; - }, [videoDimensions, isPictureInPictureSupported, canUseAutoPiPAttribute]); + }, [ + videoDimensions, + isPictureInPictureSupported, + canUseAutoPiPAttribute, + hidden, + ]); useEffect(() => { return () => { @@ -485,7 +514,10 @@ export const CameraPreviewWindow = forwardRef<

+ + ); + + const qualitySummary = `${quality.screenHeight === 2160 ? "4K" : `${quality.screenHeight}p`} · ${quality.frameRate} fps · ${quality.level === "high" ? "High" : "Standard"}`; + + const segmented = ( + value: T, + options: readonly { value: T; label: string }[], + onChange: (next: T) => void, + name: string, + ) => ( +
+ {name} + {options.map((option) => ( + + ))} +
+ ); + + const qualityRow = (label: string, control: ReactNode) => ( +
+ {label} + {control} +
+ ); + + const qualitySection = ( +
+ + {qualityOpen && ( +
+ {qualityRow( + "Screen", + segmented( + quality.screenHeight, + [ + { value: 1080, label: "1080p" }, + { value: 1440, label: "1440p" }, + { value: 2160, label: "4K" }, + ] as const, + (screenHeight) => setQuality({ screenHeight }), + "Screen resolution", + ), + )} + {qualityRow( + "Frame rate", + segmented( + quality.frameRate, + [ + { value: 30, label: "30 fps" }, + { value: 60, label: "60 fps" }, + ] as const, + (frameRate) => setQuality({ frameRate }), + "Frame rate", + ), + )} + {qualityRow( + "Camera", + segmented( + quality.cameraHeight, + [ + { value: 720, label: "720p" }, + { value: 1080, label: "1080p" }, + ] as const, + (cameraHeight) => setQuality({ cameraHeight }), + "Camera resolution", + ), + )} + {qualityRow( + "Quality", + segmented( + quality.level, + [ + { value: "standard", label: "Standard" }, + { value: "high", label: "High" }, + ] as const, + (level) => setQuality({ level }), + "Video quality", + ), + )} + {( + [ + ["noiseSuppression", "Noise suppression"], + ["echoCancellation", "Echo cancellation"], + ["autoGainControl", "Auto gain"], + ] as const + ).map(([key, label]) => + qualityRow( + label, + + setQuality({ mic: { ...quality.mic, [key]: next } }) + } + />, + ), + )} +

+ Your video uploads while you record, so higher settings need a + faster connection. The browser uses the closest size your screen and + camera support. +

+
+ )} +
+ ); + + const steps = live + ? micOnlyActive + ? [ + { + done: true, + title: "Recording your microphone", + body: "Your screen and camera aren't included.", + }, + { + done: false, + title: "Uploads when you stop", + body: "It's saved as an audio recording.", + }, + { + done: false, + title: "Stop to open it", + body: webStudioEnabled + ? "It opens in the editor with a waveform." + : "It opens as soon as it's saved.", + }, + ] + : [ + { + done: true, + title: "Your link is live", + body: "Anyone with it can watch as soon as you stop.", + }, + { + done: partsSent > 0, + title: + partsSent === 0 + ? "Uploading as you record" + : `${partsSent} ${partsSent === 1 ? "part" : "parts"} uploaded`, + body: "No export to wait for at the end.", + }, + { + done: false, + title: "Stop to open your link", + body: `Your ${trackCount} ${trackCount === 1 ? "track stays" : "tracks stay"} separate to edit.`, + }, + ] + : [ + { + done: false, + title: "Each source is its own track", + body: "Screen, camera and audio stay separate.", + }, + { + done: false, + title: "Uploads while you record", + body: "Your link works the moment you stop.", + }, + { + done: false, + title: "Arrange it in the editor", + body: "Pick a layout, trim, then save.", + }, + ]; + + const whatHappens = ( +
+ + {live ? "While you record" : "What happens"} + +
    + {steps.map((step, index) => ( +
  1. + + {step.done ? ( + + ) : ( + index + 1 + )} + + + + {step.title} + + + {step.body} + + +
    2. + ))} +
+ {!live && ( + + )} +
+ ); + + const sidebar = ( + + ); + + const overlay = + stage === "countdown" ? ( +
+ +
+

+ Recording in {countdown} +

+

+ {screenMode + ? "When it starts, switch to what you're sharing and present as normal." + : "Look at the camera and start talking when it hits zero."} +

+
+
+ + +
+
+ ) : stage === "picking" ? ( +
+ +
+

+ Choose what to share +

+

+ Pick a screen, window or tab in your browser's popup, then click + Share. +

+
+
+ ) : null; + + const studio = ( +
+
+
+
+ {live ? ( + <> + + + + {isPaused + ? "Paused" + : screenMode + ? "Switch to what you're sharing" + : "Recording"} + + + {isPaused + ? " · Resume when you're ready" + : screenMode + ? " · Come back to this tab to stop" + : " · Stop when you're done"} + + + + ) : ( + + Preview + + {showScreen && cameraEnabled + ? " · Drag the camera to place it, and hover it to resize or flip. Your video starts the same way." + : " · This is what gets recorded."} + + + )} +
+ {live && ((screenMode && !livePreview) || micOnlyActive) ? ( +
+ {cameraEnabled ? ( +
+ {cameraVideo} + {cameraOverlay(false)} +
+ ) : ( + + {!isPaused && ( + <> + + + + )} + + + + + )} +
+ + {formatClock(durationMs)} + + + {isPaused + ? "Paused. Nothing is being recorded until you resume." + : micOnlyActive + ? "Recording your microphone. It opens as an audio recording when you stop." + : `Recording your ${joinWords(recordedWordsRef.current)}. The screen preview is hidden so it stays out of your way.`} + +
+ {!micOnlyActive && ( + + )} +
+ ) : ( +
{preview}
+ )} + {!live && !showScreen && screenSupported && ( +
+ -
+ +
+ )} +
+ {transport} +
+ + {sidebar} + + {timeline} +
+ ); + + const statusView = ( + doodle: "upload" | "done" | "error", + title: string, + body: string, + extra?: ReactNode, + ) => ( +
+ +

+ {title} +

+

+ {body} +

+ {extra} +
+ ); + + const savedTracks = (done: boolean) => ( +
    + {recordedWordsRef.current.map((word) => ( +
  • + + {word === "screen" ? ( + + ) : word === "camera" ? ( + + ) : word === "mic" ? ( + + ) : ( + + )} + + {word.replace(/^./, (c) => c.toUpperCase())} + {done ? ( + + ) : ( + + )} +
    • + ))} +
+ ); + + const body = + stage === "finishing" + ? statusView( + "upload", + "Saving your recording", + micOnlyActive + ? "Your audio is being uploaded. It opens as soon as it's saved." + : "Your link is already live. It opens as soon as the last parts land.", + <> +
+ +
+ {savedTracks(false)} + , + ) + : stage === "opening" + ? statusView( + "done", + "Opening your link", + "Your recording is saved and your link is live. Every track is ready to edit.", + <> + {savedTracks(true)} + {completedShareUrl && ( + + Open your link + + )} + , + ) + : stage === "error" + ? statusView( + "error", + "Your recording didn't finish saving", + canRetryUpload + ? "It's kept here, so you can retry the upload without recording again. The recovered files are also in the bar at the top of the page." + : "Download the recovered files from the bar at the top of the page, or start a new recording.", +
+ {canRetryUpload && ( + )} - {phase === "idle" && recoveredDownloads.length > 0 && ( -
-
- Recovered recordings -
-
- {recoveredDownloads.map((download) => ( -
-
-
- {download.fileName} -
-
- {recoveredRecordingTimeFormatter.format( - new Date(download.createdAt), - )} -
-
-
- - Download - - -
-
- ))} -
-
+ + {completedShareUrl && ( + )} - - - )} - - - - {showInProgressBar && ( + +
, + ) + : studio; + + const stageArea = ( +
+ {body} + {overlay} + {confirmRestart && isRecording && ( + setConfirmRestart(false)} + onConfirm={() => { + setConfirmRestart(false); + void restartRecording(); + }} + /> + )} + {howTopic && ( + setHowTopic(null)} + /> + )} + {cameraOnlyPrompt && ( + setCameraOnlyPrompt(false)} + onAddScreen={(dontShowAgain) => { + setCameraOnlyPrompt(false); + rememberCameraOnlyChoice(dontShowAgain, "screen"); + beginShare(true); + }} + onRecordCameraOnly={(dontShowAgain) => { + setCameraOnlyPrompt(false); + rememberCameraOnlyChoice(dontShowAgain, "camera"); + void recordNow(); + }} + /> + )} + {isMicrophoneUnavailable && ( +
+
+ +
+
+ )} + {audioGuide && ( + setAudioGuide(null)} + /> + )} + {audioGuideOpen && !audioGuide && ( + setAudioGuideOpen(false)} /> + )} +
+ ); + + const outside = ( + <> + {phase === "error" && ( { if (await prepareNewRecording()) setOpen(true); @@ -472,10 +2411,71 @@ export const WebRecorderDialog = () => { {showCameraPreview && (