diff --git a/AGENTS.md b/AGENTS.md index a5109ef..b533285 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -49,6 +49,28 @@ Effect state should be treated as a LoomLarge integration boundary, not as mutable state inside Polyester CLJS agencies. Polyester should provide clean streams and snapshots for LoomLarge to consume. +## Mixer Boundary + +Do not add a CLJS runtime shaped like `tick`, `STEP`, `update(delta)`, +`requestAnimationFrame`, or interval-based curve sampling. Polyester CLJS should +emit scheduled snippets and control effects. Loom3/Three owns +`AnimationMixer.update(delta)`, clip action timing, stream events, and runtime +cleanup. + +When touching CLJS animation/gaze/blink/prosodic/lipsync/vocal code, run +`npm run test:cljs` so the mixer-boundary guard and smoke tests both execute. + +## GitHub CLI Fallbacks + +If `gh pr edit` fails because GitHub's GraphQL response includes deprecated +classic project fields, use the REST API fallback instead, for example +`gh api repos///pulls/ -X PATCH -f body="$body"`. + +When passing Markdown with backticks to `gh issue comment`, `gh pr create`, or +`gh pr edit`, use `--body-file -` with a single-quoted heredoc or assign the +body from `cat <<'EOF'`. Do not put Markdown backticks inside a double-quoted +shell argument because the shell will execute them as command substitutions. + ## PR Scope For the CLJS transition, prefer small PRs that make one architectural boundary diff --git a/package.json b/package.json index e4185ce..0288b68 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "dev:cljs": "shadow-cljs watch npm worker", "typecheck": "tsc --noEmit", "test": "vitest run", - "test:cljs": "npm run build:cljs && node scripts/smoke-cljs-blink.mjs && node scripts/smoke-cljs-azure-parity.mjs", + "test:cljs": "npm run build:cljs && node scripts/check-cljs-mixer-boundary.mjs && node scripts/smoke-cljs-blink.mjs && node scripts/smoke-cljs-azure-parity.mjs", "test:watch": "vitest", "test:coverage": "vitest run --coverage", "prepare": "npm run build", diff --git a/scripts/check-cljs-mixer-boundary.mjs b/scripts/check-cljs-mixer-boundary.mjs new file mode 100644 index 0000000..9e377c0 --- /dev/null +++ b/scripts/check-cljs-mixer-boundary.mjs @@ -0,0 +1,100 @@ +import { readdir, readFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const scriptDir = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(scriptDir, '..'); +const sourceRoot = path.join(repoRoot, 'src-cljs', 'latticework'); + +const forbiddenPatterns = [ + { + label: 'requestAnimationFrame', + pattern: /\b(?:js\/)?requestAnimationFrame\b/, + reason: 'CLJS agencies must not own a render-frame loop.', + }, + { + label: 'setInterval', + pattern: /\b(?:js\/)?setInterval\b/, + reason: 'CLJS agencies must not introduce interval-based curve evaluators.', + }, + { + label: 'AnimationMixer', + pattern: /\bAnimationMixer\b/, + reason: 'Loom3/Three owns mixer construction and advancement.', + }, + { + label: 'STEP command', + pattern: /"STEP"|\bSTEP\b|:STEP\b/, + reason: 'Polyester should emit schedule/control effects, not frame steps.', + }, + { + label: 'tick command', + pattern: /"tick"|\btick\b|:tick\b/, + reason: 'Tick/update loops belong to the host renderer.', + }, + { + label: 'frame update function', + pattern: /\(defn-?\s+(?:update-frame|advance-frame|step-frame|tick)!?\b/, + reason: 'Per-frame update helpers should stay out of CLJS agencies.', + }, +]; + +const requiredRuntimeStrings = [ + '"scheduleSnippet"', + '"updateSnippet"', + '"removeSnippet"', + '"seekSnippet"', + '"pauseSnippet"', + '"resumeSnippet"', + '"setSnippetPlaybackRate"', + '"setSnippetIntensityScale"', + '"setSnippetLoopMode"', + '"setSnippetReverse"', +]; + +const listCljsFiles = async (dir) => { + const entries = await readdir(dir, { withFileTypes: true }); + const files = await Promise.all( + entries.map((entry) => { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) return listCljsFiles(fullPath); + if (entry.isFile() && entry.name.endsWith('.cljs')) return [fullPath]; + return []; + }), + ); + return files.flat().sort(); +}; + +const stripLineComments = (source) => + source + .split('\n') + .filter((line) => !line.trimStart().startsWith(';;')) + .join('\n'); + +const relative = (file) => path.relative(repoRoot, file); + +const files = await listCljsFiles(sourceRoot); +const failures = []; + +for (const file of files) { + const source = stripLineComments(await readFile(file, 'utf8')); + + for (const check of forbiddenPatterns) { + if (check.pattern.test(source)) { + failures.push(`${relative(file)}: found ${check.label}. ${check.reason}`); + } + } +} + +const runtimeSource = await readFile(path.join(sourceRoot, 'runtime.cljs'), 'utf8'); +for (const value of requiredRuntimeStrings) { + if (!runtimeSource.includes(value)) { + failures.push(`src-cljs/latticework/runtime.cljs: missing host control effect ${value}`); + } +} + +if (failures.length > 0) { + throw new Error(`CLJS mixer boundary check failed:\n${failures.join('\n')}`); +} + +console.log(`CLJS mixer boundary passed: ${files.length} source files checked`); diff --git a/scripts/smoke-cljs-blink.mjs b/scripts/smoke-cljs-blink.mjs index 156070a..b09a12c 100644 --- a/scripts/smoke-cljs-blink.mjs +++ b/scripts/smoke-cljs-blink.mjs @@ -134,6 +134,12 @@ const animation = createAnimationAgency( setSnippetIntensityScale(name, scale) { animationEffects.push({ op: 'hostSetScale', name, scale }); }, + setSnippetLoopMode(name, mode) { + animationEffects.push({ op: 'hostSetLoop', name, mode }); + }, + setSnippetReverse(name, reverse) { + animationEffects.push({ op: 'hostSetReverse', name, reverse }); + }, onAnimationEffect(effect) { animationEffects.push(effect); }, @@ -179,6 +185,10 @@ if (!normalizedAnimation?.isPlaying || normalizedAnimation.duration !== 0.2 || n animation.setSnippetPlaybackRate('cljs_smile', 1.5); animation.setSnippetIntensityScale('cljs_smile', 0.4); animation.seek('cljs_smile', 0.1); +animation.setSnippetPlaying('cljs_smile', false); +animation.setSnippetPlaying('cljs_smile', true); +animation.setSnippetLoopMode('cljs_smile', 'repeat'); +animation.setSnippetReverse('cljs_smile', true); animation.pause(); animation.play(); @@ -202,6 +212,23 @@ if (animationState.snippets.cljs_smile.duration !== 0.3) { throw new Error(`Expected animation update to recalculate duration, received ${animationState.snippets.cljs_smile.duration}`); } +for (const op of [ + 'hostUpdateSnippet', + 'hostSeekSnippet', + 'hostPauseSnippet', + 'hostResumeSnippet', + 'hostSetRate', + 'hostSetScale', + 'hostSetLoop', + 'hostSetReverse', + 'hostPauseAll', + 'hostPlayAll', +]) { + if (!animationEffects.some((effect) => effect.op === op)) { + throw new Error(`Expected CLJS animation control effect ${op}, received ${JSON.stringify(animationEffects)}`); + } +} + animation.remove('cljs_smile'); if (!animationRemoved.includes('cljs_smile')) { throw new Error('Expected animation remove to hit host removeSnippet'); diff --git a/src/animation/MIXER_MIGRATION_README.md b/src/animation/MIXER_MIGRATION_README.md index 0db28b3..717f5b2 100644 --- a/src/animation/MIXER_MIGRATION_README.md +++ b/src/animation/MIXER_MIGRATION_README.md @@ -1,46 +1,87 @@ -# Animation Agency → AnimationMixer Migration (Work-in-Progress) - -## Goal -Replace the legacy per-frame curve scheduler (and EngineThree transition calls) with AnimationMixer-driven playback for AU/viseme snippets, including mixed morph+bone continua, while preserving the existing snippet UI (play/pause, rate, intensity, blend). - -## Current State (after latest revert) -- Legacy scheduler/machine flow is back to working baseline. -- EngineThree exposes mixer/mixerRoot/model/morph mesh accessors (added during migration attempts). -- Machine and service experiments were reverted; mixer-only playback is not active. - -## Intended Mixer-Only Design -- Build `AnimationClip` + `AnimationAction` per snippet/curve once on load/play. -- Tracks per curve: - - Morph tracks: AU numeric → `AU_TO_MORPHS`/`MORPH_VARIANTS`; non-numeric curveId → treat as morph name. - - Bone tracks: AU numeric → bone bindings from shapeDict (e.g., `BONE_AU_TO_BINDINGS`/`COMPOSITE_ROTATIONS`), create rotation tracks on model bones. - - Mixed continua (morph+bone): include both morph and bone tracks in a single action so one weight/timeScale/loop controls both sides (L/R or up/down). -- Snippet settings → action config: - - Weight ← snippetIntensityScale or mixerWeight - - TimeScale ← snippetPlaybackRate - - Loop/clamp ← snippet.loop, mixerLoopMode, mixerClampWhenFinished - - Blend mode: fade/crossfade/additive/warp using mixer helpers and face-section policies (facePart/faceArea from shapeDict). -- Lifecycle: - - On LOAD/PLAY: create/reuse actions; start play; cache per snippet/curve/binding. - - On STEP: **only** `mixer.update(dt)` (no per-frame curve sampling or EngineThree transitions). - - On STOP/REMOVE: stop actions, uncache. - -## Mapping References -- Morphs: `src/engine/arkit/shapeDict.ts` → `AU_TO_MORPHS`, `MORPH_VARIANTS`. -- Bones: same shapeDict → `BONE_AU_TO_BINDINGS`, `COMPOSITE_ROTATIONS` (and facePart/faceArea for section/channel grouping). -- Engine host accessors: `EngineThree.getAnimationMixer/getMixerRoot/getModelRoot/getMorphMeshes`. - -## What Broke in the One-Pass Attempt -- Legacy scheduler removed; machine partially rewired; mixer actions morph-only; per-frame fallback sampling introduced → UI stopped listing/playing snippets. -- Takeaway: need incremental changes with testing, not a monolithic swap. - -## Recommended Incremental Plan -1) From baseline, keep legacy on. Add SET_HOST/STEP in machine and host mixer accessors (non-breaking). Ensure UI still works. -2) Add mixer actions for morph-only curves (build on load/play, advance mixer on STEP). Keep legacy for bones/mixed. -3) Add bone tracks for AU bindings; build mixed actions (morph+bone). Verify continua stay synced. -4) Switch blend policies per face section; wire UI mixer fields to actions. -5) Remove legacy scheduler/transition calls once mixer covers all targets; clean up action caches and stop logic. - -## Notes -- The only per-frame work in mixer mode should be `mixer.update(dt)`; no curve sampling/applying AUs each frame. -- Visemes: treat like morph tracks (curveId morph names or viseme index mapping). -- Keep UI snippet list in sync by relying on machine state; avoid mutating snapshots directly. +# Animation Agency Mixer Boundary + +This note used to describe an in-progress migration back from a legacy +per-frame scheduler. The current Polyester/Latticework boundary is different: +the animation service is already mixer-first, and the CLJS agencies must stay +as planners that emit schedule/control data. + +## Current State + +- TypeScript `animationService.ts` builds Loom3 clip handles and reads playback + state from the handle event stream. +- `src-cljs/latticework/animation.cljs` stores snippet metadata and emits + `scheduleSnippet` plus control effects. +- Other CLJS agencies, including gaze, blink, prosodic, lipsync, and vocal, + build snippets or control effects as plain maps. +- The CLJS worker dispatches commands and posts ordered output maps. It does + not own a render loop. +- Loom3/Three owns `AnimationMixer.update(delta)` through the renderer host. + +There should be no Polyester CLJS `STEP`, `tick`, `update(delta)`, +`requestAnimationFrame`, or interval-based curve evaluator. Adding one would +recreate the dual-runtime problem where CLJS samples/apply curves while Loom3 is +also advancing mixer actions. + +## Ownership + +Polyester CLJS owns: + +- agency state and behavior decisions +- snippet construction +- schedule and control commands +- cleanup plans and coarse orchestration metadata +- provider normalization, such as Azure viseme timing + +Loom3/Three and the host own: + +- clip/action construction from snippet curves +- mixer frame advancement +- runtime weights, fades, loop modes, playback rate, reverse playback, and + completion +- stream events exposed by clip handles +- action cleanup and disposal + +## Host Contract + +Normal snippet playback should use the host control surface: + +- `scheduleSnippet` or `schedule` +- `updateSnippet` +- `removeSnippet` +- `seekSnippet` +- `pauseSnippet` +- `resumeSnippet` +- `setSnippetPlaybackRate` +- `setSnippetIntensityScale` +- `setSnippetLoopMode` +- `setSnippetReverse` + +The host implementation should route scheduled snippets into Loom3 clip +construction. Procedural `transitionAU` or `transitionViseme` fallbacks can +remain as legacy/dev compatibility, but they should not be the production path +for CLJS scheduled playback. + +## Guardrails + +`npm run test:cljs` now runs `scripts/check-cljs-mixer-boundary.mjs` before the +CLJS smoke tests. The boundary check scans CLJS source for frame-loop/runtime +terms and verifies that `runtime.cljs` still exposes the expected host control +effect names. + +The smoke test also verifies that the in-process CLJS animation agency forwards +schedule, update, seek, pause, resume, parameter, and global playback operations +to host callbacks. This catches accidental regressions where agency code starts +handling mixer runtime work locally instead of emitting control effects. + +## Remaining Work + +- Audit the LoomLarge host adapter that receives Polyester CLJS + `scheduleSnippet` output and confirm it routes to Loom3 clip construction in + production chat. +- Prefer `vocal.cljs` combined sentence timelines for production lipsync and + keep `lipsync.cljs` per-word scheduling as compatibility. +- Move prosodic fade plans into mixer-owned handle/weight operations where the + host supports it. Until then, document timer fallback behavior as temporary + orchestration code. +- Add coalescing or retarget/update semantics for high-frequency gaze sources + so repeated gaze target changes do not flood the mixer with short-lived clips.