diff --git a/docs/developers/cli.mdx b/docs/developers/cli.mdx index 36baba88d2..a7df085f85 100644 --- a/docs/developers/cli.mdx +++ b/docs/developers/cli.mdx @@ -37,6 +37,7 @@ remains authoritative: run `npx hyperframes --help`. | List compositions | `npx hyperframes compositions` | | Inspect keyframe behavior | `npx hyperframes keyframes` | | Compare two or more versions | `npx hyperframes compare v1/ v2/` | +| Measure against a reference video | `npx hyperframes compare . --against reference.mp4 --at 0,4,10` | ## Rendering and automation diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx index ffb25e040b..58fe7de5a1 100644 --- a/docs/packages/cli.mdx +++ b/docs/packages/cli.mdx @@ -729,6 +729,37 @@ npx hyperframes compare ./variant-a ./variant-b --labels "A,B" | `--timeout` | Render-ready timeout per variant (default 5000 ms) | | `--json` | Machine-readable results | +With `--against`, `compare` measures one composition against an external +reference video or still instead of against sibling variants. `lint` and +`check` only ever audit a composition against its own rules, so this is the +gate that catches a scene which passes everything and still looks nothing like +the thing it reproduces. + +```bash +npx hyperframes compare . --against reference.mp4 --at 0,4,10,21 --fail-under 0.95 +``` + +| Flag | Description | +| -------------- | ------------------------------------------------------------------------------ | +| `--against` | Reference video or image to measure against (requires FFmpeg) | +| `--at` | Up to 8 comma-separated sample times, sampled in both reference and replica | +| `--fail-under` | Exit non-zero when the worst sampled SSIM falls below this threshold | +| `--out` | Contact sheet path; overlays are written next to it as `-overlay-NN.png` | + +Each run writes a reference-over-replica contact sheet, a red/cyan deviation +overlay per sampled time (agreement grey, reference-only ink red, replica-only +ink cyan), and per-time numbers: `ssim`, `meanAbsDiff`, `meanSignedDiff`, and +ink bounding-box deltas `dw` / `dh` / `dcx` / `dcy` / `scale`. When +`meanSignedDiff` is close to `meanAbsDiff` the replica is uniformly lighter or +darker, which is a level shift from encoding or colour conversion rather than a +defect in the composition. + +There is no default threshold, because the floor depends on the content. A +composition compared against its own render scores 0.998 to 0.999 for flat +graphics and type, but around 0.93 (0.89 at draft quality) once photographic +video is on screen, where encode loss and browser-versus-FFmpeg colour +conversion dominate. Measure that floor first, then gate just below it. + `grade-compare` does the same for colour: candidate grades or LUTs applied to one reference frame. diff --git a/packages/cli/src/capture/compareAgainstReference.ts b/packages/cli/src/capture/compareAgainstReference.ts new file mode 100644 index 0000000000..80897cae74 --- /dev/null +++ b/packages/cli/src/capture/compareAgainstReference.ts @@ -0,0 +1,275 @@ +/** + * `hyperframes compare --against `: measure a composition against + * the artifact it is supposed to reproduce. + * + * Produces the three instruments an agent otherwise hand-builds every time: + * a reference-over-replica contact sheet, a red/cyan deviation overlay, and + * numeric ink-bounding-box + SSIM deltas per sampled time. + */ + +import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, extname, join } from "node:path"; +import sharp from "sharp"; +import { findFFmpeg, getFFmpegInstallHint } from "../browser/ffmpeg.js"; +import { createContactSheet } from "./contactSheet.js"; +import { + AUDIT_SEEK_OPTIONS, + openSettledCompositionPage, + runFfmpegOnce, + seekCompositionTimeline, +} from "./captureCompositionFrame.js"; +import { serveStaticProjectHtml } from "../utils/staticProjectServer.js"; +import { + boundsDeviation, + inkBounds, + meanAbsDiff, + meanSignedDiff, + parseSsimAll, + redCyanOverlayRaw, + type BoundsDeviation, +} from "../utils/referenceDiff.js"; + +const FFMPEG_TIMEOUT_MS = 60_000; +const VIDEO_EXTENSIONS = new Set([ + ".mp4", + ".mov", + ".m4v", + ".webm", + ".mkv", + ".avi", + ".mpeg", + ".mpg", + ".ogv", +]); + +export interface ReferenceCompareOptions { + /** Prepared replica project directory (contains index.html). */ + projectDir: string; + /** Reference video or still image. */ + referencePath: string; + /** Timeline times in seconds, sampled in both reference and replica. */ + times: number[]; + /** Contact sheet output path. */ + outPath: string; + timeoutMs: number; +} + +export interface ReferenceSample { + time: number; + /** Full-frame SSIM (1 = identical); null when ffmpeg could not measure it. */ + ssim: number | null; + meanAbsDiff: number; + /** Signed counterpart of meanAbsDiff; close to it means a uniform level shift. */ + meanSignedDiff: number; + deviation: BoundsDeviation; + overlay: string; +} + +export interface ReferenceCompareResult { + sheet: string; + samples: ReferenceSample[]; + /** Lowest SSIM across samples, or null when none could be measured. */ + worstSsim: number | null; +} + +function isVideoReference(filePath: string): boolean { + return VIDEO_EXTENSIONS.has(extname(filePath).toLowerCase()); +} + +function overlayPathFor(outPath: string, index: number): string { + const dir = dirname(outPath); + const stem = basename(outPath, extname(outPath)); + return join(dir, `${stem}-overlay-${String(index + 1).padStart(2, "0")}.png`); +} + +async function extractReferenceFrame( + ffmpegPath: string, + referencePath: string, + time: number, + outPath: string, +): Promise { + const result = await runFfmpegOnce( + ffmpegPath, + [ + "-hide_banner", + "-loglevel", + "error", + "-ss", + String(time), + "-i", + referencePath, + "-frames:v", + "1", + "-y", + outPath, + ], + FFMPEG_TIMEOUT_MS, + ); + if (result.timedOut) { + throw new Error(`ffmpeg timed out extracting the reference frame at t=${time}s`); + } + if (result.code !== 0 || !existsSync(outPath)) { + const detail = result.stderr.trim() ? `: ${result.stderr.trim()}` : ""; + throw new Error( + `ffmpeg could not extract a reference frame at t=${time}s (past the end of ${basename(referencePath)}?)${detail}`, + ); + } +} + +/** Screenshot the composition at every sampled time, reusing one browser session. */ +async function captureReplicaFrames( + options: ReferenceCompareOptions, + frameDir: string, +): Promise { + const { bundleToSingleHtml } = await import("@hyperframes/core/compiler"); + const html = await bundleToSingleHtml(options.projectDir); + const server = await serveStaticProjectHtml(options.projectDir, html); + try { + const { browser, page } = await openSettledCompositionPage(html, server.url, { + renderReadyTimeoutMs: options.timeoutMs, + renderReadyWarningSuffix: "reference comparison may be inaccurate", + }); + try { + const paths: string[] = []; + for (const [index, time] of options.times.entries()) { + // The producer bridge is the same seek target `render` drives, so a + // video-backed composition lands on the frame the render would emit. + await seekCompositionTimeline(page, time, AUDIT_SEEK_OPTIONS); + const framePath = join(frameDir, `replica-${String(index + 1).padStart(2, "0")}.png`); + await page.screenshot({ path: framePath, type: "png" }); + paths.push(framePath); + } + return paths; + } finally { + await browser.close(); + } + } finally { + await server.close(); + } +} + +async function frameSsim( + ffmpegPath: string, + referencePath: string, + replicaPath: string, +): Promise { + // ffmpeg's own ssim filter, rather than a reimplementation of the standard. + const result = await runFfmpegOnce( + ffmpegPath, + [ + "-hide_banner", + "-i", + referencePath, + "-i", + replicaPath, + "-lavfi", + "[0:v][1:v]ssim", + "-f", + "null", + "-", + ], + FFMPEG_TIMEOUT_MS, + ); + if (result.timedOut || result.code !== 0) return null; + return parseSsimAll(result.stderr); +} + +async function grayscalePlane(path: string, width: number, height: number): Promise { + const buffer = await sharp(path) + .resize(width, height, { fit: "fill" }) + .greyscale() + .raw() + .toBuffer(); + return new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); +} + +export async function compareAgainstReference( + options: ReferenceCompareOptions, +): Promise { + if (!existsSync(options.referencePath)) { + throw new Error(`Reference not found: ${options.referencePath}`); + } + const ffmpegPath = findFFmpeg(); + if (!ffmpegPath) { + throw new Error(`--against needs ffmpeg on PATH. Install it: ${getFFmpegInstallHint()}`); + } + + const workDir = mkdtempSync(join(tmpdir(), "hf-compare-against-")); + try { + const referenceFrames: string[] = []; + for (const [index, time] of options.times.entries()) { + const framePath = join(workDir, `reference-${String(index + 1).padStart(2, "0")}.png`); + if (isVideoReference(options.referencePath)) { + await extractReferenceFrame(ffmpegPath, options.referencePath, time, framePath); + } else { + // A still reference is the same target at every sampled time. + await sharp(options.referencePath).png().toFile(framePath); + } + referenceFrames.push(framePath); + } + + const replicaFrames = await captureReplicaFrames(options, workDir); + + mkdirSync(dirname(options.outPath), { recursive: true }); + const samples: ReferenceSample[] = []; + // Replicas are normalized to reference dimensions so SSIM, the overlay and + // the sheet all read the same pixels. + const normalizedReplicas: string[] = []; + + for (const [index, time] of options.times.entries()) { + const referenceFrame = referenceFrames[index]!; + const meta = await sharp(referenceFrame).metadata(); + const width = meta.width ?? 0; + const height = meta.height ?? 0; + if (width <= 0 || height <= 0) { + throw new Error(`Could not read reference frame dimensions at t=${time}s`); + } + + const normalized = join(workDir, `replica-norm-${String(index + 1).padStart(2, "0")}.png`); + await sharp(replicaFrames[index]!).resize(width, height, { fit: "fill" }).toFile(normalized); + normalizedReplicas.push(normalized); + + const referenceGray = await grayscalePlane(referenceFrame, width, height); + const replicaGray = await grayscalePlane(normalized, width, height); + + const overlay = overlayPathFor(options.outPath, index); + await sharp(redCyanOverlayRaw(referenceGray, replicaGray, width, height), { + raw: { width, height, channels: 3 }, + }) + .png() + .toFile(overlay); + + samples.push({ + time, + ssim: await frameSsim(ffmpegPath, referenceFrame, normalized), + meanAbsDiff: meanAbsDiff(referenceGray, replicaGray), + meanSignedDiff: meanSignedDiff(referenceGray, replicaGray), + deviation: boundsDeviation( + inkBounds(referenceGray, width, height), + inkBounds(replicaGray, width, height), + ), + overlay, + }); + } + + await createContactSheet([...referenceFrames, ...normalizedReplicas], options.outPath, { + cols: options.times.length, + maxImages: options.times.length * 2, + labelMode: "custom", + labels: [ + ...options.times.map((time) => `reference t=${time}s`), + ...options.times.map((time) => `replica t=${time}s`), + ], + }); + + const measured = samples.flatMap((sample) => (sample.ssim === null ? [] : [sample.ssim])); + return { + sheet: options.outPath, + samples, + worstSsim: measured.length > 0 ? Math.min(...measured) : null, + }; + } finally { + rmSync(workDir, { recursive: true, force: true }); + } +} diff --git a/packages/cli/src/commands/compare.test.ts b/packages/cli/src/commands/compare.test.ts index f5381449ed..3c4d2a66bd 100644 --- a/packages/cli/src/commands/compare.test.ts +++ b/packages/cli/src/commands/compare.test.ts @@ -1,11 +1,12 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { describe, expect, it, vi } from "vitest"; import { buildCompareSuccessPayload, capCompareVariants, parseCompareArgs, + parseReferenceCompareArgs, prepareCompareVariantProjects, } from "./compare.js"; @@ -139,3 +140,43 @@ describe("prepareCompareVariantProjects", () => { } }); }); + +describe("parseReferenceCompareArgs", () => { + it("requires exactly one composition path", () => { + expect(() => parseReferenceCompareArgs({ _: ["a", "b"], against: "ref.mp4" }, "/tmp")).toThrow( + "--against compares exactly one composition path", + ); + }); + + it("defaults to a single sample at t=0", () => { + const parsed = parseReferenceCompareArgs({ _: ["."], against: "ref.mp4" }, "/tmp"); + expect(parsed.times).toEqual([0]); + // resolve(), not a literal: Windows turns "/tmp" into "D:\tmp". + expect(parsed.referencePath).toBe(resolve("/tmp", "ref.mp4")); + expect(parsed.failUnder).toBeUndefined(); + }); + + it("parses a comma-separated sample list and an SSIM gate", () => { + const parsed = parseReferenceCompareArgs( + { _: ["."], against: "ref.mp4", at: "0,4,10.5", "fail-under": "0.9" }, + "/tmp", + ); + expect(parsed.times).toEqual([0, 4, 10.5]); + expect(parsed.failUnder).toBe(0.9); + }); + + it("rejects negative times and out-of-range thresholds", () => { + expect(() => + parseReferenceCompareArgs({ _: ["."], against: "ref.mp4", at: "0,-1" }, "/tmp"), + ).toThrow("--at must be non-negative seconds"); + expect(() => + parseReferenceCompareArgs({ _: ["."], against: "ref.mp4", "fail-under": "2" }, "/tmp"), + ).toThrow("--fail-under must be an SSIM threshold"); + }); + + it("caps the sample count", () => { + expect(() => + parseReferenceCompareArgs({ _: ["."], against: "ref.mp4", at: "1,2,3,4,5,6,7,8,9" }, "/tmp"), + ).toThrow("at most 8 times"); + }); +}); diff --git a/packages/cli/src/commands/compare.ts b/packages/cli/src/commands/compare.ts index fac16738b7..5f63da536d 100644 --- a/packages/cli/src/commands/compare.ts +++ b/packages/cli/src/commands/compare.ts @@ -4,6 +4,8 @@ import { tmpdir } from "node:os"; import { basename, dirname, extname, join } from "node:path"; import { defineCommand } from "citty"; import { createContactSheet } from "../capture/contactSheet.js"; +import { compareAgainstReference } from "../capture/compareAgainstReference.js"; +import type { BoundsDeviation } from "../utils/referenceDiff.js"; import { openSettledCompositionPage, seekCompositionTimeline, @@ -40,6 +42,17 @@ export interface ParsedCompareArgs { timeoutMs: number; } +export interface ParsedReferenceCompareArgs { + variant: CompareVariantSpec; + referencePath: string; + displayReferencePath: string; + times: number[]; + outPath: string; + failUnder?: number; + json: boolean; + timeoutMs: number; +} + export interface CompareVariantCapResult { variants: CompareVariantSpec[]; truncated: boolean; @@ -64,6 +77,10 @@ export const examples: Example[] = [ "Compare three variants at a specific timeline time", "hyperframes compare ./a ./b ./c --at 2.5 --labels classic,bold,quiet --json", ], + [ + "Measure a rebuild against the video it reproduces", + "hyperframes compare . --against reference.mp4 --at 0,4,10,21 --json", + ], ]; function defaultLabelForPath(input: string): string { @@ -151,6 +168,82 @@ export function parseCompareArgs( }; } +const MAX_REFERENCE_TIMES = 8; + +/** + * Raw citty args for the `--against` route. String flags arrive as strings and + * boolean flags as booleans, so the parse below narrows values rather than + * type-testing them. + */ +export interface ReferenceCompareCliArgs { + _?: readonly string[]; + against?: string; + out?: string; + at?: string; + "fail-under"?: string; + json?: boolean; + timeout?: string; +} + +function trimmed(value: string | undefined): string | undefined { + const text = value?.trim(); + return text ? text : undefined; +} + +function parseReferenceTimes(value: string | undefined): number[] { + const raw = trimmed(value); + if (!raw) return [0]; + const times = raw.split(",").map((entry) => { + const parsed = Number(entry.trim()); + if (!Number.isFinite(parsed) || parsed < 0) { + throw new Error("--at must be non-negative seconds (comma-separated with --against)"); + } + return parsed; + }); + if (times.length > MAX_REFERENCE_TIMES) { + throw new Error(`--at accepts at most ${MAX_REFERENCE_TIMES} times with --against`); + } + return times; +} + +function parseFailUnder(value: string | undefined): number | undefined { + const raw = trimmed(value); + if (!raw) return undefined; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed < 0 || parsed > 1) { + throw new Error("--fail-under must be an SSIM threshold between 0 and 1"); + } + return parsed; +} + +export function parseReferenceCompareArgs( + args: ReferenceCompareCliArgs, + cwd = process.cwd(), +): ParsedReferenceCompareArgs { + const paths = (args._ ?? []).map((value) => value.trim()).filter(Boolean); + if (paths.length !== 1) { + throw new Error("--against compares exactly one composition path against the reference"); + } + const reference = trimmed(args.against); + if (!reference) throw new Error("--against needs a reference video or image path"); + const input = paths[0]!; + + return { + variant: { + label: defaultLabelForPath(input), + inputPath: resolveFromBase(cwd, input), + displayPath: displayPathFromInput(cwd, input), + }, + referencePath: resolveFromBase(cwd, reference), + displayReferencePath: displayPathFromInput(cwd, reference), + times: parseReferenceTimes(args.at), + outPath: resolveFromBase(cwd, trimmed(args.out) ?? "compare.png"), + failUnder: parseFailUnder(args["fail-under"]), + json: args.json === true, + timeoutMs: Number.parseInt(trimmed(args.timeout) ?? "", 10) || DEFAULT_RENDER_READY_TIMEOUT_MS, + }; +} + export function capCompareVariants( variants: readonly CompareVariantSpec[], ): CompareVariantCapResult { @@ -345,6 +438,94 @@ async function renderCompareSheet(parsed: ParsedCompareArgs): Promise { + const prepared = prepareCompareVariantProjects([parsed.variant]); + try { + const result = await compareAgainstReference({ + projectDir: prepared[0]!.projectDir, + referencePath: parsed.referencePath, + times: parsed.times, + outPath: parsed.outPath, + timeoutMs: parsed.timeoutMs, + }); + + trackCompareSheet({ + command: "compare", + cells: parsed.times.length * 2, + truncated: false, + total: parsed.times.length * 2, + renderReadyTimedOut: false, + }); + + const gated = + parsed.failUnder !== undefined && + (result.worstSsim === null || result.worstSsim < parsed.failUnder); + + return { + ok: !gated, + sheet: result.sheet, + reference: parsed.displayReferencePath, + replica: parsed.variant.displayPath, + worstSsim: result.worstSsim, + ...(parsed.failUnder !== undefined ? { failUnder: parsed.failUnder } : {}), + samples: result.samples.map((sample) => ({ + time: sample.time, + ssim: sample.ssim, + meanAbsDiff: Number(sample.meanAbsDiff.toFixed(4)), + meanSignedDiff: Number(sample.meanSignedDiff.toFixed(4)), + deviation: sample.deviation, + overlay: sample.overlay, + })), + }; + } finally { + cleanupPreparedCompareVariants(prepared); + } +} + +function printReferenceReport(payload: ReferenceComparePayload): void { + console.log(); + console.log(c.bold(`Reference: ${payload.reference}`)); + for (const sample of payload.samples) { + const { dw, dh, dcx, dcy, scale } = sample.deviation; + const ssim = sample.ssim === null ? "n/a" : sample.ssim.toFixed(4); + const signed = sample.meanSignedDiff * 100; + console.log( + ` t=${sample.time}s SSIM ${ssim} diff ${(sample.meanAbsDiff * 100).toFixed(1)}% (bias ${signed >= 0 ? "+" : ""}${signed.toFixed(1)}%)`, + ); + console.log( + c.dim( + ` ink dw=${dw}px dh=${dh}px dcx=${dcx.toFixed(1)}px dcy=${dcy.toFixed(1)}px scale=${scale.toFixed(3)}`, + ), + ); + console.log(c.dim(` overlay ${sample.overlay}`)); + } + console.log(); + console.log(`${c.success("◇")} Contact sheet saved to ${payload.sheet}`); + if (!payload.ok) { + const worst = payload.worstSsim === null ? "unmeasured" : payload.worstSsim.toFixed(4); + console.error(`${c.error("✗")} Worst SSIM ${worst} is below --fail-under ${payload.failUnder}`); + } +} + function printJson(payload: object): void { console.log(JSON.stringify(withMeta(payload), null, 2)); } @@ -360,9 +541,19 @@ export default defineCommand({ description: "Composition project directory or .html file (pass 2+ paths)", required: false, }, + against: { + type: "string", + description: + "Reference video or image to measure one composition against (SSIM + ink-box deltas + red/cyan overlay)", + }, at: { type: "string", - description: "Timeline time in seconds to seek before screenshotting each variant", + description: + "Timeline time in seconds to seek before screenshotting each variant (comma-separated with --against)", + }, + "fail-under": { + type: "string", + description: "With --against, exit non-zero when the worst sampled SSIM falls below this", }, labels: { type: "string", @@ -389,6 +580,20 @@ export default defineCommand({ async run({ args }) { const jsonRequested = args.json === true; try { + if (trimmed(args.against)) { + const parsed = parseReferenceCompareArgs(args); + if (!parsed.json) { + console.log( + `${c.accent("◆")} Measuring ${parsed.variant.label} against ${parsed.displayReferencePath} at ${parsed.times.length} time(s)`, + ); + } + const payload = await runReferenceCompare(parsed); + if (parsed.json) printJson(payload); + else printReferenceReport(payload); + if (!payload.ok) failCommand(); + return; + } + const parsed = parseCompareArgs(args); if (!parsed.json) { console.log( diff --git a/packages/cli/src/utils/referenceDiff.test.ts b/packages/cli/src/utils/referenceDiff.test.ts new file mode 100644 index 0000000000..23117a5f75 --- /dev/null +++ b/packages/cli/src/utils/referenceDiff.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from "vitest"; +import { + boundsDeviation, + inkBounds, + meanAbsDiff, + meanSignedDiff, + parseSsimAll, + redCyanOverlayRaw, +} from "./referenceDiff.js"; + +/** Dark canvas with one bright rectangle of ink. */ +function frameWithBox( + width: number, + height: number, + box: { x: number; y: number; w: number; h: number }, +): Uint8Array { + const plane = new Uint8Array(width * height).fill(10); + for (let y = box.y; y < box.y + box.h; y++) { + for (let x = box.x; x < box.x + box.w; x++) plane[y * width + x] = 240; + } + return plane; +} + +describe("inkBounds", () => { + it("brackets the ink and ignores the background", () => { + const bounds = inkBounds(frameWithBox(40, 20, { x: 10, y: 4, w: 8, h: 6 }), 40, 20); + expect(bounds).toMatchObject({ x0: 10, y0: 4, x1: 17, y1: 9, width: 8, height: 6 }); + expect(bounds.cx).toBe(14); + expect(bounds.cy).toBe(7); + expect(bounds.empty).toBe(false); + }); + + it("reports empty for a flat frame", () => { + expect(inkBounds(new Uint8Array(40 * 20).fill(128), 40, 20).empty).toBe(true); + }); +}); + +describe("boundsDeviation", () => { + it("measures replica-minus-reference size and centre offsets", () => { + const reference = inkBounds(frameWithBox(40, 20, { x: 10, y: 4, w: 8, h: 6 }), 40, 20); + const replica = inkBounds(frameWithBox(40, 20, { x: 14, y: 4, w: 12, h: 6 }), 40, 20); + const deviation = boundsDeviation(reference, replica); + expect(deviation.dw).toBe(4); + expect(deviation.dh).toBe(0); + expect(deviation.dcx).toBe(6); + expect(deviation.dcy).toBe(0); + expect(deviation.scale).toBe(1.5); + }); + + it("reports zero scale when the reference has no ink", () => { + const empty = inkBounds(new Uint8Array(16).fill(128), 4, 4); + const replica = inkBounds(frameWithBox(4, 4, { x: 1, y: 1, w: 2, h: 2 }), 4, 4); + expect(boundsDeviation(empty, replica).scale).toBe(0); + }); +}); + +describe("meanAbsDiff", () => { + it("is 0 for identical planes and 1 for inverted ones", () => { + const plane = frameWithBox(8, 8, { x: 2, y: 2, w: 3, h: 3 }); + expect(meanAbsDiff(plane, plane)).toBe(0); + expect(meanAbsDiff(new Uint8Array(64).fill(0), new Uint8Array(64).fill(255))).toBe(1); + }); +}); + +describe("meanSignedDiff", () => { + it("matches meanAbsDiff when the replica is uniformly brighter", () => { + const reference = new Uint8Array(64).fill(100); + const replica = new Uint8Array(64).fill(110); + expect(meanSignedDiff(reference, replica)).toBeCloseTo(meanAbsDiff(reference, replica), 6); + }); + + it("cancels to ~0 when the deviation is localized in both directions", () => { + const reference = new Uint8Array(64).fill(100); + const replica = new Uint8Array(64).fill(100); + replica[0] = 200; + replica[1] = 0; + expect(meanSignedDiff(reference, replica)).toBeCloseTo(0, 6); + expect(meanAbsDiff(reference, replica)).toBeGreaterThan(0); + }); +}); + +describe("redCyanOverlayRaw", () => { + it("puts the reference in red and the replica in green+blue", () => { + const overlay = redCyanOverlayRaw(new Uint8Array([200, 0]), new Uint8Array([0, 100]), 2, 1); + expect([...overlay]).toEqual([200, 0, 0, 0, 100, 100]); + }); +}); + +describe("parseSsimAll", () => { + it("reads the last All: value ffmpeg printed", () => { + const stderr = "[Parsed_ssim_0 @ 0x1] SSIM Y:0.97 U:0.99 V:0.99 All:0.9612 (14.10)\n"; + expect(parseSsimAll(stderr)).toBeCloseTo(0.9612, 4); + }); + + it("returns null when ffmpeg printed no SSIM line", () => { + expect(parseSsimAll("Invalid argument\n")).toBeNull(); + }); +}); diff --git a/packages/cli/src/utils/referenceDiff.ts b/packages/cli/src/utils/referenceDiff.ts new file mode 100644 index 0000000000..bc7f3b20b1 --- /dev/null +++ b/packages/cli/src/utils/referenceDiff.ts @@ -0,0 +1,168 @@ +/** + * Reference-grounded frame measurements. + * + * Every other gate in the CLI is self-referential: it inspects the composition + * against its own rules and can pass a scene that renders nothing like the + * thing it is supposed to reproduce. These helpers compare a rendered replica + * frame against an external reference frame and return numbers, so "close + * enough" stops being a judgement call. + */ + +/** + * Luma distance from the frame's median before a pixel counts as ink. + * + * ponytail: a single global threshold on the median. Good for type/graphic + * frames (the case this exists for); a full-bleed photo makes every pixel ink + * and the bounds degenerate to the whole canvas. Swap in per-region stats if + * photographic frames ever need real bounds. + */ +const INK_DELTA = 32; + +export interface InkBounds { + x0: number; + y0: number; + x1: number; + y1: number; + width: number; + height: number; + cx: number; + cy: number; + empty: boolean; +} + +export interface BoundsDeviation { + /** Replica minus reference, in reference pixels. */ + dw: number; + dh: number; + dcx: number; + dcy: number; + /** Replica ink width as a fraction of reference ink width (1 = identical). */ + scale: number; +} + +const EMPTY_BOUNDS: InkBounds = { + x0: 0, + y0: 0, + x1: 0, + y1: 0, + width: 0, + height: 0, + cx: 0, + cy: 0, + empty: true, +}; + +function medianLuma(gray: Uint8Array): number { + const histogram = new Uint32Array(256); + for (const value of gray) histogram[value]! += 1; + const half = gray.length / 2; + let seen = 0; + for (let level = 0; level < 256; level++) { + seen += histogram[level]!; + if (seen >= half) return level; + } + return 0; +} + +/** Bounding box of everything that is not background, from an 8-bit grayscale plane. */ +export function inkBounds(gray: Uint8Array, width: number, height: number): InkBounds { + if (width <= 0 || height <= 0 || gray.length < width * height) return EMPTY_BOUNDS; + const median = medianLuma(gray); + let x0 = width; + let y0 = height; + let x1 = -1; + let y1 = -1; + + for (let y = 0; y < height; y++) { + const row = y * width; + for (let x = 0; x < width; x++) { + if (Math.abs(gray[row + x]! - median) <= INK_DELTA) continue; + x0 = Math.min(x0, x); + x1 = Math.max(x1, x); + y0 = Math.min(y0, y); + y1 = Math.max(y1, y); + } + } + + if (x1 < 0) return EMPTY_BOUNDS; + return { + x0, + y0, + x1, + y1, + width: x1 - x0 + 1, + height: y1 - y0 + 1, + cx: (x0 + x1 + 1) / 2, + cy: (y0 + y1 + 1) / 2, + empty: false, + }; +} + +export function boundsDeviation(reference: InkBounds, replica: InkBounds): BoundsDeviation { + return { + dw: replica.width - reference.width, + dh: replica.height - reference.height, + dcx: replica.cx - reference.cx, + dcy: replica.cy - reference.cy, + scale: reference.width > 0 ? replica.width / reference.width : 0, + }; +} + +/** Mean per-pixel luma difference, normalized to 0 (identical) .. 1 (inverted). */ +export function meanAbsDiff(reference: Uint8Array, replica: Uint8Array): number { + const length = Math.min(reference.length, replica.length); + if (length === 0) return 1; + let total = 0; + for (let i = 0; i < length; i++) total += Math.abs(reference[i]! - replica[i]!); + return total / (length * 255); +} + +/** + * Mean *signed* luma difference (replica minus reference), normalized to -1..1. + * + * Separates the two things `meanAbsDiff` sums together. A comparison whose + * signed value is close to its absolute value is uniformly lighter or darker, + * which is a level shift from encoding or colour conversion, not a structural + * error. Signed near zero with a large absolute value means the deviation is + * real and localized. + */ +export function meanSignedDiff(reference: Uint8Array, replica: Uint8Array): number { + const length = Math.min(reference.length, replica.length); + if (length === 0) return 0; + let total = 0; + for (let i = 0; i < length; i++) total += replica[i]! - reference[i]!; + return total / (length * 255); +} + +/** + * Interleaved RGB where the reference drives red and the replica drives + * green+blue: agreement reads neutral grey, reference-only ink glows red, + * replica-only ink glows cyan. + */ +export function redCyanOverlayRaw( + reference: Uint8Array, + replica: Uint8Array, + width: number, + height: number, +): Buffer { + const pixels = width * height; + const out = Buffer.allocUnsafe(pixels * 3); + for (let i = 0; i < pixels; i++) { + const ref = reference[i] ?? 0; + const rep = replica[i] ?? 0; + out[i * 3] = ref; + out[i * 3 + 1] = rep; + out[i * 3 + 2] = rep; + } + return out; +} + +/** Pull the full-frame SSIM out of ffmpeg's `ssim` filter log line. */ +export function parseSsimAll(stderr: string): number | null { + let value: number | null = null; + for (const match of stderr.matchAll(/\bAll:\s*([0-9]*\.?[0-9]+)/g)) { + const parsed = Number(match[1]); + if (Number.isFinite(parsed)) value = parsed; + } + return value; +} diff --git a/skills-manifest.json b/skills-manifest.json index cb7dbe3e27..9673d6e87f 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -18,7 +18,7 @@ "files": 4 }, "hyperframes": { - "hash": "3a3ea01fe52e1600", + "hash": "84ed304a6537276e", "files": 17 }, "hyperframes-animation": { @@ -26,7 +26,7 @@ "files": 121 }, "hyperframes-cli": { - "hash": "3eab4c5fc1ae8e26", + "hash": "89bacf4be7b179cb", "files": 11 }, "hyperframes-core": { @@ -66,7 +66,7 @@ "files": 28 }, "remotion-to-hyperframes": { - "hash": "3ecc684432b298dd", + "hash": "e34b40b26ee25ac1", "files": 70 }, "slideshow": { diff --git a/skills/hyperframes-cli/SKILL.md b/skills/hyperframes-cli/SKILL.md index 8db28f39ef..b37e56a386 100644 --- a/skills/hyperframes-cli/SKILL.md +++ b/skills/hyperframes-cli/SKILL.md @@ -38,6 +38,16 @@ ffprobe -v error -show_format out.mp4 `check` runs lint first, then uses one browser session and one seek pass to audit runtime errors, failed requests, layout, `*.motion.json` assertions, and WCAG contrast. Persistent findings gate the exit code; transient entrance or exit findings are informational. Use `--strict` to gate warnings. `validate`, `inspect`, and `layout` remain aliases for compatibility but must not appear in new instructions or scripts. +## Ground the gate in a reference when one exists + +`lint` and `check` are self-referential: they audit the composition against its own rules and never see the thing it is supposed to look like. A scene built upside down, at the wrong scale, or missing its hero element passes both. Whenever the brief comes with a reference artifact (a video being rebuilt or recut, an approved cut, a design still, the previous accepted render), add the one gate that does look outward: + +```bash +npx hyperframes compare . --against reference.mp4 --at 0,4,10,21 --json +``` + +It returns per-time SSIM, ink bounding-box deltas, a reference-over-replica contact sheet, and a red/cyan deviation overlay; `--fail-under ` makes it exit non-zero. Iterate against the numbers instead of eyeballing composites, and re-measure after every correction: the numbers are what tell you a fix helped. Full contract in `references/compare-and-batch.md`. + ## Two different preview surfaces Do not confuse these states: @@ -113,18 +123,18 @@ Keep clean-run feedback concise. For any bug or friction, capture a **reproducti The following references and owning skills are mandatory command contracts, not optional background reading. Before running a command in the table, read its matching row. -| Need | Reference | -| -------------------------------------------------------------------------------------- | ------------------------------------- | -| `init`, `capture`, `skills` | `references/init-and-scaffold.md` | -| `lint`, `check`, motion sidecars, `snapshot` | `references/lint-validate-inspect.md` | -| `compare`, `grade-compare`, variable-driven `render --batch` | `references/compare-and-batch.md` | -| `beats` for an existing project's Studio beat grid | `references/beats.md` | -| `preview`, `play`, `render`, `publish`, Studio context, feedback | `references/preview-render.md` | -| `doctor`, browser management | `references/doctor-browser.md` | -| `auth`, HeyGen-hosted cloud rendering, and template variables | `references/cloud.md` | -| AWS Lambda deployment and rendering | `references/lambda.md` | -| Google Cloud Run deployment and rendering | `references/cloudrun.md` | -| `info`, `upgrade`, `compositions`, `docs`, `benchmark`, telemetry, media preprocessing | `references/upgrade-info-misc.md` | +| Need | Reference | +| --------------------------------------------------------------------------------------------- | ------------------------------------- | +| `init`, `capture`, `skills` | `references/init-and-scaffold.md` | +| `lint`, `check`, motion sidecars, `snapshot` | `references/lint-validate-inspect.md` | +| `compare`, `compare --against `, `grade-compare`, variable-driven `render --batch` | `references/compare-and-batch.md` | +| `beats` for an existing project's Studio beat grid | `references/beats.md` | +| `preview`, `play`, `render`, `publish`, Studio context, feedback | `references/preview-render.md` | +| `doctor`, browser management | `references/doctor-browser.md` | +| `auth`, HeyGen-hosted cloud rendering, and template variables | `references/cloud.md` | +| AWS Lambda deployment and rendering | `references/lambda.md` | +| Google Cloud Run deployment and rendering | `references/cloudrun.md` | +| `info`, `upgrade`, `compositions`, `docs`, `benchmark`, telemetry, media preprocessing | `references/upgrade-info-misc.md` | For composition variables, also read `/hyperframes-core` → `references/variables-and-media.md`. For `hyperframes add` and `hyperframes catalog`, use `/hyperframes-registry`. Before `hyperframes present`, read `/slideshow`; before `hyperframes keyframes`, read `/hyperframes-keyframes`. For TTS, transcription, captions, or background removal choices, use `/media-use`. diff --git a/skills/hyperframes-cli/references/compare-and-batch.md b/skills/hyperframes-cli/references/compare-and-batch.md index 7e2a2e0dd3..a2a17cb432 100644 --- a/skills/hyperframes-cli/references/compare-and-batch.md +++ b/skills/hyperframes-cli/references/compare-and-batch.md @@ -5,6 +5,7 @@ Use these commands for deliberate visual comparison or variable-driven template ## Contents - [Compare projects or variants](#compare-projects-or-variants) +- [Measure against a reference](#measure-against-a-reference) - [Compare color grades](#compare-color-grades) - [Batch template renders](#batch-template-renders) @@ -31,7 +32,57 @@ Useful options: One sheet accepts at most 16 variants. Extra inputs are truncated with a warning; split larger comparisons into several runs. -`compare` is a visual review surface, not a quality gate. Run it when checking a baseline against a candidate, comparing implementation variants, or verifying that a repair preserves the intended look. Inspect the generated image; do not treat command success as visual approval. +Without `--against`, `compare` is a visual review surface, not a quality gate. Run it when checking a baseline against a candidate, comparing implementation variants, or verifying that a repair preserves the intended look. Inspect the generated image; do not treat command success as visual approval. + +## Measure against a reference + +Every other gate in the CLI is self-referential: `lint` and `check` audit the composition against its own rules, so a scene that renders nothing like the artifact it is supposed to reproduce still passes them. When a reference artifact exists (the video being rebuilt, an approved cut, a design still, yesterday's render), measure against it: + +```bash +npx hyperframes compare \ + --against reference.mp4 \ + --at 0,4,10,21 \ + --out compare.png \ + --json +``` + +Exactly one composition path is allowed with `--against`. The reference may be a video (frames are pulled at each `--at` time) or a still image (the same target at every time). Up to 8 times per run; `--at` defaults to `0`. + +Each run produces three instruments: + +| Artifact | Where | Reads as | +| ---------------------------- | ---------------------- | --------------------------------------------------------------------------------------------- | +| Reference-over-replica sheet | `--out` path | Row 1 reference, row 2 replica, one column per sampled time | +| Red/cyan deviation overlay | `-overlay-NN.png` | Agreement grey, reference-only ink red, replica-only ink cyan | +| Numbers | stdout and `--json` | `ssim`, `meanAbsDiff`, `meanSignedDiff`, ink-box deltas `dw` / `dh` / `dcx` / `dcy` / `scale` | + +How to read them: + +- **`ssim`** is full-frame structural similarity, 1.0 = identical. What counts as good depends on the content, so read it against the floor below rather than against 1.0. +- **`meanAbsDiff` vs `meanSignedDiff`** separates two things a single number confuses. `meanSignedDiff` is the same average without the absolute value, so when the two are close the replica is uniformly lighter or darker, which is a level shift from encoding or colour conversion and not a mistake you can fix in the composition. Signed near zero with a large absolute value means the deviation is real and localized. The CLI prints it as `diff X% (bias +Y%)`. +- **Ink-box deltas** answer "is my title the right size and in the right place": `dw`/`dh` are the replica's ink bounding box minus the reference's in reference pixels, `dcx`/`dcy` the centre offset, `scale` the width ratio. They are meaningful for type and graphic frames; a full-bleed photograph makes every pixel ink and the box degenerates to the whole canvas. +- **The overlay** localizes the deviation the numbers only total up. A flat tint across the whole frame is the level shift `meanSignedDiff` already quantified; localized red/cyan ghosting is a position, size or timing error. + +### The floor is set by content, not by your composition + +A replica is a live browser paint; a reference is a decoded compressed video. The gap between those two decode paths is a floor no correction can go below, and it depends entirely on what is on screen. Measured against their own renders: + +| Composition content | Self-comparison SSIM | Why | +| ------------------------- | ------------------------------------------- | ------------------------------------------------------------------ | +| Flat graphics and type | 0.998–0.999 | Encodes near-losslessly; a real defect shows immediately | +| Photographic video layers | ~0.93 at `--quality high`, ~0.89 at `draft` | Encode loss plus browser/FFmpeg colour conversion, visible as bias | + +So a 0.93 on a video-backed composition can be a perfect rebuild, and a 0.98 on a typographic one is a real defect. Establish the floor before you read any number: compare the composition against its own render first, then treat that value as your zero. + +Gate on it with `--fail-under `, which exits non-zero when the worst sampled SSIM falls below the threshold: + +```bash +npx hyperframes compare . --against reference.mp4 --at 0,4,10,21 --fail-under 0.95 +``` + +There is no default threshold, deliberately: pick one just below the floor you measured above. A threshold guessed before measuring passes everything and gates nothing, and one copied from a graphics-only project will fail every video-backed build for no reason. + +`--against` needs FFmpeg on PATH. ## Compare color grades diff --git a/skills/hyperframes/SKILL.md b/skills/hyperframes/SKILL.md index e447e58d4e..68976d76e8 100644 --- a/skills/hyperframes/SKILL.md +++ b/skills/hyperframes/SKILL.md @@ -73,6 +73,8 @@ Before finalizing the route, read `references/routes/.md` — one smal For fresh creation the intent layer (`references/intent-interview.md`) runs the full conversation — memory, triage, pitch round, must-haves, run-shape, hand-off — and **ends by writing `BRIEF.md`. The brief is the only routing artifact the workflow reads**; nothing later re-opens this skill or the interview. Answer every later "what did the route require?" from `BRIEF.md`. +Read the routed workflow's `SKILL.md` and nothing from its siblings. The other creation workflows cannot contribute to a route that is already decided, and reading them is pure cost: the single largest avoidable expense in a build, paid again on every turn once it is in context. If a sibling workflow looks relevant mid-build, the route was wrong: say so and re-route, rather than reading both. + ## 4. Install and enter the workflow Before reading the selected workflow, install or refresh it and the core domain skills: @@ -93,6 +95,7 @@ Use the bare name without `/`. If the command fails, surface the error; do not r | Design specs, concept, palette, typography, narration, beat planning | `/hyperframes-creative` | | Images, icons, logos, audio, captions, grades, LUTs, reusable media | `/media-use` | | Init, lint, check, snapshots, compare, batch render, Studio, render, publish, or diagnostics | `/hyperframes-cli` | +| Measuring the build against a reference video, cut, or still the brief supplies | `/hyperframes-cli` | | Registry blocks and components | `/hyperframes-registry` | | Figma assets, tokens, components, or storyboard frames as reconstructed motion | `/figma` | diff --git a/skills/remotion-to-hyperframes/SKILL.md b/skills/remotion-to-hyperframes/SKILL.md index ec7c8f1af3..a371dbd46c 100644 --- a/skills/remotion-to-hyperframes/SKILL.md +++ b/skills/remotion-to-hyperframes/SKILL.md @@ -96,6 +96,14 @@ cd ../hf-src && npx hyperframes render --skill=remotion-to-hyperframes --output Threshold: ~0.02 below `p05` of the source's complexity tier (see `eval.md`'s validated thresholds table). If the diff fails, run [`scripts/frame_strip.sh`](scripts/frame_strip.sh) to see _which_ frames diverged, then re-read the relevant timing/sequencing/media reference. +While iterating, skip the HF render and measure the live composition against the baseline directly: same SSIM number, plus ink-box deltas and a red/cyan overlay per sampled time: + +```bash +npx hyperframes compare ./hf-src --against ./remotion-src/out/baseline.mp4 --at 0,1,2,4 --json +``` + +Use it to steer corrections between full renders; the harness above stays the gate, because it scores every frame rather than a sample. + **Critical**: both renders must use matching pixel format. Set `Config.setVideoImageFormat("png")` + `Config.setColorSpace("bt709")` in the Remotion source's `remotion.config.ts` — otherwise the diff measures encoder differences (~0.05 SSIM hit), not translation fidelity. ### Step 5: Document gaps