feat(cli): measure a composition against a reference with compare --against

docs/developers/cli.mdx

@@ -37,6 +37,7 @@ remains authoritative: run `npx hyperframes <command> --help`.
 | List compositions | `npx hyperframes compositions` |
 | Inspect keyframe behavior | `npx hyperframes keyframes` |
 | Compare two or more versions | `npx hyperframes compare v1/ v2/` |
+| Measure against a reference video | `npx hyperframes compare . --against reference.mp4 --at 0,4,10` |
 
 ## Rendering and automation
 

docs/packages/cli.mdx

@@ -729,6 +729,37 @@ npx hyperframes compare ./variant-a ./variant-b --labels "A,B"
 | `--timeout` | Render-ready timeout per variant (default 5000 ms) |
 | `--json`    | Machine-readable results                           |
 
+With `--against`, `compare` measures one composition against an external
+reference video or still instead of against sibling variants. `lint` and
+`check` only ever audit a composition against its own rules, so this is the
+gate that catches a scene which passes everything and still looks nothing like
+the thing it reproduces.
+
+```bash
+npx hyperframes compare . --against reference.mp4 --at 0,4,10,21 --fail-under 0.95
+```
+
+| Flag           | Description                                                                    |
+| -------------- | ------------------------------------------------------------------------------ |
+| `--against`    | Reference video or image to measure against (requires FFmpeg)                  |
+| `--at`         | Up to 8 comma-separated sample times, sampled in both reference and replica    |
+| `--fail-under` | Exit non-zero when the worst sampled SSIM falls below this threshold           |
+| `--out`        | Contact sheet path; overlays are written next to it as `<out>-overlay-NN.png`  |
+
+Each run writes a reference-over-replica contact sheet, a red/cyan deviation
+overlay per sampled time (agreement grey, reference-only ink red, replica-only
+ink cyan), and per-time numbers: `ssim`, `meanAbsDiff`, `meanSignedDiff`, and
+ink bounding-box deltas `dw` / `dh` / `dcx` / `dcy` / `scale`. When
+`meanSignedDiff` is close to `meanAbsDiff` the replica is uniformly lighter or
+darker, which is a level shift from encoding or colour conversion rather than a
+defect in the composition.
+
+There is no default threshold, because the floor depends on the content. A
+composition compared against its own render scores 0.998 to 0.999 for flat
+graphics and type, but around 0.93 (0.89 at draft quality) once photographic
+video is on screen, where encode loss and browser-versus-FFmpeg colour
+conversion dominate. Measure that floor first, then gate just below it.
+
 `grade-compare` does the same for colour: candidate grades or LUTs applied to
 one reference frame.
 

packages/cli/src/capture/compareAgainstReference.ts

@@ -0,0 +1,275 @@
+/**
+ * `hyperframes compare --against <reference>`: measure a composition against
+ * the artifact it is supposed to reproduce.
+ *
+ * Produces the three instruments an agent otherwise hand-builds every time:
+ * a reference-over-replica contact sheet, a red/cyan deviation overlay, and
+ * numeric ink-bounding-box + SSIM deltas per sampled time.
+ */
+
+import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { basename, dirname, extname, join } from "node:path";
+import sharp from "sharp";
+import { findFFmpeg, getFFmpegInstallHint } from "../browser/ffmpeg.js";
+import { createContactSheet } from "./contactSheet.js";
+import {
+  AUDIT_SEEK_OPTIONS,
+  openSettledCompositionPage,
+  runFfmpegOnce,
+  seekCompositionTimeline,
+} from "./captureCompositionFrame.js";
+import { serveStaticProjectHtml } from "../utils/staticProjectServer.js";
+import {
+  boundsDeviation,
+  inkBounds,
+  meanAbsDiff,
+  meanSignedDiff,
+  parseSsimAll,
+  redCyanOverlayRaw,
+  type BoundsDeviation,
+} from "../utils/referenceDiff.js";
+
+const FFMPEG_TIMEOUT_MS = 60_000;
+const VIDEO_EXTENSIONS = new Set([
+  ".mp4",
+  ".mov",
+  ".m4v",
+  ".webm",
+  ".mkv",
+  ".avi",
+  ".mpeg",
+  ".mpg",
+  ".ogv",
+]);
+
+export interface ReferenceCompareOptions {
+  /** Prepared replica project directory (contains index.html). */
+  projectDir: string;
+  /** Reference video or still image. */
+  referencePath: string;
+  /** Timeline times in seconds, sampled in both reference and replica. */
+  times: number[];
+  /** Contact sheet output path. */
+  outPath: string;
+  timeoutMs: number;
+}
+
+export interface ReferenceSample {
+  time: number;
+  /** Full-frame SSIM (1 = identical); null when ffmpeg could not measure it. */
+  ssim: number | null;
+  meanAbsDiff: number;
+  /** Signed counterpart of meanAbsDiff; close to it means a uniform level shift. */
+  meanSignedDiff: number;
+  deviation: BoundsDeviation;
+  overlay: string;
+}
+
+export interface ReferenceCompareResult {
+  sheet: string;
+  samples: ReferenceSample[];
+  /** Lowest SSIM across samples, or null when none could be measured. */
+  worstSsim: number | null;
+}
+
+function isVideoReference(filePath: string): boolean {
+  return VIDEO_EXTENSIONS.has(extname(filePath).toLowerCase());
+}
+
+function overlayPathFor(outPath: string, index: number): string {
+  const dir = dirname(outPath);
+  const stem = basename(outPath, extname(outPath));
+  return join(dir, `${stem}-overlay-${String(index + 1).padStart(2, "0")}.png`);
+}
+
+async function extractReferenceFrame(
+  ffmpegPath: string,
+  referencePath: string,
+  time: number,
+  outPath: string,
+): Promise<void> {
+  const result = await runFfmpegOnce(
+    ffmpegPath,
+    [
+      "-hide_banner",
+      "-loglevel",
+      "error",
+      "-ss",
+      String(time),
+      "-i",
+      referencePath,
+      "-frames:v",
+      "1",
+      "-y",
+      outPath,
+    ],
+    FFMPEG_TIMEOUT_MS,
+  );
+  if (result.timedOut) {
+    throw new Error(`ffmpeg timed out extracting the reference frame at t=${time}s`);
+  }
+  if (result.code !== 0 || !existsSync(outPath)) {
+    const detail = result.stderr.trim() ? `: ${result.stderr.trim()}` : "";
+    throw new Error(
+      `ffmpeg could not extract a reference frame at t=${time}s (past the end of ${basename(referencePath)}?)${detail}`,
+    );
+  }
+}
+
+/** Screenshot the composition at every sampled time, reusing one browser session. */
+async function captureReplicaFrames(
+  options: ReferenceCompareOptions,
+  frameDir: string,
+): Promise<string[]> {
+  const { bundleToSingleHtml } = await import("@hyperframes/core/compiler");
+  const html = await bundleToSingleHtml(options.projectDir);
+  const server = await serveStaticProjectHtml(options.projectDir, html);
+  try {
+    const { browser, page } = await openSettledCompositionPage(html, server.url, {
+      renderReadyTimeoutMs: options.timeoutMs,
+      renderReadyWarningSuffix: "reference comparison may be inaccurate",
+    });
+    try {
+      const paths: string[] = [];
+      for (const [index, time] of options.times.entries()) {
+        // The producer bridge is the same seek target `render` drives, so a
+        // video-backed composition lands on the frame the render would emit.
+        await seekCompositionTimeline(page, time, AUDIT_SEEK_OPTIONS);
+        const framePath = join(frameDir, `replica-${String(index + 1).padStart(2, "0")}.png`);
+        await page.screenshot({ path: framePath, type: "png" });
+        paths.push(framePath);
+      }
+      return paths;
+    } finally {
+      await browser.close();
+    }
+  } finally {
+    await server.close();
+  }
+}
+
+async function frameSsim(
+  ffmpegPath: string,
+  referencePath: string,
+  replicaPath: string,
+): Promise<number | null> {
+  // ffmpeg's own ssim filter, rather than a reimplementation of the standard.
+  const result = await runFfmpegOnce(
+    ffmpegPath,
+    [
+      "-hide_banner",
+      "-i",
+      referencePath,
+      "-i",
+      replicaPath,
+      "-lavfi",
+      "[0:v][1:v]ssim",
+      "-f",
+      "null",
+      "-",
+    ],
+    FFMPEG_TIMEOUT_MS,
+  );
+  if (result.timedOut || result.code !== 0) return null;
+  return parseSsimAll(result.stderr);
+}
+
+async function grayscalePlane(path: string, width: number, height: number): Promise<Uint8Array> {
+  const buffer = await sharp(path)
+    .resize(width, height, { fit: "fill" })
+    .greyscale()
+    .raw()
+    .toBuffer();
+  return new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+}
+
+export async function compareAgainstReference(
+  options: ReferenceCompareOptions,
+): Promise<ReferenceCompareResult> {
+  if (!existsSync(options.referencePath)) {
+    throw new Error(`Reference not found: ${options.referencePath}`);
+  }
+  const ffmpegPath = findFFmpeg();
+  if (!ffmpegPath) {
+    throw new Error(`--against needs ffmpeg on PATH. Install it: ${getFFmpegInstallHint()}`);
+  }
+
+  const workDir = mkdtempSync(join(tmpdir(), "hf-compare-against-"));
+  try {
+    const referenceFrames: string[] = [];
+    for (const [index, time] of options.times.entries()) {
+      const framePath = join(workDir, `reference-${String(index + 1).padStart(2, "0")}.png`);
+      if (isVideoReference(options.referencePath)) {
+        await extractReferenceFrame(ffmpegPath, options.referencePath, time, framePath);
+      } else {
+        // A still reference is the same target at every sampled time.
+        await sharp(options.referencePath).png().toFile(framePath);
+      }
+      referenceFrames.push(framePath);
+    }
+
+    const replicaFrames = await captureReplicaFrames(options, workDir);
+
+    mkdirSync(dirname(options.outPath), { recursive: true });
+    const samples: ReferenceSample[] = [];
+    // Replicas are normalized to reference dimensions so SSIM, the overlay and
+    // the sheet all read the same pixels.
+    const normalizedReplicas: string[] = [];
+
+    for (const [index, time] of options.times.entries()) {
+      const referenceFrame = referenceFrames[index]!;
+      const meta = await sharp(referenceFrame).metadata();
+      const width = meta.width ?? 0;
+      const height = meta.height ?? 0;
+      if (width <= 0 || height <= 0) {
+        throw new Error(`Could not read reference frame dimensions at t=${time}s`);
+      }
+
+      const normalized = join(workDir, `replica-norm-${String(index + 1).padStart(2, "0")}.png`);
+      await sharp(replicaFrames[index]!).resize(width, height, { fit: "fill" }).toFile(normalized);
+      normalizedReplicas.push(normalized);
+
+      const referenceGray = await grayscalePlane(referenceFrame, width, height);
+      const replicaGray = await grayscalePlane(normalized, width, height);
+
+      const overlay = overlayPathFor(options.outPath, index);
+      await sharp(redCyanOverlayRaw(referenceGray, replicaGray, width, height), {
+        raw: { width, height, channels: 3 },
+      })
+        .png()
+        .toFile(overlay);
+
+      samples.push({
+        time,
+        ssim: await frameSsim(ffmpegPath, referenceFrame, normalized),
+        meanAbsDiff: meanAbsDiff(referenceGray, replicaGray),
+        meanSignedDiff: meanSignedDiff(referenceGray, replicaGray),
+        deviation: boundsDeviation(
+          inkBounds(referenceGray, width, height),
+          inkBounds(replicaGray, width, height),
+        ),
+        overlay,
+      });
+    }
+
+    await createContactSheet([...referenceFrames, ...normalizedReplicas], options.outPath, {
+      cols: options.times.length,
+      maxImages: options.times.length * 2,
+      labelMode: "custom",
+      labels: [
+        ...options.times.map((time) => `reference t=${time}s`),
+        ...options.times.map((time) => `replica t=${time}s`),
+      ],
+    });
+
+    const measured = samples.flatMap((sample) => (sample.ssim === null ? [] : [sample.ssim]));
+    return {
+      sheet: options.outPath,
+      samples,
+      worstSsim: measured.length > 0 ? Math.min(...measured) : null,
+    };
+  } finally {
+    rmSync(workDir, { recursive: true, force: true });
+  }
+}

packages/cli/src/commands/compare.test.ts

@@ -1,11 +1,12 @@
 import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
-import { join } from "node:path";
+import { join, resolve } from "node:path";
 import { describe, expect, it, vi } from "vitest";
 import {
   buildCompareSuccessPayload,
   capCompareVariants,
   parseCompareArgs,
+  parseReferenceCompareArgs,
   prepareCompareVariantProjects,
 } from "./compare.js";
 
@@ -139,3 +140,43 @@ describe("prepareCompareVariantProjects", () => {
     }
   });
 });
+
+describe("parseReferenceCompareArgs", () => {
+  it("requires exactly one composition path", () => {
+    expect(() => parseReferenceCompareArgs({ _: ["a", "b"], against: "ref.mp4" }, "/tmp")).toThrow(
+      "--against compares exactly one composition path",
+    );
+  });
+
+  it("defaults to a single sample at t=0", () => {
+    const parsed = parseReferenceCompareArgs({ _: ["."], against: "ref.mp4" }, "/tmp");
+    expect(parsed.times).toEqual([0]);
+    // resolve(), not a literal: Windows turns "/tmp" into "D:\tmp".
+    expect(parsed.referencePath).toBe(resolve("/tmp", "ref.mp4"));
+    expect(parsed.failUnder).toBeUndefined();
+  });
+
+  it("parses a comma-separated sample list and an SSIM gate", () => {
+    const parsed = parseReferenceCompareArgs(
+      { _: ["."], against: "ref.mp4", at: "0,4,10.5", "fail-under": "0.9" },
+      "/tmp",
+    );
+    expect(parsed.times).toEqual([0, 4, 10.5]);
+    expect(parsed.failUnder).toBe(0.9);
+  });
+
+  it("rejects negative times and out-of-range thresholds", () => {
+    expect(() =>
+      parseReferenceCompareArgs({ _: ["."], against: "ref.mp4", at: "0,-1" }, "/tmp"),
+    ).toThrow("--at must be non-negative seconds");
+    expect(() =>
+      parseReferenceCompareArgs({ _: ["."], against: "ref.mp4", "fail-under": "2" }, "/tmp"),
+    ).toThrow("--fail-under must be an SSIM threshold");
+  });
+
+  it("caps the sample count", () => {
+    expect(() =>
+      parseReferenceCompareArgs({ _: ["."], against: "ref.mp4", at: "1,2,3,4,5,6,7,8,9" }, "/tmp"),
+    ).toThrow("at most 8 times");
+  });
+});