From 3d974801c4c8db1cad3aad319ec15e70db92815c Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Wed, 9 Sep 2026 01:23:02 -0400 Subject: [PATCH 01/65] Add a Review page that audits annotations as a grid of chips New top-level Review tab (desktop: after Training; web: after Models) showing many annotations at once as cropped image chips, so a whole class (or everything carrying an attribute) can be checked across several datasets and wrong types corrected in place, without opening each sequence in the viewer. - Datasets view / Grid view toggle: pick datasets through the platform picker or listing (multicam parents expand into their cameras), then page through matching annotations in a rows x columns grid (5x4 by default, settable, with zoom in/out keeping the shape) and a context slider for the margin around each box (30% by default). - Query by type above a confidence threshold, or by attribute key/value on tracks and/or detections; sort by dataset, confidence or frame. The grid keeps its entries until the query is re-run so edits never reshuffle it. - Chips are cropped client-side from image sequences or by seeking a hidden video element (shared frame->time mapping extracted from VideoAnnotator into videoSeek.ts), through a concurrency-limited queue that renders the first box of every visible entry before any track's extra frames. Track entries then cycle through up to 8 boxes sampled along the track, with the object kept centred. - Types are edited per cell (assign, or mark correct) or for a whole page, batched and saved through saveDetections; unsaved edits are counted and guarded on navigation. - Clicking a chip opens the viewer on that dataset, seeking to the frame and selecting the track (new initialFrame / initialTrackId Viewer props read from the route query on both platforms). - Api gains an optional peekConfig for reading dataset configs without the viewer bookkeeping (desktop recents, web browse location); the web dataset store now shares its config merge with it. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01W1C4QY6hxjHaUPJfWPfQuu --- client/dive-common/apispec.ts | 16 +- .../components/Review/ReviewCell.vue | 448 ++++++++++ .../components/Review/ReviewDatasetsPanel.vue | 211 +++++ .../components/Review/ReviewGrid.vue | 81 ++ .../components/Review/ReviewPage.vue | 814 ++++++++++++++++++ client/dive-common/components/Viewer.vue | 36 + .../dive-common/review/chipRenderer.spec.ts | 26 + client/dive-common/review/chipRenderer.ts | 84 ++ client/dive-common/review/chipStore.ts | 179 ++++ client/dive-common/review/frameSource.ts | 234 +++++ client/dive-common/review/reviewItems.spec.ts | 164 ++++ client/dive-common/review/reviewItems.ts | 224 +++++ client/dive-common/review/types.ts | 102 +++ .../review/viewerNavigation.spec.ts | 15 + client/dive-common/review/viewerNavigation.ts | 45 + client/dive-common/use/useReview.spec.ts | 173 ++++ client/dive-common/use/useReview.ts | 487 +++++++++++ client/platform/desktop/frontend/api.ts | 6 + .../frontend/components/NavigationBar.vue | 3 + .../desktop/frontend/components/Recent.vue | 26 + .../frontend/components/ReviewPage.vue | 31 + .../frontend/components/ViewerLoader.vue | 6 + client/platform/desktop/router.ts | 6 + client/platform/web-girder/App.vue | 2 + .../web-girder/api/dataset.service.ts | 43 +- client/platform/web-girder/router.ts | 7 + .../platform/web-girder/store/useDataset.ts | 14 +- client/platform/web-girder/views/Home.vue | 18 + .../web-girder/views/NavigationBar.vue | 3 + client/platform/web-girder/views/Review.vue | 34 + .../web-girder/views/ViewerLoader.vue | 6 + .../components/annotators/VideoAnnotator.vue | 60 +- client/src/components/annotators/videoSeek.ts | 80 ++ docs/Review.md | 66 ++ mkdocs.yml | 1 + 35 files changed, 3677 insertions(+), 74 deletions(-) create mode 100644 client/dive-common/components/Review/ReviewCell.vue create mode 100644 client/dive-common/components/Review/ReviewDatasetsPanel.vue create mode 100644 client/dive-common/components/Review/ReviewGrid.vue create mode 100644 client/dive-common/components/Review/ReviewPage.vue create mode 100644 client/dive-common/review/chipRenderer.spec.ts create mode 100644 client/dive-common/review/chipRenderer.ts create mode 100644 client/dive-common/review/chipStore.ts create mode 100644 client/dive-common/review/frameSource.ts create mode 100644 client/dive-common/review/reviewItems.spec.ts create mode 100644 client/dive-common/review/reviewItems.ts create mode 100644 client/dive-common/review/types.ts create mode 100644 client/dive-common/review/viewerNavigation.spec.ts create mode 100644 client/dive-common/review/viewerNavigation.ts create mode 100644 client/dive-common/use/useReview.spec.ts create mode 100644 client/dive-common/use/useReview.ts create mode 100644 client/platform/desktop/frontend/components/ReviewPage.vue create mode 100644 client/platform/web-girder/views/Review.vue create mode 100644 client/src/components/annotators/videoSeek.ts create mode 100644 docs/Review.md diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index d4eed0dcd..24e17e9ac 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -469,9 +469,15 @@ interface Api { deleteScoringResult?(datasetId: string, resultId: string): Promise; /** Annotation sets, revisions or on-disk files a source on this dataset can point at. */ listScoringSources?(datasetId: string): Promise; - /** Datasets that may be named as the other side of a comparison. */ + /** + * Datasets that may be named as the other side of a comparison; also the + * dataset list the review page offers. + */ listScoringDatasets?(): Promise; - /** Open a platform dataset picker; returns null when the user cancels. */ + /** + * Open a platform dataset picker; returns null when the user cancels. + * Shared by the scoring and review pages. + */ pickScoringDataset?(excludeIds: string[]): Promise; /** Save a text export where the user chooses; resolves false when they cancel. */ saveScoringExport?(args: { filename: string; mime: string; content: string }): Promise; @@ -485,6 +491,12 @@ interface Api { ): Promise; loadConfig(datasetId: string): Promise; + /** + * loadConfig without the platform's viewer bookkeeping (desktop recents, + * web browse location), for pages that read many datasets at once such as + * Review. Callers fall back to loadConfig when absent. + */ + peekConfig?(datasetId: string): Promise; loadDetections(datasetId: string, revision?: number, set?: string): Promise; loadFrameMetadata(datasetId: string): Promise; diff --git a/client/dive-common/components/Review/ReviewCell.vue b/client/dive-common/components/Review/ReviewCell.vue new file mode 100644 index 000000000..037c13d82 --- /dev/null +++ b/client/dive-common/components/Review/ReviewCell.vue @@ -0,0 +1,448 @@ + + + + + diff --git a/client/dive-common/components/Review/ReviewDatasetsPanel.vue b/client/dive-common/components/Review/ReviewDatasetsPanel.vue new file mode 100644 index 000000000..e7bb947f0 --- /dev/null +++ b/client/dive-common/components/Review/ReviewDatasetsPanel.vue @@ -0,0 +1,211 @@ + + + + + diff --git a/client/dive-common/components/Review/ReviewGrid.vue b/client/dive-common/components/Review/ReviewGrid.vue new file mode 100644 index 000000000..153b1fa27 --- /dev/null +++ b/client/dive-common/components/Review/ReviewGrid.vue @@ -0,0 +1,81 @@ + + + + + diff --git a/client/dive-common/components/Review/ReviewPage.vue b/client/dive-common/components/Review/ReviewPage.vue new file mode 100644 index 000000000..808a6530c --- /dev/null +++ b/client/dive-common/components/Review/ReviewPage.vue @@ -0,0 +1,814 @@ + + + + + diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 6b8cbb83e..ffc48e773 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -174,6 +174,16 @@ export default defineComponent({ type: Boolean, default: false, }, + /** Deep link: frame to seek to once the media is ready (e.g. from the review grid). */ + initialFrame: { + type: Number as PropType, + default: undefined, + }, + /** Deep link: track to select once annotations are loaded. */ + initialTrackId: { + type: Number as PropType, + default: undefined, + }, }, setup(props, { emit }) { const { prompt, visible } = usePrompt(); @@ -2045,6 +2055,32 @@ export default defineComponent({ }; loadData(); + /** + * Apply a deep link (initialFrame / initialTrackId) once: after the + * annotations are loaded and the media controller reports its frame + * range, so the seek is not swallowed by the annotator's own init seek. + */ + let initialFocusApplied = false; + watch( + () => [progress.loaded, aggregateController.value.maxFrame.value] as const, + ([loaded, maxFrame]) => { + if (initialFocusApplied || !loaded) return; + if (props.initialFrame === undefined && props.initialTrackId === undefined) return; + if (props.initialFrame !== undefined && maxFrame <= 0) return; + initialFocusApplied = true; + nextTick(() => { + if (props.initialFrame !== undefined) { + handler.seekFrame(Math.min(props.initialFrame, maxFrame)); + } + if (props.initialTrackId !== undefined + && cameraStore.getAnyPossibleTrack(props.initialTrackId)) { + handler.trackSelect(props.initialTrackId, false); + } + }); + }, + { immediate: true }, + ); + const reloadAnnotations = async () => { progress.loaded = false; discardChanges(); diff --git a/client/dive-common/review/chipRenderer.spec.ts b/client/dive-common/review/chipRenderer.spec.ts new file mode 100644 index 000000000..c690c190c --- /dev/null +++ b/client/dive-common/review/chipRenderer.spec.ts @@ -0,0 +1,26 @@ +import { chipRegion, chipSizeFor } from './chipRenderer'; + +describe('chipRegion', () => { + it('is a square centred on the box, padded on every side', () => { + const region = chipRegion([10, 20, 30, 60], 0.5); + // Longer side 40, padded by 50% each side -> 80. + expect(region.side).toBe(80); + expect(region.x).toBe(20 - 40); + expect(region.y).toBe(40 - 40); + }); + + it('tolerates inverted or degenerate boxes', () => { + expect(chipRegion([30, 60, 10, 20], 0).side).toBe(40); + expect(chipRegion([5, 5, 5, 5], 0).side).toBe(1); + expect(chipRegion([0, 0, 10, 10], -1).side).toBe(10); + }); +}); + +describe('chipSizeFor', () => { + it('rounds cell sizes up to a bucket and caps at the largest', () => { + expect(chipSizeFor(100)).toBe(128); + expect(chipSizeFor(128)).toBe(128); + expect(chipSizeFor(129)).toBe(192); + expect(chipSizeFor(5000)).toBe(768); + }); +}); diff --git a/client/dive-common/review/chipRenderer.ts b/client/dive-common/review/chipRenderer.ts new file mode 100644 index 000000000..fccbf18ad --- /dev/null +++ b/client/dive-common/review/chipRenderer.ts @@ -0,0 +1,84 @@ +/** + * Crops one box out of a decoded frame into a chip image. The crop is a + * square centred on the box so the object stays centred as a track cycles + * through frames of different sizes; area past the frame edge is left dark + * rather than shifting the object off centre. + */ +import type { RectBounds } from 'vue-media-annotator/utils'; +import type { DecodedFrame } from './frameSource'; + +export interface ChipRenderOptions { + /** Context around the box as a fraction of its longer side. */ + padding: number; + /** Output edge length in pixels. */ + size: number; + /** Box outline colour; omit to draw no outline. */ + outline?: string; + /** JPEG quality. */ + quality?: number; +} + +/** Square crop region (may extend past the image) for a box with padding. */ +export function chipRegion(bounds: RectBounds, padding: number): { x: number; y: number; side: number } { + const [x1, y1, x2, y2] = bounds; + const w = Math.max(1, Math.abs(x2 - x1)); + const h = Math.max(1, Math.abs(y2 - y1)); + const side = Math.max(w, h) * (1 + 2 * Math.max(0, padding)); + const cx = (x1 + x2) / 2; + const cy = (y1 + y2) / 2; + return { x: cx - side / 2, y: cy - side / 2, side }; +} + +/** Pixel sizes chips are rendered at; cells pick the smallest that covers them. */ +export const CHIP_SIZE_BUCKETS = [128, 192, 256, 384, 512, 768]; + +export function chipSizeFor(cellPixels: number): number { + const wanted = Math.ceil(cellPixels); + return CHIP_SIZE_BUCKETS.find((size) => size >= wanted) ?? CHIP_SIZE_BUCKETS[CHIP_SIZE_BUCKETS.length - 1]; +} + +export function renderChip(frame: DecodedFrame, bounds: RectBounds, options: ChipRenderOptions): string { + const region = chipRegion(bounds, options.padding); + // Never upscale source pixels beyond 1:1 more than the bucket asks for. + const size = Math.max(16, Math.round(Math.min(options.size, Math.max(region.side, 16)))); + const canvas = document.createElement('canvas'); + canvas.width = size; + canvas.height = size; + const ctx = canvas.getContext('2d'); + if (!ctx) throw new Error('Canvas unavailable'); + ctx.fillStyle = '#101010'; + ctx.fillRect(0, 0, size, size); + const scale = size / region.side; + // Source rectangle clipped to the frame; the destination shifts by the same amount. + const sx = Math.max(0, region.x); + const sy = Math.max(0, region.y); + const ex = Math.min(frame.width, region.x + region.side); + const ey = Math.min(frame.height, region.y + region.side); + if (ex > sx && ey > sy) { + const sw = ex - sx; + const sh = ey - sy; + ctx.drawImage( + frame.source, + sx, + sy, + sw, + sh, + (sx - region.x) * scale, + (sy - region.y) * scale, + sw * scale, + sh * scale, + ); + } + if (options.outline) { + const [x1, y1, x2, y2] = bounds; + ctx.strokeStyle = options.outline; + ctx.lineWidth = Math.max(1, Math.round(size / 160)); + ctx.strokeRect( + (Math.min(x1, x2) - region.x) * scale, + (Math.min(y1, y2) - region.y) * scale, + Math.abs(x2 - x1) * scale, + Math.abs(y2 - y1) * scale, + ); + } + return canvas.toDataURL('image/jpeg', options.quality ?? 0.85); +} diff --git a/client/dive-common/review/chipStore.ts b/client/dive-common/review/chipStore.ts new file mode 100644 index 000000000..7f715507f --- /dev/null +++ b/client/dive-common/review/chipStore.ts @@ -0,0 +1,179 @@ +/** + * Reactive cache of rendered chips for review items, filled by a small + * priority queue: the first box of every requested item ("primary") is + * rendered before any of the extra frames a track cycles through + * ("sequence"), so a page paints as fast as possible and then animates. + * + * Loads run with limited concurrency: a video dataset decodes frames by + * seeking one hidden element, and image sequences would otherwise fire a + * whole page of requests at once. + */ +import { ref, set } from 'vue'; +import type { FrameSource } from './frameSource'; +import { renderChip } from './chipRenderer'; +import type { ReviewItem } from './types'; + +export interface ChipStoreOptions { + padding: number; + size: number; + outline: string; +} + +export interface ChipStoreDeps { + /** Frame access for a dataset; null when the dataset cannot be cropped. */ + frameSourceFor(datasetId: string): FrameSource | null; + concurrency?: number; +} + +interface ChipJob { + item: ReviewItem; + /** Sequence slot to fill, or null for the primary chip. */ + slot: number | null; + generation: number; +} + +export type ChipSequence = Array; + +export function createChipStore(deps: ChipStoreDeps, initial: ChipStoreOptions) { + const concurrency = deps.concurrency ?? 4; + const chips = ref>({}); + const sequences = ref>({}); + const failures = ref>({}); + let options: ChipStoreOptions = { ...initial }; + let generation = 0; + let active = 0; + const primaryQueue: ChipJob[] = []; + const sequenceQueue: ChipJob[] = []; + /** Item keys with a primary job queued or running, and its generation. */ + const pendingPrimary = new Map(); + const sequenceQueued = new Map(); + + function reset() { + generation += 1; + chips.value = {}; + sequences.value = {}; + failures.value = {}; + primaryQueue.length = 0; + sequenceQueue.length = 0; + pendingPrimary.clear(); + sequenceQueued.clear(); + } + + /** Re-render everything when the crop or resolution changes. */ + function setOptions(next: ChipStoreOptions) { + if (next.padding === options.padding && next.size === options.size && next.outline === options.outline) { + return; + } + options = { ...next }; + reset(); + } + + async function render(job: ChipJob): Promise { + const source = deps.frameSourceFor(job.item.datasetId); + if (!source) throw new Error('Media for this dataset cannot be cropped'); + const frameRef = job.slot === null ? job.item.primary : job.item.frames[job.slot]; + const frame = await source.getFrame(frameRef.frame); + return renderChip(frame, frameRef.bounds, options); + } + + function complete(job: ChipJob, dataUrl: string | null, error?: unknown) { + if (job.generation !== generation) return; + const { key } = job.item; + if (job.slot === null) { + if (dataUrl) { + set(chips.value, key, dataUrl); + } else { + set(failures.value, key, error instanceof Error ? error.message : 'Could not render this chip'); + } + } else if (dataUrl) { + const slots = sequences.value[key]; + if (slots) set(slots, job.slot, dataUrl); + } + } + + function finish(job: ChipJob) { + active -= 1; + if (job.slot === null && pendingPrimary.get(job.item.key) === job.generation) { + pendingPrimary.delete(job.item.key); + } + pump(); + } + + function start(job: ChipJob) { + active += 1; + render(job) + .then((dataUrl) => complete(job, dataUrl)) + .catch((err) => complete(job, null, err)) + .finally(() => finish(job)); + } + + function pump() { + while (active < concurrency) { + const job = primaryQueue.shift() ?? sequenceQueue.shift(); + if (!job) return; + start(job); + } + } + + /** Queue the first box of each item not already rendered or queued. */ + function ensurePrimary(items: readonly ReviewItem[]) { + items.forEach((item) => { + if (chips.value[item.key] || failures.value[item.key]) return; + if (pendingPrimary.get(item.key) === generation) return; + pendingPrimary.set(item.key, generation); + primaryQueue.push({ item, slot: null, generation }); + }); + pump(); + } + + /** + * Queue the cycling frames of track items. Call with just the visible + * page: every frame of a video dataset costs a seek. + */ + function ensureSequences(items: readonly ReviewItem[]) { + items.forEach((item) => { + if (item.frames.length < 2) return; + if (sequenceQueued.get(item.key) === generation) return; + sequenceQueued.set(item.key, generation); + // Fixed-size, null-filled so cells can show frames as they arrive. + set(sequences.value, item.key, item.frames.map((): string | null => null)); + item.frames.forEach((_, slot) => { + sequenceQueue.push({ item, slot, generation }); + }); + }); + pump(); + } + + /** Drop queued work that is no longer visible (rendered chips stay cached). */ + function trimQueues(visibleKeys: ReadonlySet) { + const keep = (job: ChipJob) => visibleKeys.has(job.item.key); + const droppedPrimary = primaryQueue.filter((job) => !keep(job)); + droppedPrimary.forEach((job) => { + if (pendingPrimary.get(job.item.key) === job.generation) pendingPrimary.delete(job.item.key); + }); + primaryQueue.splice(0, primaryQueue.length, ...primaryQueue.filter(keep)); + const droppedSequence = sequenceQueue.filter((job) => !keep(job)); + droppedSequence.forEach((job) => { + sequenceQueued.delete(job.item.key); + if (job.generation === generation) { + const slots = sequences.value; + delete slots[job.item.key]; + } + }); + sequenceQueue.splice(0, sequenceQueue.length, ...sequenceQueue.filter(keep)); + } + + return { + chips, + sequences, + failures, + setOptions, + ensurePrimary, + ensureSequences, + trimQueues, + reset, + get options() { return options; }, + }; +} + +export type ChipStore = ReturnType; diff --git a/client/dive-common/review/frameSource.ts b/client/dive-common/review/frameSource.ts new file mode 100644 index 000000000..dd843b134 --- /dev/null +++ b/client/dive-common/review/frameSource.ts @@ -0,0 +1,234 @@ +/** + * Per-dataset access to decoded frames for chip cropping. Image sequences + * load their frame images directly; videos are decoded by a hidden + * HTMLVideoElement seeking to each requested frame. + */ +import type { DatasetConfig } from 'dive-common/apispec'; +import { frameToVideoTime } from 'vue-media-annotator/components/annotators/videoSeek'; + +/** Anything drawImage accepts, with its pixel size. */ +export interface DecodedFrame { + source: CanvasImageSource; + width: number; + height: number; +} + +export interface FrameSource { + /** Frames the dataset has; null when unknown (video before metadata loads). */ + frameCount: number | null; + getFrame(frame: number): Promise; + dispose(): void; +} + +export interface FrameSourceOptions { + /** Decoded frames kept per dataset. */ + cacheSize?: number; +} + +const DefaultCacheSize = 24; + +class FrameCache { + private entries = new Map(); + + private readonly limit: number; + + constructor(limit: number) { + this.limit = limit; + } + + get(frame: number): DecodedFrame | undefined { + const hit = this.entries.get(frame); + if (hit) { + // Re-insert so the map keeps least-recently-used order. + this.entries.delete(frame); + this.entries.set(frame, hit); + } + return hit; + } + + set(frame: number, decoded: DecodedFrame) { + this.entries.delete(frame); + this.entries.set(frame, decoded); + while (this.entries.size > this.limit) { + const oldest = this.entries.keys().next().value; + if (oldest === undefined) break; + this.entries.delete(oldest); + } + } + + clear() { + this.entries.clear(); + } +} + +export function loadImage(url: string): Promise { + return new Promise((resolve, reject) => { + const image = new Image(); + // Needed so the crop canvas is not tainted; both platforms serve media with CORS headers. + image.crossOrigin = 'anonymous'; + image.onload = () => resolve(image); + image.onerror = () => reject(new Error(`Could not load ${url}`)); + image.src = url; + }); +} + +/** Dedupes concurrent requests for the same frame in front of a loader. */ +function withCache( + cache: FrameCache, + load: (frame: number) => Promise, +): (frame: number) => Promise { + const inflight = new Map>(); + return (frame: number) => { + const cached = cache.get(frame); + if (cached) return Promise.resolve(cached); + const pending = inflight.get(frame); + if (pending) return pending; + const promise = load(frame).then((decoded) => { + cache.set(frame, decoded); + return decoded; + }).finally(() => inflight.delete(frame)); + inflight.set(frame, promise); + return promise; + }; +} + +function imageFrameSource(urlFor: (frame: number) => Promise, frameCount: number | null, cacheSize: number): FrameSource { + const cache = new FrameCache(cacheSize); + const getFrame = withCache(cache, async (frame) => { + const image = await loadImage(await urlFor(frame)); + return { source: image, width: image.naturalWidth, height: image.naturalHeight }; + }); + return { + frameCount, + getFrame, + dispose: () => cache.clear(), + }; +} + +const SeekTimeoutMs = 15000; + +/** + * One hidden video element per dataset; seeks are serialized because an + * element can only sit on one frame at a time. Each decoded frame is copied + * to its own canvas so the cache survives the next seek. + */ +function videoFrameSource(config: DatasetConfig, cacheSize: number): FrameSource { + const cache = new FrameCache(cacheSize); + let video: HTMLVideoElement | null = null; + let metadata: Promise | null = null; + let queue: Promise = Promise.resolve(); + let disposed = false; + + function element(): Promise { + if (metadata) return metadata; + metadata = new Promise((resolve, reject) => { + const el = document.createElement('video'); + el.crossOrigin = 'anonymous'; + el.muted = true; + el.preload = 'auto'; + el.playsInline = true; + el.onloadedmetadata = () => resolve(el); + el.onerror = () => reject(new Error(`Could not open video for ${config.name}`)); + el.src = config.videoUrl || ''; + video = el; + }); + return metadata; + } + + function seekTo(el: HTMLVideoElement, time: number): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const timer = window.setTimeout(() => { + if (settled) return; + settled = true; + cleanup(); + reject(new Error('Timed out seeking the video')); + }, SeekTimeoutMs); + function cleanup() { + el.removeEventListener('seeked', onSeeked); + el.removeEventListener('error', onError); + window.clearTimeout(timer); + } + function onSeeked() { + if (settled) return; + settled = true; + cleanup(); + resolve(); + } + function onError() { + if (settled) return; + settled = true; + cleanup(); + reject(new Error('The video failed while seeking')); + } + el.addEventListener('seeked', onSeeked); + el.addEventListener('error', onError); + // Seeking to the time the element already sits at fires no event. + if (Math.abs(el.currentTime - time) < 1e-6 && el.readyState >= 2) { + onSeeked(); + return; + } + // eslint-disable-next-line no-param-reassign + el.currentTime = time; + }); + } + + const getFrame = withCache(cache, (frame) => { + const run = queue.then(async () => { + if (disposed) throw new Error('Frame source disposed'); + const el = await element(); + await seekTo(el, frameToVideoTime(frame, config.fps, config.originalFps ?? null)); + const canvas = document.createElement('canvas'); + canvas.width = el.videoWidth; + canvas.height = el.videoHeight; + const ctx = canvas.getContext('2d'); + if (!ctx) throw new Error('Canvas unavailable'); + ctx.drawImage(el, 0, 0); + return { source: canvas, width: canvas.width, height: canvas.height } as DecodedFrame; + }); + // Failures must not wedge the queue for later frames. + queue = run.catch(() => undefined); + return run; + }); + + return { + frameCount: null, + getFrame, + dispose: () => { + disposed = true; + cache.clear(); + if (video) { + video.removeAttribute('src'); + video.load(); + video = null; + } + metadata = null; + }, + }; +} + +/** + * Pick the loader for a dataset, or throw for media the review grid cannot + * crop (tiled large images, multicamera parents). + */ +export function createFrameSource(config: DatasetConfig, options: FrameSourceOptions = {}): FrameSource { + const cacheSize = options.cacheSize ?? DefaultCacheSize; + if (config.type === 'large-image') { + throw new Error('Tiled large-image datasets cannot be reviewed as chips yet'); + } + if (config.type === 'multi') { + throw new Error('Review the cameras of a multicamera dataset individually'); + } + if (config.type === 'video') { + if (config.videoUrl) { + return videoFrameSource(config, cacheSize); + } + throw new Error('This video has no playable media'); + } + const { imageData } = config; + return imageFrameSource(async (frame) => { + const entry = imageData[frame]; + if (!entry) throw new Error(`No image for frame ${frame}`); + return entry.url; + }, imageData.length, cacheSize); +} diff --git a/client/dive-common/review/reviewItems.spec.ts b/client/dive-common/review/reviewItems.spec.ts new file mode 100644 index 000000000..72d8c7b84 --- /dev/null +++ b/client/dive-common/review/reviewItems.spec.ts @@ -0,0 +1,164 @@ +import type { TrackData } from 'vue-media-annotator/track'; +import { + attributeMatches, + buildReviewItems, + collectAttributeKeys, + collectTypes, + matchTypePair, + sampleFrames, + sortReviewItems, +} from './reviewItems'; +import { DEFAULT_REVIEW_QUERY } from './types'; + +function track( + id: number, + pairs: [string, number][], + frames: number[], + extra: Partial = {}, +): TrackData { + return { + id, + begin: Math.min(...frames), + end: Math.max(...frames), + confidencePairs: pairs, + attributes: {}, + features: frames.map((frame) => ({ + frame, keyframe: true, bounds: [frame, 0, frame + 10, 10], + })), + ...extra, + }; +} + +describe('sampleFrames', () => { + it('keeps a single detection as one frame', () => { + const [only] = track(1, [['a', 1]], [3]).features; + expect(sampleFrames([only], 8)).toEqual([{ frame: 3, bounds: [3, 0, 13, 10] }]); + }); + + it('samples evenly including both ends and never repeats a frame', () => { + const { features } = track(1, [['a', 1]], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + const frames = sampleFrames(features, 4).map((f) => f.frame); + expect(frames).toEqual([0, 3, 7, 10]); + expect(sampleFrames(features, 50)).toHaveLength(11); + }); +}); + +describe('matchTypePair', () => { + it('uses the top pair when no type is requested', () => { + expect(matchTypePair([['fish', 0.4], ['shark', 0.9]], '', 0.5)).toEqual(['shark', 0.9]); + expect(matchTypePair([['fish', 0.4]], '', 0.5)).toBeNull(); + }); + + it('matches the named type against the threshold', () => { + expect(matchTypePair([['fish', 0.4], ['shark', 0.9]], 'fish', 0.3)).toEqual(['fish', 0.4]); + expect(matchTypePair([['fish', 0.4], ['shark', 0.9]], 'fish', 0.5)).toBeNull(); + expect(matchTypePair([['shark', 0.9]], 'fish', 0)).toBeNull(); + }); + + it('only shows untyped tracks when everything is requested', () => { + expect(matchTypePair([], '', 0)).toEqual(['', 0]); + expect(matchTypePair([], '', 0.1)).toBeNull(); + }); +}); + +describe('attributeMatches', () => { + it('requires presence, then compares as text ignoring case', () => { + expect(attributeMatches(undefined, '')).toBe(false); + expect(attributeMatches('Yes', '')).toBe(true); + expect(attributeMatches('Yes', 'yes')).toBe(true); + expect(attributeMatches(3, '3')).toBe(true); + expect(attributeMatches(true, 'false')).toBe(false); + }); +}); + +describe('buildReviewItems', () => { + const tracks = [ + track(2, [['fish', 0.8], ['shark', 0.2]], [10, 11, 12]), + track(1, [['shark', 0.95]], [5]), + track(3, [['fish', 0.05]], [0]), + { ...track(4, [['fish', 0.9]], [7]), features: [{ frame: 7, keyframe: false }] }, + ]; + + it('filters by type and threshold and orders by track id', () => { + const items = buildReviewItems('ds', tracks, { ...DEFAULT_REVIEW_QUERY, type: 'fish', threshold: 0.1 }, 8); + expect(items.map((i) => i.trackId)).toEqual([2]); + expect(items[0]).toMatchObject({ + key: 'ds#2', type: 'fish', confidence: 0.8, keyframeCount: 3, + }); + expect(items[0].primary.frame).toBe(10); + expect(items[0].frames).toHaveLength(3); + }); + + it('shows every class above the threshold when no type is picked', () => { + const items = buildReviewItems('ds', tracks, { ...DEFAULT_REVIEW_QUERY, type: '', threshold: 0.5 }, 8); + expect(items.map((i) => [i.trackId, i.type])).toEqual([[1, 'shark'], [2, 'fish']]); + }); + + it('finds track and detection attributes', () => { + const withAttributes = [ + track(1, [['fish', 1]], [0, 1], { attributes: { verified: true } }), + { + ...track(2, [['fish', 1]], [4, 5, 6]), + features: [ + { frame: 4, keyframe: true, bounds: [0, 0, 1, 1] as [number, number, number, number] }, + { + frame: 5, keyframe: true, bounds: [0, 0, 1, 1] as [number, number, number, number], attributes: { occluded: 'partial' }, + }, + { + frame: 6, keyframe: true, bounds: [0, 0, 1, 1] as [number, number, number, number], attributes: { occluded: 'partial' }, + }, + ], + }, + ]; + const query = { ...DEFAULT_REVIEW_QUERY, mode: 'attribute' as const }; + const verified = buildReviewItems('ds', withAttributes, { ...query, attributeKey: 'verified' }, 8); + expect(verified.map((i) => i.trackId)).toEqual([1]); + expect(verified[0].matchedAttribute).toEqual({ key: 'verified', value: true, scope: 'track' }); + + const occluded = buildReviewItems('ds', withAttributes, { ...query, attributeKey: 'occluded', attributeValue: 'PARTIAL' }, 8); + expect(occluded).toHaveLength(1); + expect(occluded[0]).toMatchObject({ key: 'ds#2@5', trackId: 2, type: 'fish' }); + expect(occluded[0].primary.frame).toBe(5); + expect(occluded[0].frames.map((f) => f.frame)).toEqual([5, 6]); + + const trackOnly = buildReviewItems('ds', withAttributes, { ...query, attributeKey: 'occluded', attributeScope: 'track' }, 8); + expect(trackOnly).toHaveLength(0); + }); +}); + +describe('sortReviewItems', () => { + const items = [ + ...buildReviewItems('b', [track(1, [['a', 0.5]], [9]), track(2, [['a', 0.9]], [3])], DEFAULT_REVIEW_QUERY, 8), + ...buildReviewItems('a', [track(7, [['a', 0.7]], [1])], DEFAULT_REVIEW_QUERY, 8), + ]; + + it('orders by dataset selection order, confidence or frame', () => { + const keys = (order: Parameters[1]) => ( + sortReviewItems(items, order, ['a', 'b']).map((i) => i.key) + ); + expect(keys('dataset')).toEqual(['a#7', 'b#1', 'b#2']); + expect(keys('confidence-asc')).toEqual(['b#1', 'a#7', 'b#2']); + expect(keys('confidence-desc')).toEqual(['b#2', 'a#7', 'b#1']); + expect(keys('frame')).toEqual(['a#7', 'b#2', 'b#1']); + }); +}); + +describe('vocabularies', () => { + it('collects sorted types and attribute keys', () => { + const tracks = [ + track(1, [['zeta', 0.1], ['alpha', 0.9]], [0], { attributes: { trackAttr: 1 } }), + { + ...track(2, [['beta', 1]], [0]), + features: [{ + frame: 0, keyframe: true, bounds: [0, 0, 1, 1] as [number, number, number, number], attributes: { detAttr: 'x' }, + }], + }, + ]; + expect(collectTypes(tracks)).toEqual(['alpha', 'beta', 'zeta']); + expect(collectAttributeKeys(tracks, { + defined: { + belongs: 'track', datatype: 'text', name: 'defined', key: 'track_defined', + }, + })).toEqual(['defined', 'detAttr', 'trackAttr']); + }); +}); diff --git a/client/dive-common/review/reviewItems.ts b/client/dive-common/review/reviewItems.ts new file mode 100644 index 000000000..5ff68847c --- /dev/null +++ b/client/dive-common/review/reviewItems.ts @@ -0,0 +1,224 @@ +/** + * Pure helpers that turn a dataset's tracks into review grid items for a + * query, plus the type/attribute vocabularies the query controls offer. + */ +import type { TrackData, Feature } from 'vue-media-annotator/track'; +import type { StringKeyObject } from 'vue-media-annotator/BaseAnnotation'; +import type { Attribute } from 'vue-media-annotator/use/AttributeTypes'; +import { compareTypeNames } from 'dive-common/typeHierarchy'; +import type { + ReviewFrameRef, ReviewItem, ReviewQuery, ReviewSortOrder, +} from './types'; + +/** Keyframes carrying a box, in frame order. */ +export function boxedFeatures(track: TrackData): Feature[] { + return track.features + .filter((f) => !!f.bounds) + .sort((a, b) => a.frame - b.frame); +} + +/** Up to `max` boxes evenly sampled along `features` (which must be in frame order). */ +export function sampleFrames(features: Feature[], max: number): ReviewFrameRef[] { + if (features.length === 0) return []; + const count = Math.max(1, Math.min(max, features.length)); + const chosen: ReviewFrameRef[] = []; + const seen = new Set(); + for (let i = 0; i < count; i += 1) { + const index = count === 1 ? 0 : Math.round((i * (features.length - 1)) / (count - 1)); + const feature = features[index]; + if (feature.bounds && !seen.has(feature.frame)) { + seen.add(feature.frame); + chosen.push({ frame: feature.frame, bounds: feature.bounds }); + } + } + return chosen; +} + +/** The pair a type query matches on, or null when the track does not qualify. */ +export function matchTypePair( + pairs: readonly (readonly [string, number])[], + type: string, + threshold: number, +): [string, number] | null { + if (pairs.length === 0) { + // Untyped annotations only show up when every type is requested. + return type === '' && threshold <= 0 ? ['', 0] : null; + } + // Stored pairs are normally sorted by confidence, but files are not guaranteed to be. + const candidates = type === '' ? pairs : pairs.filter(([name]) => name === type); + const best = candidates.reduce( + (acc, pair) => (acc === null || pair[1] > acc[1] ? pair : acc), + null, + ); + if (!best || best[1] < threshold) return null; + return [best[0], best[1]]; +} + +function ownAttributes(attributes: StringKeyObject | undefined): [string, unknown][] { + if (!attributes) return []; + return Object.entries(attributes).filter(([key]) => key !== 'userAttributes'); +} + +/** Attribute values compare as strings, case-insensitively; empty wanted means "present". */ +export function attributeMatches(value: unknown, wanted: string): boolean { + if (value === undefined || value === null) return false; + if (wanted === '') return true; + const asText = Array.isArray(value) ? value.map(String).join(',') : String(value); + return asText.toLowerCase() === wanted.trim().toLowerCase(); +} + +function trackTopPair(track: TrackData): [string, number] { + const top = track.confidencePairs.reduce( + (acc, pair) => (acc === null || pair[1] > acc[1] ? pair : acc), + null, + ); + return top ? [top[0], top[1]] : ['', 0]; +} + +function attributeItem( + datasetId: string, + track: TrackData, + query: ReviewQuery, + maxSequenceFrames: number, +): ReviewItem | null { + const key = query.attributeKey.trim(); + if (!key) return null; + const features = boxedFeatures(track); + if (features.length === 0) return null; + const [type, confidence] = trackTopPair(track); + const base = { + datasetId, + trackId: track.id, + keyframeCount: features.length, + type, + confidence, + }; + if (query.attributeScope !== 'detection') { + const hit = ownAttributes(track.attributes).find( + ([name, value]) => name === key && attributeMatches(value, query.attributeValue), + ); + if (hit) { + const frames = sampleFrames(features, maxSequenceFrames); + return { + ...base, + key: `${datasetId}#${track.id}`, + primary: frames[0], + frames, + matchedAttribute: { key, value: hit[1], scope: 'track' }, + }; + } + } + if (query.attributeScope !== 'track') { + const matching = features.filter((f) => ownAttributes(f.attributes).some( + ([name, value]) => name === key && attributeMatches(value, query.attributeValue), + )); + if (matching.length > 0) { + const frames = sampleFrames(matching, maxSequenceFrames); + const hit = ownAttributes(matching[0].attributes).find(([name]) => name === key); + return { + ...base, + key: `${datasetId}#${track.id}@${matching[0].frame}`, + primary: frames[0], + frames, + matchedAttribute: { key, value: hit ? hit[1] : undefined, scope: 'detection' }, + }; + } + } + return null; +} + +function typeItem( + datasetId: string, + track: TrackData, + query: ReviewQuery, + maxSequenceFrames: number, +): ReviewItem | null { + const pair = matchTypePair(track.confidencePairs, query.type, query.threshold); + if (!pair) return null; + const features = boxedFeatures(track); + if (features.length === 0) return null; + const frames = sampleFrames(features, maxSequenceFrames); + return { + key: `${datasetId}#${track.id}`, + datasetId, + trackId: track.id, + primary: frames[0], + frames, + keyframeCount: features.length, + type: pair[0], + confidence: pair[1], + }; +} + +/** Every grid item one dataset contributes to a query, in track id order. */ +export function buildReviewItems( + datasetId: string, + tracks: Iterable, + query: ReviewQuery, + maxSequenceFrames: number, +): ReviewItem[] { + const items: ReviewItem[] = []; + Array.from(tracks) + .sort((a, b) => a.id - b.id) + .forEach((track) => { + const item = query.mode === 'attribute' + ? attributeItem(datasetId, track, query, maxSequenceFrames) + : typeItem(datasetId, track, query, maxSequenceFrames); + if (item) items.push(item); + }); + return items; +} + +export function sortReviewItems( + items: ReviewItem[], + order: ReviewSortOrder, + datasetOrder: readonly string[], +): ReviewItem[] { + const rank = new Map(datasetOrder.map((id, index) => [id, index])); + const byDataset = (a: ReviewItem, b: ReviewItem) => ( + (rank.get(a.datasetId) ?? Infinity) - (rank.get(b.datasetId) ?? Infinity) + ); + const sorted = [...items]; + switch (order) { + case 'confidence-asc': + sorted.sort((a, b) => a.confidence - b.confidence || byDataset(a, b) || a.trackId - b.trackId); + break; + case 'confidence-desc': + sorted.sort((a, b) => b.confidence - a.confidence || byDataset(a, b) || a.trackId - b.trackId); + break; + case 'frame': + sorted.sort((a, b) => byDataset(a, b) || a.primary.frame - b.primary.frame || a.trackId - b.trackId); + break; + default: + sorted.sort((a, b) => byDataset(a, b) || a.trackId - b.trackId); + } + return sorted; +} + +/** Every type named by any confidence pair, in type order. */ +export function collectTypes(tracks: Iterable): string[] { + const types = new Set(); + Array.from(tracks).forEach((track) => { + track.confidencePairs.forEach(([type]) => { if (type) types.add(type); }); + }); + return Array.from(types).sort(compareTypeNames); +} + +/** + * Attribute keys the query control can offer: those defined in the dataset + * configuration plus any actually present on tracks or detections. + */ +export function collectAttributeKeys( + tracks: Iterable, + definitions: Readonly> | undefined, +): string[] { + const keys = new Set(); + Object.values(definitions || {}).forEach((attribute) => keys.add(attribute.name)); + Array.from(tracks).forEach((track) => { + ownAttributes(track.attributes).forEach(([key]) => keys.add(key)); + track.features.forEach((feature) => { + ownAttributes(feature.attributes).forEach(([key]) => keys.add(key)); + }); + }); + return Array.from(keys).sort((a, b) => a.localeCompare(b)); +} diff --git a/client/dive-common/review/types.ts b/client/dive-common/review/types.ts new file mode 100644 index 000000000..df8362e96 --- /dev/null +++ b/client/dive-common/review/types.ts @@ -0,0 +1,102 @@ +/** + * Review mode: contract shared by the review page, the chip loader and the + * platform shells. The page shows many detections at once as cropped chips + * so a user can audit and correct their types (or find them by attribute) + * without opening every sequence in the viewer. + */ +import type { AnnotationId } from 'vue-media-annotator/BaseAnnotation'; +import type { RectBounds } from 'vue-media-annotator/utils'; + +export type ReviewQueryMode = 'type' | 'attribute'; + +/** Where an attribute query looks for the attribute. */ +export type ReviewAttributeScope = 'any' | 'track' | 'detection'; + +export interface ReviewQuery { + mode: ReviewQueryMode; + /** Type mode: the class to show; empty for every class. */ + type: string; + /** Type mode: minimum confidence of the matching pair (0 shows everything). */ + threshold: number; + /** Attribute mode: the attribute key to look for. */ + attributeKey: string; + /** Attribute mode: required value (string compared); empty just requires presence. */ + attributeValue: string; + attributeScope: ReviewAttributeScope; +} + +export type ReviewSortOrder = 'dataset' | 'confidence-asc' | 'confidence-desc' | 'frame'; + +/** One box of a track shown in a chip, in the dataset's own frame numbers. */ +export interface ReviewFrameRef { + frame: number; + bounds: RectBounds; +} + +/** + * One grid entry: a track (or one detection of it) in one dataset. Built + * once per query run; the live type is read from the review service so + * edits show without the grid reshuffling. + */ +export interface ReviewItem { + /** Stable within a query run: `${datasetId}#${trackId}` or with `@frame`. */ + key: string; + datasetId: string; + trackId: AnnotationId; + /** The box shown first: the matched detection, or the track's first keyframe. */ + primary: ReviewFrameRef; + /** + * Boxes sampled along the track for the cycling animation, primary first. + * A single entry means a static detection. + */ + frames: ReviewFrameRef[]; + /** Number of keyframes in the track. */ + keyframeCount: number; + /** Type and confidence of the pair the query matched on (top pair otherwise). */ + type: string; + confidence: number; + /** Attribute mode: what matched. */ + matchedAttribute?: { + key: string; + value: unknown; + scope: 'track' | 'detection'; + }; +} + +/** Grid presentation settings, persisted per browser. */ +export interface ReviewGridSettings { + columns: number; + rows: number; + /** + * Context around the box as a fraction of its longer side: 0.3 shows the + * box plus 30% of its size on each side. + */ + padding: number; + /** Milliseconds between frames of a cycling track. */ + cycleIntervalMs: number; + /** Most frames sampled along a track. */ + maxSequenceFrames: number; +} + +export const DEFAULT_REVIEW_QUERY: ReviewQuery = { + mode: 'type', + type: '', + threshold: 0.1, + attributeKey: '', + attributeValue: '', + attributeScope: 'any', +}; + +export const DEFAULT_REVIEW_GRID: ReviewGridSettings = { + columns: 5, + rows: 4, + padding: 0.3, + cycleIntervalMs: 400, + maxSequenceFrames: 8, +}; + +export const REVIEW_GRID_LIMITS = { + columns: [1, 12] as const, + rows: [1, 10] as const, + padding: [0, 3] as const, +}; diff --git a/client/dive-common/review/viewerNavigation.spec.ts b/client/dive-common/review/viewerNavigation.spec.ts new file mode 100644 index 000000000..9ccd2de8a --- /dev/null +++ b/client/dive-common/review/viewerNavigation.spec.ts @@ -0,0 +1,15 @@ +import { parseViewerFocus, reviewViewerLocation } from './viewerNavigation'; + +describe('review viewer navigation', () => { + it('round-trips a frame and track through the query', () => { + const location = reviewViewerLocation('abc', { frame: 12, trackId: 7 }); + expect(location).toEqual({ name: 'viewer', params: { id: 'abc' }, query: { frame: '12', track: '7' } }); + expect(parseViewerFocus(location.query)).toEqual({ frame: 12, trackId: 7 }); + }); + + it('ignores missing or malformed values', () => { + expect(reviewViewerLocation('abc', {}).query).toEqual({}); + expect(parseViewerFocus({ frame: 'x', track: ['3'] })).toEqual({ trackId: 3 }); + expect(parseViewerFocus({ frame: '-1' })).toEqual({}); + }); +}); diff --git a/client/dive-common/review/viewerNavigation.ts b/client/dive-common/review/viewerNavigation.ts new file mode 100644 index 000000000..995da442e --- /dev/null +++ b/client/dive-common/review/viewerNavigation.ts @@ -0,0 +1,45 @@ +/** + * Deep links from the review grid into the annotation viewer: the viewer + * route plus query parameters naming the frame to seek to and the track to + * select once media is ready. + */ +export const VIEWER_FRAME_QUERY = 'frame'; +export const VIEWER_TRACK_QUERY = 'track'; + +export interface ViewerFocus { + frame?: number; + trackId?: number; +} + +export interface ReviewViewerLocation { + name: string; + params: Record; + query: Record; +} + +/** Both platforms name their single-dataset viewer route `viewer` with an `id` param. */ +export function reviewViewerLocation( + datasetId: string, + focus: ViewerFocus, +): ReviewViewerLocation { + const query: Record = {}; + if (focus.frame !== undefined) query[VIEWER_FRAME_QUERY] = String(focus.frame); + if (focus.trackId !== undefined) query[VIEWER_TRACK_QUERY] = String(focus.trackId); + return { name: 'viewer', params: { id: datasetId }, query }; +} + +function integerParam(value: unknown): number | undefined { + const text = Array.isArray(value) ? value[0] : value; + if (typeof text !== 'string' || !/^-?\d+$/.test(text)) return undefined; + return Number(text); +} + +/** Read the focus back out of a route query (unknown values are ignored). */ +export function parseViewerFocus(query: Record): ViewerFocus { + const frame = integerParam(query[VIEWER_FRAME_QUERY]); + const trackId = integerParam(query[VIEWER_TRACK_QUERY]); + const focus: ViewerFocus = {}; + if (frame !== undefined && frame >= 0) focus.frame = frame; + if (trackId !== undefined) focus.trackId = trackId; + return focus; +} diff --git a/client/dive-common/use/useReview.spec.ts b/client/dive-common/use/useReview.spec.ts new file mode 100644 index 000000000..76e63bb39 --- /dev/null +++ b/client/dive-common/use/useReview.spec.ts @@ -0,0 +1,173 @@ +import { nextTick } from 'vue'; +import type { TrackData } from 'vue-media-annotator/track'; +import type { DatasetConfig } from 'dive-common/apispec'; +import { createReviewService, ReviewApi } from './useReview'; + +vi.mock('dive-common/review/frameSource', () => ({ + createFrameSource: vi.fn(() => ({ + frameCount: 1, + getFrame: vi.fn(), + dispose: vi.fn(), + })), +})); + +function track(id: number, pairs: [string, number][], frames: number[]): TrackData { + return { + id, + begin: Math.min(...frames), + end: Math.max(...frames), + confidencePairs: pairs, + attributes: {}, + features: frames.map((frame) => ({ + frame, keyframe: true, bounds: [0, 0, 10, 10], + })), + }; +} + +function config(id: string, overrides: Partial = {}): DatasetConfig { + return { + id, + name: `Dataset ${id}`, + type: 'image-sequence', + fps: 1, + createdAt: '', + subType: null, + multiCamMedia: null, + imageData: [{ url: 'a.jpg', filename: 'a.jpg' }], + videoUrl: undefined, + ...overrides, + } as DatasetConfig; +} + +function makeApi(tracksById: Record, overrides: Partial = {}): ReviewApi { + return { + loadConfig: vi.fn(async (id: string) => config(id)), + // Fresh copies each call, as a real platform returns them. + loadDetections: vi.fn(async (id: string) => ({ + version: 2, tracks: JSON.parse(JSON.stringify(tracksById[id] || [])), groups: [], sets: [], + })), + saveDetections: vi.fn(async () => undefined), + listScoringDatasets: vi.fn(async () => [{ id: 'a', name: 'Alpha' }, { id: 'b', name: 'Beta' }]), + ...overrides, + }; +} + +describe('createReviewService', () => { + it('loads datasets, prefers peekConfig, and builds items for a query', async () => { + const peekConfig = vi.fn(async (id: string) => config(id)); + const api = makeApi({ + a: [track(1, [['fish', 0.9]], [0, 1, 2]), track(2, [['shark', 0.3]], [4])], + }, { peekConfig }); + const service = createReviewService({ api }); + await service.refreshAvailable(); + await service.addDatasets(['a', 'a']); + expect(peekConfig).toHaveBeenCalledTimes(1); + expect(api.loadConfig).not.toHaveBeenCalled(); + expect(service.datasets.value).toHaveLength(1); + expect(service.datasets.value[0]).toMatchObject({ + id: 'a', name: 'Alpha', status: 'ready', trackCount: 2, croppable: true, + }); + expect(service.types.value).toEqual(['fish', 'shark']); + expect(service.stale.value).toBe(true); + + service.query.threshold = 0.5; + service.runQuery(); + expect(service.stale.value).toBe(false); + expect(service.items.value.map((i) => i.key)).toEqual(['a#1']); + expect(service.items.value[0].frames).toHaveLength(3); + }); + + it('expands a multicamera parent into its cameras', async () => { + const api = makeApi({ 'm/left': [track(1, [['fish', 1]], [0])], 'm/right': [] }, { + loadConfig: vi.fn(async (id: string) => (id === 'm' + ? config(id, { + type: 'multi', + multiCamMedia: { + defaultDisplay: 'left', + cameras: { + left: { type: 'image-sequence', imageData: [], videoUrl: '' }, + right: { type: 'image-sequence', imageData: [], videoUrl: '' }, + }, + }, + }) + : config(id))), + }); + const service = createReviewService({ api }); + await service.addDataset('m', { id: 'm', name: 'Rig' }); + expect(service.datasets.value.map((d) => [d.id, d.name, d.status])).toEqual([ + ['m/left', 'Rig (left)', 'ready'], + ['m/right', 'Rig (right)', 'ready'], + ]); + }); + + it('reassigns and accepts types, tracks pending edits, and saves them', async () => { + const api = makeApi({ + a: [track(1, [['fish', 0.6], ['shark', 0.4]], [0]), track(2, [['fish', 0.9]], [0])], + }); + const service = createReviewService({ api }); + await service.addDataset('a'); + service.query.type = 'fish'; + service.query.threshold = 0; + service.runQuery(); + const [first, second] = service.items.value; + + service.assignType(first, 'shark'); + expect(service.currentType(first)).toEqual({ type: 'shark', confidence: 1 }); + expect(service.trackOf('a', 1)?.confidencePairs).toEqual([['shark', 1]]); + expect(service.isPending(first)).toBe(true); + expect(service.pendingCount.value).toBe(1); + // The grid keeps the item until the query is run again. + expect(service.items.value).toHaveLength(2); + + service.acceptType(second); + expect(service.trackOf('a', 2)?.confidencePairs).toEqual([['fish', 1]]); + expect(service.pendingCount.value).toBe(2); + + await service.save(); + expect(api.saveDetections).toHaveBeenCalledTimes(1); + const [datasetId, args] = (api.saveDetections as ReturnType).mock.calls[0]; + expect(datasetId).toBe('a'); + expect(args.tracks.upsert.map((t: TrackData) => t.id).sort()).toEqual([1, 2]); + expect(args.groups).toEqual({ upsert: [], delete: [] }); + expect(service.pendingCount.value).toBe(0); + expect(service.error.value).toBeNull(); + }); + + it('reports a failed save and keeps the edits pending', async () => { + const api = makeApi({ a: [track(1, [['fish', 0.6]], [0])] }, { + saveDetections: vi.fn(async () => { throw new Error('disk full'); }), + }); + const service = createReviewService({ api }); + await service.addDataset('a'); + service.runQuery(); + service.assignType(service.items.value[0], 'shark'); + await service.save(); + expect(service.error.value).toBe('disk full'); + expect(service.pendingCount.value).toBe(1); + }); + + it('discards edits by reloading the changed datasets', async () => { + const api = makeApi({ a: [track(1, [['fish', 0.6]], [0])] }); + const service = createReviewService({ api }); + await service.addDataset('a'); + service.runQuery(); + service.assignType(service.items.value[0], 'shark'); + await service.discardChanges(); + expect(api.loadDetections).toHaveBeenCalledTimes(2); + expect(service.pendingCount.value).toBe(0); + expect(service.trackOf('a', 1)?.confidencePairs).toEqual([['fish', 0.6]]); + }); + + it('marks a dataset that failed to load and lets it be removed', async () => { + const api = makeApi({}, { + loadDetections: vi.fn(async () => { throw new Error('missing'); }), + }); + const service = createReviewService({ api }); + await service.addDataset('a'); + expect(service.datasets.value[0]).toMatchObject({ status: 'error', error: 'missing' }); + service.removeDataset('a'); + await nextTick(); + expect(service.datasets.value).toHaveLength(0); + expect(service.types.value).toEqual([]); + }); +}); diff --git a/client/dive-common/use/useReview.ts b/client/dive-common/use/useReview.ts new file mode 100644 index 000000000..51730b8d7 --- /dev/null +++ b/client/dive-common/use/useReview.ts @@ -0,0 +1,487 @@ +/** + * State behind the Review page: the datasets under review (with their + * tracks held in memory), the query, the grid settings, the rendered + * chips, and the type edits waiting to be saved back. + */ +import { + computed, inject, provide, reactive, ref, Ref, watch, +} from 'vue'; +import type { Api, DatasetConfig } from 'dive-common/apispec'; +import type { ScoringDatasetSummary } from 'dive-common/scoring/types'; +import type { TrackData } from 'vue-media-annotator/track'; +import type { AnnotationId, ConfidencePair } from 'vue-media-annotator/BaseAnnotation'; +import { + acceptPairAsCorrect, compileHierarchy, reassignPairs, TypeHierarchyIndex, +} from 'dive-common/typeHierarchy'; +import { createFrameSource, FrameSource } from 'dive-common/review/frameSource'; +import { createChipStore, ChipStore } from 'dive-common/review/chipStore'; +import { + buildReviewItems, collectAttributeKeys, collectTypes, sortReviewItems, +} from 'dive-common/review/reviewItems'; +import { + DEFAULT_REVIEW_GRID, + DEFAULT_REVIEW_QUERY, + REVIEW_GRID_LIMITS, + ReviewGridSettings, + ReviewItem, + ReviewQuery, + ReviewSortOrder, +} from 'dive-common/review/types'; + +const GRID_STORAGE_KEY = 'dive.review.grid'; +const CHIP_OUTLINE = '#00e5ff'; + +export type ReviewApi = Pick; + +export interface ReviewServiceDeps { + api: ReviewApi; +} + +export type ReviewDatasetStatus = 'loading' | 'ready' | 'error'; + +export interface ReviewDataset { + id: string; + name: string; + type?: string; + status: ReviewDatasetStatus; + error?: string; + trackCount: number; + /** True when the media can be cropped into chips. */ + croppable: boolean; +} + +export interface ReviewService { + datasets: Readonly>; + available: Readonly>; + query: ReviewQuery; + grid: ReviewGridSettings; + sort: Ref; + items: Readonly>; + /** Bumps whenever tracks load or change; computeds that read tracks depend on it. */ + dataRevision: Readonly>; + /** True once tracks or the query changed after the last run. */ + stale: Readonly>; + types: Readonly>; + attributeKeys: Readonly>; + pendingCount: Readonly>; + saving: Readonly>; + loading: Readonly>; + error: Readonly>; + chipStore: ChipStore; + datasetName(id: string): string; + refreshAvailable(): Promise; + addDataset(id: string, summary?: ScoringDatasetSummary): Promise; + addDatasets(ids: string[]): Promise; + removeDataset(id: string): void; + reloadDataset(id: string): Promise; + runQuery(): void; + trackOf(datasetId: string, trackId: AnnotationId): TrackData | undefined; + /** The item's live top type/confidence after any edits. */ + currentType(item: ReviewItem): { type: string; confidence: number }; + isPending(item: ReviewItem): boolean; + assignType(item: ReviewItem, type: string): void; + acceptType(item: ReviewItem): void; + save(): Promise; + discardChanges(): Promise; + clearError(): void; + dispose(): void; +} + +function loadGridSettings(): ReviewGridSettings { + try { + const raw = window.localStorage.getItem(GRID_STORAGE_KEY); + if (raw) { + const parsed = JSON.parse(raw); + if (parsed && typeof parsed === 'object') { + return { ...DEFAULT_REVIEW_GRID, ...parsed }; + } + } + } catch { + // Storage may be unavailable; defaults are fine. + } + return { ...DEFAULT_REVIEW_GRID }; +} + +function storeGridSettings(grid: ReviewGridSettings) { + try { + window.localStorage.setItem(GRID_STORAGE_KEY, JSON.stringify(grid)); + } catch { + // Ignore storage failures. + } +} + +export function clampGrid(grid: ReviewGridSettings): ReviewGridSettings { + const clamp = (value: number, [min, max]: readonly [number, number], fallback: number) => ( + Number.isFinite(value) ? Math.min(max, Math.max(min, value)) : fallback + ); + return { + ...grid, + columns: Math.round(clamp(grid.columns, REVIEW_GRID_LIMITS.columns, DEFAULT_REVIEW_GRID.columns)), + rows: Math.round(clamp(grid.rows, REVIEW_GRID_LIMITS.rows, DEFAULT_REVIEW_GRID.rows)), + padding: clamp(grid.padding, REVIEW_GRID_LIMITS.padding, DEFAULT_REVIEW_GRID.padding), + }; +} + +interface LoadedDataset { + config: DatasetConfig; + tracks: Map; + hierarchy: TypeHierarchyIndex; + frameSource: FrameSource | null; + pending: Set; +} + +function topPair(pairs: readonly ConfidencePair[]): { type: string; confidence: number } { + const top = pairs.reduce( + (acc, pair) => (acc === null || pair[1] > acc[1] ? pair : acc), + null, + ); + return top ? { type: top[0], confidence: top[1] } : { type: '', confidence: 0 }; +} + +export function createReviewService(deps: ReviewServiceDeps): ReviewService { + const { api } = deps; + const datasets = ref([]); + const available = ref([]); + const query = reactive({ ...DEFAULT_REVIEW_QUERY }); + const grid = reactive(clampGrid(loadGridSettings())); + const sort = ref('dataset'); + const items = ref([]); + const dataRevision = ref(0); + const stale = ref(false); + const saving = ref(false); + const loading = ref(false); + const error = ref(null); + /** Tracks and media, deliberately outside Vue reactivity (they can be large). */ + const loaded = new Map(); + /** Loads still in flight, so a removal during load is honoured. */ + let loadGeneration = 0; + + watch(grid, () => storeGridSettings({ ...grid }), { deep: true }); + watch(query, () => { stale.value = true; }, { deep: true }); + + const chipStore = createChipStore({ + frameSourceFor: (datasetId) => loaded.get(datasetId)?.frameSource ?? null, + }, { padding: grid.padding, size: 256, outline: CHIP_OUTLINE }); + + function fail(reason: unknown, fallback: string) { + const message = reason instanceof Error ? reason.message : String(reason || fallback); + error.value = message || fallback; + } + + function datasetName(id: string) { + return datasets.value.find((d) => d.id === id)?.name + || available.value.find((d) => d.id === id)?.name + || id; + } + + function entry(id: string) { + return datasets.value.find((d) => d.id === id); + } + + function patch(id: string, changes: Partial) { + datasets.value = datasets.value.map((d) => (d.id === id ? { ...d, ...changes } : d)); + } + + async function refreshAvailable() { + if (!api.listScoringDatasets) return; + try { + available.value = await api.listScoringDatasets(); + } catch (err) { + fail(err, 'Could not list datasets'); + } + } + + function loadConfig(id: string) { + return api.peekConfig ? api.peekConfig(id) : api.loadConfig(id); + } + + function frameSourceFor(config: DatasetConfig): FrameSource | null { + try { + return createFrameSource(config); + } catch { + return null; + } + } + + function dropLoaded(id: string) { + const existing = loaded.get(id); + if (existing) { + existing.frameSource?.dispose(); + loaded.delete(id); + } + } + + async function load(id: string, generation: number) { + loading.value = true; + try { + const config = await loadConfig(id); + if (generation !== loadGeneration || !entry(id)) return; + if (config.type === 'multi') { + // Review the cameras of a multicamera dataset as separate sequences. + const cameras = Object.keys(config.multiCamMedia?.cameras || {}); + const parentName = entry(id)?.name || config.name; + datasets.value = datasets.value.filter((d) => d.id !== id); + await Promise.all(cameras.map((camera) => addDataset(`${id}/${camera}`, { + id: `${id}/${camera}`, name: `${parentName} (${camera})`, type: config.multiCamMedia?.cameras[camera]?.type, + }))); + return; + } + const detections = await api.loadDetections(id); + if (generation !== loadGeneration || !entry(id)) return; + dropLoaded(id); + const tracks = new Map(); + detections.tracks.forEach((track) => tracks.set(track.id, track)); + const frameSource = frameSourceFor(config); + loaded.set(id, { + config, + tracks, + hierarchy: compileHierarchy(config.typeHierarchy || {}), + frameSource, + pending: new Set(), + }); + patch(id, { + status: 'ready', + error: undefined, + name: entry(id)?.name || config.name, + type: config.type, + trackCount: tracks.size, + croppable: frameSource !== null, + }); + dataRevision.value += 1; + stale.value = true; + } catch (err) { + if (generation !== loadGeneration || !entry(id)) return; + patch(id, { + status: 'error', + error: err instanceof Error ? err.message : String(err), + }); + } finally { + loading.value = datasets.value.some((d) => d.status === 'loading'); + } + } + + async function addDataset(id: string, summary?: ScoringDatasetSummary) { + if (!id || entry(id)) return; + datasets.value = [...datasets.value, { + id, + name: summary?.name || datasetName(id), + type: summary?.type, + status: 'loading', + trackCount: 0, + croppable: false, + }]; + await load(id, loadGeneration); + } + + async function addDatasets(ids: string[]) { + const unique = Array.from(new Set(ids.filter(Boolean))); + await Promise.all(unique.map((id) => addDataset(id))); + } + + function removeDataset(id: string) { + datasets.value = datasets.value.filter((d) => d.id !== id); + dropLoaded(id); + loadGeneration += 1; + dataRevision.value += 1; + stale.value = true; + } + + async function reloadDataset(id: string) { + if (!entry(id)) return; + patch(id, { status: 'loading', error: undefined }); + await load(id, loadGeneration); + } + + function allTracks(): TrackData[] { + const all: TrackData[] = []; + loaded.forEach((dataset) => all.push(...dataset.tracks.values())); + return all; + } + + /** Read inside a computed so it re-runs when tracks load or change. */ + function dependOnData(): number { + return dataRevision.value; + } + + const types = computed(() => { + dependOnData(); + return collectTypes(allTracks()); + }); + + const attributeKeys = computed(() => { + dependOnData(); + const keys = new Set(); + loaded.forEach((dataset) => { + collectAttributeKeys(dataset.tracks.values(), dataset.config.attributes).forEach((k) => keys.add(k)); + }); + return Array.from(keys).sort((a, b) => a.localeCompare(b)); + }); + + function runQuery() { + const order = datasets.value.map((d) => d.id); + const built: ReviewItem[] = []; + order.forEach((id) => { + const dataset = loaded.get(id); + if (dataset) { + built.push(...buildReviewItems(id, dataset.tracks.values(), { ...query }, grid.maxSequenceFrames)); + } + }); + items.value = sortReviewItems(built, sort.value, order); + stale.value = false; + } + + watch(sort, () => { + items.value = sortReviewItems(items.value, sort.value, datasets.value.map((d) => d.id)); + }); + + function trackOf(datasetId: string, trackId: AnnotationId) { + return loaded.get(datasetId)?.tracks.get(trackId); + } + + function currentType(item: ReviewItem) { + const track = trackOf(item.datasetId, item.trackId); + if (!track) return { type: item.type, confidence: item.confidence }; + if (query.mode === 'type' && query.type) { + // Keep showing the queried pair while it still exists, so an edit that + // demotes it is visible as such. + const pair = track.confidencePairs.find(([type]) => type === query.type); + const top = topPair(track.confidencePairs); + if (pair && top.type === query.type) return { type: pair[0], confidence: pair[1] }; + return top; + } + return topPair(track.confidencePairs); + } + + function isPending(item: ReviewItem) { + return loaded.get(item.datasetId)?.pending.has(item.trackId) ?? false; + } + + const pendingCount = computed(() => { + dependOnData(); + let count = 0; + loaded.forEach((dataset) => { count += dataset.pending.size; }); + return count; + }); + + function updatePairs(item: ReviewItem, update: (pairs: ConfidencePair[], hierarchy: TypeHierarchyIndex) => ConfidencePair[]) { + const dataset = loaded.get(item.datasetId); + const track = dataset?.tracks.get(item.trackId); + if (!dataset || !track) return; + const next = update(track.confidencePairs.map(([t, c]) => [t, c] as ConfidencePair), dataset.hierarchy); + track.confidencePairs = next; + dataset.pending.add(item.trackId); + dataRevision.value += 1; + } + + function assignType(item: ReviewItem, type: string) { + const trimmed = type.trim(); + if (!trimmed) return; + const current = currentType(item); + if (current.type === trimmed && current.confidence >= 1) return; + updatePairs(item, (pairs, hierarchy) => reassignPairs( + hierarchy, + pairs, + current.type || trimmed, + trimmed, + 1, + )); + } + + function acceptType(item: ReviewItem) { + const current = currentType(item); + if (!current.type) return; + updatePairs(item, (pairs, hierarchy) => acceptPairAsCorrect(hierarchy, pairs, current.type)); + } + + async function save() { + if (saving.value) return; + saving.value = true; + error.value = null; + try { + const targets = Array.from(loaded.entries()).filter(([, d]) => d.pending.size > 0); + const results = await Promise.allSettled(targets.map(async ([id, dataset]) => { + const upsert = Array.from(dataset.pending) + .map((trackId) => dataset.tracks.get(trackId)) + .filter((t): t is TrackData => !!t); + await api.saveDetections(id, { + tracks: { upsert, delete: [] }, + groups: { upsert: [], delete: [] }, + }); + dataset.pending.clear(); + })); + const failed = results.find((r) => r.status === 'rejected') as PromiseRejectedResult | undefined; + if (failed) throw failed.reason; + } catch (err) { + fail(err, 'Could not save the changed annotations'); + } finally { + dataRevision.value += 1; + saving.value = false; + } + } + + async function discardChanges() { + const dirty = Array.from(loaded.entries()).filter(([, d]) => d.pending.size > 0).map(([id]) => id); + await Promise.all(dirty.map((id) => reloadDataset(id))); + } + + function clearError() { + error.value = null; + } + + function dispose() { + loadGeneration += 1; + loaded.forEach((dataset) => dataset.frameSource?.dispose()); + loaded.clear(); + chipStore.reset(); + } + + return { + datasets, + available, + query, + grid, + sort, + items, + dataRevision, + stale, + types, + attributeKeys, + pendingCount, + saving, + loading, + error, + chipStore, + datasetName, + refreshAvailable, + addDataset, + addDatasets, + removeDataset, + reloadDataset, + runQuery, + trackOf, + currentType, + isPending, + assignType, + acceptType, + save, + discardChanges, + clearError, + dispose, + }; +} + +const ReviewSymbol = Symbol('review'); + +export function provideReview(service: ReviewService) { + provide(ReviewSymbol, service); +} + +export function useReview(): ReviewService { + const service = inject(ReviewSymbol, null); + if (!service) { + throw new Error('Review service not provided'); + } + return service; +} diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index 825633d6d..23daa131b 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -815,6 +815,11 @@ async function loadConfig(id: string) { return { ...data, calibration: data.multiCam?.calibration ?? null }; } +/** loadConfig without the recents bookkeeping the stateful wrapper adds. */ +function peekConfig(id: string) { + return loadConfig(id); +} + let scoringAnnotationPreviewFile: string | null = null; /** One-shot annotation file to load in the viewer (from scoring result links). */ @@ -928,6 +933,7 @@ export { exportScoringPdf, /* Standard Specification APIs */ loadConfig, + peekConfig, loadDetections, loadFrameMetadata, getPipelineList, diff --git a/client/platform/desktop/frontend/components/NavigationBar.vue b/client/platform/desktop/frontend/components/NavigationBar.vue index f3907c753..97c98209b 100644 --- a/client/platform/desktop/frontend/components/NavigationBar.vue +++ b/client/platform/desktop/frontend/components/NavigationBar.vue @@ -28,6 +28,9 @@ export default defineComponent({ Trainingmdi-brain + + Reviewmdi-view-grid-outline + Pipelinemdi-pipe diff --git a/client/platform/desktop/frontend/components/Recent.vue b/client/platform/desktop/frontend/components/Recent.vue index 27630ab0b..040fca447 100644 --- a/client/platform/desktop/frontend/components/Recent.vue +++ b/client/platform/desktop/frontend/components/Recent.vue @@ -265,6 +265,10 @@ export default defineComponent({ function scoreSelected() { router.push({ name: 'scoring', query: selectedIdsQuery() }); } + + function reviewSelected() { + router.push({ name: 'review', query: selectedIdsQuery() }); + } function getTypeIcon(recent: JsonConfigCache) { if (recent.subType) { if (recent.subType === 'stereo') { @@ -360,6 +364,7 @@ export default defineComponent({ runPipelineOnSelected, runTrainingOnSelected, scoreSelected, + reviewSelected, isSelected, toggleSelected, toggleSelectAll, @@ -649,6 +654,27 @@ export default defineComponent({ Score the selected datasets against ground truth + + + Review the selected datasets' annotations as a grid +