From 3d974801c4c8db1cad3aad319ec15e70db92815c Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Wed, 9 Sep 2026 01:23:02 -0400 Subject: [PATCH 01/27] Add a Review page that audits annotations as a grid of chips New top-level Review tab (desktop: after Training; web: after Models) showing many annotations at once as cropped image chips, so a whole class (or everything carrying an attribute) can be checked across several datasets and wrong types corrected in place, without opening each sequence in the viewer. - Datasets view / Grid view toggle: pick datasets through the platform picker or listing (multicam parents expand into their cameras), then page through matching annotations in a rows x columns grid (5x4 by default, settable, with zoom in/out keeping the shape) and a context slider for the margin around each box (30% by default). - Query by type above a confidence threshold, or by attribute key/value on tracks and/or detections; sort by dataset, confidence or frame. The grid keeps its entries until the query is re-run so edits never reshuffle it. - Chips are cropped client-side from image sequences or by seeking a hidden video element (shared frame->time mapping extracted from VideoAnnotator into videoSeek.ts), through a concurrency-limited queue that renders the first box of every visible entry before any track's extra frames. Track entries then cycle through up to 8 boxes sampled along the track, with the object kept centred. - Types are edited per cell (assign, or mark correct) or for a whole page, batched and saved through saveDetections; unsaved edits are counted and guarded on navigation. - Clicking a chip opens the viewer on that dataset, seeking to the frame and selecting the track (new initialFrame / initialTrackId Viewer props read from the route query on both platforms). - Api gains an optional peekConfig for reading dataset configs without the viewer bookkeeping (desktop recents, web browse location); the web dataset store now shares its config merge with it. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01W1C4QY6hxjHaUPJfWPfQuu --- client/dive-common/apispec.ts | 16 +- .../components/Review/ReviewCell.vue | 448 ++++++++++ .../components/Review/ReviewDatasetsPanel.vue | 211 +++++ .../components/Review/ReviewGrid.vue | 81 ++ .../components/Review/ReviewPage.vue | 814 ++++++++++++++++++ client/dive-common/components/Viewer.vue | 36 + .../dive-common/review/chipRenderer.spec.ts | 26 + client/dive-common/review/chipRenderer.ts | 84 ++ client/dive-common/review/chipStore.ts | 179 ++++ client/dive-common/review/frameSource.ts | 234 +++++ client/dive-common/review/reviewItems.spec.ts | 164 ++++ client/dive-common/review/reviewItems.ts | 224 +++++ client/dive-common/review/types.ts | 102 +++ .../review/viewerNavigation.spec.ts | 15 + client/dive-common/review/viewerNavigation.ts | 45 + client/dive-common/use/useReview.spec.ts | 173 ++++ client/dive-common/use/useReview.ts | 487 +++++++++++ client/platform/desktop/frontend/api.ts | 6 + .../frontend/components/NavigationBar.vue | 3 + .../desktop/frontend/components/Recent.vue | 26 + .../frontend/components/ReviewPage.vue | 31 + .../frontend/components/ViewerLoader.vue | 6 + client/platform/desktop/router.ts | 6 + client/platform/web-girder/App.vue | 2 + .../web-girder/api/dataset.service.ts | 43 +- client/platform/web-girder/router.ts | 7 + .../platform/web-girder/store/useDataset.ts | 14 +- client/platform/web-girder/views/Home.vue | 18 + .../web-girder/views/NavigationBar.vue | 3 + client/platform/web-girder/views/Review.vue | 34 + .../web-girder/views/ViewerLoader.vue | 6 + .../components/annotators/VideoAnnotator.vue | 60 +- client/src/components/annotators/videoSeek.ts | 80 ++ docs/Review.md | 66 ++ mkdocs.yml | 1 + 35 files changed, 3677 insertions(+), 74 deletions(-) create mode 100644 client/dive-common/components/Review/ReviewCell.vue create mode 100644 client/dive-common/components/Review/ReviewDatasetsPanel.vue create mode 100644 client/dive-common/components/Review/ReviewGrid.vue create mode 100644 client/dive-common/components/Review/ReviewPage.vue create mode 100644 client/dive-common/review/chipRenderer.spec.ts create mode 100644 client/dive-common/review/chipRenderer.ts create mode 100644 client/dive-common/review/chipStore.ts create mode 100644 client/dive-common/review/frameSource.ts create mode 100644 client/dive-common/review/reviewItems.spec.ts create mode 100644 client/dive-common/review/reviewItems.ts create mode 100644 client/dive-common/review/types.ts create mode 100644 client/dive-common/review/viewerNavigation.spec.ts create mode 100644 client/dive-common/review/viewerNavigation.ts create mode 100644 client/dive-common/use/useReview.spec.ts create mode 100644 client/dive-common/use/useReview.ts create mode 100644 client/platform/desktop/frontend/components/ReviewPage.vue create mode 100644 client/platform/web-girder/views/Review.vue create mode 100644 client/src/components/annotators/videoSeek.ts create mode 100644 docs/Review.md diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index d4eed0dcd..24e17e9ac 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -469,9 +469,15 @@ interface Api { deleteScoringResult?(datasetId: string, resultId: string): Promise; /** Annotation sets, revisions or on-disk files a source on this dataset can point at. */ listScoringSources?(datasetId: string): Promise; - /** Datasets that may be named as the other side of a comparison. */ + /** + * Datasets that may be named as the other side of a comparison; also the + * dataset list the review page offers. + */ listScoringDatasets?(): Promise; - /** Open a platform dataset picker; returns null when the user cancels. */ + /** + * Open a platform dataset picker; returns null when the user cancels. + * Shared by the scoring and review pages. + */ pickScoringDataset?(excludeIds: string[]): Promise; /** Save a text export where the user chooses; resolves false when they cancel. */ saveScoringExport?(args: { filename: string; mime: string; content: string }): Promise; @@ -485,6 +491,12 @@ interface Api { ): Promise; loadConfig(datasetId: string): Promise; + /** + * loadConfig without the platform's viewer bookkeeping (desktop recents, + * web browse location), for pages that read many datasets at once such as + * Review. Callers fall back to loadConfig when absent. + */ + peekConfig?(datasetId: string): Promise; loadDetections(datasetId: string, revision?: number, set?: string): Promise; loadFrameMetadata(datasetId: string): Promise; diff --git a/client/dive-common/components/Review/ReviewCell.vue b/client/dive-common/components/Review/ReviewCell.vue new file mode 100644 index 000000000..037c13d82 --- /dev/null +++ b/client/dive-common/components/Review/ReviewCell.vue @@ -0,0 +1,448 @@ + + + + + diff --git a/client/dive-common/components/Review/ReviewDatasetsPanel.vue b/client/dive-common/components/Review/ReviewDatasetsPanel.vue new file mode 100644 index 000000000..e7bb947f0 --- /dev/null +++ b/client/dive-common/components/Review/ReviewDatasetsPanel.vue @@ -0,0 +1,211 @@ + + + + + diff --git a/client/dive-common/components/Review/ReviewGrid.vue b/client/dive-common/components/Review/ReviewGrid.vue new file mode 100644 index 000000000..153b1fa27 --- /dev/null +++ b/client/dive-common/components/Review/ReviewGrid.vue @@ -0,0 +1,81 @@ + + + + + diff --git a/client/dive-common/components/Review/ReviewPage.vue b/client/dive-common/components/Review/ReviewPage.vue new file mode 100644 index 000000000..808a6530c --- /dev/null +++ b/client/dive-common/components/Review/ReviewPage.vue @@ -0,0 +1,814 @@ + + + + + diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index 6b8cbb83e..ffc48e773 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -174,6 +174,16 @@ export default defineComponent({ type: Boolean, default: false, }, + /** Deep link: frame to seek to once the media is ready (e.g. from the review grid). */ + initialFrame: { + type: Number as PropType, + default: undefined, + }, + /** Deep link: track to select once annotations are loaded. */ + initialTrackId: { + type: Number as PropType, + default: undefined, + }, }, setup(props, { emit }) { const { prompt, visible } = usePrompt(); @@ -2045,6 +2055,32 @@ export default defineComponent({ }; loadData(); + /** + * Apply a deep link (initialFrame / initialTrackId) once: after the + * annotations are loaded and the media controller reports its frame + * range, so the seek is not swallowed by the annotator's own init seek. + */ + let initialFocusApplied = false; + watch( + () => [progress.loaded, aggregateController.value.maxFrame.value] as const, + ([loaded, maxFrame]) => { + if (initialFocusApplied || !loaded) return; + if (props.initialFrame === undefined && props.initialTrackId === undefined) return; + if (props.initialFrame !== undefined && maxFrame <= 0) return; + initialFocusApplied = true; + nextTick(() => { + if (props.initialFrame !== undefined) { + handler.seekFrame(Math.min(props.initialFrame, maxFrame)); + } + if (props.initialTrackId !== undefined + && cameraStore.getAnyPossibleTrack(props.initialTrackId)) { + handler.trackSelect(props.initialTrackId, false); + } + }); + }, + { immediate: true }, + ); + const reloadAnnotations = async () => { progress.loaded = false; discardChanges(); diff --git a/client/dive-common/review/chipRenderer.spec.ts b/client/dive-common/review/chipRenderer.spec.ts new file mode 100644 index 000000000..c690c190c --- /dev/null +++ b/client/dive-common/review/chipRenderer.spec.ts @@ -0,0 +1,26 @@ +import { chipRegion, chipSizeFor } from './chipRenderer'; + +describe('chipRegion', () => { + it('is a square centred on the box, padded on every side', () => { + const region = chipRegion([10, 20, 30, 60], 0.5); + // Longer side 40, padded by 50% each side -> 80. + expect(region.side).toBe(80); + expect(region.x).toBe(20 - 40); + expect(region.y).toBe(40 - 40); + }); + + it('tolerates inverted or degenerate boxes', () => { + expect(chipRegion([30, 60, 10, 20], 0).side).toBe(40); + expect(chipRegion([5, 5, 5, 5], 0).side).toBe(1); + expect(chipRegion([0, 0, 10, 10], -1).side).toBe(10); + }); +}); + +describe('chipSizeFor', () => { + it('rounds cell sizes up to a bucket and caps at the largest', () => { + expect(chipSizeFor(100)).toBe(128); + expect(chipSizeFor(128)).toBe(128); + expect(chipSizeFor(129)).toBe(192); + expect(chipSizeFor(5000)).toBe(768); + }); +}); diff --git a/client/dive-common/review/chipRenderer.ts b/client/dive-common/review/chipRenderer.ts new file mode 100644 index 000000000..fccbf18ad --- /dev/null +++ b/client/dive-common/review/chipRenderer.ts @@ -0,0 +1,84 @@ +/** + * Crops one box out of a decoded frame into a chip image. The crop is a + * square centred on the box so the object stays centred as a track cycles + * through frames of different sizes; area past the frame edge is left dark + * rather than shifting the object off centre. + */ +import type { RectBounds } from 'vue-media-annotator/utils'; +import type { DecodedFrame } from './frameSource'; + +export interface ChipRenderOptions { + /** Context around the box as a fraction of its longer side. */ + padding: number; + /** Output edge length in pixels. */ + size: number; + /** Box outline colour; omit to draw no outline. */ + outline?: string; + /** JPEG quality. */ + quality?: number; +} + +/** Square crop region (may extend past the image) for a box with padding. */ +export function chipRegion(bounds: RectBounds, padding: number): { x: number; y: number; side: number } { + const [x1, y1, x2, y2] = bounds; + const w = Math.max(1, Math.abs(x2 - x1)); + const h = Math.max(1, Math.abs(y2 - y1)); + const side = Math.max(w, h) * (1 + 2 * Math.max(0, padding)); + const cx = (x1 + x2) / 2; + const cy = (y1 + y2) / 2; + return { x: cx - side / 2, y: cy - side / 2, side }; +} + +/** Pixel sizes chips are rendered at; cells pick the smallest that covers them. */ +export const CHIP_SIZE_BUCKETS = [128, 192, 256, 384, 512, 768]; + +export function chipSizeFor(cellPixels: number): number { + const wanted = Math.ceil(cellPixels); + return CHIP_SIZE_BUCKETS.find((size) => size >= wanted) ?? CHIP_SIZE_BUCKETS[CHIP_SIZE_BUCKETS.length - 1]; +} + +export function renderChip(frame: DecodedFrame, bounds: RectBounds, options: ChipRenderOptions): string { + const region = chipRegion(bounds, options.padding); + // Never upscale source pixels beyond 1:1 more than the bucket asks for. + const size = Math.max(16, Math.round(Math.min(options.size, Math.max(region.side, 16)))); + const canvas = document.createElement('canvas'); + canvas.width = size; + canvas.height = size; + const ctx = canvas.getContext('2d'); + if (!ctx) throw new Error('Canvas unavailable'); + ctx.fillStyle = '#101010'; + ctx.fillRect(0, 0, size, size); + const scale = size / region.side; + // Source rectangle clipped to the frame; the destination shifts by the same amount. + const sx = Math.max(0, region.x); + const sy = Math.max(0, region.y); + const ex = Math.min(frame.width, region.x + region.side); + const ey = Math.min(frame.height, region.y + region.side); + if (ex > sx && ey > sy) { + const sw = ex - sx; + const sh = ey - sy; + ctx.drawImage( + frame.source, + sx, + sy, + sw, + sh, + (sx - region.x) * scale, + (sy - region.y) * scale, + sw * scale, + sh * scale, + ); + } + if (options.outline) { + const [x1, y1, x2, y2] = bounds; + ctx.strokeStyle = options.outline; + ctx.lineWidth = Math.max(1, Math.round(size / 160)); + ctx.strokeRect( + (Math.min(x1, x2) - region.x) * scale, + (Math.min(y1, y2) - region.y) * scale, + Math.abs(x2 - x1) * scale, + Math.abs(y2 - y1) * scale, + ); + } + return canvas.toDataURL('image/jpeg', options.quality ?? 0.85); +} diff --git a/client/dive-common/review/chipStore.ts b/client/dive-common/review/chipStore.ts new file mode 100644 index 000000000..7f715507f --- /dev/null +++ b/client/dive-common/review/chipStore.ts @@ -0,0 +1,179 @@ +/** + * Reactive cache of rendered chips for review items, filled by a small + * priority queue: the first box of every requested item ("primary") is + * rendered before any of the extra frames a track cycles through + * ("sequence"), so a page paints as fast as possible and then animates. + * + * Loads run with limited concurrency: a video dataset decodes frames by + * seeking one hidden element, and image sequences would otherwise fire a + * whole page of requests at once. + */ +import { ref, set } from 'vue'; +import type { FrameSource } from './frameSource'; +import { renderChip } from './chipRenderer'; +import type { ReviewItem } from './types'; + +export interface ChipStoreOptions { + padding: number; + size: number; + outline: string; +} + +export interface ChipStoreDeps { + /** Frame access for a dataset; null when the dataset cannot be cropped. */ + frameSourceFor(datasetId: string): FrameSource | null; + concurrency?: number; +} + +interface ChipJob { + item: ReviewItem; + /** Sequence slot to fill, or null for the primary chip. */ + slot: number | null; + generation: number; +} + +export type ChipSequence = Array; + +export function createChipStore(deps: ChipStoreDeps, initial: ChipStoreOptions) { + const concurrency = deps.concurrency ?? 4; + const chips = ref>({}); + const sequences = ref>({}); + const failures = ref>({}); + let options: ChipStoreOptions = { ...initial }; + let generation = 0; + let active = 0; + const primaryQueue: ChipJob[] = []; + const sequenceQueue: ChipJob[] = []; + /** Item keys with a primary job queued or running, and its generation. */ + const pendingPrimary = new Map(); + const sequenceQueued = new Map(); + + function reset() { + generation += 1; + chips.value = {}; + sequences.value = {}; + failures.value = {}; + primaryQueue.length = 0; + sequenceQueue.length = 0; + pendingPrimary.clear(); + sequenceQueued.clear(); + } + + /** Re-render everything when the crop or resolution changes. */ + function setOptions(next: ChipStoreOptions) { + if (next.padding === options.padding && next.size === options.size && next.outline === options.outline) { + return; + } + options = { ...next }; + reset(); + } + + async function render(job: ChipJob): Promise { + const source = deps.frameSourceFor(job.item.datasetId); + if (!source) throw new Error('Media for this dataset cannot be cropped'); + const frameRef = job.slot === null ? job.item.primary : job.item.frames[job.slot]; + const frame = await source.getFrame(frameRef.frame); + return renderChip(frame, frameRef.bounds, options); + } + + function complete(job: ChipJob, dataUrl: string | null, error?: unknown) { + if (job.generation !== generation) return; + const { key } = job.item; + if (job.slot === null) { + if (dataUrl) { + set(chips.value, key, dataUrl); + } else { + set(failures.value, key, error instanceof Error ? error.message : 'Could not render this chip'); + } + } else if (dataUrl) { + const slots = sequences.value[key]; + if (slots) set(slots, job.slot, dataUrl); + } + } + + function finish(job: ChipJob) { + active -= 1; + if (job.slot === null && pendingPrimary.get(job.item.key) === job.generation) { + pendingPrimary.delete(job.item.key); + } + pump(); + } + + function start(job: ChipJob) { + active += 1; + render(job) + .then((dataUrl) => complete(job, dataUrl)) + .catch((err) => complete(job, null, err)) + .finally(() => finish(job)); + } + + function pump() { + while (active < concurrency) { + const job = primaryQueue.shift() ?? sequenceQueue.shift(); + if (!job) return; + start(job); + } + } + + /** Queue the first box of each item not already rendered or queued. */ + function ensurePrimary(items: readonly ReviewItem[]) { + items.forEach((item) => { + if (chips.value[item.key] || failures.value[item.key]) return; + if (pendingPrimary.get(item.key) === generation) return; + pendingPrimary.set(item.key, generation); + primaryQueue.push({ item, slot: null, generation }); + }); + pump(); + } + + /** + * Queue the cycling frames of track items. Call with just the visible + * page: every frame of a video dataset costs a seek. + */ + function ensureSequences(items: readonly ReviewItem[]) { + items.forEach((item) => { + if (item.frames.length < 2) return; + if (sequenceQueued.get(item.key) === generation) return; + sequenceQueued.set(item.key, generation); + // Fixed-size, null-filled so cells can show frames as they arrive. + set(sequences.value, item.key, item.frames.map((): string | null => null)); + item.frames.forEach((_, slot) => { + sequenceQueue.push({ item, slot, generation }); + }); + }); + pump(); + } + + /** Drop queued work that is no longer visible (rendered chips stay cached). */ + function trimQueues(visibleKeys: ReadonlySet) { + const keep = (job: ChipJob) => visibleKeys.has(job.item.key); + const droppedPrimary = primaryQueue.filter((job) => !keep(job)); + droppedPrimary.forEach((job) => { + if (pendingPrimary.get(job.item.key) === job.generation) pendingPrimary.delete(job.item.key); + }); + primaryQueue.splice(0, primaryQueue.length, ...primaryQueue.filter(keep)); + const droppedSequence = sequenceQueue.filter((job) => !keep(job)); + droppedSequence.forEach((job) => { + sequenceQueued.delete(job.item.key); + if (job.generation === generation) { + const slots = sequences.value; + delete slots[job.item.key]; + } + }); + sequenceQueue.splice(0, sequenceQueue.length, ...sequenceQueue.filter(keep)); + } + + return { + chips, + sequences, + failures, + setOptions, + ensurePrimary, + ensureSequences, + trimQueues, + reset, + get options() { return options; }, + }; +} + +export type ChipStore = ReturnType; diff --git a/client/dive-common/review/frameSource.ts b/client/dive-common/review/frameSource.ts new file mode 100644 index 000000000..dd843b134 --- /dev/null +++ b/client/dive-common/review/frameSource.ts @@ -0,0 +1,234 @@ +/** + * Per-dataset access to decoded frames for chip cropping. Image sequences + * load their frame images directly; videos are decoded by a hidden + * HTMLVideoElement seeking to each requested frame. + */ +import type { DatasetConfig } from 'dive-common/apispec'; +import { frameToVideoTime } from 'vue-media-annotator/components/annotators/videoSeek'; + +/** Anything drawImage accepts, with its pixel size. */ +export interface DecodedFrame { + source: CanvasImageSource; + width: number; + height: number; +} + +export interface FrameSource { + /** Frames the dataset has; null when unknown (video before metadata loads). */ + frameCount: number | null; + getFrame(frame: number): Promise; + dispose(): void; +} + +export interface FrameSourceOptions { + /** Decoded frames kept per dataset. */ + cacheSize?: number; +} + +const DefaultCacheSize = 24; + +class FrameCache { + private entries = new Map(); + + private readonly limit: number; + + constructor(limit: number) { + this.limit = limit; + } + + get(frame: number): DecodedFrame | undefined { + const hit = this.entries.get(frame); + if (hit) { + // Re-insert so the map keeps least-recently-used order. + this.entries.delete(frame); + this.entries.set(frame, hit); + } + return hit; + } + + set(frame: number, decoded: DecodedFrame) { + this.entries.delete(frame); + this.entries.set(frame, decoded); + while (this.entries.size > this.limit) { + const oldest = this.entries.keys().next().value; + if (oldest === undefined) break; + this.entries.delete(oldest); + } + } + + clear() { + this.entries.clear(); + } +} + +export function loadImage(url: string): Promise { + return new Promise((resolve, reject) => { + const image = new Image(); + // Needed so the crop canvas is not tainted; both platforms serve media with CORS headers. + image.crossOrigin = 'anonymous'; + image.onload = () => resolve(image); + image.onerror = () => reject(new Error(`Could not load ${url}`)); + image.src = url; + }); +} + +/** Dedupes concurrent requests for the same frame in front of a loader. */ +function withCache( + cache: FrameCache, + load: (frame: number) => Promise, +): (frame: number) => Promise { + const inflight = new Map>(); + return (frame: number) => { + const cached = cache.get(frame); + if (cached) return Promise.resolve(cached); + const pending = inflight.get(frame); + if (pending) return pending; + const promise = load(frame).then((decoded) => { + cache.set(frame, decoded); + return decoded; + }).finally(() => inflight.delete(frame)); + inflight.set(frame, promise); + return promise; + }; +} + +function imageFrameSource(urlFor: (frame: number) => Promise, frameCount: number | null, cacheSize: number): FrameSource { + const cache = new FrameCache(cacheSize); + const getFrame = withCache(cache, async (frame) => { + const image = await loadImage(await urlFor(frame)); + return { source: image, width: image.naturalWidth, height: image.naturalHeight }; + }); + return { + frameCount, + getFrame, + dispose: () => cache.clear(), + }; +} + +const SeekTimeoutMs = 15000; + +/** + * One hidden video element per dataset; seeks are serialized because an + * element can only sit on one frame at a time. Each decoded frame is copied + * to its own canvas so the cache survives the next seek. + */ +function videoFrameSource(config: DatasetConfig, cacheSize: number): FrameSource { + const cache = new FrameCache(cacheSize); + let video: HTMLVideoElement | null = null; + let metadata: Promise | null = null; + let queue: Promise = Promise.resolve(); + let disposed = false; + + function element(): Promise { + if (metadata) return metadata; + metadata = new Promise((resolve, reject) => { + const el = document.createElement('video'); + el.crossOrigin = 'anonymous'; + el.muted = true; + el.preload = 'auto'; + el.playsInline = true; + el.onloadedmetadata = () => resolve(el); + el.onerror = () => reject(new Error(`Could not open video for ${config.name}`)); + el.src = config.videoUrl || ''; + video = el; + }); + return metadata; + } + + function seekTo(el: HTMLVideoElement, time: number): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const timer = window.setTimeout(() => { + if (settled) return; + settled = true; + cleanup(); + reject(new Error('Timed out seeking the video')); + }, SeekTimeoutMs); + function cleanup() { + el.removeEventListener('seeked', onSeeked); + el.removeEventListener('error', onError); + window.clearTimeout(timer); + } + function onSeeked() { + if (settled) return; + settled = true; + cleanup(); + resolve(); + } + function onError() { + if (settled) return; + settled = true; + cleanup(); + reject(new Error('The video failed while seeking')); + } + el.addEventListener('seeked', onSeeked); + el.addEventListener('error', onError); + // Seeking to the time the element already sits at fires no event. + if (Math.abs(el.currentTime - time) < 1e-6 && el.readyState >= 2) { + onSeeked(); + return; + } + // eslint-disable-next-line no-param-reassign + el.currentTime = time; + }); + } + + const getFrame = withCache(cache, (frame) => { + const run = queue.then(async () => { + if (disposed) throw new Error('Frame source disposed'); + const el = await element(); + await seekTo(el, frameToVideoTime(frame, config.fps, config.originalFps ?? null)); + const canvas = document.createElement('canvas'); + canvas.width = el.videoWidth; + canvas.height = el.videoHeight; + const ctx = canvas.getContext('2d'); + if (!ctx) throw new Error('Canvas unavailable'); + ctx.drawImage(el, 0, 0); + return { source: canvas, width: canvas.width, height: canvas.height } as DecodedFrame; + }); + // Failures must not wedge the queue for later frames. + queue = run.catch(() => undefined); + return run; + }); + + return { + frameCount: null, + getFrame, + dispose: () => { + disposed = true; + cache.clear(); + if (video) { + video.removeAttribute('src'); + video.load(); + video = null; + } + metadata = null; + }, + }; +} + +/** + * Pick the loader for a dataset, or throw for media the review grid cannot + * crop (tiled large images, multicamera parents). + */ +export function createFrameSource(config: DatasetConfig, options: FrameSourceOptions = {}): FrameSource { + const cacheSize = options.cacheSize ?? DefaultCacheSize; + if (config.type === 'large-image') { + throw new Error('Tiled large-image datasets cannot be reviewed as chips yet'); + } + if (config.type === 'multi') { + throw new Error('Review the cameras of a multicamera dataset individually'); + } + if (config.type === 'video') { + if (config.videoUrl) { + return videoFrameSource(config, cacheSize); + } + throw new Error('This video has no playable media'); + } + const { imageData } = config; + return imageFrameSource(async (frame) => { + const entry = imageData[frame]; + if (!entry) throw new Error(`No image for frame ${frame}`); + return entry.url; + }, imageData.length, cacheSize); +} diff --git a/client/dive-common/review/reviewItems.spec.ts b/client/dive-common/review/reviewItems.spec.ts new file mode 100644 index 000000000..72d8c7b84 --- /dev/null +++ b/client/dive-common/review/reviewItems.spec.ts @@ -0,0 +1,164 @@ +import type { TrackData } from 'vue-media-annotator/track'; +import { + attributeMatches, + buildReviewItems, + collectAttributeKeys, + collectTypes, + matchTypePair, + sampleFrames, + sortReviewItems, +} from './reviewItems'; +import { DEFAULT_REVIEW_QUERY } from './types'; + +function track( + id: number, + pairs: [string, number][], + frames: number[], + extra: Partial = {}, +): TrackData { + return { + id, + begin: Math.min(...frames), + end: Math.max(...frames), + confidencePairs: pairs, + attributes: {}, + features: frames.map((frame) => ({ + frame, keyframe: true, bounds: [frame, 0, frame + 10, 10], + })), + ...extra, + }; +} + +describe('sampleFrames', () => { + it('keeps a single detection as one frame', () => { + const [only] = track(1, [['a', 1]], [3]).features; + expect(sampleFrames([only], 8)).toEqual([{ frame: 3, bounds: [3, 0, 13, 10] }]); + }); + + it('samples evenly including both ends and never repeats a frame', () => { + const { features } = track(1, [['a', 1]], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + const frames = sampleFrames(features, 4).map((f) => f.frame); + expect(frames).toEqual([0, 3, 7, 10]); + expect(sampleFrames(features, 50)).toHaveLength(11); + }); +}); + +describe('matchTypePair', () => { + it('uses the top pair when no type is requested', () => { + expect(matchTypePair([['fish', 0.4], ['shark', 0.9]], '', 0.5)).toEqual(['shark', 0.9]); + expect(matchTypePair([['fish', 0.4]], '', 0.5)).toBeNull(); + }); + + it('matches the named type against the threshold', () => { + expect(matchTypePair([['fish', 0.4], ['shark', 0.9]], 'fish', 0.3)).toEqual(['fish', 0.4]); + expect(matchTypePair([['fish', 0.4], ['shark', 0.9]], 'fish', 0.5)).toBeNull(); + expect(matchTypePair([['shark', 0.9]], 'fish', 0)).toBeNull(); + }); + + it('only shows untyped tracks when everything is requested', () => { + expect(matchTypePair([], '', 0)).toEqual(['', 0]); + expect(matchTypePair([], '', 0.1)).toBeNull(); + }); +}); + +describe('attributeMatches', () => { + it('requires presence, then compares as text ignoring case', () => { + expect(attributeMatches(undefined, '')).toBe(false); + expect(attributeMatches('Yes', '')).toBe(true); + expect(attributeMatches('Yes', 'yes')).toBe(true); + expect(attributeMatches(3, '3')).toBe(true); + expect(attributeMatches(true, 'false')).toBe(false); + }); +}); + +describe('buildReviewItems', () => { + const tracks = [ + track(2, [['fish', 0.8], ['shark', 0.2]], [10, 11, 12]), + track(1, [['shark', 0.95]], [5]), + track(3, [['fish', 0.05]], [0]), + { ...track(4, [['fish', 0.9]], [7]), features: [{ frame: 7, keyframe: false }] }, + ]; + + it('filters by type and threshold and orders by track id', () => { + const items = buildReviewItems('ds', tracks, { ...DEFAULT_REVIEW_QUERY, type: 'fish', threshold: 0.1 }, 8); + expect(items.map((i) => i.trackId)).toEqual([2]); + expect(items[0]).toMatchObject({ + key: 'ds#2', type: 'fish', confidence: 0.8, keyframeCount: 3, + }); + expect(items[0].primary.frame).toBe(10); + expect(items[0].frames).toHaveLength(3); + }); + + it('shows every class above the threshold when no type is picked', () => { + const items = buildReviewItems('ds', tracks, { ...DEFAULT_REVIEW_QUERY, type: '', threshold: 0.5 }, 8); + expect(items.map((i) => [i.trackId, i.type])).toEqual([[1, 'shark'], [2, 'fish']]); + }); + + it('finds track and detection attributes', () => { + const withAttributes = [ + track(1, [['fish', 1]], [0, 1], { attributes: { verified: true } }), + { + ...track(2, [['fish', 1]], [4, 5, 6]), + features: [ + { frame: 4, keyframe: true, bounds: [0, 0, 1, 1] as [number, number, number, number] }, + { + frame: 5, keyframe: true, bounds: [0, 0, 1, 1] as [number, number, number, number], attributes: { occluded: 'partial' }, + }, + { + frame: 6, keyframe: true, bounds: [0, 0, 1, 1] as [number, number, number, number], attributes: { occluded: 'partial' }, + }, + ], + }, + ]; + const query = { ...DEFAULT_REVIEW_QUERY, mode: 'attribute' as const }; + const verified = buildReviewItems('ds', withAttributes, { ...query, attributeKey: 'verified' }, 8); + expect(verified.map((i) => i.trackId)).toEqual([1]); + expect(verified[0].matchedAttribute).toEqual({ key: 'verified', value: true, scope: 'track' }); + + const occluded = buildReviewItems('ds', withAttributes, { ...query, attributeKey: 'occluded', attributeValue: 'PARTIAL' }, 8); + expect(occluded).toHaveLength(1); + expect(occluded[0]).toMatchObject({ key: 'ds#2@5', trackId: 2, type: 'fish' }); + expect(occluded[0].primary.frame).toBe(5); + expect(occluded[0].frames.map((f) => f.frame)).toEqual([5, 6]); + + const trackOnly = buildReviewItems('ds', withAttributes, { ...query, attributeKey: 'occluded', attributeScope: 'track' }, 8); + expect(trackOnly).toHaveLength(0); + }); +}); + +describe('sortReviewItems', () => { + const items = [ + ...buildReviewItems('b', [track(1, [['a', 0.5]], [9]), track(2, [['a', 0.9]], [3])], DEFAULT_REVIEW_QUERY, 8), + ...buildReviewItems('a', [track(7, [['a', 0.7]], [1])], DEFAULT_REVIEW_QUERY, 8), + ]; + + it('orders by dataset selection order, confidence or frame', () => { + const keys = (order: Parameters[1]) => ( + sortReviewItems(items, order, ['a', 'b']).map((i) => i.key) + ); + expect(keys('dataset')).toEqual(['a#7', 'b#1', 'b#2']); + expect(keys('confidence-asc')).toEqual(['b#1', 'a#7', 'b#2']); + expect(keys('confidence-desc')).toEqual(['b#2', 'a#7', 'b#1']); + expect(keys('frame')).toEqual(['a#7', 'b#2', 'b#1']); + }); +}); + +describe('vocabularies', () => { + it('collects sorted types and attribute keys', () => { + const tracks = [ + track(1, [['zeta', 0.1], ['alpha', 0.9]], [0], { attributes: { trackAttr: 1 } }), + { + ...track(2, [['beta', 1]], [0]), + features: [{ + frame: 0, keyframe: true, bounds: [0, 0, 1, 1] as [number, number, number, number], attributes: { detAttr: 'x' }, + }], + }, + ]; + expect(collectTypes(tracks)).toEqual(['alpha', 'beta', 'zeta']); + expect(collectAttributeKeys(tracks, { + defined: { + belongs: 'track', datatype: 'text', name: 'defined', key: 'track_defined', + }, + })).toEqual(['defined', 'detAttr', 'trackAttr']); + }); +}); diff --git a/client/dive-common/review/reviewItems.ts b/client/dive-common/review/reviewItems.ts new file mode 100644 index 000000000..5ff68847c --- /dev/null +++ b/client/dive-common/review/reviewItems.ts @@ -0,0 +1,224 @@ +/** + * Pure helpers that turn a dataset's tracks into review grid items for a + * query, plus the type/attribute vocabularies the query controls offer. + */ +import type { TrackData, Feature } from 'vue-media-annotator/track'; +import type { StringKeyObject } from 'vue-media-annotator/BaseAnnotation'; +import type { Attribute } from 'vue-media-annotator/use/AttributeTypes'; +import { compareTypeNames } from 'dive-common/typeHierarchy'; +import type { + ReviewFrameRef, ReviewItem, ReviewQuery, ReviewSortOrder, +} from './types'; + +/** Keyframes carrying a box, in frame order. */ +export function boxedFeatures(track: TrackData): Feature[] { + return track.features + .filter((f) => !!f.bounds) + .sort((a, b) => a.frame - b.frame); +} + +/** Up to `max` boxes evenly sampled along `features` (which must be in frame order). */ +export function sampleFrames(features: Feature[], max: number): ReviewFrameRef[] { + if (features.length === 0) return []; + const count = Math.max(1, Math.min(max, features.length)); + const chosen: ReviewFrameRef[] = []; + const seen = new Set(); + for (let i = 0; i < count; i += 1) { + const index = count === 1 ? 0 : Math.round((i * (features.length - 1)) / (count - 1)); + const feature = features[index]; + if (feature.bounds && !seen.has(feature.frame)) { + seen.add(feature.frame); + chosen.push({ frame: feature.frame, bounds: feature.bounds }); + } + } + return chosen; +} + +/** The pair a type query matches on, or null when the track does not qualify. */ +export function matchTypePair( + pairs: readonly (readonly [string, number])[], + type: string, + threshold: number, +): [string, number] | null { + if (pairs.length === 0) { + // Untyped annotations only show up when every type is requested. + return type === '' && threshold <= 0 ? ['', 0] : null; + } + // Stored pairs are normally sorted by confidence, but files are not guaranteed to be. + const candidates = type === '' ? pairs : pairs.filter(([name]) => name === type); + const best = candidates.reduce( + (acc, pair) => (acc === null || pair[1] > acc[1] ? pair : acc), + null, + ); + if (!best || best[1] < threshold) return null; + return [best[0], best[1]]; +} + +function ownAttributes(attributes: StringKeyObject | undefined): [string, unknown][] { + if (!attributes) return []; + return Object.entries(attributes).filter(([key]) => key !== 'userAttributes'); +} + +/** Attribute values compare as strings, case-insensitively; empty wanted means "present". */ +export function attributeMatches(value: unknown, wanted: string): boolean { + if (value === undefined || value === null) return false; + if (wanted === '') return true; + const asText = Array.isArray(value) ? value.map(String).join(',') : String(value); + return asText.toLowerCase() === wanted.trim().toLowerCase(); +} + +function trackTopPair(track: TrackData): [string, number] { + const top = track.confidencePairs.reduce( + (acc, pair) => (acc === null || pair[1] > acc[1] ? pair : acc), + null, + ); + return top ? [top[0], top[1]] : ['', 0]; +} + +function attributeItem( + datasetId: string, + track: TrackData, + query: ReviewQuery, + maxSequenceFrames: number, +): ReviewItem | null { + const key = query.attributeKey.trim(); + if (!key) return null; + const features = boxedFeatures(track); + if (features.length === 0) return null; + const [type, confidence] = trackTopPair(track); + const base = { + datasetId, + trackId: track.id, + keyframeCount: features.length, + type, + confidence, + }; + if (query.attributeScope !== 'detection') { + const hit = ownAttributes(track.attributes).find( + ([name, value]) => name === key && attributeMatches(value, query.attributeValue), + ); + if (hit) { + const frames = sampleFrames(features, maxSequenceFrames); + return { + ...base, + key: `${datasetId}#${track.id}`, + primary: frames[0], + frames, + matchedAttribute: { key, value: hit[1], scope: 'track' }, + }; + } + } + if (query.attributeScope !== 'track') { + const matching = features.filter((f) => ownAttributes(f.attributes).some( + ([name, value]) => name === key && attributeMatches(value, query.attributeValue), + )); + if (matching.length > 0) { + const frames = sampleFrames(matching, maxSequenceFrames); + const hit = ownAttributes(matching[0].attributes).find(([name]) => name === key); + return { + ...base, + key: `${datasetId}#${track.id}@${matching[0].frame}`, + primary: frames[0], + frames, + matchedAttribute: { key, value: hit ? hit[1] : undefined, scope: 'detection' }, + }; + } + } + return null; +} + +function typeItem( + datasetId: string, + track: TrackData, + query: ReviewQuery, + maxSequenceFrames: number, +): ReviewItem | null { + const pair = matchTypePair(track.confidencePairs, query.type, query.threshold); + if (!pair) return null; + const features = boxedFeatures(track); + if (features.length === 0) return null; + const frames = sampleFrames(features, maxSequenceFrames); + return { + key: `${datasetId}#${track.id}`, + datasetId, + trackId: track.id, + primary: frames[0], + frames, + keyframeCount: features.length, + type: pair[0], + confidence: pair[1], + }; +} + +/** Every grid item one dataset contributes to a query, in track id order. */ +export function buildReviewItems( + datasetId: string, + tracks: Iterable, + query: ReviewQuery, + maxSequenceFrames: number, +): ReviewItem[] { + const items: ReviewItem[] = []; + Array.from(tracks) + .sort((a, b) => a.id - b.id) + .forEach((track) => { + const item = query.mode === 'attribute' + ? attributeItem(datasetId, track, query, maxSequenceFrames) + : typeItem(datasetId, track, query, maxSequenceFrames); + if (item) items.push(item); + }); + return items; +} + +export function sortReviewItems( + items: ReviewItem[], + order: ReviewSortOrder, + datasetOrder: readonly string[], +): ReviewItem[] { + const rank = new Map(datasetOrder.map((id, index) => [id, index])); + const byDataset = (a: ReviewItem, b: ReviewItem) => ( + (rank.get(a.datasetId) ?? Infinity) - (rank.get(b.datasetId) ?? Infinity) + ); + const sorted = [...items]; + switch (order) { + case 'confidence-asc': + sorted.sort((a, b) => a.confidence - b.confidence || byDataset(a, b) || a.trackId - b.trackId); + break; + case 'confidence-desc': + sorted.sort((a, b) => b.confidence - a.confidence || byDataset(a, b) || a.trackId - b.trackId); + break; + case 'frame': + sorted.sort((a, b) => byDataset(a, b) || a.primary.frame - b.primary.frame || a.trackId - b.trackId); + break; + default: + sorted.sort((a, b) => byDataset(a, b) || a.trackId - b.trackId); + } + return sorted; +} + +/** Every type named by any confidence pair, in type order. */ +export function collectTypes(tracks: Iterable): string[] { + const types = new Set(); + Array.from(tracks).forEach((track) => { + track.confidencePairs.forEach(([type]) => { if (type) types.add(type); }); + }); + return Array.from(types).sort(compareTypeNames); +} + +/** + * Attribute keys the query control can offer: those defined in the dataset + * configuration plus any actually present on tracks or detections. + */ +export function collectAttributeKeys( + tracks: Iterable, + definitions: Readonly> | undefined, +): string[] { + const keys = new Set(); + Object.values(definitions || {}).forEach((attribute) => keys.add(attribute.name)); + Array.from(tracks).forEach((track) => { + ownAttributes(track.attributes).forEach(([key]) => keys.add(key)); + track.features.forEach((feature) => { + ownAttributes(feature.attributes).forEach(([key]) => keys.add(key)); + }); + }); + return Array.from(keys).sort((a, b) => a.localeCompare(b)); +} diff --git a/client/dive-common/review/types.ts b/client/dive-common/review/types.ts new file mode 100644 index 000000000..df8362e96 --- /dev/null +++ b/client/dive-common/review/types.ts @@ -0,0 +1,102 @@ +/** + * Review mode: contract shared by the review page, the chip loader and the + * platform shells. The page shows many detections at once as cropped chips + * so a user can audit and correct their types (or find them by attribute) + * without opening every sequence in the viewer. + */ +import type { AnnotationId } from 'vue-media-annotator/BaseAnnotation'; +import type { RectBounds } from 'vue-media-annotator/utils'; + +export type ReviewQueryMode = 'type' | 'attribute'; + +/** Where an attribute query looks for the attribute. */ +export type ReviewAttributeScope = 'any' | 'track' | 'detection'; + +export interface ReviewQuery { + mode: ReviewQueryMode; + /** Type mode: the class to show; empty for every class. */ + type: string; + /** Type mode: minimum confidence of the matching pair (0 shows everything). */ + threshold: number; + /** Attribute mode: the attribute key to look for. */ + attributeKey: string; + /** Attribute mode: required value (string compared); empty just requires presence. */ + attributeValue: string; + attributeScope: ReviewAttributeScope; +} + +export type ReviewSortOrder = 'dataset' | 'confidence-asc' | 'confidence-desc' | 'frame'; + +/** One box of a track shown in a chip, in the dataset's own frame numbers. */ +export interface ReviewFrameRef { + frame: number; + bounds: RectBounds; +} + +/** + * One grid entry: a track (or one detection of it) in one dataset. Built + * once per query run; the live type is read from the review service so + * edits show without the grid reshuffling. + */ +export interface ReviewItem { + /** Stable within a query run: `${datasetId}#${trackId}` or with `@frame`. */ + key: string; + datasetId: string; + trackId: AnnotationId; + /** The box shown first: the matched detection, or the track's first keyframe. */ + primary: ReviewFrameRef; + /** + * Boxes sampled along the track for the cycling animation, primary first. + * A single entry means a static detection. + */ + frames: ReviewFrameRef[]; + /** Number of keyframes in the track. */ + keyframeCount: number; + /** Type and confidence of the pair the query matched on (top pair otherwise). */ + type: string; + confidence: number; + /** Attribute mode: what matched. */ + matchedAttribute?: { + key: string; + value: unknown; + scope: 'track' | 'detection'; + }; +} + +/** Grid presentation settings, persisted per browser. */ +export interface ReviewGridSettings { + columns: number; + rows: number; + /** + * Context around the box as a fraction of its longer side: 0.3 shows the + * box plus 30% of its size on each side. + */ + padding: number; + /** Milliseconds between frames of a cycling track. */ + cycleIntervalMs: number; + /** Most frames sampled along a track. */ + maxSequenceFrames: number; +} + +export const DEFAULT_REVIEW_QUERY: ReviewQuery = { + mode: 'type', + type: '', + threshold: 0.1, + attributeKey: '', + attributeValue: '', + attributeScope: 'any', +}; + +export const DEFAULT_REVIEW_GRID: ReviewGridSettings = { + columns: 5, + rows: 4, + padding: 0.3, + cycleIntervalMs: 400, + maxSequenceFrames: 8, +}; + +export const REVIEW_GRID_LIMITS = { + columns: [1, 12] as const, + rows: [1, 10] as const, + padding: [0, 3] as const, +}; diff --git a/client/dive-common/review/viewerNavigation.spec.ts b/client/dive-common/review/viewerNavigation.spec.ts new file mode 100644 index 000000000..9ccd2de8a --- /dev/null +++ b/client/dive-common/review/viewerNavigation.spec.ts @@ -0,0 +1,15 @@ +import { parseViewerFocus, reviewViewerLocation } from './viewerNavigation'; + +describe('review viewer navigation', () => { + it('round-trips a frame and track through the query', () => { + const location = reviewViewerLocation('abc', { frame: 12, trackId: 7 }); + expect(location).toEqual({ name: 'viewer', params: { id: 'abc' }, query: { frame: '12', track: '7' } }); + expect(parseViewerFocus(location.query)).toEqual({ frame: 12, trackId: 7 }); + }); + + it('ignores missing or malformed values', () => { + expect(reviewViewerLocation('abc', {}).query).toEqual({}); + expect(parseViewerFocus({ frame: 'x', track: ['3'] })).toEqual({ trackId: 3 }); + expect(parseViewerFocus({ frame: '-1' })).toEqual({}); + }); +}); diff --git a/client/dive-common/review/viewerNavigation.ts b/client/dive-common/review/viewerNavigation.ts new file mode 100644 index 000000000..995da442e --- /dev/null +++ b/client/dive-common/review/viewerNavigation.ts @@ -0,0 +1,45 @@ +/** + * Deep links from the review grid into the annotation viewer: the viewer + * route plus query parameters naming the frame to seek to and the track to + * select once media is ready. + */ +export const VIEWER_FRAME_QUERY = 'frame'; +export const VIEWER_TRACK_QUERY = 'track'; + +export interface ViewerFocus { + frame?: number; + trackId?: number; +} + +export interface ReviewViewerLocation { + name: string; + params: Record; + query: Record; +} + +/** Both platforms name their single-dataset viewer route `viewer` with an `id` param. */ +export function reviewViewerLocation( + datasetId: string, + focus: ViewerFocus, +): ReviewViewerLocation { + const query: Record = {}; + if (focus.frame !== undefined) query[VIEWER_FRAME_QUERY] = String(focus.frame); + if (focus.trackId !== undefined) query[VIEWER_TRACK_QUERY] = String(focus.trackId); + return { name: 'viewer', params: { id: datasetId }, query }; +} + +function integerParam(value: unknown): number | undefined { + const text = Array.isArray(value) ? value[0] : value; + if (typeof text !== 'string' || !/^-?\d+$/.test(text)) return undefined; + return Number(text); +} + +/** Read the focus back out of a route query (unknown values are ignored). */ +export function parseViewerFocus(query: Record): ViewerFocus { + const frame = integerParam(query[VIEWER_FRAME_QUERY]); + const trackId = integerParam(query[VIEWER_TRACK_QUERY]); + const focus: ViewerFocus = {}; + if (frame !== undefined && frame >= 0) focus.frame = frame; + if (trackId !== undefined) focus.trackId = trackId; + return focus; +} diff --git a/client/dive-common/use/useReview.spec.ts b/client/dive-common/use/useReview.spec.ts new file mode 100644 index 000000000..76e63bb39 --- /dev/null +++ b/client/dive-common/use/useReview.spec.ts @@ -0,0 +1,173 @@ +import { nextTick } from 'vue'; +import type { TrackData } from 'vue-media-annotator/track'; +import type { DatasetConfig } from 'dive-common/apispec'; +import { createReviewService, ReviewApi } from './useReview'; + +vi.mock('dive-common/review/frameSource', () => ({ + createFrameSource: vi.fn(() => ({ + frameCount: 1, + getFrame: vi.fn(), + dispose: vi.fn(), + })), +})); + +function track(id: number, pairs: [string, number][], frames: number[]): TrackData { + return { + id, + begin: Math.min(...frames), + end: Math.max(...frames), + confidencePairs: pairs, + attributes: {}, + features: frames.map((frame) => ({ + frame, keyframe: true, bounds: [0, 0, 10, 10], + })), + }; +} + +function config(id: string, overrides: Partial = {}): DatasetConfig { + return { + id, + name: `Dataset ${id}`, + type: 'image-sequence', + fps: 1, + createdAt: '', + subType: null, + multiCamMedia: null, + imageData: [{ url: 'a.jpg', filename: 'a.jpg' }], + videoUrl: undefined, + ...overrides, + } as DatasetConfig; +} + +function makeApi(tracksById: Record, overrides: Partial = {}): ReviewApi { + return { + loadConfig: vi.fn(async (id: string) => config(id)), + // Fresh copies each call, as a real platform returns them. + loadDetections: vi.fn(async (id: string) => ({ + version: 2, tracks: JSON.parse(JSON.stringify(tracksById[id] || [])), groups: [], sets: [], + })), + saveDetections: vi.fn(async () => undefined), + listScoringDatasets: vi.fn(async () => [{ id: 'a', name: 'Alpha' }, { id: 'b', name: 'Beta' }]), + ...overrides, + }; +} + +describe('createReviewService', () => { + it('loads datasets, prefers peekConfig, and builds items for a query', async () => { + const peekConfig = vi.fn(async (id: string) => config(id)); + const api = makeApi({ + a: [track(1, [['fish', 0.9]], [0, 1, 2]), track(2, [['shark', 0.3]], [4])], + }, { peekConfig }); + const service = createReviewService({ api }); + await service.refreshAvailable(); + await service.addDatasets(['a', 'a']); + expect(peekConfig).toHaveBeenCalledTimes(1); + expect(api.loadConfig).not.toHaveBeenCalled(); + expect(service.datasets.value).toHaveLength(1); + expect(service.datasets.value[0]).toMatchObject({ + id: 'a', name: 'Alpha', status: 'ready', trackCount: 2, croppable: true, + }); + expect(service.types.value).toEqual(['fish', 'shark']); + expect(service.stale.value).toBe(true); + + service.query.threshold = 0.5; + service.runQuery(); + expect(service.stale.value).toBe(false); + expect(service.items.value.map((i) => i.key)).toEqual(['a#1']); + expect(service.items.value[0].frames).toHaveLength(3); + }); + + it('expands a multicamera parent into its cameras', async () => { + const api = makeApi({ 'm/left': [track(1, [['fish', 1]], [0])], 'm/right': [] }, { + loadConfig: vi.fn(async (id: string) => (id === 'm' + ? config(id, { + type: 'multi', + multiCamMedia: { + defaultDisplay: 'left', + cameras: { + left: { type: 'image-sequence', imageData: [], videoUrl: '' }, + right: { type: 'image-sequence', imageData: [], videoUrl: '' }, + }, + }, + }) + : config(id))), + }); + const service = createReviewService({ api }); + await service.addDataset('m', { id: 'm', name: 'Rig' }); + expect(service.datasets.value.map((d) => [d.id, d.name, d.status])).toEqual([ + ['m/left', 'Rig (left)', 'ready'], + ['m/right', 'Rig (right)', 'ready'], + ]); + }); + + it('reassigns and accepts types, tracks pending edits, and saves them', async () => { + const api = makeApi({ + a: [track(1, [['fish', 0.6], ['shark', 0.4]], [0]), track(2, [['fish', 0.9]], [0])], + }); + const service = createReviewService({ api }); + await service.addDataset('a'); + service.query.type = 'fish'; + service.query.threshold = 0; + service.runQuery(); + const [first, second] = service.items.value; + + service.assignType(first, 'shark'); + expect(service.currentType(first)).toEqual({ type: 'shark', confidence: 1 }); + expect(service.trackOf('a', 1)?.confidencePairs).toEqual([['shark', 1]]); + expect(service.isPending(first)).toBe(true); + expect(service.pendingCount.value).toBe(1); + // The grid keeps the item until the query is run again. + expect(service.items.value).toHaveLength(2); + + service.acceptType(second); + expect(service.trackOf('a', 2)?.confidencePairs).toEqual([['fish', 1]]); + expect(service.pendingCount.value).toBe(2); + + await service.save(); + expect(api.saveDetections).toHaveBeenCalledTimes(1); + const [datasetId, args] = (api.saveDetections as ReturnType).mock.calls[0]; + expect(datasetId).toBe('a'); + expect(args.tracks.upsert.map((t: TrackData) => t.id).sort()).toEqual([1, 2]); + expect(args.groups).toEqual({ upsert: [], delete: [] }); + expect(service.pendingCount.value).toBe(0); + expect(service.error.value).toBeNull(); + }); + + it('reports a failed save and keeps the edits pending', async () => { + const api = makeApi({ a: [track(1, [['fish', 0.6]], [0])] }, { + saveDetections: vi.fn(async () => { throw new Error('disk full'); }), + }); + const service = createReviewService({ api }); + await service.addDataset('a'); + service.runQuery(); + service.assignType(service.items.value[0], 'shark'); + await service.save(); + expect(service.error.value).toBe('disk full'); + expect(service.pendingCount.value).toBe(1); + }); + + it('discards edits by reloading the changed datasets', async () => { + const api = makeApi({ a: [track(1, [['fish', 0.6]], [0])] }); + const service = createReviewService({ api }); + await service.addDataset('a'); + service.runQuery(); + service.assignType(service.items.value[0], 'shark'); + await service.discardChanges(); + expect(api.loadDetections).toHaveBeenCalledTimes(2); + expect(service.pendingCount.value).toBe(0); + expect(service.trackOf('a', 1)?.confidencePairs).toEqual([['fish', 0.6]]); + }); + + it('marks a dataset that failed to load and lets it be removed', async () => { + const api = makeApi({}, { + loadDetections: vi.fn(async () => { throw new Error('missing'); }), + }); + const service = createReviewService({ api }); + await service.addDataset('a'); + expect(service.datasets.value[0]).toMatchObject({ status: 'error', error: 'missing' }); + service.removeDataset('a'); + await nextTick(); + expect(service.datasets.value).toHaveLength(0); + expect(service.types.value).toEqual([]); + }); +}); diff --git a/client/dive-common/use/useReview.ts b/client/dive-common/use/useReview.ts new file mode 100644 index 000000000..51730b8d7 --- /dev/null +++ b/client/dive-common/use/useReview.ts @@ -0,0 +1,487 @@ +/** + * State behind the Review page: the datasets under review (with their + * tracks held in memory), the query, the grid settings, the rendered + * chips, and the type edits waiting to be saved back. + */ +import { + computed, inject, provide, reactive, ref, Ref, watch, +} from 'vue'; +import type { Api, DatasetConfig } from 'dive-common/apispec'; +import type { ScoringDatasetSummary } from 'dive-common/scoring/types'; +import type { TrackData } from 'vue-media-annotator/track'; +import type { AnnotationId, ConfidencePair } from 'vue-media-annotator/BaseAnnotation'; +import { + acceptPairAsCorrect, compileHierarchy, reassignPairs, TypeHierarchyIndex, +} from 'dive-common/typeHierarchy'; +import { createFrameSource, FrameSource } from 'dive-common/review/frameSource'; +import { createChipStore, ChipStore } from 'dive-common/review/chipStore'; +import { + buildReviewItems, collectAttributeKeys, collectTypes, sortReviewItems, +} from 'dive-common/review/reviewItems'; +import { + DEFAULT_REVIEW_GRID, + DEFAULT_REVIEW_QUERY, + REVIEW_GRID_LIMITS, + ReviewGridSettings, + ReviewItem, + ReviewQuery, + ReviewSortOrder, +} from 'dive-common/review/types'; + +const GRID_STORAGE_KEY = 'dive.review.grid'; +const CHIP_OUTLINE = '#00e5ff'; + +export type ReviewApi = Pick; + +export interface ReviewServiceDeps { + api: ReviewApi; +} + +export type ReviewDatasetStatus = 'loading' | 'ready' | 'error'; + +export interface ReviewDataset { + id: string; + name: string; + type?: string; + status: ReviewDatasetStatus; + error?: string; + trackCount: number; + /** True when the media can be cropped into chips. */ + croppable: boolean; +} + +export interface ReviewService { + datasets: Readonly>; + available: Readonly>; + query: ReviewQuery; + grid: ReviewGridSettings; + sort: Ref; + items: Readonly>; + /** Bumps whenever tracks load or change; computeds that read tracks depend on it. */ + dataRevision: Readonly>; + /** True once tracks or the query changed after the last run. */ + stale: Readonly>; + types: Readonly>; + attributeKeys: Readonly>; + pendingCount: Readonly>; + saving: Readonly>; + loading: Readonly>; + error: Readonly>; + chipStore: ChipStore; + datasetName(id: string): string; + refreshAvailable(): Promise; + addDataset(id: string, summary?: ScoringDatasetSummary): Promise; + addDatasets(ids: string[]): Promise; + removeDataset(id: string): void; + reloadDataset(id: string): Promise; + runQuery(): void; + trackOf(datasetId: string, trackId: AnnotationId): TrackData | undefined; + /** The item's live top type/confidence after any edits. */ + currentType(item: ReviewItem): { type: string; confidence: number }; + isPending(item: ReviewItem): boolean; + assignType(item: ReviewItem, type: string): void; + acceptType(item: ReviewItem): void; + save(): Promise; + discardChanges(): Promise; + clearError(): void; + dispose(): void; +} + +function loadGridSettings(): ReviewGridSettings { + try { + const raw = window.localStorage.getItem(GRID_STORAGE_KEY); + if (raw) { + const parsed = JSON.parse(raw); + if (parsed && typeof parsed === 'object') { + return { ...DEFAULT_REVIEW_GRID, ...parsed }; + } + } + } catch { + // Storage may be unavailable; defaults are fine. + } + return { ...DEFAULT_REVIEW_GRID }; +} + +function storeGridSettings(grid: ReviewGridSettings) { + try { + window.localStorage.setItem(GRID_STORAGE_KEY, JSON.stringify(grid)); + } catch { + // Ignore storage failures. + } +} + +export function clampGrid(grid: ReviewGridSettings): ReviewGridSettings { + const clamp = (value: number, [min, max]: readonly [number, number], fallback: number) => ( + Number.isFinite(value) ? Math.min(max, Math.max(min, value)) : fallback + ); + return { + ...grid, + columns: Math.round(clamp(grid.columns, REVIEW_GRID_LIMITS.columns, DEFAULT_REVIEW_GRID.columns)), + rows: Math.round(clamp(grid.rows, REVIEW_GRID_LIMITS.rows, DEFAULT_REVIEW_GRID.rows)), + padding: clamp(grid.padding, REVIEW_GRID_LIMITS.padding, DEFAULT_REVIEW_GRID.padding), + }; +} + +interface LoadedDataset { + config: DatasetConfig; + tracks: Map; + hierarchy: TypeHierarchyIndex; + frameSource: FrameSource | null; + pending: Set; +} + +function topPair(pairs: readonly ConfidencePair[]): { type: string; confidence: number } { + const top = pairs.reduce( + (acc, pair) => (acc === null || pair[1] > acc[1] ? pair : acc), + null, + ); + return top ? { type: top[0], confidence: top[1] } : { type: '', confidence: 0 }; +} + +export function createReviewService(deps: ReviewServiceDeps): ReviewService { + const { api } = deps; + const datasets = ref([]); + const available = ref([]); + const query = reactive({ ...DEFAULT_REVIEW_QUERY }); + const grid = reactive(clampGrid(loadGridSettings())); + const sort = ref('dataset'); + const items = ref([]); + const dataRevision = ref(0); + const stale = ref(false); + const saving = ref(false); + const loading = ref(false); + const error = ref(null); + /** Tracks and media, deliberately outside Vue reactivity (they can be large). */ + const loaded = new Map(); + /** Loads still in flight, so a removal during load is honoured. */ + let loadGeneration = 0; + + watch(grid, () => storeGridSettings({ ...grid }), { deep: true }); + watch(query, () => { stale.value = true; }, { deep: true }); + + const chipStore = createChipStore({ + frameSourceFor: (datasetId) => loaded.get(datasetId)?.frameSource ?? null, + }, { padding: grid.padding, size: 256, outline: CHIP_OUTLINE }); + + function fail(reason: unknown, fallback: string) { + const message = reason instanceof Error ? reason.message : String(reason || fallback); + error.value = message || fallback; + } + + function datasetName(id: string) { + return datasets.value.find((d) => d.id === id)?.name + || available.value.find((d) => d.id === id)?.name + || id; + } + + function entry(id: string) { + return datasets.value.find((d) => d.id === id); + } + + function patch(id: string, changes: Partial) { + datasets.value = datasets.value.map((d) => (d.id === id ? { ...d, ...changes } : d)); + } + + async function refreshAvailable() { + if (!api.listScoringDatasets) return; + try { + available.value = await api.listScoringDatasets(); + } catch (err) { + fail(err, 'Could not list datasets'); + } + } + + function loadConfig(id: string) { + return api.peekConfig ? api.peekConfig(id) : api.loadConfig(id); + } + + function frameSourceFor(config: DatasetConfig): FrameSource | null { + try { + return createFrameSource(config); + } catch { + return null; + } + } + + function dropLoaded(id: string) { + const existing = loaded.get(id); + if (existing) { + existing.frameSource?.dispose(); + loaded.delete(id); + } + } + + async function load(id: string, generation: number) { + loading.value = true; + try { + const config = await loadConfig(id); + if (generation !== loadGeneration || !entry(id)) return; + if (config.type === 'multi') { + // Review the cameras of a multicamera dataset as separate sequences. + const cameras = Object.keys(config.multiCamMedia?.cameras || {}); + const parentName = entry(id)?.name || config.name; + datasets.value = datasets.value.filter((d) => d.id !== id); + await Promise.all(cameras.map((camera) => addDataset(`${id}/${camera}`, { + id: `${id}/${camera}`, name: `${parentName} (${camera})`, type: config.multiCamMedia?.cameras[camera]?.type, + }))); + return; + } + const detections = await api.loadDetections(id); + if (generation !== loadGeneration || !entry(id)) return; + dropLoaded(id); + const tracks = new Map(); + detections.tracks.forEach((track) => tracks.set(track.id, track)); + const frameSource = frameSourceFor(config); + loaded.set(id, { + config, + tracks, + hierarchy: compileHierarchy(config.typeHierarchy || {}), + frameSource, + pending: new Set(), + }); + patch(id, { + status: 'ready', + error: undefined, + name: entry(id)?.name || config.name, + type: config.type, + trackCount: tracks.size, + croppable: frameSource !== null, + }); + dataRevision.value += 1; + stale.value = true; + } catch (err) { + if (generation !== loadGeneration || !entry(id)) return; + patch(id, { + status: 'error', + error: err instanceof Error ? err.message : String(err), + }); + } finally { + loading.value = datasets.value.some((d) => d.status === 'loading'); + } + } + + async function addDataset(id: string, summary?: ScoringDatasetSummary) { + if (!id || entry(id)) return; + datasets.value = [...datasets.value, { + id, + name: summary?.name || datasetName(id), + type: summary?.type, + status: 'loading', + trackCount: 0, + croppable: false, + }]; + await load(id, loadGeneration); + } + + async function addDatasets(ids: string[]) { + const unique = Array.from(new Set(ids.filter(Boolean))); + await Promise.all(unique.map((id) => addDataset(id))); + } + + function removeDataset(id: string) { + datasets.value = datasets.value.filter((d) => d.id !== id); + dropLoaded(id); + loadGeneration += 1; + dataRevision.value += 1; + stale.value = true; + } + + async function reloadDataset(id: string) { + if (!entry(id)) return; + patch(id, { status: 'loading', error: undefined }); + await load(id, loadGeneration); + } + + function allTracks(): TrackData[] { + const all: TrackData[] = []; + loaded.forEach((dataset) => all.push(...dataset.tracks.values())); + return all; + } + + /** Read inside a computed so it re-runs when tracks load or change. */ + function dependOnData(): number { + return dataRevision.value; + } + + const types = computed(() => { + dependOnData(); + return collectTypes(allTracks()); + }); + + const attributeKeys = computed(() => { + dependOnData(); + const keys = new Set(); + loaded.forEach((dataset) => { + collectAttributeKeys(dataset.tracks.values(), dataset.config.attributes).forEach((k) => keys.add(k)); + }); + return Array.from(keys).sort((a, b) => a.localeCompare(b)); + }); + + function runQuery() { + const order = datasets.value.map((d) => d.id); + const built: ReviewItem[] = []; + order.forEach((id) => { + const dataset = loaded.get(id); + if (dataset) { + built.push(...buildReviewItems(id, dataset.tracks.values(), { ...query }, grid.maxSequenceFrames)); + } + }); + items.value = sortReviewItems(built, sort.value, order); + stale.value = false; + } + + watch(sort, () => { + items.value = sortReviewItems(items.value, sort.value, datasets.value.map((d) => d.id)); + }); + + function trackOf(datasetId: string, trackId: AnnotationId) { + return loaded.get(datasetId)?.tracks.get(trackId); + } + + function currentType(item: ReviewItem) { + const track = trackOf(item.datasetId, item.trackId); + if (!track) return { type: item.type, confidence: item.confidence }; + if (query.mode === 'type' && query.type) { + // Keep showing the queried pair while it still exists, so an edit that + // demotes it is visible as such. + const pair = track.confidencePairs.find(([type]) => type === query.type); + const top = topPair(track.confidencePairs); + if (pair && top.type === query.type) return { type: pair[0], confidence: pair[1] }; + return top; + } + return topPair(track.confidencePairs); + } + + function isPending(item: ReviewItem) { + return loaded.get(item.datasetId)?.pending.has(item.trackId) ?? false; + } + + const pendingCount = computed(() => { + dependOnData(); + let count = 0; + loaded.forEach((dataset) => { count += dataset.pending.size; }); + return count; + }); + + function updatePairs(item: ReviewItem, update: (pairs: ConfidencePair[], hierarchy: TypeHierarchyIndex) => ConfidencePair[]) { + const dataset = loaded.get(item.datasetId); + const track = dataset?.tracks.get(item.trackId); + if (!dataset || !track) return; + const next = update(track.confidencePairs.map(([t, c]) => [t, c] as ConfidencePair), dataset.hierarchy); + track.confidencePairs = next; + dataset.pending.add(item.trackId); + dataRevision.value += 1; + } + + function assignType(item: ReviewItem, type: string) { + const trimmed = type.trim(); + if (!trimmed) return; + const current = currentType(item); + if (current.type === trimmed && current.confidence >= 1) return; + updatePairs(item, (pairs, hierarchy) => reassignPairs( + hierarchy, + pairs, + current.type || trimmed, + trimmed, + 1, + )); + } + + function acceptType(item: ReviewItem) { + const current = currentType(item); + if (!current.type) return; + updatePairs(item, (pairs, hierarchy) => acceptPairAsCorrect(hierarchy, pairs, current.type)); + } + + async function save() { + if (saving.value) return; + saving.value = true; + error.value = null; + try { + const targets = Array.from(loaded.entries()).filter(([, d]) => d.pending.size > 0); + const results = await Promise.allSettled(targets.map(async ([id, dataset]) => { + const upsert = Array.from(dataset.pending) + .map((trackId) => dataset.tracks.get(trackId)) + .filter((t): t is TrackData => !!t); + await api.saveDetections(id, { + tracks: { upsert, delete: [] }, + groups: { upsert: [], delete: [] }, + }); + dataset.pending.clear(); + })); + const failed = results.find((r) => r.status === 'rejected') as PromiseRejectedResult | undefined; + if (failed) throw failed.reason; + } catch (err) { + fail(err, 'Could not save the changed annotations'); + } finally { + dataRevision.value += 1; + saving.value = false; + } + } + + async function discardChanges() { + const dirty = Array.from(loaded.entries()).filter(([, d]) => d.pending.size > 0).map(([id]) => id); + await Promise.all(dirty.map((id) => reloadDataset(id))); + } + + function clearError() { + error.value = null; + } + + function dispose() { + loadGeneration += 1; + loaded.forEach((dataset) => dataset.frameSource?.dispose()); + loaded.clear(); + chipStore.reset(); + } + + return { + datasets, + available, + query, + grid, + sort, + items, + dataRevision, + stale, + types, + attributeKeys, + pendingCount, + saving, + loading, + error, + chipStore, + datasetName, + refreshAvailable, + addDataset, + addDatasets, + removeDataset, + reloadDataset, + runQuery, + trackOf, + currentType, + isPending, + assignType, + acceptType, + save, + discardChanges, + clearError, + dispose, + }; +} + +const ReviewSymbol = Symbol('review'); + +export function provideReview(service: ReviewService) { + provide(ReviewSymbol, service); +} + +export function useReview(): ReviewService { + const service = inject(ReviewSymbol, null); + if (!service) { + throw new Error('Review service not provided'); + } + return service; +} diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index 825633d6d..23daa131b 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -815,6 +815,11 @@ async function loadConfig(id: string) { return { ...data, calibration: data.multiCam?.calibration ?? null }; } +/** loadConfig without the recents bookkeeping the stateful wrapper adds. */ +function peekConfig(id: string) { + return loadConfig(id); +} + let scoringAnnotationPreviewFile: string | null = null; /** One-shot annotation file to load in the viewer (from scoring result links). */ @@ -928,6 +933,7 @@ export { exportScoringPdf, /* Standard Specification APIs */ loadConfig, + peekConfig, loadDetections, loadFrameMetadata, getPipelineList, diff --git a/client/platform/desktop/frontend/components/NavigationBar.vue b/client/platform/desktop/frontend/components/NavigationBar.vue index f3907c753..97c98209b 100644 --- a/client/platform/desktop/frontend/components/NavigationBar.vue +++ b/client/platform/desktop/frontend/components/NavigationBar.vue @@ -28,6 +28,9 @@ export default defineComponent({ Trainingmdi-brain + + Reviewmdi-view-grid-outline + Pipelinemdi-pipe diff --git a/client/platform/desktop/frontend/components/Recent.vue b/client/platform/desktop/frontend/components/Recent.vue index 27630ab0b..040fca447 100644 --- a/client/platform/desktop/frontend/components/Recent.vue +++ b/client/platform/desktop/frontend/components/Recent.vue @@ -265,6 +265,10 @@ export default defineComponent({ function scoreSelected() { router.push({ name: 'scoring', query: selectedIdsQuery() }); } + + function reviewSelected() { + router.push({ name: 'review', query: selectedIdsQuery() }); + } function getTypeIcon(recent: JsonConfigCache) { if (recent.subType) { if (recent.subType === 'stereo') { @@ -360,6 +364,7 @@ export default defineComponent({ runPipelineOnSelected, runTrainingOnSelected, scoreSelected, + reviewSelected, isSelected, toggleSelected, toggleSelectAll, @@ -649,6 +654,27 @@ export default defineComponent({ Score the selected datasets against ground truth + + + Review the selected datasets' annotations as a grid + - Open in the annotation viewer + Open in the annotation viewer at this frame @@ -317,7 +771,7 @@ export default defineComponent({ >
{{ attributeText || subtitle }} @@ -329,6 +783,7 @@ export default defineComponent({ diff --git a/client/dive-common/components/Review/ReviewGridControls.vue b/client/dive-common/components/Review/ReviewGridControls.vue index 5b9a44dfc..af3600a32 100644 --- a/client/dive-common/components/Review/ReviewGridControls.vue +++ b/client/dive-common/components/Review/ReviewGridControls.vue @@ -103,7 +103,6 @@ export default defineComponent({ :max="limits.padding[1]" step="0.05" hide-details - dense class="padding-slider mr-3" @input="$emit('set-padding', Number($event))" /> @@ -142,12 +141,12 @@ export default defineComponent({ } .grid-number { - max-width: 62px; - font-size: 13px; + max-width: 66px; + font-size: 14px; } .padding-slider { - min-width: 110px; - max-width: 180px; + min-width: 170px; + max-width: 260px; } diff --git a/client/dive-common/components/Review/ReviewPage.vue b/client/dive-common/components/Review/ReviewPage.vue index a19bfafda..dd75642bc 100644 --- a/client/dive-common/components/Review/ReviewPage.vue +++ b/client/dive-common/components/Review/ReviewPage.vue @@ -7,15 +7,21 @@ import { useApi } from 'dive-common/apispec'; import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; import { createReviewService, provideReview } from 'dive-common/use/useReview'; import { useReviewGrid } from 'dive-common/review/useReviewGrid'; +import { cellScaleFor } from 'dive-common/review/gridSettings'; import { ReviewItem, ReviewSortOrder } from 'dive-common/review/types'; import type { ViewerFocus } from 'dive-common/review/viewerNavigation'; import ReviewDatasetsPanel from './ReviewDatasetsPanel.vue'; import ReviewGrid from './ReviewGrid.vue'; import ReviewGridControls from './ReviewGridControls.vue'; -import ReviewCell from './ReviewCell.vue'; +import ReviewCell, { ReviewCellGeometryEdit } from './ReviewCell.vue'; const TYPE_LIST_ID = 'reviewTypeOptions'; +/** Base footer height of a cell at scale 1 (type field plus caption). */ +const CELL_FOOTER_BASE_PX = 52; + +type ReviewView = 'results' | 'datasets'; + const SORT_OPTIONS: { value: ReviewSortOrder; text: string }[] = [ { value: 'dataset', text: 'Dataset, track id' }, { value: 'confidence-asc', text: 'Confidence, low first' }, @@ -52,14 +58,17 @@ export default defineComponent({ provideReview(review); const { prompt } = usePrompt(); - const view = ref<'datasets' | 'grid'>('datasets'); + const view = ref('results'); const pageTypeInput = ref(''); - const gridActive = computed(() => view.value === 'grid'); + const gridActive = computed(() => view.value === 'results'); + const cellScale = computed(() => cellScaleFor(review.grid.columns, review.grid.rows)); + const footerPx = computed(() => Math.round(CELL_FOOTER_BASE_PX * cellScale.value)); const grid = useReviewGrid({ items: review.items, grid: review.grid, chipStore: review.chipStore, active: gridActive, + footerPx, }); const showDatasetNames = computed(() => review.datasets.value.length > 1); @@ -87,6 +96,7 @@ export default defineComponent({ title: `${review.datasetName(item.datasetId)} · track ${item.trackId} · frame ${item.primary.frame}`, subtitle: subtitleBits.join(' · '), attributeText, + frames: item.frames, }; }); }); @@ -104,21 +114,32 @@ export default defineComponent({ function show() { review.runQuery(); - view.value = 'grid'; + view.value = 'results'; } - function setView(next: 'datasets' | 'grid') { - if (next === 'grid' && review.stale.value) { + function setView(next: ReviewView) { + if (next === 'results' && review.stale.value) { review.runQuery(); } view.value = next; } - function openItem(item: ReviewItem) { - const focus: ViewerFocus = { frame: item.primary.frame, trackId: item.trackId }; + /** Open the viewer on the frame the cell is showing (its first frame otherwise). */ + function openItem(item: ReviewItem, frame?: number) { + const focus: ViewerFocus = { frame: frame ?? item.primary.frame, trackId: item.trackId }; emit('open-viewer', item.datasetId, focus); } + function applyGeometry(item: ReviewItem, edit: ReviewCellGeometryEdit) { + review.updateGeometry(item, edit.frame, { + bounds: edit.bounds, + polygons: edit.polygons, + head: edit.head, + tail: edit.tail, + }); + grid.ensureVisible(); + } + function openDataset(datasetId: string) { emit('open-viewer', datasetId, {}); } @@ -204,6 +225,7 @@ export default defineComponent({ view, grid, cells, + cellScale, typeItems, typeListId: TYPE_LIST_ID, sortOptions: SORT_OPTIONS, @@ -215,6 +237,7 @@ export default defineComponent({ setView, openItem, openDataset, + applyGeometry, applyTypeToPage, acceptPage, discard, @@ -235,33 +258,32 @@ export default defineComponent({ > - mdi-database + mdi-view-grid - Datasets - ({{ review.datasets.value.length }}) + Results - mdi-view-grid + mdi-database - Grid + Datasets + ({{ review.datasets.value.length }}) - diff --git a/client/dive-common/review/reviewItems.spec.ts b/client/dive-common/review/reviewItems.spec.ts index f61801126..fb80296e8 100644 --- a/client/dive-common/review/reviewItems.spec.ts +++ b/client/dive-common/review/reviewItems.spec.ts @@ -6,6 +6,8 @@ import { collectTypes, cycleIntervalFor, frameGeometry, + groupReviewItems, + interpolateBounds, matchTypePair, sampleFrames, sortReviewItems, @@ -213,3 +215,42 @@ describe('cycleIntervalFor', () => { expect(cycleIntervalFor([...one, { frame: 6, bounds: [0, 0, 1, 1] }], 0, 400)).toBe(400); }); }); + +describe('interpolateBounds', () => { + it('holds the nearest box at the ends and interpolates between keyframes', () => { + const t = track(1, [['fish', 1]], [0, 10]); + expect(interpolateBounds(t, 5)).toEqual([5, 0, 15, 10]); + expect(interpolateBounds(t, -3)).toEqual([0, 0, 10, 10]); + expect(interpolateBounds(t, 20)).toEqual([10, 0, 20, 10]); + expect(interpolateBounds(t, 10)).toEqual([10, 0, 20, 10]); + expect(interpolateBounds({ ...t, features: [] }, 3)).toBeNull(); + }); +}); + +describe('groupReviewItems', () => { + it('joins a track across the cameras of a rig with aligned frames and box-less gaps', () => { + const left = track(7, [['fish', 1]], [0, 4]); + const right = track(7, [['fish', 1]], [4, 8]); + const query = { ...DEFAULT_REVIEW_QUERY, threshold: 0 }; + const items = [ + ...buildReviewItems('rig/right', [right], query, 8), + ...buildReviewItems('rig/left', [left], query, 8), + ...buildReviewItems('solo', [track(1, [['fish', 1]], [2])], query, 8), + ]; + const membership = (id: string) => (id.startsWith('rig/') + ? { parent: 'rig', camera: id.slice(4), rank: id === 'rig/left' ? 0 : 1 } + : undefined); + const tracks: Record = { 'rig/left': left, 'rig/right': right }; + const entries = groupReviewItems(items, membership, (item) => tracks[item.datasetId], 8); + + expect(entries.map((e) => e.key)).toEqual(['rig#7', 'solo#1']); + const rig = entries[0]; + expect(rig.labels).toEqual(['left', 'right']); + expect(rig.items.map((i) => i.datasetId)).toEqual(['rig/left', 'rig/right']); + // Both cameras show frames 0, 4 and 8; the sides without a detection are interpolated. + expect(rig.items[0].frames.map((f) => [f.frame, f.missing ?? false])).toEqual([[0, false], [4, false], [8, true]]); + expect(rig.items[1].frames.map((f) => [f.frame, f.missing ?? false])).toEqual([[0, true], [4, false], [8, false]]); + expect(rig.items[0].frames[2].bounds).toEqual([4, 0, 14, 10]); + expect(entries[1].labels).toEqual(['']); + }); +}); diff --git a/client/dive-common/review/reviewItems.ts b/client/dive-common/review/reviewItems.ts index 534d8acb2..6574bbcec 100644 --- a/client/dive-common/review/reviewItems.ts +++ b/client/dive-common/review/reviewItems.ts @@ -6,8 +6,9 @@ import type { TrackData, Feature } from 'vue-media-annotator/track'; import type { StringKeyObject } from 'vue-media-annotator/BaseAnnotation'; import type { Attribute } from 'vue-media-annotator/use/AttributeTypes'; import { compareTypeNames } from 'dive-common/typeHierarchy'; +import type { RectBounds } from 'vue-media-annotator/utils'; import type { - ReviewFrameGeometry, ReviewFrameRef, ReviewItem, ReviewPolygon, ReviewQuery, ReviewSortOrder, + ReviewEntry, ReviewFrameGeometry, ReviewFrameRef, ReviewItem, ReviewPolygon, ReviewQuery, ReviewSortOrder, } from './types'; function isPoint(value: unknown): value is [number, number] { @@ -247,6 +248,116 @@ export function sortReviewItems( return sorted; } +/** Which multicamera parent and camera a dataset id belongs to, if any. */ +export interface CameraMembership { + parent: string; + camera: string; + /** Position of the camera in the rig's display order. */ + rank: number; +} + +/** + * Box a track would have on `frame` in one camera, interpolated between + * its nearest keyframes with boxes (held at the ends), or null when the + * track has no boxes there at all. + */ +export function interpolateBounds(track: TrackData, frame: number): RectBounds | null { + const features = boxedFeatures(track); + if (features.length === 0) return null; + const exact = features.find((f) => f.frame === frame); + if (exact?.bounds) return exact.bounds; + const before = [...features].reverse().find((f) => f.frame < frame); + const after = features.find((f) => f.frame > frame); + if (before?.bounds && after?.bounds) { + const t = (frame - before.frame) / (after.frame - before.frame); + return before.bounds.map((v, i) => v + ((after.bounds as RectBounds)[i] - v) * t) as RectBounds; + } + return (before?.bounds ?? after?.bounds) ?? null; +} + +/** + * Give the items of a multicamera entry the same frames: the union of their + * keyframes sampled once, with a camera that lacks a detection on a frame + * getting an interpolated, box-less reference there. + */ +export function alignCameraFrames( + items: ReviewItem[], + trackOf: (item: ReviewItem) => TrackData | undefined, + maxSequenceFrames: number, +): ReviewItem[] { + if (items.length < 2) return items; + const byFrame = new Map(); + items.forEach((item) => { + const track = trackOf(item); + if (!track) return; + boxedFeatures(track).forEach((feature) => { + if (!byFrame.has(feature.frame)) byFrame.set(feature.frame, feature); + }); + }); + const union = Array.from(byFrame.values()).sort((a, b) => a.frame - b.frame); + // Detection matches keep their own frame first so the query hit stays visible. + const sampled = sampleFrames(union, maxSequenceFrames).map((ref) => ref.frame); + const anchor = items.find((item) => item.key.includes('@'))?.primary.frame; + if (anchor !== undefined && !sampled.includes(anchor)) sampled.unshift(anchor); + + return items.map((item) => { + const track = trackOf(item); + if (!track) return item; + const frames: ReviewFrameRef[] = []; + sampled.forEach((frame) => { + const feature = track.features.find((f) => f.frame === frame && f.bounds); + if (feature) { + const ref = frameRefFor(feature); + if (ref) frames.push(ref); + return; + } + const bounds = interpolateBounds(track, frame); + if (bounds) frames.push({ frame, bounds, missing: true }); + }); + if (frames.length === 0) return item; + return { + ...item, primary: frames[0], frames, keyframeCount: boxedFeatures(track).length, + }; + }); +} + +/** + * Group items into grid entries: one per track, holding an item per camera + * the track appears in (in rig order) for multicamera datasets. + */ +export function groupReviewItems( + items: readonly ReviewItem[], + membershipOf: (datasetId: string) => CameraMembership | undefined, + trackOf: (item: ReviewItem) => TrackData | undefined, + maxSequenceFrames: number, +): ReviewEntry[] { + const entries: ReviewEntry[] = []; + const byKey = new Map(); + items.forEach((item) => { + const membership = membershipOf(item.datasetId); + if (!membership) { + entries.push({ key: item.key, items: [item], labels: [''] }); + return; + } + const key = item.key.replace(item.datasetId, membership.parent); + let group = byKey.get(key); + if (!group) { + group = { items: [], ranks: [], labels: [] }; + byKey.set(key, group); + entries.push({ key, items: group.items, labels: group.labels }); + } + // Keep cameras in rig order whatever order the items arrived in. + let at = group.ranks.findIndex((rank) => rank > membership.rank); + if (at < 0) at = group.ranks.length; + group.items.splice(at, 0, item); + group.ranks.splice(at, 0, membership.rank); + group.labels.splice(at, 0, membership.camera); + }); + return entries.map((entry) => (entry.items.length > 1 + ? { ...entry, items: alignCameraFrames(entry.items, trackOf, maxSequenceFrames) } + : entry)); +} + /** Every type named by any confidence pair, in type order. */ export function collectTypes(tracks: Iterable): string[] { const types = new Set(); diff --git a/client/dive-common/review/types.ts b/client/dive-common/review/types.ts index 8e23d1bff..1f5a4442e 100644 --- a/client/dive-common/review/types.ts +++ b/client/dive-common/review/types.ts @@ -41,6 +41,12 @@ export interface ReviewFrameGeometry { export interface ReviewFrameRef extends ReviewFrameGeometry { frame: number; bounds: RectBounds; + /** + * The track has no detection on this frame in this camera; `bounds` is + * interpolated from its neighbours so the chip can still be cropped there, + * and no box is drawn. + */ + missing?: boolean; } /** @@ -73,6 +79,18 @@ export interface ReviewItem { }; } +/** + * One grid entry: a track shown once per camera it appears in. Single + * camera datasets have one item per entry; the cameras of a multicamera + * dataset share the entry, with their items' frames aligned. + */ +export interface ReviewEntry { + key: string; + items: ReviewItem[]; + /** Camera name per item; empty strings for single-camera entries. */ + labels: string[]; +} + /** Grid presentation settings, persisted per browser. */ export interface ReviewGridSettings { columns: number; diff --git a/client/dive-common/review/useReviewGrid.ts b/client/dive-common/review/useReviewGrid.ts index f598fc487..a69105af0 100644 --- a/client/dive-common/review/useReviewGrid.ts +++ b/client/dive-common/review/useReviewGrid.ts @@ -26,8 +26,11 @@ export function chipAspectFor(cellWidth: number, cellHeight: number, footerPx = return Math.round(ratio * 10) / 10; } -export interface ReviewGridOptions { - items: Ref; +export interface ReviewGridOptions { + /** What the grid pages over: review items, or entries holding several. */ + items: Ref; + /** Items whose chips an entry needs; defaults to the entry being an item. */ + chipItemsOf?: (entry: T) => readonly ReviewItem[]; /** Reactive grid settings (mutated in place by the setters below). */ grid: ReviewGridSettings; chipStore: ChipStore; @@ -42,10 +45,12 @@ export interface ReviewGridOptions { /** How long paging must be idle before chips load, so skipped pages never render. */ const PAGE_SETTLE_MS = 250; -export function useReviewGrid(options: ReviewGridOptions) { +export function useReviewGrid(options: ReviewGridOptions) { const { items, grid, chipStore, active, } = options; + const chipItemsOf = options.chipItemsOf ?? ((entry: T) => [entry as unknown as ReviewItem]); + const chipItems = (entries: readonly T[]) => entries.flatMap((entry) => chipItemsOf(entry)); const page = ref(0); const cellSize = ref({ width: 0, height: 0 }); @@ -56,8 +61,8 @@ export function useReviewGrid(options: ReviewGridOptions) { function ensureVisible() { if (!active.value) return; - const visible = pageItems.value; - const prefetch = nextPageItems.value; + const visible = chipItems(pageItems.value); + const prefetch = chipItems(nextPageItems.value); chipStore.trimQueues(new Set([...visible, ...prefetch].map((i) => i.key))); chipStore.ensurePrimary([...visible, ...prefetch]); chipStore.ensureSequences(visible); @@ -83,7 +88,7 @@ export function useReviewGrid(options: ReviewGridOptions) { const ensureVisibleSettled = debounce(ensureVisible, PAGE_SETTLE_MS); function onPageChanged() { if (!active.value) return; - chipStore.trimQueues(new Set(pageItems.value.map((i) => i.key))); + chipStore.trimQueues(new Set(chipItems(pageItems.value).map((i) => i.key))); ensureVisibleSettled(); } diff --git a/client/dive-common/use/useReview.spec.ts b/client/dive-common/use/useReview.spec.ts index da351fdf4..bc8f0d526 100644 --- a/client/dive-common/use/useReview.spec.ts +++ b/client/dive-common/use/useReview.spec.ts @@ -159,6 +159,49 @@ describe('createReviewService', () => { expect(feature?.geometry?.features.map((g) => (g.properties as { key: string }).key)).toEqual(['head']); }); + it('groups a track across cameras, adds boxes where a side is missing, and deletes tracks', async () => { + const api = makeApi({ 'm/left': [track(3, [['fish', 1]], [0, 4])], 'm/right': [track(3, [['fish', 1]], [4])] }, { + loadConfig: vi.fn(async (id: string) => (id === 'm' + ? config(id, { + type: 'multi', + multiCamMedia: { + defaultDisplay: 'left', + cameras: { + left: { type: 'image-sequence', imageData: [], videoUrl: '' }, + right: { type: 'image-sequence', imageData: [], videoUrl: '' }, + }, + }, + }) + : config(id))), + }); + const service = createReviewService({ api }); + await service.addDataset('m', { id: 'm', name: 'Rig' }); + service.query.threshold = 0; + service.runQuery(); + + expect(service.entries.value).toHaveLength(1); + const [entry] = service.entries.value; + expect(entry.labels).toEqual(['left', 'right']); + expect(service.parentOf('m/right')).toBe('m'); + const right = entry.items[1]; + expect(right.frames.map((f) => f.missing ?? false)).toEqual([true, false]); + + service.addKeyframe(right, 0, right.frames[0].bounds); + expect(service.trackOf('m/right', 3)?.features.map((f) => f.frame)).toEqual([0, 4]); + expect(service.trackOf('m/right', 3)?.begin).toBe(0); + expect(service.entries.value[0].items[1].frames.every((f) => !f.missing)).toBe(true); + expect(service.pendingCount.value).toBe(1); + + service.deleteTrack(entry.items[0]); + service.deleteTrack(entry.items[1]); + expect(service.entries.value).toHaveLength(0); + expect(service.pendingCount.value).toBe(2); + await service.save(); + const { calls } = (api.saveDetections as ReturnType).mock; + expect(calls.map(([id, args]) => [id, args.tracks.delete])).toEqual([['m/left', [3]], ['m/right', [3]]]); + expect(service.pendingCount.value).toBe(0); + }); + it('reports a failed save and keeps the edits pending', async () => { const api = makeApi({ a: [track(1, [['fish', 0.6]], [0])] }, { saveDetections: vi.fn(async () => { throw new Error('disk full'); }), diff --git a/client/dive-common/use/useReview.ts b/client/dive-common/use/useReview.ts index 917b37ab3..c3c9af8dc 100644 --- a/client/dive-common/use/useReview.ts +++ b/client/dive-common/use/useReview.ts @@ -19,11 +19,13 @@ import { import { createFrameSource, FrameSource } from 'dive-common/review/frameSource'; import { createChipStore, ChipStore } from 'dive-common/review/chipStore'; import { - buildReviewItems, collectAttributeKeys, collectTypes, frameRefFor, sortReviewItems, + buildReviewItems, CameraMembership, collectAttributeKeys, collectTypes, frameRefFor, groupReviewItems, + sortReviewItems, } from 'dive-common/review/reviewItems'; import { usePersistentGridSettings } from 'dive-common/review/gridSettings'; import { DEFAULT_REVIEW_QUERY, + ReviewEntry, ReviewGridSettings, ReviewItem, ReviewPolygon, @@ -69,6 +71,8 @@ export interface ReviewService { grid: ReviewGridSettings; sort: Ref; items: Readonly>; + /** Items grouped into grid entries (one per track across its cameras). */ + entries: Readonly>; /** Bumps whenever tracks load or change; computeds that read tracks depend on it. */ dataRevision: Readonly>; /** True once tracks or the query changed after the last run. */ @@ -94,6 +98,12 @@ export interface ReviewService { colorFor(type: string): string; /** Frames per second the dataset is annotated at, or 0 when unknown. */ datasetFps(id: string): number; + /** The multicamera parent a camera dataset was expanded from, or the id itself. */ + parentOf(id: string): string; + /** Add a keyframe with a box to a track, e.g. where one camera lacks a detection. */ + addKeyframe(item: ReviewItem, frame: number, bounds: RectBounds): void; + /** Remove the track behind an item; written on the next save. */ + deleteTrack(item: ReviewItem): void; isPending(item: ReviewItem): boolean; assignType(item: ReviewItem, type: string): void; acceptType(item: ReviewItem): void; @@ -111,6 +121,8 @@ interface LoadedDataset { hierarchy: TypeHierarchyIndex; frameSource: FrameSource | null; pending: Set; + /** Tracks removed here and not yet deleted on the platform. */ + deleted: Set; } type GeoFeature = NonNullable['features'][number]; @@ -205,6 +217,8 @@ export function createReviewService(deps: ReviewServiceDeps): ReviewService { let loadGeneration = 0; /** Type colours as the annotator assigns them, seeded from each dataset's custom styles. */ const styles = new StyleManager({ markChangesPending: () => undefined }); + /** Camera datasets expanded from a multicamera parent. */ + const memberships = new Map(); // Query changes apply as soon as they settle; the grid only reshuffles // for those, never for edits made in it. @@ -278,6 +292,7 @@ export function createReviewService(deps: ReviewServiceDeps): ReviewService { const cameras = Object.keys(config.multiCamMedia?.cameras || {}); const parentName = entry(id)?.name || config.name; datasets.value = datasets.value.filter((d) => d.id !== id); + cameras.forEach((camera, rank) => memberships.set(`${id}/${camera}`, { parent: id, camera, rank })); await Promise.all(cameras.map((camera) => addDataset(`${id}/${camera}`, { id: `${id}/${camera}`, name: `${parentName} (${camera})`, type: config.multiCamMedia?.cameras[camera]?.type, }))); @@ -298,6 +313,7 @@ export function createReviewService(deps: ReviewServiceDeps): ReviewService { hierarchy: compileHierarchy(config.typeHierarchy || {}), frameSource, pending: new Set(), + deleted: new Set(), }); patch(id, { status: 'ready', @@ -399,6 +415,20 @@ export function createReviewService(deps: ReviewServiceDeps): ReviewService { return loaded.get(datasetId)?.tracks.get(trackId); } + function parentOf(id: string) { + return memberships.get(id)?.parent ?? id; + } + + const entries = computed(() => { + dependOnData(); + return groupReviewItems( + items.value, + (datasetId) => memberships.get(datasetId), + (item) => trackOf(item.datasetId, item.trackId), + grid.maxSequenceFrames, + ); + }); + function currentType(item: ReviewItem) { const track = trackOf(item.datasetId, item.trackId); if (!track) return { type: item.type, confidence: item.confidence }; @@ -429,7 +459,7 @@ export function createReviewService(deps: ReviewServiceDeps): ReviewService { const pendingCount = computed(() => { dependOnData(); let count = 0; - loaded.forEach((dataset) => { count += dataset.pending.size; }); + loaded.forEach((dataset) => { count += dataset.pending.size + dataset.deleted.size; }); return count; }); @@ -463,6 +493,42 @@ export function createReviewService(deps: ReviewServiceDeps): ReviewService { updatePairs(item, (pairs, hierarchy) => acceptPairAsCorrect(hierarchy, pairs, current.type)); } + function deleteTrack(item: ReviewItem) { + const dataset = loaded.get(item.datasetId); + if (!dataset || !dataset.tracks.has(item.trackId)) return; + dataset.tracks.delete(item.trackId); + dataset.pending.delete(item.trackId); + dataset.deleted.add(item.trackId); + // The entry leaves the grid at once; everything else stays put. + items.value = items.value.filter( + (other) => !(other.datasetId === item.datasetId && other.trackId === item.trackId), + ); + dataRevision.value += 1; + } + + function addKeyframe(item: ReviewItem, frame: number, bounds: RectBounds) { + const dataset = loaded.get(item.datasetId); + const track = dataset?.tracks.get(item.trackId); + if (!dataset || !track || track.features.some((f) => f.frame === frame && f.bounds)) return; + const [x1, y1, x2, y2] = bounds; + const previous = [...track.features].reverse().find((f) => f.frame < frame); + const feature: Feature = { + frame, + keyframe: true, + interpolate: previous?.interpolate ?? false, + bounds: [ + Math.round(Math.min(x1, x2)), Math.round(Math.min(y1, y2)), + Math.round(Math.max(x1, x2)), Math.round(Math.max(y1, y2)), + ], + }; + track.features = [...track.features.filter((f) => f.frame !== frame), feature] + .sort((a, b) => a.frame - b.frame); + track.begin = Math.min(track.begin, frame); + track.end = Math.max(track.end, frame); + dataset.pending.add(item.trackId); + dataRevision.value += 1; + } + function updateGeometry(item: ReviewItem, frame: number, edit: ReviewGeometryEdit) { const dataset = loaded.get(item.datasetId); const track = dataset?.tracks.get(item.trackId); @@ -501,16 +567,18 @@ export function createReviewService(deps: ReviewServiceDeps): ReviewService { saving.value = true; error.value = null; try { - const targets = Array.from(loaded.entries()).filter(([, d]) => d.pending.size > 0); + const targets = Array.from(loaded.entries()) + .filter(([, d]) => d.pending.size > 0 || d.deleted.size > 0); const results = await Promise.allSettled(targets.map(async ([id, dataset]) => { const upsert = Array.from(dataset.pending) .map((trackId) => dataset.tracks.get(trackId)) .filter((t): t is TrackData => !!t); await api.saveDetections(id, { - tracks: { upsert, delete: [] }, + tracks: { upsert, delete: Array.from(dataset.deleted) }, groups: { upsert: [], delete: [] }, }); dataset.pending.clear(); + dataset.deleted.clear(); })); const failed = results.find((r) => r.status === 'rejected') as PromiseRejectedResult | undefined; if (failed) throw failed.reason; @@ -523,7 +591,8 @@ export function createReviewService(deps: ReviewServiceDeps): ReviewService { } async function discardChanges() { - const dirty = Array.from(loaded.entries()).filter(([, d]) => d.pending.size > 0).map(([id]) => id); + const dirty = Array.from(loaded.entries()) + .filter(([, d]) => d.pending.size > 0 || d.deleted.size > 0).map(([id]) => id); await Promise.all(dirty.map((id) => reloadDataset(id))); } @@ -546,6 +615,7 @@ export function createReviewService(deps: ReviewServiceDeps): ReviewService { grid, sort, items, + entries, dataRevision, stale, types, @@ -566,6 +636,9 @@ export function createReviewService(deps: ReviewServiceDeps): ReviewService { currentType, colorFor, datasetFps, + parentOf, + addKeyframe, + deleteTrack, isPending, assignType, acceptType, diff --git a/docs/Review.md b/docs/Review.md index 928b2c25b..0333c11a9 100644 --- a/docs/Review.md +++ b/docs/Review.md @@ -37,12 +37,16 @@ The **Results** panel opens by default and shows the annotations that match the * the dataset name (when more than one is loaded), track id, and frame or frame count; * any polygon outline and head/tail points the detection carries, drawn over the chip. -Hover an entry for three actions (they grow under the mouse): **mark correct** (sets the shown type's confidence to 1 and drops other candidate types), **edit geometry**, and **open in viewer**. Double clicking the image also opens the annotation viewer on that dataset, seeks to the frame the entry is showing, and selects the track. +Hover an entry for its actions (they grow under the mouse): **mark correct** (sets the shown type's confidence to 1 and drops other candidate types), **delete** (a red X; the annotation is removed on the next save), **edit geometry**, and **open in viewer**. Double clicking the image also opens the annotation viewer on that dataset, seeks to the frame the entry is showing, and selects the track. Tracks cycle through their sampled frames at the dataset's real-time rate (sparser samples wait proportionally longer, so a loop lasts about as long as the track does). The arrows in the filmstrip badge step through them by hand, which pauses the cycling on that frame until the play button resumes it; starting an edit pauses it too. The type field and caption grow a little as the grid shows fewer entries, so a 3 by 3 grid is comfortably readable while a dense grid stays compact. +### Stereo and multi-camera datasets + +Each camera of a multi-camera (or stereo) dataset is loaded as its own sequence, but a track that appears in several cameras is one entry, showing a chip per camera side by side with the camera named on it. The chips show the same frames on every side. Where one camera has no detection on a frame the track has elsewhere, that side is still cropped at a position interpolated from its own neighbouring boxes and shows **no box**; its **add box** action creates a detection there, at the interpolated position, ready to be adjusted. The type field applies to the track in every camera, and opening the viewer opens the whole rig. + ### Editing boxes, polygons and points in place Right click an entry (or use its edit action) to adjust the frame it is showing without opening the viewer. The cycling pauses on that frame, the box gains the same handles the annotator uses (in the type's colour, red while dragged), and any polygon vertices and head/tail points can be dragged too. The mouse wheel zooms into the chip about the cursor and dragging empty space pans it, as in the annotator; the zoom stays until you wheel back out. Right click again, or press **Enter** or **Apply**, to keep the change; **Esc** or **Cancel** drops it. The chip keeps its crop after an edit, with the box drawn over it at its new position. Edits are held with the type edits until you **Save**; when auto-save is enabled in the settings, review edits are saved after the same delay the annotator uses. From 78aef8688edb2e231c1bb179a513fb6cea856cba Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Wed, 9 Sep 2026 17:30:42 -0400 Subject: [PATCH 10/27] Review grid: cameras of an entry cycle in step, with frame controls kept clear of the camera label Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01W1C4QY6hxjHaUPJfWPfQuu --- .../components/Review/ReviewCell.vue | 74 ++++++++++++++++++- .../components/Review/ReviewChip.vue | 33 ++++++++- 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/client/dive-common/components/Review/ReviewCell.vue b/client/dive-common/components/Review/ReviewCell.vue index b785fabef..d2f41e3b7 100644 --- a/client/dive-common/components/Review/ReviewCell.vue +++ b/client/dive-common/components/Review/ReviewCell.vue @@ -1,6 +1,6 @@ + + + + diff --git a/client/dive-common/datasetPicker.spec.ts b/client/dive-common/datasetPicker.spec.ts new file mode 100644 index 000000000..9a7d03417 --- /dev/null +++ b/client/dive-common/datasetPicker.spec.ts @@ -0,0 +1,27 @@ +import { filterDatasetRows, selectableIds } from './datasetPicker'; + +const rows = [ + { + id: 'a', name: 'Amchitka East', type: 'image-sequence', fps: 5, + }, + { + id: 'b', name: 'Bering clip', type: 'video', fps: 30, + }, + { id: 'c', name: 'Caton rig', type: 'multi' }, +]; + +describe('filterDatasetRows', () => { + it('matches any listed field, ignoring case and surrounding space', () => { + expect(filterDatasetRows(rows, ' VIDEO ').map((r) => r.id)).toEqual(['b']); + expect(filterDatasetRows(rows, 'ri').map((r) => r.id)).toEqual(['b', 'c']); + expect(filterDatasetRows(rows, '30', ['fps']).map((r) => r.id)).toEqual(['b']); + expect(filterDatasetRows(rows, '')).toHaveLength(3); + }); +}); + +describe('selectableIds', () => { + it('leaves out what is already selected', () => { + expect(selectableIds(rows, ['b'])).toEqual(['a', 'c']); + expect(selectableIds(filterDatasetRows(rows, 'rig'), ['c'])).toEqual([]); + }); +}); diff --git a/client/dive-common/datasetPicker.ts b/client/dive-common/datasetPicker.ts new file mode 100644 index 000000000..a3b1bc558 --- /dev/null +++ b/client/dive-common/datasetPicker.ts @@ -0,0 +1,38 @@ +/** + * Rows and filtering behind the shared dataset picker, kept free of Vue so + * the search behaviour is testable and identical on every page. + */ + +/** A dataset offered for selection; extra fields feed extra table columns. */ +export interface DatasetPickerRow { + id: string; + name: string; + type?: string; + [column: string]: unknown; +} + +/** + * Rows whose listed fields contain the search text (case-insensitive, + * whitespace-trimmed). An empty search keeps everything. + */ +export function filterDatasetRows( + rows: readonly T[], + search: string, + fields: readonly string[] = ['name', 'type'], +): T[] { + const needle = search.trim().toLowerCase(); + if (!needle) return [...rows]; + return rows.filter((row) => fields.some((field) => { + const value = row[field]; + return value !== undefined && value !== null && String(value).toLowerCase().includes(needle); + })); +} + +/** Ids of the listed rows not yet selected: what "select all" adds. */ +export function selectableIds( + rows: readonly T[], + selectedIds: readonly string[], +): string[] { + const selected = new Set(selectedIds); + return rows.filter((row) => !selected.has(row.id)).map((row) => row.id); +} From 055661a0c7f86c7fdba73083a2689ea2d99e70c7 Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Wed, 9 Sep 2026 22:44:58 -0400 Subject: [PATCH 13/27] Use DatasetPicker on the Training and Pipelines pages Both pages offered their datasets through their own data table with a plus button (Pipelines also had its own search and select all); they now share the picker, which keeps the fps column and Training's view button through the picker's headers and row-actions slot. Training's staged items are typed as the cache rows they always were. The picker's search field is clearable, which yields null; filter on an empty string in that case. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01W1C4QY6hxjHaUPJfWPfQuu --- .../dive-common/components/DatasetPicker.vue | 12 ++- client/dive-common/datasetPicker.spec.ts | 1 + client/dive-common/datasetPicker.ts | 5 +- .../frontend/components/MultiPipeline.vue | 91 +++---------------- .../frontend/components/MultiTrainingMenu.vue | 82 ++++++----------- 5 files changed, 54 insertions(+), 137 deletions(-) diff --git a/client/dive-common/components/DatasetPicker.vue b/client/dive-common/components/DatasetPicker.vue index 5f86382ce..5d3d0a94a 100644 --- a/client/dive-common/components/DatasetPicker.vue +++ b/client/dive-common/components/DatasetPicker.vue @@ -66,10 +66,11 @@ export default defineComponent({ }, }, setup(props, { emit }) { - const search = ref(''); + /** Null when the field's clear button is used. */ + const search = ref(''); const searchFields = computed(() => props.headers.map((h) => h.value)); - const listed = computed(() => filterDatasetRows(props.items, search.value, searchFields.value)); + const listed = computed(() => filterDatasetRows(props.items, search.value ?? '', searchFields.value)); const addable = computed(() => selectableIds(listed.value, props.selectedIds)); const selected = computed(() => new Set(props.selectedIds)); @@ -80,6 +81,10 @@ export default defineComponent({ }, ]); + function rowClass(item: DatasetPickerRow) { + return selected.value.has(item.id) ? 'picker-row-selected' : ''; + } + function addAll() { if (addable.value.length) emit('add-many', addable.value); } @@ -90,6 +95,7 @@ export default defineComponent({ addable, selected, tableHeaders, + rowClass, addAll, clientSettings, itemsPerPageOptions, @@ -172,7 +178,7 @@ export default defineComponent({ :footer-props="{ itemsPerPageOptions }" :hide-default-footer="compact && listed.length <= clientSettings.rowsPerPage" :no-data-text="items.length ? 'Nothing matches the search.' : noDataText" - :item-class="(item) => (selected.has(item.id) ? 'picker-row-selected' : '')" + :item-class="rowClass" class="picker-table" > diff --git a/client/platform/desktop/frontend/components/MultiTrainingMenu.vue b/client/platform/desktop/frontend/components/MultiTrainingMenu.vue index cf38b2cac..62b194a55 100644 --- a/client/platform/desktop/frontend/components/MultiTrainingMenu.vue +++ b/client/platform/desktop/frontend/components/MultiTrainingMenu.vue @@ -12,11 +12,12 @@ import { watch, } from 'vue'; import { - DatasetConfig, Pipelines, TrainingConfigs, useApi, Pipe, + Pipelines, TrainingConfigs, useApi, Pipe, } from 'dive-common/apispec'; import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; import { itemsPerPageOptions, simplifyTrainingName } from 'dive-common/constants'; import { clientSettings } from 'dive-common/store/settings'; +import DatasetPicker from 'dive-common/components/DatasetPicker.vue'; import { useRoute, useRouter } from 'vue-router/composables'; import { DesktopJob, RunTraining } from 'platform/desktop/constants'; @@ -31,6 +32,7 @@ function joinPath(dir: string, filename: string) { } export default defineComponent({ + components: { DatasetPicker }, setup() { const { runTraining, getPipelineList, deleteTrainedPipeline, getTrainingConfigurations, exportTrainedPipeline, @@ -128,7 +130,7 @@ export default defineComponent({ ]; const data = reactive({ - stagedItems: {} as Record, + stagedItems: {} as Record, trainingOutputName: '', selectedTrainingConfig: 'foo.whatever', fineTuneTraining: false, @@ -186,7 +188,7 @@ export default defineComponent({ return []; }); - function toggleStaged(meta: DatasetConfig) { + function toggleStaged(meta: JsonConfigCache) { if (data.stagedItems[meta.id]) { del(data.stagedItems, meta.id); } else { @@ -194,16 +196,18 @@ export default defineComponent({ } } const availableItems = computed(() => Object.values(datasets.value) - .filter((item) => item.subType === null) - .map((item) => ({ - ...item, - included: item.id in data.stagedItems, - }))); + .filter((item) => item.subType === null)); const stagedItems = computed(() => Object.values(data.stagedItems)); - function getAvailableItemClass({ included }: JsonConfigCache & { included: boolean }) { - return included ? 'disabled-row' : ''; + const stagedIds = computed(() => Object.keys(data.stagedItems)); + + /** Stage the picked datasets that are not staged yet. */ + function stageIds(ids: string[]) { + ids.forEach((id) => { + const meta = datasets.value[id]; + if (meta && !data.stagedItems[id]) toggleStaged(meta); + }); } const isReadyToTrain = computed(() => ( @@ -343,7 +347,8 @@ export default defineComponent({ labelFile, clearLabelText, toggleStaged, - getAvailableItemClass, + stagedIds, + stageIds, deleteModel, exportModel, simplifyTrainingName, @@ -375,17 +380,7 @@ export default defineComponent({ }, available: { items: availableItems, - headers: headersTmpl.concat({ - text: 'View', - value: 'view', - sortable: false, - width: 80, - }, { - text: 'Include', - value: 'action', - sortable: false, - width: 80, - }), + headers: headersTmpl, }, staged: { items: stagedItems, @@ -582,37 +577,28 @@ export default defineComponent({ These datasets meet the requirements for the chosen training configuration. - - - - From 6d850ed12848ea9cda6c3c68946c031c7f8f9916 Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Wed, 9 Sep 2026 22:52:08 -0400 Subject: [PATCH 14/27] DatasetPicker: remove all, per-row remove, and hold the list in place Selected rows now show a clickable check that removes them, and a "Remove all (n)" button drops every listed selection, mirroring select all. Adding or removing keeps the picker at the same place on screen: the pages list the selection above it, so growing that list used to push the picker down under the user's pointer. Training and Pipelines section titles and descriptions lose their card padding so they line up with the tables and the picker under them. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01W1C4QY6hxjHaUPJfWPfQuu --- .../dive-common/components/DatasetPicker.vue | 107 ++++++++++++++++-- .../frontend/components/MultiPipeline.vue | 26 +++-- .../frontend/components/MultiTrainingMenu.vue | 25 ++-- 3 files changed, 134 insertions(+), 24 deletions(-) diff --git a/client/dive-common/components/DatasetPicker.vue b/client/dive-common/components/DatasetPicker.vue index 5d3d0a94a..1e3581711 100644 --- a/client/dive-common/components/DatasetPicker.vue +++ b/client/dive-common/components/DatasetPicker.vue @@ -1,6 +1,6 @@