diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index 16d42c657..e6164e192 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -471,9 +471,15 @@ interface Api { deleteScoringResult?(datasetId: string, resultId: string): Promise; /** Annotation sets, revisions or on-disk files a source on this dataset can point at. */ listScoringSources?(datasetId: string): Promise; - /** Datasets that may be named as the other side of a comparison. */ + /** + * Datasets that may be named as the other side of a comparison; also the + * dataset list the review page offers. + */ listScoringDatasets?(): Promise; - /** Open a platform dataset picker; returns null when the user cancels. */ + /** + * Open a platform dataset picker; returns null when the user cancels. + * Shared by the scoring and review pages. + */ pickScoringDataset?(excludeIds: string[]): Promise; /** Save a text export where the user chooses; resolves false when they cancel. */ saveScoringExport?(args: { filename: string; mime: string; content: string }): Promise; @@ -487,6 +493,12 @@ interface Api { ): Promise; loadConfig(datasetId: string): Promise; + /** + * loadConfig without the platform's viewer bookkeeping (desktop recents, + * web browse location), for pages that read many datasets at once such as + * Review. Callers fall back to loadConfig when absent. + */ + peekConfig?(datasetId: string): Promise; loadDetections(datasetId: string, revision?: number, set?: string): Promise; loadFrameMetadata(datasetId: string): Promise; diff --git a/client/dive-common/components/Review/ReviewCell.vue b/client/dive-common/components/Review/ReviewCell.vue new file mode 100644 index 000000000..17b0e2ac7 --- /dev/null +++ b/client/dive-common/components/Review/ReviewCell.vue @@ -0,0 +1,359 @@ + + + + + diff --git a/client/dive-common/components/Review/ReviewChip.vue b/client/dive-common/components/Review/ReviewChip.vue new file mode 100644 index 000000000..2c58a8462 --- /dev/null +++ b/client/dive-common/components/Review/ReviewChip.vue @@ -0,0 +1,1292 @@ + + + + + diff --git a/client/dive-common/components/Review/ReviewDatasetsPanel.vue b/client/dive-common/components/Review/ReviewDatasetsPanel.vue new file mode 100644 index 000000000..6e3fa67ee --- /dev/null +++ b/client/dive-common/components/Review/ReviewDatasetsPanel.vue @@ -0,0 +1,200 @@ + + + + + diff --git a/client/dive-common/components/Review/ReviewGrid.vue b/client/dive-common/components/Review/ReviewGrid.vue new file mode 100644 index 000000000..153b1fa27 --- /dev/null +++ b/client/dive-common/components/Review/ReviewGrid.vue @@ -0,0 +1,81 @@ + + + + + diff --git a/client/dive-common/components/Review/ReviewGridControls.vue b/client/dive-common/components/Review/ReviewGridControls.vue new file mode 100644 index 000000000..29d47de15 --- /dev/null +++ b/client/dive-common/components/Review/ReviewGridControls.vue @@ -0,0 +1,157 @@ + + + + + diff --git a/client/dive-common/components/Review/ReviewPage.spec.ts b/client/dive-common/components/Review/ReviewPage.spec.ts new file mode 100644 index 000000000..fd37f2fad --- /dev/null +++ b/client/dive-common/components/Review/ReviewPage.spec.ts @@ -0,0 +1,111 @@ +import { shallowMount } from '@vue/test-utils'; +import Vue, { ComponentOptions, CreateElement, nextTick } from 'vue'; +import type { NavigationGuard } from 'vue-router'; +import type { DatasetConfig } from 'dive-common/apispec'; +import type { ReviewService } from 'dive-common/use/useReview'; +import { takeReviewSession } from 'dive-common/review/reviewSession'; +import ReviewPage from './ReviewPage.vue'; + +const mocks = vi.hoisted(() => ({ + guard: null as NavigationGuard | null, + loadDetections: vi.fn(), + saveDetections: vi.fn(async () => undefined), +})); +vi.mock('vue-router/composables', () => ({ + onBeforeRouteLeave: (guard: NavigationGuard) => { mocks.guard = guard; }, +})); +vi.mock('dive-common/apispec', () => ({ + useApi: () => ({ + loadConfig: async (id: string) => ({ + id, + name: id, + type: 'image-sequence', + fps: 1, + imageData: [], + videoUrl: undefined, + createdAt: '', + subType: null, + multiCamMedia: null, + } as DatasetConfig), + loadDetections: mocks.loadDetections, + saveDetections: mocks.saveDetections, + }), +})); +vi.mock('dive-common/vue-utilities/prompt-service', () => ({ usePrompt: () => ({ prompt: vi.fn() }) })); +vi.mock('dive-common/store/settings', () => ({ + clientSettings: { autoSaveSettings: { enabled: false, delaySeconds: 60 } }, +})); +vi.mock('dive-common/review/frameSource', () => ({ createFrameSource: () => null })); +vi.mock('dive-common/components/UserSettingsDialog.vue', () => ({ default: {} })); +vi.mock('./ReviewDatasetsPanel.vue', () => ({ default: {} })); +vi.mock('./ReviewGrid.vue', () => ({ default: {} })); +vi.mock('./ReviewGridControls.vue', () => ({ default: {} })); +vi.mock('./ReviewCell.vue', () => ({ default: {} })); + +interface PageState { + review: ReviewService; + resolveLeave(choice: 'save' | 'discard' | 'cancel'): void; + leaveDialog: boolean; +} + +function mountPage() { + // Exercise the real setup/lifecycle and guard, without rendering Vuetify/media. + return shallowMount({ ...(ReviewPage as unknown as ComponentOptions), render: (h: CreateElement) => h('div') }); +} + +beforeEach(() => { + takeReviewSession()?.review.dispose(); + mocks.loadDetections.mockReset().mockResolvedValue({ + tracks: [{ + id: 1, + begin: 0, + end: 0, + confidencePairs: [['fish', 0.9]], + attributes: {}, + features: [{ frame: 0, keyframe: true, bounds: [0, 0, 10, 10] }], + }], + groups: [], + sets: [], + version: 2, + }); +}); + +it('stays mounted with pending edits when Discard and Leave fails', async () => { + const wrapper = mountPage(); + const page = wrapper.vm as unknown as PageState; + await page.review.addDataset('a'); + page.review.assignType(page.review.items.value[0], 'shark'); + mocks.loadDetections.mockRejectedValue(new Error('offline')); + const next = vi.fn(); + const navigation = mocks.guard!({} as never, {} as never, next); + expect(page.leaveDialog).toBe(true); + page.resolveLeave('discard'); + await navigation; + expect(next).toHaveBeenCalledWith(false); + expect(page.review.pendingCount.value).toBe(1); + expect(page.review.error.value).toContain('Could not discard'); + wrapper.destroy(); +}); + +it('refreshes data and guards edits after leaving and remounting Review', async () => { + const first = mountPage(); + const original = first.vm as unknown as PageState; + await original.review.addDataset('a'); + await mocks.guard!({} as never, {} as never, vi.fn()); + first.destroy(); + const second = mountPage(); + const resumed = second.vm as unknown as PageState; + await new Promise((resolve) => setTimeout(resolve, 0)); + await nextTick(); + expect(resumed.review).toBe(original.review); + expect(mocks.loadDetections).toHaveBeenCalledTimes(2); + resumed.review.assignType(resumed.review.items.value[0], 'shark'); + expect(resumed.review.pendingCount.value).toBe(1); + const next = vi.fn(); + const navigation = mocks.guard!({} as never, {} as never, next); + expect(resumed.leaveDialog).toBe(true); + resumed.resolveLeave('cancel'); + await navigation; + expect(next).toHaveBeenCalledWith(false); + second.destroy(); +}); diff --git a/client/dive-common/components/Review/ReviewPage.vue b/client/dive-common/components/Review/ReviewPage.vue new file mode 100644 index 000000000..bc9054f74 --- /dev/null +++ b/client/dive-common/components/Review/ReviewPage.vue @@ -0,0 +1,944 @@ + + + + + diff --git a/client/dive-common/components/Review/ReviewTypeField.vue b/client/dive-common/components/Review/ReviewTypeField.vue new file mode 100644 index 000000000..4cbe54290 --- /dev/null +++ b/client/dive-common/components/Review/ReviewTypeField.vue @@ -0,0 +1,309 @@ + + + + + diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index e2c3f1e8b..75b3710e4 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -175,6 +175,16 @@ export default defineComponent({ type: Boolean, default: false, }, + /** Deep link: frame to seek to once the media is ready (e.g. from the review grid). */ + initialFrame: { + type: Number as PropType, + default: undefined, + }, + /** Deep link: track to select once annotations are loaded. */ + initialTrackId: { + type: Number as PropType, + default: undefined, + }, }, setup(props, { emit }) { const { prompt, visible } = usePrompt(); @@ -2033,6 +2043,32 @@ export default defineComponent({ }; loadData(); + /** + * Apply a deep link (initialFrame / initialTrackId) once: after the + * annotations are loaded and the media controller reports its frame + * range, so the seek is not swallowed by the annotator's own init seek. + */ + let initialFocusApplied = false; + watch( + () => [progress.loaded, aggregateController.value.maxFrame.value] as const, + ([loaded, maxFrame]) => { + if (initialFocusApplied || !loaded) return; + if (props.initialFrame === undefined && props.initialTrackId === undefined) return; + if (props.initialFrame !== undefined && maxFrame <= 0) return; + initialFocusApplied = true; + nextTick(() => { + if (props.initialFrame !== undefined) { + handler.seekFrame(Math.min(props.initialFrame, maxFrame)); + } + if (props.initialTrackId !== undefined + && cameraStore.getAnyPossibleTrack(props.initialTrackId)) { + handler.trackSelect(props.initialTrackId, false); + } + }); + }, + { immediate: true }, + ); + const reloadAnnotations = async () => { progress.loaded = false; discardChanges(); diff --git a/client/dive-common/review/chipRenderer.spec.ts b/client/dive-common/review/chipRenderer.spec.ts new file mode 100644 index 000000000..1903cf9f2 --- /dev/null +++ b/client/dive-common/review/chipRenderer.spec.ts @@ -0,0 +1,66 @@ +import { + chipRegion, chipScale, chipSizeFor, toChipPoint, toImagePoint, +} from './chipRenderer'; + +describe('chipRegion', () => { + it('is a square centred on the box, padded on every side', () => { + const region = chipRegion([10, 20, 30, 60], 0.5); + // Longer side 40, padded by 50% each side -> 80. + expect(region).toEqual({ + x: 20 - 40, y: 40 - 40, width: 80, height: 80, + }); + }); + + it('extends the square to the requested aspect ratio, keeping the box centred', () => { + const wide = chipRegion([10, 20, 30, 60], 0, 2); + expect(wide).toEqual({ + x: 20 - 40, y: 40 - 20, width: 80, height: 40, + }); + const tall = chipRegion([10, 20, 30, 60], 0, 0.5); + expect(tall).toEqual({ + x: 20 - 20, y: 40 - 40, width: 40, height: 80, + }); + expect(chipRegion([0, 0, 10, 10], 0, Number.NaN).width).toBe(10); + }); + + it('tolerates inverted or degenerate boxes', () => { + expect(chipRegion([30, 60, 10, 20], 0).width).toBe(40); + expect(chipRegion([5, 5, 5, 5], 0).width).toBe(1); + expect(chipRegion([0, 0, 10, 10], -1).width).toBe(10); + }); +}); + +describe('chipSizeFor', () => { + it('rounds cell sizes up to a bucket and caps at the largest', () => { + expect(chipSizeFor(100)).toBe(128); + expect(chipSizeFor(128)).toBe(128); + expect(chipSizeFor(129)).toBe(192); + expect(chipSizeFor(5000)).toBe(768); + }); +}); + +describe('chipScale', () => { + it('fills the requested size, upscaling small crops and downscaling large ones', () => { + expect(chipScale({ + x: 0, y: 0, width: 32, height: 32, + }, 256)).toBe(8); + expect(chipScale({ + x: 0, y: 0, width: 1024, height: 512, + }, 256)).toBe(0.25); + }); +}); + +describe('chip point mapping', () => { + it('round-trips between image and chip coordinates', () => { + const transform = { + region: { + x: 100, y: 50, width: 40, height: 20, + }, + scale: 4, + width: 160, + height: 80, + }; + expect(toChipPoint(transform, 110, 60)).toEqual([40, 40]); + expect(toImagePoint(transform, 40, 40)).toEqual([110, 60]); + }); +}); diff --git a/client/dive-common/review/chipRenderer.ts b/client/dive-common/review/chipRenderer.ts new file mode 100644 index 000000000..33ed44207 --- /dev/null +++ b/client/dive-common/review/chipRenderer.ts @@ -0,0 +1,145 @@ +/** + * Crops one box out of a decoded frame into a chip image. The crop is a + * square centred on the box so the object stays centred as a track cycles + * through frames of different sizes; area past the frame edge is left dark + * rather than shifting the object off centre. + */ +import type { RectBounds } from 'vue-media-annotator/utils'; +import type { DecodedFrame } from './frameSource'; + +export interface ChipRenderOptions { + /** Context around the box as a fraction of its longer side. */ + padding: number; + /** Longer output edge in pixels. */ + size: number; + /** Output width / height; the crop region widens or heightens to match. */ + aspect?: number; + /** Box outline colour; omit to draw no outline. */ + outline?: string; + /** JPEG quality. */ + quality?: number; +} + +export interface ChipRegion { + x: number; + y: number; + width: number; + height: number; +} + +/** How a rendered chip maps to the frame: chip px = (image px - region.x) * scale. */ +export interface ChipTransform { + region: ChipRegion; + scale: number; + /** Chip image size in pixels. */ + width: number; + height: number; +} + +export interface RenderedChip { + dataUrl: string; + transform: ChipTransform; +} + +/** Chip pixel coordinates of an image point. */ +export function toChipPoint(transform: ChipTransform, x: number, y: number): [number, number] { + return [(x - transform.region.x) * transform.scale, (y - transform.region.y) * transform.scale]; +} + +/** Image coordinates of a chip pixel. */ +export function toImagePoint(transform: ChipTransform, x: number, y: number): [number, number] { + return [transform.region.x + x / transform.scale, transform.region.y + y / transform.scale]; +} + +/** + * Crop region (may extend past the image) for a box with padding: the + * padded square around the box, widened or heightened to the requested + * aspect ratio so the object stays centred whatever the cell's shape. + */ +export function chipRegion(bounds: RectBounds, padding: number, aspect = 1): ChipRegion { + const [x1, y1, x2, y2] = bounds; + const w = Math.max(1, Math.abs(x2 - x1)); + const h = Math.max(1, Math.abs(y2 - y1)); + const side = Math.max(w, h) * (1 + 2 * Math.max(0, padding)); + const ratio = aspect > 0 && Number.isFinite(aspect) ? aspect : 1; + const width = ratio >= 1 ? side * ratio : side; + const height = ratio >= 1 ? side : side / ratio; + const cx = (x1 + x2) / 2; + const cy = (y1 + y2) / 2; + return { + x: cx - width / 2, y: cy - height / 2, width, height, + }; +} + +/** Pixel sizes chips are rendered at; cells pick the smallest that covers them. */ +export const CHIP_SIZE_BUCKETS = [128, 192, 256, 384, 512, 768]; + +export function chipSizeFor(cellPixels: number): number { + const wanted = Math.ceil(cellPixels); + return CHIP_SIZE_BUCKETS.find((size) => size >= wanted) ?? CHIP_SIZE_BUCKETS[CHIP_SIZE_BUCKETS.length - 1]; +} + +/** + * Scale from frame pixels to chip pixels: the crop's longer side fills the + * requested size. Small crops are upscaled so the chip is rendered once at + * the cell's resolution with high-quality resampling, instead of the browser + * stretching a tiny image (and its compression artifacts) on every paint. + */ +export function chipScale(region: ChipRegion, size: number): number { + const longest = Math.max(1, Math.max(region.width, region.height)); + return Math.max(16, size) / longest; +} + +export function renderChip(frame: DecodedFrame, bounds: RectBounds, options: ChipRenderOptions): RenderedChip { + const region = chipRegion(bounds, options.padding, options.aspect); + const scale = chipScale(region, options.size); + const canvas = document.createElement('canvas'); + canvas.width = Math.max(1, Math.round(region.width * scale)); + canvas.height = Math.max(1, Math.round(region.height * scale)); + const ctx = canvas.getContext('2d'); + if (!ctx) throw new Error('Canvas unavailable'); + ctx.imageSmoothingEnabled = true; + ctx.imageSmoothingQuality = 'high'; + ctx.fillStyle = '#101010'; + ctx.fillRect(0, 0, canvas.width, canvas.height); + // Source rectangle clipped to the frame; the destination shifts by the same amount. + const sx = Math.max(0, region.x); + const sy = Math.max(0, region.y); + const ex = Math.min(frame.width, region.x + region.width); + const ey = Math.min(frame.height, region.y + region.height); + if (ex > sx && ey > sy) { + const sw = ex - sx; + const sh = ey - sy; + ctx.drawImage( + frame.source, + sx, + sy, + sw, + sh, + (sx - region.x) * scale, + (sy - region.y) * scale, + sw * scale, + sh * scale, + ); + } + if (options.outline) { + const [x1, y1, x2, y2] = bounds; + ctx.strokeStyle = options.outline; + ctx.lineWidth = Math.max(1, Math.round(Math.max(canvas.width, canvas.height) / 160)); + ctx.strokeRect( + (Math.min(x1, x2) - region.x) * scale, + (Math.min(y1, y2) - region.y) * scale, + Math.abs(x2 - x1) * scale, + Math.abs(y2 - y1) * scale, + ); + } + const transform: ChipTransform = { + region, scale, width: canvas.width, height: canvas.height, + }; + // Upscaled crops hold few source pixels; keep them lossless so the little + // detail there is does not pick up compression noise. + const dataUrl = scale > 1 + ? canvas.toDataURL('image/png') + : canvas.toDataURL('image/jpeg', options.quality ?? 0.92); + return { dataUrl, transform }; +} diff --git a/client/dive-common/review/chipStore.ts b/client/dive-common/review/chipStore.ts new file mode 100644 index 000000000..4ebe84933 --- /dev/null +++ b/client/dive-common/review/chipStore.ts @@ -0,0 +1,211 @@ +/** + * Reactive cache of rendered chips for review items, filled by a small + * priority queue: the first box of every requested item ("primary") is + * rendered before any of the extra frames a track cycles through + * ("sequence"), so a page paints as fast as possible and then animates. + * + * Loads run with limited concurrency: a video dataset decodes frames by + * seeking one hidden element, and image sequences would otherwise fire a + * whole page of requests at once. + */ +import { ref, set, del } from 'vue'; +import type { FrameSource } from './frameSource'; +import { renderChip, ChipTransform, RenderedChip } from './chipRenderer'; +import type { ReviewItem } from './types'; + +export interface ChipStoreOptions { + padding: number; + size: number; + /** Chip width / height, matching the cells the chips are shown in. */ + aspect: number; + outline: string; +} + +export interface ChipStoreDeps { + /** Frame access for a dataset; null when the dataset cannot be cropped. */ + frameSourceFor(datasetId: string): FrameSource | null; + concurrency?: number; +} + +interface ChipJob { + item: ReviewItem; + /** Sequence slot to fill, or null for the primary chip. */ + slot: number | null; + generation: number; +} + +export type ChipSequence = Array; +export type ChipTransformSequence = Array; + +export function createChipStore(deps: ChipStoreDeps, initial: ChipStoreOptions) { + const concurrency = deps.concurrency ?? 4; + const chips = ref>({}); + const sequences = ref>({}); + const failures = ref>({}); + /** How each rendered chip maps to its frame, for overlays and editing. */ + const transforms = ref>({}); + const sequenceTransforms = ref>({}); + let options: ChipStoreOptions = { ...initial }; + let generation = 0; + let active = 0; + const primaryQueue: ChipJob[] = []; + const sequenceQueue: ChipJob[] = []; + /** Item keys with a primary job queued or running, and its generation. */ + const pendingPrimary = new Map(); + const sequenceQueued = new Map(); + + function reset() { + generation += 1; + chips.value = {}; + sequences.value = {}; + failures.value = {}; + transforms.value = {}; + sequenceTransforms.value = {}; + primaryQueue.length = 0; + sequenceQueue.length = 0; + pendingPrimary.clear(); + sequenceQueued.clear(); + } + + /** Re-render everything when the crop or resolution changes. */ + function setOptions(next: ChipStoreOptions) { + if (next.padding === options.padding && next.size === options.size + && next.aspect === options.aspect && next.outline === options.outline) { + return; + } + options = { ...next }; + reset(); + } + + async function render(job: ChipJob): Promise { + const source = deps.frameSourceFor(job.item.datasetId); + if (!source) throw new Error('Media for this dataset cannot be cropped'); + const frameRef = job.slot === null ? job.item.primary : job.item.frames[job.slot]; + const frame = await source.getFrame(frameRef.frame); + return renderChip(frame, frameRef.bounds, options); + } + + function complete(job: ChipJob, rendered: RenderedChip | null, error?: unknown) { + if (job.generation !== generation) return; + const { key } = job.item; + if (job.slot === null) { + if (rendered) { + set(chips.value, key, rendered.dataUrl); + set(transforms.value, key, rendered.transform); + } else { + set(failures.value, key, error instanceof Error ? error.message : 'Could not render this chip'); + } + } else if (rendered) { + const slots = sequences.value[key]; + if (slots) set(slots, job.slot, rendered.dataUrl); + const slotTransforms = sequenceTransforms.value[key]; + if (slotTransforms) set(slotTransforms, job.slot, rendered.transform); + } + } + + function finish(job: ChipJob) { + active -= 1; + if (job.slot === null && pendingPrimary.get(job.item.key) === job.generation) { + pendingPrimary.delete(job.item.key); + } + pump(); + } + + function start(job: ChipJob) { + active += 1; + render(job) + .then((rendered) => complete(job, rendered)) + .catch((err) => complete(job, null, err)) + .finally(() => finish(job)); + } + + function pump() { + while (active < concurrency) { + const job = primaryQueue.shift() ?? sequenceQueue.shift(); + if (!job) return; + start(job); + } + } + + /** Queue the first box of each item not already rendered or queued. */ + function ensurePrimary(items: readonly ReviewItem[]) { + items.forEach((item) => { + if (chips.value[item.key] || failures.value[item.key]) return; + if (pendingPrimary.get(item.key) === generation) return; + pendingPrimary.set(item.key, generation); + primaryQueue.push({ item, slot: null, generation }); + }); + pump(); + } + + /** + * Queue the cycling frames of track items. Call with just the visible + * page: every frame of a video dataset costs a seek. + */ + function ensureSequences(items: readonly ReviewItem[]) { + items.forEach((item) => { + if (item.frames.length < 2) return; + if (sequenceQueued.get(item.key) === generation) return; + sequenceQueued.set(item.key, generation); + // Fixed-size, null-filled so cells can show frames as they arrive. + set(sequences.value, item.key, item.frames.map((): string | null => null)); + set(sequenceTransforms.value, item.key, item.frames.map((): ChipTransform | null => null)); + item.frames.forEach((_, slot) => { + sequenceQueue.push({ item, slot, generation }); + }); + }); + pump(); + } + + /** Drop queued work that is no longer visible (rendered chips stay cached). */ + function trimQueues(visibleKeys: ReadonlySet) { + const keep = (job: ChipJob) => visibleKeys.has(job.item.key); + const droppedPrimary = primaryQueue.filter((job) => !keep(job)); + droppedPrimary.forEach((job) => { + if (pendingPrimary.get(job.item.key) === job.generation) pendingPrimary.delete(job.item.key); + }); + primaryQueue.splice(0, primaryQueue.length, ...primaryQueue.filter(keep)); + const droppedSequence = sequenceQueue.filter((job) => !keep(job)); + droppedSequence.forEach((job) => { + sequenceQueued.delete(job.item.key); + if (job.generation === generation) { + del(sequences.value, job.item.key); + del(sequenceTransforms.value, job.item.key); + } + }); + sequenceQueue.splice(0, sequenceQueue.length, ...sequenceQueue.filter(keep)); + } + + /** + * Forget an item's chips (its boxes changed) so the next ensure call + * renders them again. Queued work for the item is dropped too. + */ + function invalidate(key: string) { + del(chips.value, key); + del(failures.value, key); + del(transforms.value, key); + del(sequences.value, key); + del(sequenceTransforms.value, key); + pendingPrimary.delete(key); + sequenceQueued.delete(key); + primaryQueue.splice(0, primaryQueue.length, ...primaryQueue.filter((job) => job.item.key !== key)); + sequenceQueue.splice(0, sequenceQueue.length, ...sequenceQueue.filter((job) => job.item.key !== key)); + } + + return { + chips, + sequences, + failures, + transforms, + sequenceTransforms, + setOptions, + ensurePrimary, + ensureSequences, + trimQueues, + invalidate, + reset, + get options() { return options; }, + }; +} + +export type ChipStore = ReturnType; diff --git a/client/dive-common/review/frameSource.ts b/client/dive-common/review/frameSource.ts new file mode 100644 index 000000000..dd843b134 --- /dev/null +++ b/client/dive-common/review/frameSource.ts @@ -0,0 +1,234 @@ +/** + * Per-dataset access to decoded frames for chip cropping. Image sequences + * load their frame images directly; videos are decoded by a hidden + * HTMLVideoElement seeking to each requested frame. + */ +import type { DatasetConfig } from 'dive-common/apispec'; +import { frameToVideoTime } from 'vue-media-annotator/components/annotators/videoSeek'; + +/** Anything drawImage accepts, with its pixel size. */ +export interface DecodedFrame { + source: CanvasImageSource; + width: number; + height: number; +} + +export interface FrameSource { + /** Frames the dataset has; null when unknown (video before metadata loads). */ + frameCount: number | null; + getFrame(frame: number): Promise; + dispose(): void; +} + +export interface FrameSourceOptions { + /** Decoded frames kept per dataset. */ + cacheSize?: number; +} + +const DefaultCacheSize = 24; + +class FrameCache { + private entries = new Map(); + + private readonly limit: number; + + constructor(limit: number) { + this.limit = limit; + } + + get(frame: number): DecodedFrame | undefined { + const hit = this.entries.get(frame); + if (hit) { + // Re-insert so the map keeps least-recently-used order. + this.entries.delete(frame); + this.entries.set(frame, hit); + } + return hit; + } + + set(frame: number, decoded: DecodedFrame) { + this.entries.delete(frame); + this.entries.set(frame, decoded); + while (this.entries.size > this.limit) { + const oldest = this.entries.keys().next().value; + if (oldest === undefined) break; + this.entries.delete(oldest); + } + } + + clear() { + this.entries.clear(); + } +} + +export function loadImage(url: string): Promise { + return new Promise((resolve, reject) => { + const image = new Image(); + // Needed so the crop canvas is not tainted; both platforms serve media with CORS headers. + image.crossOrigin = 'anonymous'; + image.onload = () => resolve(image); + image.onerror = () => reject(new Error(`Could not load ${url}`)); + image.src = url; + }); +} + +/** Dedupes concurrent requests for the same frame in front of a loader. */ +function withCache( + cache: FrameCache, + load: (frame: number) => Promise, +): (frame: number) => Promise { + const inflight = new Map>(); + return (frame: number) => { + const cached = cache.get(frame); + if (cached) return Promise.resolve(cached); + const pending = inflight.get(frame); + if (pending) return pending; + const promise = load(frame).then((decoded) => { + cache.set(frame, decoded); + return decoded; + }).finally(() => inflight.delete(frame)); + inflight.set(frame, promise); + return promise; + }; +} + +function imageFrameSource(urlFor: (frame: number) => Promise, frameCount: number | null, cacheSize: number): FrameSource { + const cache = new FrameCache(cacheSize); + const getFrame = withCache(cache, async (frame) => { + const image = await loadImage(await urlFor(frame)); + return { source: image, width: image.naturalWidth, height: image.naturalHeight }; + }); + return { + frameCount, + getFrame, + dispose: () => cache.clear(), + }; +} + +const SeekTimeoutMs = 15000; + +/** + * One hidden video element per dataset; seeks are serialized because an + * element can only sit on one frame at a time. Each decoded frame is copied + * to its own canvas so the cache survives the next seek. + */ +function videoFrameSource(config: DatasetConfig, cacheSize: number): FrameSource { + const cache = new FrameCache(cacheSize); + let video: HTMLVideoElement | null = null; + let metadata: Promise | null = null; + let queue: Promise = Promise.resolve(); + let disposed = false; + + function element(): Promise { + if (metadata) return metadata; + metadata = new Promise((resolve, reject) => { + const el = document.createElement('video'); + el.crossOrigin = 'anonymous'; + el.muted = true; + el.preload = 'auto'; + el.playsInline = true; + el.onloadedmetadata = () => resolve(el); + el.onerror = () => reject(new Error(`Could not open video for ${config.name}`)); + el.src = config.videoUrl || ''; + video = el; + }); + return metadata; + } + + function seekTo(el: HTMLVideoElement, time: number): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const timer = window.setTimeout(() => { + if (settled) return; + settled = true; + cleanup(); + reject(new Error('Timed out seeking the video')); + }, SeekTimeoutMs); + function cleanup() { + el.removeEventListener('seeked', onSeeked); + el.removeEventListener('error', onError); + window.clearTimeout(timer); + } + function onSeeked() { + if (settled) return; + settled = true; + cleanup(); + resolve(); + } + function onError() { + if (settled) return; + settled = true; + cleanup(); + reject(new Error('The video failed while seeking')); + } + el.addEventListener('seeked', onSeeked); + el.addEventListener('error', onError); + // Seeking to the time the element already sits at fires no event. + if (Math.abs(el.currentTime - time) < 1e-6 && el.readyState >= 2) { + onSeeked(); + return; + } + // eslint-disable-next-line no-param-reassign + el.currentTime = time; + }); + } + + const getFrame = withCache(cache, (frame) => { + const run = queue.then(async () => { + if (disposed) throw new Error('Frame source disposed'); + const el = await element(); + await seekTo(el, frameToVideoTime(frame, config.fps, config.originalFps ?? null)); + const canvas = document.createElement('canvas'); + canvas.width = el.videoWidth; + canvas.height = el.videoHeight; + const ctx = canvas.getContext('2d'); + if (!ctx) throw new Error('Canvas unavailable'); + ctx.drawImage(el, 0, 0); + return { source: canvas, width: canvas.width, height: canvas.height } as DecodedFrame; + }); + // Failures must not wedge the queue for later frames. + queue = run.catch(() => undefined); + return run; + }); + + return { + frameCount: null, + getFrame, + dispose: () => { + disposed = true; + cache.clear(); + if (video) { + video.removeAttribute('src'); + video.load(); + video = null; + } + metadata = null; + }, + }; +} + +/** + * Pick the loader for a dataset, or throw for media the review grid cannot + * crop (tiled large images, multicamera parents). + */ +export function createFrameSource(config: DatasetConfig, options: FrameSourceOptions = {}): FrameSource { + const cacheSize = options.cacheSize ?? DefaultCacheSize; + if (config.type === 'large-image') { + throw new Error('Tiled large-image datasets cannot be reviewed as chips yet'); + } + if (config.type === 'multi') { + throw new Error('Review the cameras of a multicamera dataset individually'); + } + if (config.type === 'video') { + if (config.videoUrl) { + return videoFrameSource(config, cacheSize); + } + throw new Error('This video has no playable media'); + } + const { imageData } = config; + return imageFrameSource(async (frame) => { + const entry = imageData[frame]; + if (!entry) throw new Error(`No image for frame ${frame}`); + return entry.url; + }, imageData.length, cacheSize); +} diff --git a/client/dive-common/review/gridSettings.spec.ts b/client/dive-common/review/gridSettings.spec.ts new file mode 100644 index 000000000..be1a0abb3 --- /dev/null +++ b/client/dive-common/review/gridSettings.spec.ts @@ -0,0 +1,33 @@ +import { cellScaleFor, clampGrid } from './gridSettings'; +import { DEFAULT_REVIEW_GRID } from './types'; + +describe('clampGrid', () => { + it('rounds and clamps columns and rows and clamps padding', () => { + expect(clampGrid({ + ...DEFAULT_REVIEW_GRID, columns: 40, rows: 0.4, padding: -2, + })).toMatchObject({ + columns: 12, rows: 1, padding: 0, + }); + expect(clampGrid({ + ...DEFAULT_REVIEW_GRID, columns: 3.6, rows: 2.2, padding: 0.5, + })).toMatchObject({ + columns: 4, rows: 2, padding: 0.5, + }); + }); + + it('falls back to defaults for non-numbers', () => { + expect(clampGrid({ ...DEFAULT_REVIEW_GRID, columns: Number.NaN, rows: Infinity })).toMatchObject({ + columns: DEFAULT_REVIEW_GRID.columns, rows: DEFAULT_REVIEW_GRID.rows, + }); + }); +}); + +describe('cellScaleFor', () => { + it('grows footer text as the grid shows fewer cells, within limits', () => { + expect(cellScaleFor(5, 4)).toBeCloseTo(1.12, 2); + expect(cellScaleFor(3, 3)).toBe(1.35); + expect(cellScaleFor(1, 1)).toBe(1.35); + expect(cellScaleFor(8, 6)).toBe(1); + expect(cellScaleFor(12, 10)).toBe(1); + }); +}); diff --git a/client/dive-common/review/gridSettings.ts b/client/dive-common/review/gridSettings.ts new file mode 100644 index 000000000..cc10bdbb7 --- /dev/null +++ b/client/dive-common/review/gridSettings.ts @@ -0,0 +1,60 @@ +/** + * Grid presentation settings (columns, rows, context margin) shared by every + * chip grid in the app and remembered per browser. + */ +import { reactive, watch } from 'vue'; +import { DEFAULT_REVIEW_GRID, REVIEW_GRID_LIMITS, ReviewGridSettings } from './types'; + +/** + * Size factor for a cell's footer text: a 5x4 grid reads about a tenth + * larger than the base size, a 3x3 grid about a third larger, and dense + * grids stay at the base size. + */ +export function cellScaleFor(columns: number, rows: number): number { + const cells = Math.max(1, columns * rows); + return Math.round(Math.min(1.35, Math.max(1, 5 / Math.sqrt(cells))) * 100) / 100; +} + +const GRID_STORAGE_KEY = 'dive.review.grid'; + +export function loadGridSettings(): ReviewGridSettings { + try { + const raw = window.localStorage.getItem(GRID_STORAGE_KEY); + if (raw) { + const parsed = JSON.parse(raw); + if (parsed && typeof parsed === 'object') { + return { ...DEFAULT_REVIEW_GRID, ...parsed }; + } + } + } catch { + // Storage may be unavailable; defaults are fine. + } + return { ...DEFAULT_REVIEW_GRID }; +} + +export function storeGridSettings(grid: ReviewGridSettings) { + try { + window.localStorage.setItem(GRID_STORAGE_KEY, JSON.stringify(grid)); + } catch { + // Ignore storage failures. + } +} + +export function clampGrid(grid: ReviewGridSettings): ReviewGridSettings { + const clamp = (value: number, [min, max]: readonly [number, number], fallback: number) => ( + Number.isFinite(value) ? Math.min(max, Math.max(min, value)) : fallback + ); + return { + ...grid, + columns: Math.round(clamp(grid.columns, REVIEW_GRID_LIMITS.columns, DEFAULT_REVIEW_GRID.columns)), + rows: Math.round(clamp(grid.rows, REVIEW_GRID_LIMITS.rows, DEFAULT_REVIEW_GRID.rows)), + padding: clamp(grid.padding, REVIEW_GRID_LIMITS.padding, DEFAULT_REVIEW_GRID.padding), + }; +} + +/** A reactive settings object seeded from storage and written back on change. */ +export function usePersistentGridSettings(): ReviewGridSettings { + const grid = reactive(clampGrid(loadGridSettings())); + watch(grid, () => storeGridSettings({ ...grid }), { deep: true }); + return grid; +} diff --git a/client/dive-common/review/reviewItems.spec.ts b/client/dive-common/review/reviewItems.spec.ts new file mode 100644 index 000000000..ae6d0d952 --- /dev/null +++ b/client/dive-common/review/reviewItems.spec.ts @@ -0,0 +1,277 @@ +import type { TrackData } from 'vue-media-annotator/track'; +import { + attributeMatches, + buildReviewItems, + collectAttributeKeys, + collectTypes, + cycleIntervalFor, + frameGeometry, + groupReviewItems, + interpolateBounds, + matchTypePair, + sampleFrames, + sortReviewItems, +} from './reviewItems'; +import { DEFAULT_REVIEW_QUERY } from './types'; + +function track( + id: number, + pairs: [string, number][], + frames: number[], + extra: Partial = {}, +): TrackData { + return { + id, + begin: Math.min(...frames), + end: Math.max(...frames), + confidencePairs: pairs, + attributes: {}, + features: frames.map((frame) => ({ + frame, keyframe: true, bounds: [frame, 0, frame + 10, 10], + })), + ...extra, + }; +} + +describe('sampleFrames', () => { + it('keeps a single detection as one frame', () => { + const [only] = track(1, [['a', 1]], [3]).features; + expect(sampleFrames([only], 8)).toEqual([{ frame: 3, bounds: [3, 0, 13, 10] }]); + }); + + it('samples evenly including both ends and never repeats a frame', () => { + const { features } = track(1, [['a', 1]], [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + const frames = sampleFrames(features, 4).map((f) => f.frame); + expect(frames).toEqual([0, 3, 7, 10]); + expect(sampleFrames(features, 50)).toHaveLength(11); + }); +}); + +describe('matchTypePair', () => { + it('uses the top pair when no type is requested', () => { + expect(matchTypePair([['fish', 0.4], ['shark', 0.9]], '', 0.5)).toEqual(['shark', 0.9]); + expect(matchTypePair([['fish', 0.4]], '', 0.5)).toBeNull(); + }); + + it('matches the named type against the threshold', () => { + expect(matchTypePair([['fish', 0.4], ['shark', 0.9]], 'fish', 0.3)).toEqual(['fish', 0.4]); + expect(matchTypePair([['fish', 0.4], ['shark', 0.9]], 'fish', 0.5)).toBeNull(); + expect(matchTypePair([['shark', 0.9]], 'fish', 0)).toBeNull(); + }); + + it('only shows untyped tracks when everything is requested', () => { + expect(matchTypePair([], '', 0)).toEqual(['', 0]); + expect(matchTypePair([], '', 0.1)).toBeNull(); + }); +}); + +describe('attributeMatches', () => { + it('requires presence, then compares as text ignoring case', () => { + expect(attributeMatches(undefined, '')).toBe(false); + expect(attributeMatches('Yes', '')).toBe(true); + expect(attributeMatches('Yes', 'yes')).toBe(true); + expect(attributeMatches(3, '3')).toBe(true); + expect(attributeMatches(true, 'false')).toBe(false); + }); +}); + +describe('buildReviewItems', () => { + const tracks = [ + track(2, [['fish', 0.8], ['shark', 0.2]], [10, 11, 12]), + track(1, [['shark', 0.95]], [5]), + track(3, [['fish', 0.05]], [0]), + { ...track(4, [['fish', 0.9]], [7]), features: [{ frame: 7, keyframe: false }] }, + ]; + + it('filters by type and threshold and orders by track id', () => { + const items = buildReviewItems('ds', tracks, { ...DEFAULT_REVIEW_QUERY, type: 'fish', threshold: 0.1 }, 8); + expect(items.map((i) => i.trackId)).toEqual([2]); + expect(items[0]).toMatchObject({ + key: 'ds#2', type: 'fish', confidence: 0.8, keyframeCount: 3, + }); + expect(items[0].primary.frame).toBe(10); + expect(items[0].frames).toHaveLength(3); + }); + + it('shows every class above the threshold when no type is picked', () => { + const items = buildReviewItems('ds', tracks, { ...DEFAULT_REVIEW_QUERY, type: '', threshold: 0.5 }, 8); + expect(items.map((i) => [i.trackId, i.type])).toEqual([[1, 'shark'], [2, 'fish']]); + }); + + it('finds track and detection attributes', () => { + const withAttributes = [ + track(1, [['fish', 1]], [0, 1], { attributes: { verified: true } }), + { + ...track(2, [['fish', 1]], [4, 5, 6]), + features: [ + { frame: 4, keyframe: true, bounds: [0, 0, 1, 1] as [number, number, number, number] }, + { + frame: 5, keyframe: true, bounds: [0, 0, 1, 1] as [number, number, number, number], attributes: { occluded: 'partial' }, + }, + { + frame: 6, keyframe: true, bounds: [0, 0, 1, 1] as [number, number, number, number], attributes: { occluded: 'partial' }, + }, + ], + }, + ]; + const query = { ...DEFAULT_REVIEW_QUERY, mode: 'attribute' as const }; + const verified = buildReviewItems('ds', withAttributes, { ...query, attributeKey: 'verified' }, 8); + expect(verified.map((i) => i.trackId)).toEqual([1]); + expect(verified[0].matchedAttribute).toEqual({ key: 'verified', value: true, scope: 'track' }); + + const occluded = buildReviewItems('ds', withAttributes, { ...query, attributeKey: 'occluded', attributeValue: 'PARTIAL' }, 8); + expect(occluded).toHaveLength(1); + expect(occluded[0]).toMatchObject({ key: 'ds#2@5', trackId: 2, type: 'fish' }); + expect(occluded[0].primary.frame).toBe(5); + expect(occluded[0].frames.map((f) => f.frame)).toEqual([5, 6]); + + const trackOnly = buildReviewItems('ds', withAttributes, { ...query, attributeKey: 'occluded', attributeScope: 'track' }, 8); + expect(trackOnly).toHaveLength(0); + }); +}); + +describe('sortReviewItems', () => { + const items = [ + ...buildReviewItems('b', [track(1, [['a', 0.5]], [9]), track(2, [['a', 0.9]], [3])], DEFAULT_REVIEW_QUERY, 8), + ...buildReviewItems('a', [track(7, [['a', 0.7]], [1])], DEFAULT_REVIEW_QUERY, 8), + ]; + + it('orders by dataset selection order, confidence or frame', () => { + const keys = (order: Parameters[1]) => ( + sortReviewItems(items, order, ['a', 'b']).map((i) => i.key) + ); + expect(keys('dataset')).toEqual(['a#7', 'b#1', 'b#2']); + expect(keys('confidence-asc')).toEqual(['b#1', 'a#7', 'b#2']); + expect(keys('confidence-desc')).toEqual(['b#2', 'a#7', 'b#1']); + expect(keys('frame')).toEqual(['a#7', 'b#2', 'b#1']); + }); +}); + +describe('vocabularies', () => { + it('collects sorted types and attribute keys', () => { + const tracks = [ + track(1, [['zeta', 0.1], ['alpha', 0.9]], [0], { attributes: { trackAttr: 1 } }), + { + ...track(2, [['beta', 1]], [0]), + features: [{ + frame: 0, keyframe: true, bounds: [0, 0, 1, 1] as [number, number, number, number], attributes: { detAttr: 'x' }, + }], + }, + ]; + expect(collectTypes(tracks)).toEqual(['alpha', 'beta', 'zeta']); + expect(collectTypes(tracks, 0.5)).toEqual(['alpha', 'beta']); + expect(collectAttributeKeys(tracks, { + defined: { + belongs: 'track', datatype: 'text', name: 'defined', key: 'track_defined', + }, + })).toEqual(['defined', 'detAttr', 'trackAttr']); + }); +}); + +describe('frameGeometry', () => { + it('reads polygons and head/tail points from the GeoJSON features', () => { + const geometry = frameGeometry({ + frame: 2, + bounds: [0, 0, 10, 10], + geometry: { + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + properties: { key: '' }, + geometry: { type: 'Polygon', coordinates: [[[1, 1], [9, 1], [9, 9], [1, 1]]] }, + }, + { type: 'Feature', properties: { key: 'head' }, geometry: { type: 'Point', coordinates: [3, 4] } }, + { type: 'Feature', properties: { key: 'tail' }, geometry: { type: 'Point', coordinates: [7, 8] } }, + { type: 'Feature', properties: { key: 'HeadTails' }, geometry: { type: 'LineString', coordinates: [[3, 4], [7, 8]] } }, + ], + }, + }); + expect(geometry).toEqual({ + polygons: [[[1, 1], [9, 1], [9, 9]]], + head: [3, 4], + tail: [7, 8], + }); + }); + + it('falls back to the feature head/tail fields and leaves plain boxes bare', () => { + expect(frameGeometry({ frame: 0, bounds: [0, 0, 1, 1], head: [1, 2] })).toEqual({ head: [1, 2] }); + expect(frameGeometry({ frame: 0, bounds: [0, 0, 1, 1] })).toEqual({}); + }); +}); + +describe('cycleIntervalFor', () => { + it('plays consecutive frames at the dataset rate and sparse samples proportionally slower', () => { + const consecutive = [0, 1, 2, 3].map((frame) => ({ frame, bounds: [0, 0, 1, 1] as [number, number, number, number] })); + expect(cycleIntervalFor(consecutive, 10, 400)).toBe(100); + const sparse = [0, 30, 60, 90].map((frame) => ({ frame, bounds: [0, 0, 1, 1] as [number, number, number, number] })); + expect(cycleIntervalFor(sparse, 30, 400)).toBe(1000); + expect(cycleIntervalFor(sparse, 1, 400)).toBe(2000); + expect(cycleIntervalFor(consecutive, 120, 400)).toBe(33); + }); + + it('falls back when the rate is unknown or there is nothing to cycle', () => { + const one = [{ frame: 5, bounds: [0, 0, 1, 1] as [number, number, number, number] }]; + expect(cycleIntervalFor(one, 30, 400)).toBe(400); + expect(cycleIntervalFor([...one, { frame: 6, bounds: [0, 0, 1, 1] }], 0, 400)).toBe(400); + }); +}); + +describe('interpolateBounds', () => { + it('holds the nearest box at the ends and interpolates between keyframes', () => { + const t = track(1, [['fish', 1]], [0, 10]); + expect(interpolateBounds(t, 5)).toEqual([5, 0, 15, 10]); + expect(interpolateBounds(t, -3)).toEqual([0, 0, 10, 10]); + expect(interpolateBounds(t, 20)).toEqual([10, 0, 20, 10]); + expect(interpolateBounds(t, 10)).toEqual([10, 0, 20, 10]); + expect(interpolateBounds({ ...t, features: [] }, 3)).toBeNull(); + }); +}); + +describe('groupReviewItems', () => { + it('joins a track across the cameras of a rig with aligned frames and box-less gaps', () => { + const left = track(7, [['fish', 1]], [0, 4]); + const right = track(7, [['fish', 1]], [4, 8]); + const query = { ...DEFAULT_REVIEW_QUERY, threshold: 0 }; + const items = [ + ...buildReviewItems('rig/right', [right], query, 8), + ...buildReviewItems('rig/left', [left], query, 8), + ...buildReviewItems('solo', [track(1, [['fish', 1]], [2])], query, 8), + ]; + const membership = (id: string) => (id.startsWith('rig/') + ? { parent: 'rig', camera: id.slice(4), rank: id === 'rig/left' ? 0 : 1 } + : undefined); + const tracks: Record = { 'rig/left': left, 'rig/right': right }; + const entries = groupReviewItems(items, membership, (item) => tracks[item.datasetId], 8); + + expect(entries.map((e) => e.key)).toEqual(['rig#7', 'solo#1']); + const rig = entries[0]; + expect(rig.labels).toEqual(['left', 'right']); + expect(rig.items.map((i) => i.datasetId)).toEqual(['rig/left', 'rig/right']); + // Both cameras show frames 0, 4 and 8; the sides without a detection are interpolated. + expect(rig.items[0].frames.map((f) => [f.frame, f.missing ?? false])).toEqual([[0, false], [4, false], [8, true]]); + expect(rig.items[1].frames.map((f) => [f.frame, f.missing ?? false])).toEqual([[0, true], [4, false], [8, false]]); + expect(rig.items[0].frames[2].bounds).toEqual([4, 0, 14, 10]); + expect(entries[1].labels).toEqual(['']); + }); +}); + +it('groups detection matches from different frames into the same camera track', () => { + const left = track(7, [['fish', 1]], [0]); + const right = track(7, [['fish', 1]], [4]); + left.features[0].attributes = { checked: true }; + right.features[0].attributes = { checked: true }; + const query = { ...DEFAULT_REVIEW_QUERY, mode: 'attribute' as const, attributeKey: 'checked' }; + const items = [ + ...buildReviewItems('rig/left', [left], query, 8), + ...buildReviewItems('rig/right', [right], query, 8), + ]; + const entries = groupReviewItems( + items, + (id) => ({ parent: 'rig', camera: id, rank: id === 'rig/left' ? 0 : 1 }), + (item) => (item.datasetId === 'rig/left' ? left : right), + 8, + ); + expect(entries).toHaveLength(1); + expect(entries[0].items).toHaveLength(2); +}); diff --git a/client/dive-common/review/reviewItems.ts b/client/dive-common/review/reviewItems.ts new file mode 100644 index 000000000..0a99f4bad --- /dev/null +++ b/client/dive-common/review/reviewItems.ts @@ -0,0 +1,394 @@ +/** + * Pure helpers that turn a dataset's tracks into review grid items for a + * query, plus the type/attribute vocabularies the query controls offer. + */ +import type { TrackData, Feature } from 'vue-media-annotator/track'; +import type { StringKeyObject } from 'vue-media-annotator/BaseAnnotation'; +import type { Attribute } from 'vue-media-annotator/use/AttributeTypes'; +import { compareTypeNames } from 'dive-common/typeHierarchy'; +import type { RectBounds } from 'vue-media-annotator/utils'; +import type { + ReviewEntry, ReviewFrameGeometry, ReviewFrameRef, ReviewItem, ReviewPolygon, ReviewQuery, ReviewSortOrder, +} from './types'; + +function isPoint(value: unknown): value is [number, number] { + return Array.isArray(value) && value.length >= 2 + && Number.isFinite(value[0]) && Number.isFinite(value[1]); +} + +/** + * Polygons and head/tail points of a keyframe. Points come from the + * GeoJSON features keyed "head"/"tail" (how DIVE stores them), falling back + * to the feature's own head/tail fields. + */ +export function frameGeometry(feature: Feature): ReviewFrameGeometry { + const geometry: ReviewFrameGeometry = {}; + const polygons: ReviewPolygon[] = []; + (feature.geometry?.features || []).forEach((geo) => { + const key = (geo.properties as { key?: unknown } | null)?.key; + if (geo.geometry.type === 'Polygon') { + const ring = (geo.geometry.coordinates[0] || []).filter(isPoint).map(([x, y]) => [x, y] as [number, number]); + if (ring.length > 1 && ring[0][0] === ring[ring.length - 1][0] && ring[0][1] === ring[ring.length - 1][1]) { + ring.pop(); + } + if (ring.length >= 3) polygons.push(ring); + } else if (geo.geometry.type === 'Point' && (key === 'head' || key === 'tail')) { + const point = geo.geometry.coordinates; + if (isPoint(point)) geometry[key] = [point[0], point[1]]; + } + }); + if (!geometry.head && isPoint(feature.head)) geometry.head = [feature.head[0], feature.head[1]]; + if (!geometry.tail && isPoint(feature.tail)) geometry.tail = [feature.tail[0], feature.tail[1]]; + if (polygons.length) geometry.polygons = polygons; + return geometry; +} + +/** A frame reference for a keyframe, carrying its extra geometry when present. */ +export function frameRefFor(feature: Feature): ReviewFrameRef | null { + if (!feature.bounds) return null; + return { frame: feature.frame, bounds: feature.bounds, ...frameGeometry(feature) }; +} + +/** Keyframes carrying a box, in frame order. */ +export function boxedFeatures(track: TrackData): Feature[] { + return track.features + .filter((f) => !!f.bounds) + .sort((a, b) => a.frame - b.frame); +} + +/** Up to `max` boxes evenly sampled along `features` (which must be in frame order). */ +export function sampleFrames(features: Feature[], max: number): ReviewFrameRef[] { + if (features.length === 0) return []; + const count = Math.max(1, Math.min(max, features.length)); + const chosen: ReviewFrameRef[] = []; + const seen = new Set(); + for (let i = 0; i < count; i += 1) { + const index = count === 1 ? 0 : Math.round((i * (features.length - 1)) / (count - 1)); + const feature = features[index]; + const ref = frameRefFor(feature); + if (ref && !seen.has(feature.frame)) { + seen.add(feature.frame); + chosen.push(ref); + } + } + return chosen; +} + +/** + * Milliseconds between the sampled frames of a track so that cycling + * through them plays at the dataset's real-time rate: consecutive frames + * advance every 1/fps seconds, and sparser samples wait proportionally + * longer. Clamped so a loop is neither a strobe nor a slideshow. + */ +export function cycleIntervalFor(frames: readonly ReviewFrameRef[], fps: number, fallbackMs: number): number { + if (frames.length < 2 || !(fps > 0)) return fallbackMs; + const span = frames[frames.length - 1].frame - frames[0].frame; + const stride = Math.max(1, span / (frames.length - 1)); + return Math.min(2000, Math.max(33, Math.round((stride / fps) * 1000))); +} + +/** The pair a type query matches on, or null when the track does not qualify. */ +export function matchTypePair( + pairs: readonly (readonly [string, number])[], + type: string, + threshold: number, +): [string, number] | null { + if (pairs.length === 0) { + // Untyped annotations only show up when every type is requested. + return type === '' && threshold <= 0 ? ['', 0] : null; + } + // Stored pairs are normally sorted by confidence, but files are not guaranteed to be. + const candidates = type === '' ? pairs : pairs.filter(([name]) => name === type); + const best = candidates.reduce( + (acc, pair) => (acc === null || pair[1] > acc[1] ? pair : acc), + null, + ); + if (!best || best[1] < threshold) return null; + return [best[0], best[1]]; +} + +function ownAttributes(attributes: StringKeyObject | undefined): [string, unknown][] { + if (!attributes) return []; + return Object.entries(attributes).filter(([key]) => key !== 'userAttributes'); +} + +/** Attribute values compare as strings, case-insensitively; empty wanted means "present". */ +export function attributeMatches(value: unknown, wanted: string): boolean { + if (value === undefined || value === null) return false; + if (wanted === '') return true; + const asText = Array.isArray(value) ? value.map(String).join(',') : String(value); + return asText.toLowerCase() === wanted.trim().toLowerCase(); +} + +function trackTopPair(track: TrackData): [string, number] { + const top = track.confidencePairs.reduce( + (acc, pair) => (acc === null || pair[1] > acc[1] ? pair : acc), + null, + ); + return top ? [top[0], top[1]] : ['', 0]; +} + +function attributeItem( + datasetId: string, + track: TrackData, + query: ReviewQuery, + maxSequenceFrames: number, +): ReviewItem | null { + const key = query.attributeKey.trim(); + if (!key) return null; + const features = boxedFeatures(track); + if (features.length === 0) return null; + const [type, confidence] = trackTopPair(track); + const base = { + datasetId, + trackId: track.id, + keyframeCount: features.length, + type, + confidence, + }; + if (query.attributeScope !== 'detection') { + const hit = ownAttributes(track.attributes).find( + ([name, value]) => name === key && attributeMatches(value, query.attributeValue), + ); + if (hit) { + const frames = sampleFrames(features, maxSequenceFrames); + return { + ...base, + key: `${datasetId}#${track.id}`, + primary: frames[0], + frames, + matchedAttribute: { key, value: hit[1], scope: 'track' }, + }; + } + } + if (query.attributeScope !== 'track') { + const matching = features.filter((f) => ownAttributes(f.attributes).some( + ([name, value]) => name === key && attributeMatches(value, query.attributeValue), + )); + if (matching.length > 0) { + const frames = sampleFrames(matching, maxSequenceFrames); + const hit = ownAttributes(matching[0].attributes).find(([name]) => name === key); + return { + ...base, + key: `${datasetId}#${track.id}@${matching[0].frame}`, + primary: frames[0], + frames, + matchedAttribute: { key, value: hit ? hit[1] : undefined, scope: 'detection' }, + }; + } + } + return null; +} + +function typeItem( + datasetId: string, + track: TrackData, + query: ReviewQuery, + maxSequenceFrames: number, +): ReviewItem | null { + const pair = matchTypePair(track.confidencePairs, query.type, query.threshold); + if (!pair) return null; + const features = boxedFeatures(track); + if (features.length === 0) return null; + const frames = sampleFrames(features, maxSequenceFrames); + return { + key: `${datasetId}#${track.id}`, + datasetId, + trackId: track.id, + primary: frames[0], + frames, + keyframeCount: features.length, + type: pair[0], + confidence: pair[1], + }; +} + +/** Every grid item one dataset contributes to a query, in track id order. */ +export function buildReviewItems( + datasetId: string, + tracks: Iterable, + query: ReviewQuery, + maxSequenceFrames: number, +): ReviewItem[] { + const items: ReviewItem[] = []; + Array.from(tracks) + .sort((a, b) => a.id - b.id) + .forEach((track) => { + const item = query.mode === 'attribute' + ? attributeItem(datasetId, track, query, maxSequenceFrames) + : typeItem(datasetId, track, query, maxSequenceFrames); + if (item) items.push(item); + }); + return items; +} + +export function sortReviewItems( + items: ReviewItem[], + order: ReviewSortOrder, + datasetOrder: readonly string[], +): ReviewItem[] { + const rank = new Map(datasetOrder.map((id, index) => [id, index])); + const byDataset = (a: ReviewItem, b: ReviewItem) => ( + (rank.get(a.datasetId) ?? Infinity) - (rank.get(b.datasetId) ?? Infinity) + ); + const sorted = [...items]; + switch (order) { + case 'confidence-asc': + sorted.sort((a, b) => a.confidence - b.confidence || byDataset(a, b) || a.trackId - b.trackId); + break; + case 'confidence-desc': + sorted.sort((a, b) => b.confidence - a.confidence || byDataset(a, b) || a.trackId - b.trackId); + break; + case 'frame': + sorted.sort((a, b) => byDataset(a, b) || a.primary.frame - b.primary.frame || a.trackId - b.trackId); + break; + default: + sorted.sort((a, b) => byDataset(a, b) || a.trackId - b.trackId); + } + return sorted; +} + +/** Which multicamera parent and camera a dataset id belongs to, if any. */ +export interface CameraMembership { + parent: string; + camera: string; + /** Position of the camera in the rig's display order. */ + rank: number; +} + +/** + * Box a track would have on `frame` in one camera, interpolated between + * its nearest keyframes with boxes (held at the ends), or null when the + * track has no boxes there at all. + */ +export function interpolateBounds(track: TrackData, frame: number): RectBounds | null { + const features = boxedFeatures(track); + if (features.length === 0) return null; + const exact = features.find((f) => f.frame === frame); + if (exact?.bounds) return exact.bounds; + const before = [...features].reverse().find((f) => f.frame < frame); + const after = features.find((f) => f.frame > frame); + if (before?.bounds && after?.bounds) { + const t = (frame - before.frame) / (after.frame - before.frame); + return before.bounds.map((v, i) => v + ((after.bounds as RectBounds)[i] - v) * t) as RectBounds; + } + return (before?.bounds ?? after?.bounds) ?? null; +} + +/** + * Give the items of a multicamera entry the same frames: the union of their + * keyframes sampled once, with a camera that lacks a detection on a frame + * getting an interpolated, box-less reference there. + */ +export function alignCameraFrames( + items: ReviewItem[], + trackOf: (item: ReviewItem) => TrackData | undefined, + maxSequenceFrames: number, +): ReviewItem[] { + if (items.length < 2) return items; + const byFrame = new Map(); + items.forEach((item) => { + const track = trackOf(item); + if (!track) return; + boxedFeatures(track).forEach((feature) => { + if (!byFrame.has(feature.frame)) byFrame.set(feature.frame, feature); + }); + }); + const union = Array.from(byFrame.values()).sort((a, b) => a.frame - b.frame); + // Detection matches keep their own frame first so the query hit stays visible. + const sampled = sampleFrames(union, maxSequenceFrames).map((ref) => ref.frame); + const anchor = items.find((item) => item.key.includes('@'))?.primary.frame; + if (anchor !== undefined && !sampled.includes(anchor)) sampled.unshift(anchor); + + return items.map((item) => { + const track = trackOf(item); + if (!track) return item; + const frames: ReviewFrameRef[] = []; + sampled.forEach((frame) => { + const feature = track.features.find((f) => f.frame === frame && f.bounds); + if (feature) { + const ref = frameRefFor(feature); + if (ref) frames.push(ref); + return; + } + const bounds = interpolateBounds(track, frame); + if (bounds) frames.push({ frame, bounds, missing: true }); + }); + if (frames.length === 0) return item; + return { + ...item, primary: frames[0], frames, keyframeCount: boxedFeatures(track).length, + }; + }); +} + +/** + * Group items into grid entries: one per track, holding an item per camera + * the track appears in (in rig order) for multicamera datasets. + */ +export function groupReviewItems( + items: readonly ReviewItem[], + membershipOf: (datasetId: string) => CameraMembership | undefined, + trackOf: (item: ReviewItem) => TrackData | undefined, + maxSequenceFrames: number, +): ReviewEntry[] { + const entries: ReviewEntry[] = []; + const byKey = new Map(); + items.forEach((item) => { + const membership = membershipOf(item.datasetId); + if (!membership) { + entries.push({ key: item.key, items: [item], labels: [''] }); + return; + } + const key = `${membership.parent}#${item.trackId}`; + let group = byKey.get(key); + if (!group) { + group = { items: [], ranks: [], labels: [] }; + byKey.set(key, group); + entries.push({ key, items: group.items, labels: group.labels }); + } + // Keep cameras in rig order whatever order the items arrived in. + let at = group.ranks.findIndex((rank) => rank > membership.rank); + if (at < 0) at = group.ranks.length; + group.items.splice(at, 0, item); + group.ranks.splice(at, 0, membership.rank); + group.labels.splice(at, 0, membership.camera); + }); + return entries.map((entry) => (entry.items.length > 1 + ? { ...entry, items: alignCameraFrames(entry.items, trackOf, maxSequenceFrames) } + : entry)); +} + +/** Every type named by any confidence pair, in type order. */ +/** + * Types present on the tracks, sorted; with a threshold, only those some + * track carries at that confidence or above, so a type query built from + * the list always finds something. + */ +export function collectTypes(tracks: Iterable, threshold = 0): string[] { + const types = new Set(); + Array.from(tracks).forEach((track) => { + track.confidencePairs.forEach(([type, confidence]) => { + if (type && confidence >= threshold) types.add(type); + }); + }); + return Array.from(types).sort(compareTypeNames); +} + +/** + * Attribute keys the query control can offer: those defined in the dataset + * configuration plus any actually present on tracks or detections. + */ +export function collectAttributeKeys( + tracks: Iterable, + definitions: Readonly> | undefined, +): string[] { + const keys = new Set(); + Object.values(definitions || {}).forEach((attribute) => keys.add(attribute.name)); + Array.from(tracks).forEach((track) => { + ownAttributes(track.attributes).forEach(([key]) => keys.add(key)); + track.features.forEach((feature) => { + ownAttributes(feature.attributes).forEach(([key]) => keys.add(key)); + }); + }); + return Array.from(keys).sort((a, b) => a.localeCompare(b)); +} diff --git a/client/dive-common/review/reviewSession.spec.ts b/client/dive-common/review/reviewSession.spec.ts new file mode 100644 index 000000000..da8750a01 --- /dev/null +++ b/client/dive-common/review/reviewSession.spec.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest'; +import { ref } from 'vue'; +import type { ReviewService } from 'dive-common/use/useReview'; +import { + holdReviewSession, peekReviewSession, sessionKey, shouldResume, takeReviewSession, + useReviewSessionHeld, +} from './reviewSession'; + +function session(datasetIds: string[], pending = 0) { + return { + review: { pendingCount: ref(pending) } as unknown as ReviewService, + view: 'results' as const, + page: 3, + datasetKey: sessionKey(datasetIds), + }; +} + +describe('review session', () => { + it('is handed over once', () => { + expect(takeReviewSession()).toBeNull(); + const held = session(['a']); + holdReviewSession(held); + expect(takeReviewSession()).toBe(held); + expect(takeReviewSession()).toBeNull(); + }); + + it('can be peeked without taking ownership', () => { + expect(peekReviewSession()).toBeNull(); + const heldFlag = useReviewSessionHeld(); + expect(heldFlag.value).toBe(false); + const held = session(['a']); + holdReviewSession(held); + expect(peekReviewSession()).toBe(held); + expect(heldFlag.value).toBe(true); + expect(takeReviewSession()).toBe(held); + expect(peekReviewSession()).toBeNull(); + expect(heldFlag.value).toBe(false); + }); + + it('resumes for a plain visit, the same selection, or unsaved edits', () => { + expect(shouldResume(session(['a', 'b']), [])).toBe(true); + expect(shouldResume(session(['a', 'b']), ['b', 'a'])).toBe(true); + expect(shouldResume(session(['a', 'b']), ['c'])).toBe(false); + expect(shouldResume(session(['a', 'b'], 2), ['c'])).toBe(true); + }); +}); diff --git a/client/dive-common/review/reviewSession.ts b/client/dive-common/review/reviewSession.ts new file mode 100644 index 000000000..55a383f40 --- /dev/null +++ b/client/dive-common/review/reviewSession.ts @@ -0,0 +1,59 @@ +import { computed, shallowRef, type ComputedRef } from 'vue'; +import type { ReviewService } from 'dive-common/use/useReview'; + +/** + * The review page hands its state here when it is left and takes it back + * when it is next shown, so navigating away and back resumes the same + * datasets and page (Results when any are loaded, otherwise Datasets). + * Unsaved edits are resolved (saved or discarded) before the leave + * completes. Only one session is held; a fresh start (new datasets + * picked in the library) replaces it. + */ +export interface ReviewSession { + review: ReviewService; + view: 'results' | 'datasets'; + page: number; + /** Key of the datasets the session was started with, see `sessionKey`. */ + datasetKey: string; +} + +/** Shallow: the service itself must not be deeply unwrapped. */ +const held = shallowRef(null); + +export function sessionKey(ids: readonly string[]): string { + return [...ids].sort().join('\n'); +} + +export function holdReviewSession(session: ReviewSession) { + held.value = session; +} + +/** The held session, if any; the caller now owns it. */ +export function takeReviewSession(): ReviewSession | null { + const session = held.value; + held.value = null; + return session; +} + +/** Look at the held session without taking ownership. */ +export function peekReviewSession(): ReviewSession | null { + return held.value; +} + +/** True while a review session is parked (e.g. the user opened the viewer). */ +export function useReviewSessionHeld(): ComputedRef { + return computed(() => held.value !== null); +} + +/** + * Whether a held session should be resumed for a page opened with + * `initialIds`: always when the page is opened plainly, or when the same + * datasets are asked for again. A held session that still has pending edits + * (e.g. after a hard refresh path that could not prompt) is also resumed so + * those edits are not dropped by a new library selection. + */ +export function shouldResume(session: ReviewSession, initialIds: readonly string[]): boolean { + if (initialIds.length === 0) return true; + if (session.review.pendingCount.value > 0) return true; + return session.datasetKey === sessionKey(initialIds); +} diff --git a/client/dive-common/review/types.ts b/client/dive-common/review/types.ts new file mode 100644 index 000000000..1f5a4442e --- /dev/null +++ b/client/dive-common/review/types.ts @@ -0,0 +1,130 @@ +/** + * Review mode: contract shared by the review page, the chip loader and the + * platform shells. The page shows many detections at once as cropped chips + * so a user can audit and correct their types (or find them by attribute) + * without opening every sequence in the viewer. + */ +import type { AnnotationId } from 'vue-media-annotator/BaseAnnotation'; +import type { RectBounds } from 'vue-media-annotator/utils'; + +export type ReviewQueryMode = 'type' | 'attribute'; + +/** Where an attribute query looks for the attribute. */ +export type ReviewAttributeScope = 'any' | 'track' | 'detection'; + +export interface ReviewQuery { + mode: ReviewQueryMode; + /** Type mode: the class to show; empty for every class. */ + type: string; + /** Type mode: minimum confidence of the matching pair (0 shows everything). */ + threshold: number; + /** Attribute mode: the attribute key to look for. */ + attributeKey: string; + /** Attribute mode: required value (string compared); empty just requires presence. */ + attributeValue: string; + attributeScope: ReviewAttributeScope; +} + +export type ReviewSortOrder = 'dataset' | 'confidence-asc' | 'confidence-desc' | 'frame'; + +/** A polygon outline in image coordinates: [[x, y], ...] without a closing repeat. */ +export type ReviewPolygon = [number, number][]; + +/** Extra geometry a detection may carry besides its box. */ +export interface ReviewFrameGeometry { + polygons?: ReviewPolygon[]; + head?: [number, number]; + tail?: [number, number]; +} + +/** One box of a track shown in a chip, in the dataset's own frame numbers. */ +export interface ReviewFrameRef extends ReviewFrameGeometry { + frame: number; + bounds: RectBounds; + /** + * The track has no detection on this frame in this camera; `bounds` is + * interpolated from its neighbours so the chip can still be cropped there, + * and no box is drawn. + */ + missing?: boolean; +} + +/** + * One grid entry: a track (or one detection of it) in one dataset. Built + * once per query run; the live type is read from the review service so + * edits show without the grid reshuffling. + */ +export interface ReviewItem { + /** Stable within a query run: `${datasetId}#${trackId}` or with `@frame`. */ + key: string; + datasetId: string; + trackId: AnnotationId; + /** The box shown first: the matched detection, or the track's first keyframe. */ + primary: ReviewFrameRef; + /** + * Boxes sampled along the track for the cycling animation, primary first. + * A single entry means a static detection. + */ + frames: ReviewFrameRef[]; + /** Number of keyframes in the track. */ + keyframeCount: number; + /** Type and confidence of the pair the query matched on (top pair otherwise). */ + type: string; + confidence: number; + /** Attribute mode: what matched. */ + matchedAttribute?: { + key: string; + value: unknown; + scope: 'track' | 'detection'; + }; +} + +/** + * One grid entry: a track shown once per camera it appears in. Single + * camera datasets have one item per entry; the cameras of a multicamera + * dataset share the entry, with their items' frames aligned. + */ +export interface ReviewEntry { + key: string; + items: ReviewItem[]; + /** Camera name per item; empty strings for single-camera entries. */ + labels: string[]; +} + +/** Grid presentation settings, persisted per browser. */ +export interface ReviewGridSettings { + columns: number; + rows: number; + /** + * Context around the box as a fraction of its longer side: 0.3 shows the + * box plus 30% of its size on each side. + */ + padding: number; + /** Milliseconds between frames of a cycling track. */ + cycleIntervalMs: number; + /** Most frames sampled along a track. */ + maxSequenceFrames: number; +} + +export const DEFAULT_REVIEW_QUERY: ReviewQuery = { + mode: 'type', + type: '', + threshold: 0.1, + attributeKey: '', + attributeValue: '', + attributeScope: 'any', +}; + +export const DEFAULT_REVIEW_GRID: ReviewGridSettings = { + columns: 5, + rows: 4, + padding: 0.3, + cycleIntervalMs: 400, + maxSequenceFrames: 8, +}; + +export const REVIEW_GRID_LIMITS = { + columns: [1, 12] as const, + rows: [1, 10] as const, + padding: [0, 3] as const, +}; diff --git a/client/dive-common/review/useReviewGrid.ts b/client/dive-common/review/useReviewGrid.ts new file mode 100644 index 000000000..a69105af0 --- /dev/null +++ b/client/dive-common/review/useReviewGrid.ts @@ -0,0 +1,181 @@ +/** + * Paging, zoom and chip-loading behaviour shared by every chip grid: the + * review page and any other view that shows {@link ReviewItem}s in a + * {@link ReviewGrid}. Owns the current page, keeps the chip store rendering + * at the cells' resolution and aspect ratio, and loads chips for the + * visible page (plus a prefetch of the next one). + */ +import { + computed, ref, Ref, unref, watch, +} from 'vue'; +import { debounce } from 'lodash'; +import type { ChipStore } from './chipStore'; +import { chipSizeFor } from './chipRenderer'; +import { clampGrid } from './gridSettings'; +import { REVIEW_GRID_LIMITS, ReviewGridSettings, ReviewItem } from './types'; + +const CHIP_OUTLINE = '#00e5ff'; +/** Height of a cell's footer (type field and caption), excluded from the image area. */ +const CELL_FOOTER_PX = 46; + +/** Chip aspect for a cell, coarsened so window resizes rarely force a re-render. */ +export function chipAspectFor(cellWidth: number, cellHeight: number, footerPx = CELL_FOOTER_PX): number { + const imageHeight = cellHeight - footerPx; + if (cellWidth <= 0 || imageHeight <= 0) return 1; + const ratio = Math.min(3, Math.max(1 / 3, cellWidth / imageHeight)); + return Math.round(ratio * 10) / 10; +} + +export interface ReviewGridOptions { + /** What the grid pages over: review items, or entries holding several. */ + items: Ref; + /** Items whose chips an entry needs; defaults to the entry being an item. */ + chipItemsOf?: (entry: T) => readonly ReviewItem[]; + /** Reactive grid settings (mutated in place by the setters below). */ + grid: ReviewGridSettings; + chipStore: ChipStore; + /** Whether the grid is on screen; chips only load and keys only page while true. */ + active: Ref; + /** Footer height to exclude from the chip aspect ratio. */ + footerPx?: number | Ref; + /** Box outline burned into the chips; empty draws none (the cell overlays it). */ + outline?: string; +} + +/** How long paging must be idle before chips load, so skipped pages never render. */ +const PAGE_SETTLE_MS = 250; + +export function useReviewGrid(options: ReviewGridOptions) { + const { + items, grid, chipStore, active, + } = options; + const chipItemsOf = options.chipItemsOf ?? ((entry: T) => [entry as unknown as ReviewItem]); + const chipItems = (entries: readonly T[]) => entries.flatMap((entry) => chipItemsOf(entry)); + const page = ref(0); + const cellSize = ref({ width: 0, height: 0 }); + + const perPage = computed(() => grid.columns * grid.rows); + const pageCount = computed(() => Math.max(1, Math.ceil(items.value.length / perPage.value))); + const pageItems = computed(() => items.value.slice(page.value * perPage.value, (page.value + 1) * perPage.value)); + const nextPageItems = computed(() => items.value.slice((page.value + 1) * perPage.value, (page.value + 2) * perPage.value)); + + function ensureVisible() { + if (!active.value) return; + const visible = chipItems(pageItems.value); + const prefetch = chipItems(nextPageItems.value); + chipStore.trimQueues(new Set([...visible, ...prefetch].map((i) => i.key))); + chipStore.ensurePrimary([...visible, ...prefetch]); + chipStore.ensureSequences(visible); + } + + const applyChipOptions = debounce(() => { + const pixels = Math.max(cellSize.value.width, cellSize.value.height) + * (window.devicePixelRatio || 1); + if (pixels <= 0) return; + chipStore.setOptions({ + padding: grid.padding, + size: chipSizeFor(pixels), + aspect: chipAspectFor(cellSize.value.width, cellSize.value.height, unref(options.footerPx)), + outline: options.outline ?? CHIP_OUTLINE, + }); + ensureVisible(); + }, 200); + + /** + * Rapid paging only loads the page landed on: queued work for pages + * passed over is dropped at once, and new work waits for paging to settle. + */ + const ensureVisibleSettled = debounce(ensureVisible, PAGE_SETTLE_MS); + function onPageChanged() { + if (!active.value) return; + chipStore.trimQueues(new Set(chipItems(pageItems.value).map((i) => i.key))); + ensureVisibleSettled(); + } + + watch(() => [grid.padding, cellSize.value.width, cellSize.value.height, unref(options.footerPx)], applyChipOptions); + watch(page, onPageChanged); + watch(active, ensureVisible); + watch(items, () => { + page.value = 0; + ensureVisible(); + }); + watch(perPage, () => { + page.value = Math.min(page.value, pageCount.value - 1); + ensureVisible(); + }); + + function goToPage(next: number) { + page.value = Math.min(Math.max(0, next), pageCount.value - 1); + } + + function setColumns(value: number) { + Object.assign(grid, clampGrid({ ...grid, columns: Number(value) })); + } + + function setRows(value: number) { + Object.assign(grid, clampGrid({ ...grid, rows: Number(value) })); + } + + function setPadding(value: number) { + Object.assign(grid, clampGrid({ ...grid, padding: Number(value) })); + } + + /** Fewer, larger cells (-1) or more, smaller cells (+1), keeping the shape. */ + function zoom(direction: 1 | -1) { + const ratio = grid.rows / grid.columns; + const columns = grid.columns + direction; + const rows = Math.max(1, Math.round(columns * ratio)); + Object.assign(grid, clampGrid({ ...grid, columns, rows })); + } + + const canZoomIn = computed(() => grid.columns > REVIEW_GRID_LIMITS.columns[0]); + const canZoomOut = computed(() => grid.columns < REVIEW_GRID_LIMITS.columns[1]); + + function isTypingTarget(target: EventTarget | null) { + if (!(target instanceof HTMLElement)) return false; + return ['INPUT', 'TEXTAREA', 'SELECT'].includes(target.tagName) || target.isContentEditable; + } + + /** Arrow / page keys page the grid; returns true when the key was consumed. */ + function handleKeydown(event: KeyboardEvent): boolean { + if (!active.value || isTypingTarget(event.target)) return false; + if (event.key === 'ArrowLeft' || event.key === 'PageUp') { + goToPage(page.value - 1); + } else if (event.key === 'ArrowRight' || event.key === 'PageDown') { + goToPage(page.value + 1); + } else if (event.key === 'Home') { + goToPage(0); + } else if (event.key === 'End') { + goToPage(pageCount.value - 1); + } else { + return false; + } + event.preventDefault(); + return true; + } + + function dispose() { + applyChipOptions.cancel(); + ensureVisibleSettled.cancel(); + } + + return { + page, + pageCount, + perPage, + pageItems, + cellSize, + ensureVisible, + goToPage, + setColumns, + setRows, + setPadding, + zoom, + canZoomIn, + canZoomOut, + handleKeydown, + dispose, + }; +} + +export type ReviewGridController = ReturnType; diff --git a/client/dive-common/review/viewerNavigation.spec.ts b/client/dive-common/review/viewerNavigation.spec.ts new file mode 100644 index 000000000..9ccd2de8a --- /dev/null +++ b/client/dive-common/review/viewerNavigation.spec.ts @@ -0,0 +1,15 @@ +import { parseViewerFocus, reviewViewerLocation } from './viewerNavigation'; + +describe('review viewer navigation', () => { + it('round-trips a frame and track through the query', () => { + const location = reviewViewerLocation('abc', { frame: 12, trackId: 7 }); + expect(location).toEqual({ name: 'viewer', params: { id: 'abc' }, query: { frame: '12', track: '7' } }); + expect(parseViewerFocus(location.query)).toEqual({ frame: 12, trackId: 7 }); + }); + + it('ignores missing or malformed values', () => { + expect(reviewViewerLocation('abc', {}).query).toEqual({}); + expect(parseViewerFocus({ frame: 'x', track: ['3'] })).toEqual({ trackId: 3 }); + expect(parseViewerFocus({ frame: '-1' })).toEqual({}); + }); +}); diff --git a/client/dive-common/review/viewerNavigation.ts b/client/dive-common/review/viewerNavigation.ts new file mode 100644 index 000000000..995da442e --- /dev/null +++ b/client/dive-common/review/viewerNavigation.ts @@ -0,0 +1,45 @@ +/** + * Deep links from the review grid into the annotation viewer: the viewer + * route plus query parameters naming the frame to seek to and the track to + * select once media is ready. + */ +export const VIEWER_FRAME_QUERY = 'frame'; +export const VIEWER_TRACK_QUERY = 'track'; + +export interface ViewerFocus { + frame?: number; + trackId?: number; +} + +export interface ReviewViewerLocation { + name: string; + params: Record; + query: Record; +} + +/** Both platforms name their single-dataset viewer route `viewer` with an `id` param. */ +export function reviewViewerLocation( + datasetId: string, + focus: ViewerFocus, +): ReviewViewerLocation { + const query: Record = {}; + if (focus.frame !== undefined) query[VIEWER_FRAME_QUERY] = String(focus.frame); + if (focus.trackId !== undefined) query[VIEWER_TRACK_QUERY] = String(focus.trackId); + return { name: 'viewer', params: { id: datasetId }, query }; +} + +function integerParam(value: unknown): number | undefined { + const text = Array.isArray(value) ? value[0] : value; + if (typeof text !== 'string' || !/^-?\d+$/.test(text)) return undefined; + return Number(text); +} + +/** Read the focus back out of a route query (unknown values are ignored). */ +export function parseViewerFocus(query: Record): ViewerFocus { + const frame = integerParam(query[VIEWER_FRAME_QUERY]); + const trackId = integerParam(query[VIEWER_TRACK_QUERY]); + const focus: ViewerFocus = {}; + if (frame !== undefined && frame >= 0) focus.frame = frame; + if (trackId !== undefined) focus.trackId = trackId; + return focus; +} diff --git a/client/dive-common/use/useReview.spec.ts b/client/dive-common/use/useReview.spec.ts new file mode 100644 index 000000000..ddb43d716 --- /dev/null +++ b/client/dive-common/use/useReview.spec.ts @@ -0,0 +1,436 @@ +import { effectScope, nextTick } from 'vue'; +import type { TrackData } from 'vue-media-annotator/track'; +import type { DatasetConfig } from 'dive-common/apispec'; +import { createReviewService, ReviewApi } from './useReview'; + +vi.mock('dive-common/review/frameSource', () => ({ + createFrameSource: vi.fn(() => ({ + frameCount: 1, + getFrame: vi.fn(), + dispose: vi.fn(), + })), +})); + +function track(id: number, pairs: [string, number][], frames: number[]): TrackData { + return { + id, + begin: Math.min(...frames), + end: Math.max(...frames), + confidencePairs: pairs, + attributes: {}, + features: frames.map((frame) => ({ + frame, keyframe: true, bounds: [0, 0, 10, 10], + })), + }; +} + +function config(id: string, overrides: Partial = {}): DatasetConfig { + return { + id, + name: `Dataset ${id}`, + type: 'image-sequence', + fps: 1, + createdAt: '', + subType: null, + multiCamMedia: null, + imageData: [{ url: 'a.jpg', filename: 'a.jpg' }], + videoUrl: undefined, + ...overrides, + } as DatasetConfig; +} + +function makeApi(tracksById: Record, overrides: Partial = {}): ReviewApi { + return { + loadConfig: vi.fn(async (id: string) => config(id)), + // Fresh copies each call, as a real platform returns them. + loadDetections: vi.fn(async (id: string) => ({ + version: 2, tracks: JSON.parse(JSON.stringify(tracksById[id] || [])), groups: [], sets: [], + })), + saveDetections: vi.fn(async () => undefined), + listScoringDatasets: vi.fn(async () => [{ id: 'a', name: 'Alpha' }, { id: 'b', name: 'Beta' }]), + ...overrides, + }; +} + +describe('createReviewService', () => { + it('keeps deferred datasets queued until loadQueued', async () => { + const api = makeApi({ a: [track(1, [['fish', 0.9]], [0])] }); + const service = createReviewService({ api }); + await service.addDataset('a', { id: 'a', name: 'Alpha' }, { defer: true }); + expect(service.datasets.value.map((d) => d.status)).toEqual(['queued']); + expect(api.loadDetections).not.toHaveBeenCalled(); + await service.loadQueued(); + expect(service.datasets.value.map((d) => [d.status, d.trackCount])).toEqual([['ready', 1]]); + expect(api.loadDetections).toHaveBeenCalledTimes(1); + }); + + it('loads datasets, prefers peekConfig, and builds items for a query', async () => { + const peekConfig = vi.fn(async (id: string) => config(id)); + const api = makeApi({ + a: [track(1, [['fish', 0.9]], [0, 1, 2]), track(2, [['shark', 0.3]], [4])], + }, { peekConfig }); + const service = createReviewService({ api }); + await service.refreshAvailable(); + await service.addDatasets(['a', 'a']); + expect(peekConfig).toHaveBeenCalledTimes(1); + expect(api.loadConfig).not.toHaveBeenCalled(); + expect(service.datasets.value).toHaveLength(1); + expect(service.datasets.value[0]).toMatchObject({ + id: 'a', name: 'Alpha', status: 'ready', trackCount: 2, croppable: true, + }); + expect(service.types.value).toEqual(['fish', 'shark']); + expect(service.items.value.map((i) => i.key)).toEqual(['a#1', 'a#2']); + + service.query.threshold = 0.5; + service.runQuery(); + expect(service.stale.value).toBe(false); + expect(service.items.value.map((i) => i.key)).toEqual(['a#1']); + expect(service.items.value[0].frames).toHaveLength(3); + }); + + it('expands a multicamera parent into its cameras', async () => { + const api = makeApi({ 'm/left': [track(1, [['fish', 1]], [0])], 'm/right': [] }, { + loadConfig: vi.fn(async (id: string) => (id === 'm' + ? config(id, { + type: 'multi', + multiCamMedia: { + defaultDisplay: 'left', + cameras: { + left: { type: 'image-sequence', imageData: [], videoUrl: '' }, + right: { type: 'image-sequence', imageData: [], videoUrl: '' }, + }, + }, + }) + : config(id))), + }); + const service = createReviewService({ api }); + await service.addDataset('m', { id: 'm', name: 'Rig' }); + expect(service.datasets.value.map((d) => [d.id, d.name, d.status])).toEqual([ + ['m/left', 'Rig (left)', 'ready'], + ['m/right', 'Rig (right)', 'ready'], + ]); + }); + + it('reassigns and accepts types, tracks pending edits, and saves them', async () => { + const api = makeApi({ + a: [track(1, [['fish', 0.6], ['shark', 0.4]], [0]), track(2, [['fish', 0.9]], [0])], + }); + const service = createReviewService({ api }); + await service.addDataset('a'); + service.query.type = 'fish'; + service.query.threshold = 0; + service.runQuery(); + const first = service.items.value.find((i) => i.trackId === 1)!; + const second = service.items.value.find((i) => i.trackId === 2)!; + + service.assignType(first, 'shark'); + expect(service.currentType(first)).toEqual({ type: 'shark', confidence: 1 }); + expect(service.trackOf('a', 1)?.confidencePairs).toEqual([['shark', 1]]); + expect(service.isPending(first)).toBe(true); + expect(service.pendingCount.value).toBe(1); + // The grid keeps the item until the query is run again. + expect(service.items.value).toHaveLength(2); + + service.acceptType(second); + expect(service.trackOf('a', 2)?.confidencePairs).toEqual([['fish', 1]]); + expect(service.pendingCount.value).toBe(2); + + await service.save(); + expect(api.saveDetections).toHaveBeenCalledTimes(1); + const [datasetId, args] = (api.saveDetections as ReturnType).mock.calls[0]; + expect(datasetId).toBe('a'); + expect(args.tracks.upsert.map((t: TrackData) => t.id).sort()).toEqual([1, 2]); + expect(args.groups).toEqual({ upsert: [], delete: [] }); + expect(service.pendingCount.value).toBe(0); + expect(service.error.value).toBeNull(); + }); + + it('edits a keyframe box and points, marks the track pending, and refreshes the item', async () => { + const api = makeApi({ a: [track(1, [['fish', 0.6]], [0, 4])] }); + const service = createReviewService({ api }); + await service.addDataset('a'); + service.query.type = ''; + service.query.threshold = 0; + service.runQuery(); + const [item] = service.items.value; + expect(item.primary.bounds).toEqual([0, 0, 10, 10]); + + service.updateGeometry(item, 4, { bounds: [22.4, 8, 2.6, 30], head: [5, 5], tail: [9, 9] }); + const feature = service.trackOf('a', 1)?.features.find((f) => f.frame === 4); + expect(feature?.bounds).toEqual([3, 8, 22, 30]); + expect(feature?.head).toEqual([5, 5]); + const keys = feature?.geometry?.features.map((g) => (g.properties as { key: string }).key).sort(); + expect(keys).toEqual(['HeadTails', 'head', 'tail']); + expect(item.frames.find((f) => f.frame === 4)).toMatchObject({ bounds: [3, 8, 22, 30], head: [5, 5], tail: [9, 9] }); + expect(item.primary.bounds).toEqual([0, 0, 10, 10]); + expect(service.isPending(item)).toBe(true); + + service.updateGeometry(item, 4, { tail: null }); + expect(feature?.tail).toBeUndefined(); + expect(feature?.geometry?.features.map((g) => (g.properties as { key: string }).key)).toEqual(['head']); + }); + + it('groups a track across cameras, adds boxes where a side is missing, and deletes tracks', async () => { + const api = makeApi({ 'm/left': [track(3, [['fish', 1]], [0, 4])], 'm/right': [track(3, [['fish', 1]], [4])] }, { + loadConfig: vi.fn(async (id: string) => (id === 'm' + ? config(id, { + type: 'multi', + multiCamMedia: { + defaultDisplay: 'left', + cameras: { + left: { type: 'image-sequence', imageData: [], videoUrl: '' }, + right: { type: 'image-sequence', imageData: [], videoUrl: '' }, + }, + }, + }) + : config(id))), + }); + const service = createReviewService({ api }); + await service.addDataset('m', { id: 'm', name: 'Rig' }); + service.query.threshold = 0; + service.runQuery(); + + expect(service.entries.value).toHaveLength(1); + const [entry] = service.entries.value; + expect(entry.labels).toEqual(['left', 'right']); + expect(service.parentOf('m/right')).toBe('m'); + const right = entry.items[1]; + expect(right.frames.map((f) => f.missing ?? false)).toEqual([true, false]); + + service.addKeyframe(right, 0, right.frames[0].bounds); + expect(service.trackOf('m/right', 3)?.features.map((f) => f.frame)).toEqual([0, 4]); + expect(service.trackOf('m/right', 3)?.begin).toBe(0); + expect(service.entries.value[0].items[1].frames.every((f) => !f.missing)).toBe(true); + expect(service.pendingCount.value).toBe(1); + + service.deleteTrack(entry.items[0]); + service.deleteTrack(entry.items[1]); + expect(service.entries.value).toHaveLength(0); + expect(service.pendingCount.value).toBe(2); + await service.save(); + const { calls } = (api.saveDetections as ReturnType).mock; + expect(calls.map(([id, args]) => [id, args.tracks.delete])).toEqual([['m/left', [3]], ['m/right', [3]]]); + expect(service.pendingCount.value).toBe(0); + }); + + it('reports a failed save and keeps the edits pending', async () => { + const api = makeApi({ a: [track(1, [['fish', 0.6]], [0])] }, { + saveDetections: vi.fn(async () => { throw new Error('disk full'); }), + }); + const service = createReviewService({ api }); + await service.addDataset('a'); + service.runQuery(); + service.assignType(service.items.value[0], 'shark'); + await service.save(); + expect(service.error.value).toBe('disk full'); + expect(service.pendingCount.value).toBe(1); + }); + + it('keeps edits and deletions made while a save is in flight', async () => { + let resolveFirst: (() => void) | undefined; + const api = makeApi({ + a: [track(1, [['fish', 0.6]], [0]), track(2, [['shark', 0.4]], [0]), track(3, [['ray', 0.5]], [0])], + }, { + saveDetections: vi.fn(() => new Promise((resolve) => { resolveFirst = resolve; })), + }); + const service = createReviewService({ api }); + await service.addDataset('a'); + service.query.threshold = 0; + service.runQuery(); + const first = service.items.value.find((i) => i.trackId === 1)!; + const second = service.items.value.find((i) => i.trackId === 2)!; + const third = service.items.value.find((i) => i.trackId === 3)!; + + service.assignType(first, 'shark'); + const savePromise = service.save(); + await Promise.resolve(); + expect(api.saveDetections).toHaveBeenCalledTimes(1); + expect((api.saveDetections as ReturnType).mock.calls[0][1].tracks.upsert + .map((t: TrackData) => t.id)).toEqual([1]); + + // A later edit of another track, a re-edit of the submitted track, and a + // deletion must all survive acknowledgement of the first request. + service.assignType(second, 'fish'); + service.assignType(first, 'ray'); + service.deleteTrack(third); + expect(service.pendingCount.value).toBe(3); + + resolveFirst?.(); + await savePromise; + expect(service.pendingCount.value).toBe(3); + expect(service.isPending(first)).toBe(true); + expect(service.isPending(second)).toBe(true); + expect(service.trackOf('a', 1)?.confidencePairs).toEqual([['ray', 1]]); + + (api.saveDetections as ReturnType).mockResolvedValue(undefined); + await service.save(); + expect(api.saveDetections).toHaveBeenCalledTimes(2); + const secondArgs = (api.saveDetections as ReturnType).mock.calls[1][1]; + expect(secondArgs.tracks.upsert.map((t: TrackData) => t.id).sort()).toEqual([1, 2]); + expect(secondArgs.tracks.delete).toEqual([3]); + expect(service.pendingCount.value).toBe(0); + }); + + it('sends a snapshot so mid-flight edits do not alter the in-flight payload', async () => { + let resolveFirst: (() => void) | undefined; + const api = makeApi({ a: [track(1, [['fish', 0.6]], [0])] }, { + saveDetections: vi.fn(() => new Promise((resolve) => { resolveFirst = resolve; })), + }); + const service = createReviewService({ api }); + await service.addDataset('a'); + service.runQuery(); + const [item] = service.items.value; + service.assignType(item, 'shark'); + const savePromise = service.save(); + await Promise.resolve(); + service.assignType(item, 'ray'); + resolveFirst?.(); + await savePromise; + + const firstUpsert = (api.saveDetections as ReturnType).mock.calls[0][1] + .tracks.upsert[0] as TrackData; + expect(firstUpsert.confidencePairs).toEqual([['shark', 1]]); + expect(service.trackOf('a', 1)?.confidencePairs).toEqual([['ray', 1]]); + expect(service.pendingCount.value).toBe(1); + }); + + it('discards edits by reloading the changed datasets', async () => { + const api = makeApi({ a: [track(1, [['fish', 0.6]], [0])] }); + const service = createReviewService({ api }); + await service.addDataset('a'); + service.runQuery(); + service.assignType(service.items.value[0], 'shark'); + await service.discardChanges(); + expect(api.loadDetections).toHaveBeenCalledTimes(2); + expect(service.pendingCount.value).toBe(0); + expect(service.trackOf('a', 1)?.confidencePairs).toEqual([['fish', 0.6]]); + }); + + it('marks a dataset that failed to load and lets it be removed', async () => { + const api = makeApi({}, { + loadDetections: vi.fn(async () => { throw new Error('missing'); }), + }); + const service = createReviewService({ api }); + await service.addDataset('a'); + expect(service.datasets.value[0]).toMatchObject({ status: 'error', error: 'missing' }); + service.removeDataset('a'); + await nextTick(); + expect(service.datasets.value).toHaveLength(0); + expect(service.types.value).toEqual([]); + }); +}); +describe('review session regressions', () => { + it('does not strand another dataset when one is removed during loading', async () => { + let finish!: (value: Awaited>) => void; + const api = makeApi({}, { + loadDetections: vi.fn(() => new Promise>>((resolve) => { finish = resolve; })), + }); + const service = createReviewService({ api }); + await service.addDataset('a', undefined, { defer: true }); + const loading = service.addDataset('b'); + await Promise.resolve(); + service.removeDataset('a'); + finish({ + tracks: [], groups: [], sets: [], version: 2, + }); + await loading; + expect(service.datasets.value[0].status).toBe('ready'); + service.dispose(); + }); + + it('edits both cameras when only one meets the type threshold', async () => { + const api = makeApi({ + 'm/left': [track(1, [['fish', 0.9]], [0])], + 'm/right': [track(1, [['fish', 0.2]], [0])], + }, { + loadConfig: vi.fn(async (id) => (id === 'm' ? config(id, { + type: 'multi', + multiCamMedia: { + defaultDisplay: 'left', + cameras: { + left: { type: 'image-sequence', imageData: [], videoUrl: '' }, + right: { type: 'image-sequence', imageData: [], videoUrl: '' }, + }, + }, + }) : config(id))), + }); + const service = createReviewService({ api }); + await service.addDataset('m'); + service.query.type = 'fish'; + service.query.threshold = 0.5; + service.runQuery(); + expect(service.entries.value[0].items).toHaveLength(2); + service.entries.value[0].items.forEach((item) => service.assignType(item, 'shark')); + await service.save(); + const { calls } = (api.saveDetections as ReturnType).mock; + expect(calls.map(([id, args]) => [id, args.tracks.upsert[0].confidencePairs])).toEqual([ + ['m/left', [['shark', 1]]], ['m/right', [['shark', 1]]], + ]); + service.dispose(); + }); + + it('keeps query watches and dirty counts active after the owner unmounts', async () => { + const scope = effectScope(); + const service = scope.run(() => createReviewService({ + api: makeApi({ a: [track(1, [['fish', 0.9]], [0]), track(2, [['shark', 0.8]], [0])] }), + }))!; + await service.addDataset('a'); + expect(service.pendingCount.value).toBe(0); + scope.stop(); + service.query.type = 'shark'; + await nextTick(); + await new Promise((resolve) => setTimeout(resolve, 250)); + expect(service.items.value.map((item) => item.trackId)).toEqual([2]); + service.assignType(service.items.value[0], 'ray'); + expect(service.pendingCount.value).toBe(1); + service.dispose(); + }); + + it('reports failure and retains edits when discard cannot reload', async () => { + const api = makeApi({ a: [track(1, [['fish', 0.9]], [0])] }); + const service = createReviewService({ api }); + await service.addDataset('a'); + service.assignType(service.items.value[0], 'shark'); + api.loadDetections = vi.fn(async () => { throw new Error('offline'); }); + expect(await service.discardChanges()).toBe(false); + expect(service.pendingCount.value).toBe(1); + expect(service.datasets.value[0].status).toBe('error'); + service.dispose(); + }); +}); + +it('refreshes viewer changes before another review edit is saved', async () => { + const stored = { a: [track(1, [['fish', 0.9]], [0])] }; + const api = makeApi(stored); + const service = createReviewService({ api }); + await service.addDataset('a'); + stored.a[0].features[0].bounds = [20, 20, 40, 40]; + expect(await service.refreshOnResume()).toBe(true); + service.assignType(service.items.value[0], 'shark'); + await service.save(); + const args = (api.saveDetections as ReturnType).mock.calls[0][1]; + expect(args.tracks.upsert[0].features[0].bounds).toEqual([20, 20, 40, 40]); + service.dispose(); +}); + +it('does not expose stale tracks if the resume refresh fails', async () => { + const api = makeApi({ a: [track(1, [['fish', 0.9]], [0])] }); + const service = createReviewService({ api }); + await service.addDataset('a'); + (api.loadDetections as ReturnType).mockRejectedValue(new Error('offline')); + expect(await service.refreshOnResume()).toBe(false); + expect(service.items.value).toEqual([]); + expect(service.trackOf('a', 1)).toBeUndefined(); + expect(service.datasets.value[0].status).toBe('error'); + service.dispose(); +}); + +it('preserves dirty data instead of replacing it on resume', async () => { + const service = createReviewService({ api: makeApi({ a: [track(1, [['fish', 0.9]], [0])] }) }); + await service.addDataset('a'); + service.assignType(service.items.value[0], 'shark'); + expect(await service.refreshOnResume()).toBe(false); + expect(service.pendingCount.value).toBe(1); + expect(service.trackOf('a', 1)?.confidencePairs).toEqual([['shark', 1]]); + service.dispose(); +}); diff --git a/client/dive-common/use/useReview.ts b/client/dive-common/use/useReview.ts new file mode 100644 index 000000000..dbfd67405 --- /dev/null +++ b/client/dive-common/use/useReview.ts @@ -0,0 +1,774 @@ +/** + * State behind the Review page: the datasets under review (with their + * tracks held in memory), the query, the grid settings, the rendered + * chips, and the type edits waiting to be saved back. + */ +import { + computed, effectScope, inject, provide, reactive, ref, Ref, watch, +} from 'vue'; +import { debounce } from 'lodash'; +import type { Api, DatasetConfig } from 'dive-common/apispec'; +import type { ScoringDatasetSummary } from 'dive-common/scoring/types'; +import type { Feature, TrackData } from 'vue-media-annotator/track'; +import type { AnnotationId, ConfidencePair } from 'vue-media-annotator/BaseAnnotation'; +import type { RectBounds } from 'vue-media-annotator/utils'; +import StyleManager from 'vue-media-annotator/StyleManager'; +import { + acceptPairAsCorrect, compileHierarchy, reassignPairs, TypeHierarchyIndex, +} from 'dive-common/typeHierarchy'; +import { createFrameSource, FrameSource } from 'dive-common/review/frameSource'; +import { createChipStore, ChipStore } from 'dive-common/review/chipStore'; +import { + buildReviewItems, CameraMembership, collectAttributeKeys, collectTypes, frameRefFor, groupReviewItems, + sortReviewItems, +} from 'dive-common/review/reviewItems'; +import { usePersistentGridSettings } from 'dive-common/review/gridSettings'; +import { + DEFAULT_REVIEW_QUERY, + ReviewEntry, + ReviewGridSettings, + ReviewItem, + ReviewPolygon, + ReviewQuery, + ReviewSortOrder, +} from 'dive-common/review/types'; + +const CHIP_OUTLINE = '#00e5ff'; + +/** Geometry edits for one keyframe; omitted fields are left alone, null removes a point. */ +export interface ReviewGeometryEdit { + bounds?: RectBounds; + polygons?: ReviewPolygon[]; + head?: [number, number] | null; + tail?: [number, number] | null; +} + +export type ReviewApi = Pick; + +export interface ReviewServiceDeps { + api: ReviewApi; +} + +/** `queued` datasets were picked but load only once results are wanted. */ +export type ReviewDatasetStatus = 'queued' | 'loading' | 'ready' | 'error'; + +export interface ReviewDataset { + id: string; + name: string; + type?: string; + status: ReviewDatasetStatus; + error?: string; + trackCount: number; + /** True when the media can be cropped into chips. */ + croppable: boolean; +} + +export interface ReviewService { + datasets: Readonly>; + available: Readonly>; + query: ReviewQuery; + grid: ReviewGridSettings; + sort: Ref; + items: Readonly>; + /** Items grouped into grid entries (one per track across its cameras). */ + entries: Readonly>; + /** Bumps whenever tracks load or change; computeds that read tracks depend on it. */ + dataRevision: Readonly>; + /** True once tracks or the query changed after the last run. */ + stale: Readonly>; + types: Readonly>; + knownTypes: Readonly>; + attributeKeys: Readonly>; + pendingCount: Readonly>; + saving: Readonly>; + loading: Readonly>; + error: Readonly>; + chipStore: ChipStore; + datasetName(id: string): string; + refreshAvailable(): Promise; + addDataset(id: string, summary?: ScoringDatasetSummary, options?: { defer?: boolean }): Promise; + loadQueued(): Promise; + addDatasets(ids: string[]): Promise; + removeDataset(id: string): void; + reloadDataset(id: string): Promise; + runQuery(): void; + trackOf(datasetId: string, trackId: AnnotationId): TrackData | undefined; + /** The item's live top type/confidence after any edits. */ + currentType(item: ReviewItem): { type: string; confidence: number }; + /** The colour the annotator draws a type in (custom dataset styles applied). */ + colorFor(type: string): string; + /** Frames per second the dataset is annotated at, or 0 when unknown. */ + datasetFps(id: string): number; + /** The multicamera parent a camera dataset was expanded from, or the id itself. */ + parentOf(id: string): string; + /** Add a keyframe with a box to a track, e.g. where one camera lacks a detection. */ + addKeyframe(item: ReviewItem, frame: number, bounds: RectBounds): void; + /** Remove the track behind an item; written on the next save. */ + deleteTrack(item: ReviewItem): void; + isPending(item: ReviewItem): boolean; + assignType(item: ReviewItem, type: string): void; + acceptType(item: ReviewItem): void; + /** Change a keyframe's box, polygons or head/tail points; re-renders the item's chips. */ + updateGeometry(item: ReviewItem, frame: number, edit: ReviewGeometryEdit): void; + save(): Promise; + discardChanges(): Promise; + refreshOnResume(): Promise; + clearError(): void; + dispose(): void; +} + +interface LoadedDataset { + config: DatasetConfig; + tracks: Map; + hierarchy: TypeHierarchyIndex; + frameSource: FrameSource | null; + pending: Set; + /** + * Monotonic edit version per pending track. A save snapshots these and only + * acknowledges a track when the version still matches, so an edit made while + * the request is in flight stays dirty. + */ + pendingVersions: Map; + /** Tracks removed here and not yet deleted on the platform. */ + deleted: Set; +} + +type GeoFeature = NonNullable['features'][number]; + +function featureKey(geo: GeoFeature): unknown { + return (geo.properties as { key?: unknown } | null)?.key; +} + +/** Write a head/tail point into the keyframe the way DIVE stores it. */ +function setKeypoint(feature: Feature, key: 'head' | 'tail', point: [number, number] | null) { + const collection = feature.geometry || { type: 'FeatureCollection' as const, features: [] }; + const remaining = collection.features.filter( + (geo) => !(geo.geometry.type === 'Point' && featureKey(geo) === key), + ); + if (point) { + remaining.push({ + type: 'Feature', + properties: { key }, + geometry: { type: 'Point', coordinates: [point[0], point[1]] }, + }); + } + collection.features = remaining; + // eslint-disable-next-line no-param-reassign + feature.geometry = collection; + if (point) { + // eslint-disable-next-line no-param-reassign + feature[key] = [point[0], point[1]]; + } else { + // eslint-disable-next-line no-param-reassign + delete feature[key]; + } +} + +/** Keep the head-to-tail line in step with its end points. */ +function syncHeadTailLine(feature: Feature) { + const collection = feature.geometry; + if (!collection) return; + const head = collection.features.find((g) => g.geometry.type === 'Point' && featureKey(g) === 'head'); + const tail = collection.features.find((g) => g.geometry.type === 'Point' && featureKey(g) === 'tail'); + collection.features = collection.features.filter( + (geo) => !(geo.geometry.type === 'LineString' && featureKey(geo) === 'HeadTails'), + ); + if (head && tail && head.geometry.type === 'Point' && tail.geometry.type === 'Point') { + collection.features.push({ + type: 'Feature', + properties: { key: 'HeadTails' }, + geometry: { type: 'LineString', coordinates: [head.geometry.coordinates, tail.geometry.coordinates] }, + }); + } +} + +/** Replace the outer rings of the keyframe's polygons, in order. */ +function setPolygons(feature: Feature, polygons: ReviewPolygon[]) { + const collection = feature.geometry; + if (!collection) return; + let index = 0; + collection.features.forEach((geo) => { + if (geo.geometry.type !== 'Polygon') return; + const ring = polygons[index]; + index += 1; + if (!ring || ring.length < 3) return; + const holes = geo.geometry.coordinates.slice(1); + // eslint-disable-next-line no-param-reassign + geo.geometry.coordinates = [[...ring.map(([x, y]) => [x, y]), [ring[0][0], ring[0][1]]], ...holes]; + }); +} + +function topPair(pairs: readonly ConfidencePair[]): { type: string; confidence: number } { + const top = pairs.reduce( + (acc, pair) => (acc === null || pair[1] > acc[1] ? pair : acc), + null, + ); + return top ? { type: top[0], confidence: top[1] } : { type: '', confidence: 0 }; +} + +export function createReviewService(deps: ReviewServiceDeps): ReviewService { + const scope = effectScope(true); + const service = scope.run(() => createScopedReviewService(deps))!; + const { dispose } = service; + service.dispose = () => { + dispose(); + scope.stop(); + }; + return service; +} + +function createScopedReviewService(deps: ReviewServiceDeps): ReviewService { + const { api } = deps; + const datasets = ref([]); + const available = ref([]); + const query = reactive({ ...DEFAULT_REVIEW_QUERY }); + const grid = usePersistentGridSettings(); + const sort = ref('confidence-desc'); + const items = ref([]); + const dataRevision = ref(0); + const stale = ref(false); + const saving = ref(false); + const loading = ref(false); + const error = ref(null); + /** Tracks and media, deliberately outside Vue reactivity (they can be large). */ + const loaded = new Map(); + /** Loads still in flight, so a removal during load is honoured. */ + const loadTokens = new Map(); + /** Type colours as the annotator assigns them, seeded from each dataset's custom styles. */ + const styles = new StyleManager({ markChangesPending: () => undefined }); + /** Camera datasets expanded from a multicamera parent. */ + const memberships = new Map(); + + // Query changes apply as soon as they settle; the grid only reshuffles + // for those, never for edits made in it. + const runQuerySettled = debounce(() => runQuery(), 200); + watch(query, () => { + stale.value = true; + runQuerySettled(); + }, { deep: true }); + + const chipStore = createChipStore({ + frameSourceFor: (datasetId) => loaded.get(datasetId)?.frameSource ?? null, + }, { + padding: grid.padding, size: 256, aspect: 1, outline: CHIP_OUTLINE, + }); + + function fail(reason: unknown, fallback: string) { + const message = reason instanceof Error ? reason.message : String(reason || fallback); + error.value = message || fallback; + } + + function datasetName(id: string) { + return datasets.value.find((d) => d.id === id)?.name + || available.value.find((d) => d.id === id)?.name + || id; + } + + function entry(id: string) { + return datasets.value.find((d) => d.id === id); + } + + function patch(id: string, changes: Partial) { + datasets.value = datasets.value.map((d) => (d.id === id ? { ...d, ...changes } : d)); + } + + async function refreshAvailable() { + if (!api.listScoringDatasets) return; + try { + available.value = await api.listScoringDatasets(); + } catch (err) { + fail(err, 'Could not list datasets'); + } + } + + function loadConfig(id: string) { + return api.peekConfig ? api.peekConfig(id) : api.loadConfig(id); + } + + function frameSourceFor(config: DatasetConfig): FrameSource | null { + try { + return createFrameSource(config); + } catch { + return null; + } + } + + function dropLoaded(id: string) { + const existing = loaded.get(id); + if (existing) { + existing.frameSource?.dispose(); + loaded.delete(id); + } + } + + async function load(id: string) { + const token = Symbol(id); + loadTokens.set(id, token); + const isCurrent = () => loadTokens.get(id) === token && !!entry(id); + loading.value = true; + try { + const config = await loadConfig(id); + if (!isCurrent()) return; + if (config.type === 'multi') { + // Review the cameras of a multicamera dataset as separate sequences. + const cameras = Object.keys(config.multiCamMedia?.cameras || {}); + const parentName = entry(id)?.name || config.name; + datasets.value = datasets.value.filter((d) => d.id !== id); + cameras.forEach((camera, rank) => memberships.set(`${id}/${camera}`, { parent: id, camera, rank })); + await Promise.all(cameras.map((camera) => addDataset(`${id}/${camera}`, { + id: `${id}/${camera}`, name: `${parentName} (${camera})`, type: config.multiCamMedia?.cameras[camera]?.type, + }))); + return; + } + const detections = await api.loadDetections(id); + if (!isCurrent()) return; + dropLoaded(id); + const tracks = new Map(); + detections.tracks.forEach((track) => tracks.set(track.id, track)); + const frameSource = frameSourceFor(config); + if (config.customTypeStyling) { + styles.populateTypeStyles({ ...styles.customStyles.value, ...config.customTypeStyling }); + } + loaded.set(id, { + config, + tracks, + hierarchy: compileHierarchy(config.typeHierarchy || {}), + frameSource, + pending: new Set(), + pendingVersions: new Map(), + deleted: new Set(), + }); + patch(id, { + status: 'ready', + error: undefined, + name: entry(id)?.name || config.name, + type: config.type, + trackCount: tracks.size, + croppable: frameSource !== null, + }); + dataRevision.value += 1; + // Newly loaded tracks join the grid without any further action. + runQuery(); + } catch (err) { + if (!isCurrent()) return; + patch(id, { + status: 'error', + error: err instanceof Error ? err.message : String(err), + }); + } finally { + loading.value = datasets.value.some((d) => d.status === 'loading'); + } + } + + /** + * Add a dataset; with `defer` it only joins the list and loads on the + * next `loadQueued`, so picking many datasets costs nothing until the + * results are actually wanted. + */ + async function addDataset(id: string, summary?: ScoringDatasetSummary, options: { defer?: boolean } = {}) { + if (!id || entry(id)) return; + datasets.value = [...datasets.value, { + id, + name: summary?.name || datasetName(id), + type: summary?.type, + status: options.defer ? 'queued' : 'loading', + trackCount: 0, + croppable: false, + }]; + if (options.defer) return; + await load(id); + } + + /** Load every queued dataset; annotations are read and the query rerun as each arrives. */ + async function loadQueued() { + const queued = datasets.value.filter((d) => d.status === 'queued').map((d) => d.id); + queued.forEach((id) => patch(id, { status: 'loading' })); + await Promise.all(queued.map((id) => load(id))); + } + + async function addDatasets(ids: string[]) { + const unique = Array.from(new Set(ids.filter(Boolean))); + await Promise.all(unique.map((id) => addDataset(id))); + } + + function removeDataset(id: string) { + datasets.value = datasets.value.filter((d) => d.id !== id); + dropLoaded(id); + loadTokens.delete(id); + dataRevision.value += 1; + runQuery(); + } + + async function reloadDataset(id: string) { + if (!entry(id)) return; + patch(id, { status: 'loading', error: undefined }); + await load(id); + } + + function allTracks(): TrackData[] { + const all: TrackData[] = []; + loaded.forEach((dataset) => all.push(...dataset.tracks.values())); + return all; + } + + /** Read inside a computed so it re-runs when tracks load or change. */ + function dependOnData(): number { + return dataRevision.value; + } + + /** Types the query can ask for: present on the selected datasets at the current threshold. */ + const types = computed(() => { + dependOnData(); + return collectTypes(allTracks(), query.threshold); + }); + + /** Every type on the selected datasets, offered when assigning a type. */ + const knownTypes = computed(() => { + dependOnData(); + return collectTypes(allTracks()); + }); + + const attributeKeys = computed(() => { + dependOnData(); + const keys = new Set(); + loaded.forEach((dataset) => { + collectAttributeKeys(dataset.tracks.values(), dataset.config.attributes).forEach((k) => keys.add(k)); + }); + return Array.from(keys).sort((a, b) => a.localeCompare(b)); + }); + + function runQuery() { + const order = datasets.value.map((d) => d.id); + const built: ReviewItem[] = []; + order.forEach((id) => { + const dataset = loaded.get(id); + if (dataset) { + built.push(...buildReviewItems(id, dataset.tracks.values(), { ...query }, grid.maxSequenceFrames)); + } + }); + // A matching camera selects the logical track. Include every other camera + // of that track, even when its label/confidence/attributes do not match. + const matched = new Map>(); + built.forEach((item) => { + const parent = memberships.get(item.datasetId)?.parent; + if (!parent) return; + if (!matched.has(parent)) matched.set(parent, new Set()); + matched.get(parent)!.add(item.trackId); + }); + const included = new Set(built.map((item) => `${item.datasetId}#${item.trackId}`)); + loaded.forEach((dataset, id) => { + const parent = memberships.get(id)?.parent; + const trackIds = parent ? matched.get(parent) : undefined; + if (!trackIds) return; + const cameras = buildReviewItems(id, dataset.tracks.values(), { + ...DEFAULT_REVIEW_QUERY, mode: 'type', type: '', threshold: 0, + }, grid.maxSequenceFrames); + cameras.forEach((item) => { + if (trackIds.has(item.trackId) && !included.has(`${id}#${item.trackId}`)) built.push(item); + }); + }); + items.value = sortReviewItems(built, sort.value, order); + stale.value = false; + } + + watch(sort, () => { + items.value = sortReviewItems(items.value, sort.value, datasets.value.map((d) => d.id)); + }); + + function trackOf(datasetId: string, trackId: AnnotationId) { + return loaded.get(datasetId)?.tracks.get(trackId); + } + + function parentOf(id: string) { + return memberships.get(id)?.parent ?? id; + } + + const entries = computed(() => { + dependOnData(); + return groupReviewItems( + items.value, + (datasetId) => memberships.get(datasetId), + (item) => trackOf(item.datasetId, item.trackId), + grid.maxSequenceFrames, + ); + }); + + function currentType(item: ReviewItem) { + const track = trackOf(item.datasetId, item.trackId); + if (!track) return { type: item.type, confidence: item.confidence }; + if (query.mode === 'type' && query.type) { + // Keep showing the queried pair while it still exists, so an edit that + // demotes it is visible as such. + const pair = track.confidencePairs.find(([type]) => type === query.type); + const top = topPair(track.confidencePairs); + if (pair && top.type === query.type) return { type: pair[0], confidence: pair[1] }; + return top; + } + return topPair(track.confidencePairs); + } + + function colorFor(type: string) { + return styles.typeStyling.value.color(type); + } + + function datasetFps(id: string) { + const fps = Number(loaded.get(id)?.config.fps); + return Number.isFinite(fps) && fps > 0 ? fps : 0; + } + + function isPending(item: ReviewItem) { + return loaded.get(item.datasetId)?.pending.has(item.trackId) ?? false; + } + + const pendingCount = computed(() => { + dependOnData(); + let count = 0; + loaded.forEach((dataset) => { count += dataset.pending.size + dataset.deleted.size; }); + return count; + }); + + /** Mark a track dirty and bump its edit version so in-flight saves cannot clear it. */ + function markPending(dataset: LoadedDataset, trackId: AnnotationId) { + dataset.pending.add(trackId); + dataset.pendingVersions.set(trackId, (dataset.pendingVersions.get(trackId) ?? 0) + 1); + } + + function updatePairs(item: ReviewItem, update: (pairs: ConfidencePair[], hierarchy: TypeHierarchyIndex) => ConfidencePair[]) { + const dataset = loaded.get(item.datasetId); + const track = dataset?.tracks.get(item.trackId); + if (!dataset || !track) return; + const next = update(track.confidencePairs.map(([t, c]) => [t, c] as ConfidencePair), dataset.hierarchy); + track.confidencePairs = next; + markPending(dataset, item.trackId); + dataRevision.value += 1; + } + + function assignType(item: ReviewItem, type: string) { + const trimmed = type.trim(); + if (!trimmed) return; + const current = currentType(item); + if (current.type === trimmed && current.confidence >= 1) return; + updatePairs(item, (pairs, hierarchy) => reassignPairs( + hierarchy, + pairs, + current.type || trimmed, + trimmed, + 1, + )); + } + + function acceptType(item: ReviewItem) { + const current = currentType(item); + if (!current.type) return; + updatePairs(item, (pairs, hierarchy) => acceptPairAsCorrect(hierarchy, pairs, current.type)); + } + + function deleteTrack(item: ReviewItem) { + const dataset = loaded.get(item.datasetId); + if (!dataset || !dataset.tracks.has(item.trackId)) return; + dataset.tracks.delete(item.trackId); + dataset.pending.delete(item.trackId); + dataset.pendingVersions.delete(item.trackId); + dataset.deleted.add(item.trackId); + // The entry leaves the grid at once; everything else stays put. + items.value = items.value.filter( + (other) => !(other.datasetId === item.datasetId && other.trackId === item.trackId), + ); + dataRevision.value += 1; + } + + function addKeyframe(item: ReviewItem, frame: number, bounds: RectBounds) { + const dataset = loaded.get(item.datasetId); + const track = dataset?.tracks.get(item.trackId); + if (!dataset || !track || track.features.some((f) => f.frame === frame && f.bounds)) return; + const [x1, y1, x2, y2] = bounds; + const previous = [...track.features].reverse().find((f) => f.frame < frame); + const feature: Feature = { + frame, + keyframe: true, + interpolate: previous?.interpolate ?? false, + bounds: [ + Math.round(Math.min(x1, x2)), Math.round(Math.min(y1, y2)), + Math.round(Math.max(x1, x2)), Math.round(Math.max(y1, y2)), + ], + }; + track.features = [...track.features.filter((f) => f.frame !== frame), feature] + .sort((a, b) => a.frame - b.frame); + track.begin = Math.min(track.begin, frame); + track.end = Math.max(track.end, frame); + markPending(dataset, item.trackId); + dataRevision.value += 1; + } + + function updateGeometry(item: ReviewItem, frame: number, edit: ReviewGeometryEdit) { + const dataset = loaded.get(item.datasetId); + const track = dataset?.tracks.get(item.trackId); + const feature = track?.features.find((f) => f.frame === frame); + if (!dataset || !track || !feature) return; + + if (edit.bounds) { + const [x1, y1, x2, y2] = edit.bounds; + feature.bounds = [ + Math.round(Math.min(x1, x2)), Math.round(Math.min(y1, y2)), + Math.round(Math.max(x1, x2)), Math.round(Math.max(y1, y2)), + ]; + } + if (edit.head !== undefined) setKeypoint(feature, 'head', edit.head); + if (edit.tail !== undefined) setKeypoint(feature, 'tail', edit.tail); + if (edit.head !== undefined || edit.tail !== undefined) syncHeadTailLine(feature); + if (edit.polygons) setPolygons(feature, edit.polygons); + + // The grid item mirrors the keyframe; refresh it so overlays and the + // re-rendered chip follow the edit. + const refreshed = frameRefFor(feature); + if (refreshed) { + item.frames.forEach((ref) => { + if (ref.frame === frame) Object.assign(ref, refreshed); + }); + if (item.primary.frame === frame) Object.assign(item.primary, refreshed); + } + markPending(dataset, item.trackId); + dataRevision.value += 1; + // The chip keeps its crop: the box is drawn over it, so the view does + // not jump when an edit lands. + } + + async function save() { + if (saving.value) return; + saving.value = true; + error.value = null; + try { + const targets = Array.from(loaded.entries()) + .filter(([, d]) => d.pending.size > 0 || d.deleted.size > 0); + // Snapshot payloads and versions before awaiting so mid-flight edits + // neither mutate what we send nor get cleared on success. + const results = await Promise.allSettled(targets.map(async ([id, dataset]) => { + const submittedVersions = new Map(); + const upsert: TrackData[] = []; + dataset.pending.forEach((trackId) => { + const track = dataset.tracks.get(trackId); + if (!track) return; + submittedVersions.set(trackId, dataset.pendingVersions.get(trackId) ?? 0); + upsert.push(JSON.parse(JSON.stringify(track)) as TrackData); + }); + const submittedDeleted = Array.from(dataset.deleted); + await api.saveDetections(id, { + tracks: { upsert, delete: submittedDeleted }, + groups: { upsert: [], delete: [] }, + }); + submittedVersions.forEach((version, trackId) => { + if (dataset.pendingVersions.get(trackId) === version) { + dataset.pending.delete(trackId); + dataset.pendingVersions.delete(trackId); + } + }); + submittedDeleted.forEach((trackId) => { + dataset.deleted.delete(trackId); + }); + })); + const failed = results.find((r) => r.status === 'rejected') as PromiseRejectedResult | undefined; + if (failed) throw failed.reason; + } catch (err) { + fail(err, 'Could not save the changed annotations'); + } finally { + dataRevision.value += 1; + saving.value = false; + } + } + + async function discardChanges() { + const dirty = Array.from(loaded.entries()) + .filter(([, d]) => d.pending.size > 0 || d.deleted.size > 0).map(([id]) => id); + await Promise.all(dirty.map((id) => reloadDataset(id))); + const discarded = dirty.every((id) => !loaded.get(id)?.pending.size && !loaded.get(id)?.deleted.size); + if (!discarded) error.value = 'Could not discard all changes. Retry when the datasets are available.'; + return discarded; + } + + /** Reload clean snapshots before the resumed page can edit them. */ + async function refreshOnResume() { + if (pendingCount.value > 0 || saving.value) { + error.value = 'Save or discard pending changes before refreshing the review session.'; + return false; + } + const ids = datasets.value.filter((dataset) => dataset.status !== 'queued').map((dataset) => dataset.id); + ids.forEach((id) => dropLoaded(id)); + chipStore.reset(); + dataRevision.value += 1; + runQuery(); + await Promise.all(ids.map((id) => reloadDataset(id))); + return ids.every((id) => entry(id)?.status === 'ready'); + } + + function clearError() { + error.value = null; + } + + function dispose() { + runQuerySettled.cancel(); + loadTokens.clear(); + loaded.forEach((dataset) => dataset.frameSource?.dispose()); + loaded.clear(); + chipStore.reset(); + } + + return { + datasets, + available, + query, + grid, + sort, + items, + entries, + dataRevision, + stale, + types, + knownTypes, + attributeKeys, + pendingCount, + saving, + loading, + error, + chipStore, + datasetName, + refreshAvailable, + addDataset, + addDatasets, + loadQueued, + removeDataset, + reloadDataset, + runQuery, + trackOf, + currentType, + colorFor, + datasetFps, + parentOf, + addKeyframe, + deleteTrack, + isPending, + assignType, + acceptType, + updateGeometry, + save, + discardChanges, + refreshOnResume, + clearError, + dispose, + }; +} + +const ReviewSymbol = Symbol('review'); + +export function provideReview(service: ReviewService) { + provide(ReviewSymbol, service); +} + +export function useReview(): ReviewService { + const service = inject(ReviewSymbol, null); + if (!service) { + throw new Error('Review service not provided'); + } + return service; +} diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index 26e14b08e..ad1a48489 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -826,6 +826,11 @@ async function loadConfig(id: string) { return { ...data, calibration: data.multiCam?.calibration ?? null }; } +/** loadConfig without the recents bookkeeping the stateful wrapper adds. */ +function peekConfig(id: string) { + return loadConfig(id); +} + let scoringAnnotationPreviewFile: string | null = null; /** One-shot annotation file to load in the viewer (from scoring result links). */ @@ -939,6 +944,7 @@ export { exportScoringPdf, /* Standard Specification APIs */ loadConfig, + peekConfig, loadDetections, loadFrameMetadata, getPipelineList, diff --git a/client/platform/desktop/frontend/components/NavigationBar.vue b/client/platform/desktop/frontend/components/NavigationBar.vue index 04ff20a0e..e78ecfb3c 100644 --- a/client/platform/desktop/frontend/components/NavigationBar.vue +++ b/client/platform/desktop/frontend/components/NavigationBar.vue @@ -31,6 +31,9 @@ export default defineComponent({ Trainingmdi-brain + + Reviewmdi-view-grid-outline + Scoringmdi-chart-box-outline diff --git a/client/platform/desktop/frontend/components/Recent.vue b/client/platform/desktop/frontend/components/Recent.vue index 27630ab0b..040fca447 100644 --- a/client/platform/desktop/frontend/components/Recent.vue +++ b/client/platform/desktop/frontend/components/Recent.vue @@ -265,6 +265,10 @@ export default defineComponent({ function scoreSelected() { router.push({ name: 'scoring', query: selectedIdsQuery() }); } + + function reviewSelected() { + router.push({ name: 'review', query: selectedIdsQuery() }); + } function getTypeIcon(recent: JsonConfigCache) { if (recent.subType) { if (recent.subType === 'stereo') { @@ -360,6 +364,7 @@ export default defineComponent({ runPipelineOnSelected, runTrainingOnSelected, scoreSelected, + reviewSelected, isSelected, toggleSelected, toggleSelectAll, @@ -649,6 +654,27 @@ export default defineComponent({ Score the selected datasets against ground truth + + + Review the selected datasets' annotations as a grid +