Video search on top of the Review grid (rebase of #1743)

client/dive-common/apispec.ts

@@ -768,6 +768,96 @@ export interface RefineDetectionsResponse {
   detections?: TextQueryDetection[];
 }
 
+/**
+ * Video Search / IQR (rapid model generation) Types
+ */
+
+export type VideoSearchIndexMethod = 'detections' | 'tracking' | 'existing' | 'frames';
+
+/**
+ * One indexed media stream (video/sequence identifier) in the shared search
+ * database. All database rows key on this identifier, so a dataset can be
+ * added, updated, or removed from the index independently.
+ */
+export interface VideoSearchStreamEntry {
+  datasetId: string;
+  method: VideoSearchIndexMethod;
+  // frame rate the media was indexed at (video only; must match dataset fps)
+  fps?: number;
+  createdAt: string;
+}
+
+/** How a search index stores descriptors: per-stream files or an embedded PostgreSQL database. */
+export type SearchIndexBackend = 'files' | 'postgres';
+
+/** Sidecar metadata describing the shared search index. */
+export interface VideoSearchIndexMeta {
+  version: number;
+  /** Storage backend the index was built with; absent on indexes from before it was recorded (postgres). */
+  backend?: SearchIndexBackend;
+  // stream identifier (as reported in query results) -> source dataset
+  streams: Record<string, VideoSearchStreamEntry>;
+}
+
+export interface VideoSearchIndexStatus {
+  /** The shared index exists and is queryable (ITQ files present). */
+  built: boolean;
+  /** This dataset has been ingested into the index. */
+  indexed: boolean;
+  /** The stream entry for this dataset, when indexed. */
+  stream?: VideoSearchStreamEntry & { streamName: string };
+  /** Total datasets in the shared index. */
+  datasetCount: number;
+  meta?: VideoSearchIndexMeta;
+}
+
+export interface VideoSearchTrackState {
+  frame: number;
+  bbox?: [number, number, number, number];
+}
+
+export interface VideoSearchResultTrack {
+  id: number;
+  states: VideoSearchTrackState[];
+}
+
+/** One ranked similarity result returned by the query service. */
+export interface VideoSearchResult {
+  /** Unique reference for adjudication: "<session>:<instance_id>" */
+  ref: string;
+  /** Federated session index this result came from */
+  session: number;
+  /** Index directory this result came from */
+  index_dir: string;
+  instance_id: number;
+  query_id: string;
+  stream_id: string;
+  relevancy_score: number;
+  start_frame: number | null;
+  end_frame: number | null;
+  tracks: VideoSearchResultTrack[];
+}
+
+/** One dataset present in the shared search index. */
+export interface VideoSearchIndexInfo {
+  /** Stream identifier query results report for this dataset */
+  streamName: string;
+  datasetId: string;
+  /** Dataset display name */
+  name: string;
+  /** How the entry was built. */
+  method: VideoSearchIndexMethod;
+}
+
+export interface VideoSearchQueryResponse {
+  success: boolean;
+  error?: string;
+  descriptor_count?: number;
+  model_available?: boolean;
+  results?: VideoSearchResult[];
+  feedback_requests?: VideoSearchResult[];
+}
+
 export {
   provideApi,
   useApi,

client/dive-common/components/DatasetPicker.vue

@@ -182,7 +182,7 @@ export default defineComponent({
       </v-btn>
       <v-btn
         small
-        color="success"
+        color="grey lighten-1"
         outlined
         :disabled="addable.length === 0"
         class="ml-2"
@@ -193,7 +193,7 @@ export default defineComponent({
           small
           left
         >
-          mdi-check-all
+          mdi-select-all
         </v-icon>
         Select all{{ addable.length ? ` (${addable.length})` : '' }}
       </v-btn>

client/dive-common/components/Review/ReviewCell.vue

@@ -141,6 +141,11 @@ export default defineComponent({
       type: String,
       default: '#00e5ff',
     },
+    /** Outline the cell as accepted or rejected (e.g. search adjudication). */
+    highlight: {
+      type: String as PropType<'' | 'positive' | 'negative'>,
+      default: '',
+    },
   },
   setup(props, { emit }) {
     /** Chips currently in edit mode, to outline the whole entry. */
@@ -244,7 +249,12 @@ export default defineComponent({
 <template>
   <div
     class="review-cell"
-    :class="{ 'cell-pending': pending, 'cell-editing': editingCount > 0 }"
+    :class="{
+      'cell-pending': pending,
+      'cell-editing': editingCount > 0,
+      'cell-positive': highlight === 'positive',
+      'cell-negative': highlight === 'negative',
+    }"
     :style="{ '--cell-scale': scale }"
   >
     <div class="cell-views">
@@ -286,9 +296,12 @@ export default defineComponent({
       >
         <template
           v-if="$scopedSlots.actions"
-          #actions
+          #actions="scope"
         >
-          <slot name="actions" />
+          <slot
+            name="actions"
+            v-bind="scope"
+          />
         </template>
       </ReviewChip>
     </div>
@@ -331,6 +344,14 @@ export default defineComponent({
   &.cell-editing {
     border-color: #90caf9;
   }
+
+  &.cell-positive {
+    border-color: #4caf50;
+  }
+
+  &.cell-negative {
+    border-color: #f44336;
+  }
 }
 
 .cell-views {

client/dive-common/components/Review/ReviewChip.vue

@@ -8,7 +8,8 @@ import type { ReviewFrameRef, ReviewPolygon } from 'dive-common/review/types';
 
 /** Geometry of one frame while it is being edited, in image coordinates. */
 interface GeometryDraft {
-  bounds: RectBounds;
+  /** Null for entries without a box (whole-frame results). */
+  bounds: RectBounds | null;
   polygons: ReviewPolygon[];
   head: [number, number] | null;
   tail: [number, number] | null;
@@ -26,7 +27,7 @@ export interface ReviewChipGeometryEdit {
   /** Sequence slot edited (0 is the primary frame). */
   slot: number;
   frame: number;
-  bounds: RectBounds;
+  bounds: RectBounds | null;
   polygons: ReviewPolygon[];
   head: [number, number] | null;
   tail: [number, number] | null;
@@ -339,7 +340,7 @@ export default defineComponent({
 
     const boxRect = computed(() => {
       const g = shownGeometry.value;
-      if (!g) return null;
+      if (!g || !g.bounds) return null;
       const [x1, y1, x2, y2] = g.bounds;
       const a = chip([Math.min(x1, x2), Math.min(y1, y2)]);
       const b = chip([Math.max(x1, x2), Math.max(y1, y2)]);
@@ -399,7 +400,7 @@ export default defineComponent({
 
     function cloneDraft(g: GeometryDraft): GeometryDraft {
       return {
-        bounds: [...g.bounds] as RectBounds,
+        bounds: g.bounds ? [...g.bounds] as RectBounds : null,
         polygons: g.polygons.map((polygon) => polygon.map((p) => [p[0], p[1]] as [number, number])),
         head: g.head ? [g.head[0], g.head[1]] : null,
         tail: g.tail ? [g.tail[0], g.tail[1]] : null,
@@ -409,7 +410,7 @@ export default defineComponent({
     /** Turn the interpolated position into a real box the user can then adjust. */
     function addBox() {
       const frame = currentFrame.value;
-      if (!props.editable || !frame || !frame.missing) return;
+      if (!props.editable || !frame || !frame.missing || !frame.bounds) return;
       emit('add-box', { frame: frame.frame, bounds: frame.bounds });
     }
 
@@ -605,7 +606,7 @@ export default defineComponent({
       const dy = current[1] - drag.startImage[1];
       const { target, startDraft } = drag;
       if (target.kind === 'box') {
-        draft.value.bounds = moveBox(target.handle, startDraft.bounds, dx, dy);
+        if (startDraft.bounds) draft.value.bounds = moveBox(target.handle, startDraft.bounds, dx, dy);
       } else if (target.kind === 'vertex') {
         const origin = startDraft.polygons[target.polygon]?.[target.vertex];
         if (origin) {
@@ -967,7 +968,11 @@ export default defineComponent({
       class="cell-actions"
       @click.stop
     >
-      <slot name="actions">
+      <slot
+        name="actions"
+        :begin-edit="beginEdit"
+        :frame="currentFrame ? currentFrame.frame : undefined"
+      >
         <v-tooltip
           bottom
           open-delay="600"

client/dive-common/components/Review/ReviewPage.vue

@@ -188,7 +188,7 @@ export default defineComponent({
       const item = entry.items[viewIndex];
       if (!item) return;
       review.updateGeometry(item, edit.frame, {
-        bounds: edit.bounds,
+        bounds: edit.bounds ?? undefined,
         polygons: edit.polygons,
         head: edit.head,
         tail: edit.tail,

client/dive-common/review/chipRenderer.spec.ts

@@ -1,5 +1,5 @@
 import {
-  chipRegion, chipScale, chipSizeFor, toChipPoint, toImagePoint,
+  chipRegion, chipScale, chipSizeFor, frameRegion, toChipPoint, toImagePoint,
 } from './chipRenderer';
 
 describe('chipRegion', () => {
@@ -39,6 +39,20 @@ describe('chipSizeFor', () => {
   });
 });
 
+describe('frameRegion', () => {
+  it('covers the whole frame, padding the shorter direction to the aspect', () => {
+    expect(frameRegion(100, 50, 2)).toEqual({
+      x: 0, y: 0, width: 100, height: 50,
+    });
+    expect(frameRegion(100, 50, 1)).toEqual({
+      x: 0, y: -25, width: 100, height: 100,
+    });
+    expect(frameRegion(100, 50, 4)).toEqual({
+      x: -50, y: 0, width: 200, height: 50,
+    });
+  });
+});
+
 describe('chipScale', () => {
   it('fills the requested size, upscaling small crops and downscaling large ones', () => {
     expect(chipScale({

client/dive-common/review/chipRenderer.ts

@@ -79,6 +79,18 @@ export function chipSizeFor(cellPixels: number): number {
   return CHIP_SIZE_BUCKETS.find((size) => size >= wanted) ?? CHIP_SIZE_BUCKETS[CHIP_SIZE_BUCKETS.length - 1];
 }
 
+/** Crop region covering a whole frame at the requested aspect ratio, centred. */
+export function frameRegion(width: number, height: number, aspect = 1): ChipRegion {
+  const ratio = aspect > 0 && Number.isFinite(aspect) ? aspect : 1;
+  const w = Math.max(1, width);
+  const h = Math.max(1, height);
+  const regionWidth = w / h >= ratio ? w : h * ratio;
+  const regionHeight = w / h >= ratio ? w / ratio : h;
+  return {
+    x: (w - regionWidth) / 2, y: (h - regionHeight) / 2, width: regionWidth, height: regionHeight,
+  };
+}
+
 /**
  * Scale from frame pixels to chip pixels: the crop's longer side fills the
  * requested size. Small crops are upscaled so the chip is rendered once at
@@ -90,8 +102,15 @@ export function chipScale(region: ChipRegion, size: number): number {
   return Math.max(16, size) / longest;
 }
 
-export function renderChip(frame: DecodedFrame, bounds: RectBounds, options: ChipRenderOptions): RenderedChip {
-  const region = chipRegion(bounds, options.padding, options.aspect);
+/** Render a box crop, or the whole frame when no box is given. */
+export function renderChip(
+  frame: DecodedFrame,
+  bounds: RectBounds | null,
+  options: ChipRenderOptions,
+): RenderedChip {
+  const region = bounds
+    ? chipRegion(bounds, options.padding, options.aspect)
+    : frameRegion(frame.width, frame.height, options.aspect);
   const scale = chipScale(region, options.size);
   const canvas = document.createElement('canvas');
   canvas.width = Math.max(1, Math.round(region.width * scale));
@@ -122,7 +141,7 @@ export function renderChip(frame: DecodedFrame, bounds: RectBounds, options: Chi
       sh * scale,
     );
   }
-  if (options.outline) {
+  if (options.outline && bounds) {
     const [x1, y1, x2, y2] = bounds;
     ctx.strokeStyle = options.outline;
     ctx.lineWidth = Math.max(1, Math.round(Math.max(canvas.width, canvas.height) / 160));

client/dive-common/review/chipStore.ts

@@ -23,7 +23,7 @@ export interface ChipStoreOptions {
 
 export interface ChipStoreDeps {
   /** Frame access for a dataset; null when the dataset cannot be cropped. */
-  frameSourceFor(datasetId: string): FrameSource | null;
+  frameSourceFor(datasetId: string): FrameSource | null | Promise<FrameSource | null>;
   concurrency?: number;
   /** Retain recent rendered items when paging; visible items are always kept. */
   cacheSize?: number;
@@ -80,7 +80,7 @@ export function createChipStore(deps: ChipStoreDeps, initial: ChipStoreOptions)
   }
 
   async function render(job: ChipJob): Promise<RenderedChip> {
-    const source = deps.frameSourceFor(job.item.datasetId);
+    const source = await deps.frameSourceFor(job.item.datasetId);
     if (!source) throw new Error('Media for this dataset cannot be cropped');
     const frameRef = job.slot === null ? job.item.primary : job.item.frames[job.slot];
     const frame = await source.getFrame(frameRef.frame);

client/dive-common/review/frameSource.ts

@@ -290,3 +290,45 @@ export function createFrameSource(config: DatasetConfig, options: FrameSourceOpt
     return entry.url;
   }, imageData.length, cacheSize, cacheBytes);
 }
+
+/**
+ * Lazily creates one frame source per dataset from its config, for chip
+ * stores whose items can name any dataset (e.g. cross-dataset search
+ * results). A dataset whose config cannot be loaded or whose media cannot
+ * be cropped resolves to null.
+ */
+export function createFrameSourceRegistry(
+  loadConfig: (datasetId: string) => Promise<DatasetConfig>,
+  options: FrameSourceOptions = {},
+) {
+  const sources = new Map<string, Promise<FrameSource | null>>();
+
+  function frameSourceFor(datasetId: string): Promise<FrameSource | null> {
+    if (!datasetId) return Promise.resolve(null);
+    let pending = sources.get(datasetId);
+    if (!pending) {
+      pending = loadConfig(datasetId)
+        .then((config) => {
+          try {
+            return createFrameSource(config, options);
+          } catch {
+            return null;
+          }
+        })
+        .catch(() => null);
+      sources.set(datasetId, pending);
+    }
+    return pending;
+  }
+
+  function dispose() {
+    sources.forEach((pending) => {
+      pending.then((source) => source?.dispose()).catch(() => undefined);
+    });
+    sources.clear();
+  }
+
+  return { frameSourceFor, dispose };
+}
+
+export type FrameSourceRegistry = ReturnType<typeof createFrameSourceRegistry>;