diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index e781ba9db..6f2f91ba7 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -13,6 +13,14 @@ import type { } from 'vue-media-annotator/alignedView/CameraRegistrationStore'; import type { CameraRole } from 'dive-common/pipelineCameraOrder'; import type { PercentileStretch } from 'vue-media-annotator/use/useImageEnhancements'; +import type { + ScoringDatasetSummary, + ScoringJobArgs, + ScoringPair, + ScoringResult, + ScoringResultSummary, + ScoringSourceOptions, +} from 'dive-common/scoring/types'; type DatasetType = 'image-sequence' | 'video' | 'multi' | 'large-image'; type MultiTrackRecord = Record; @@ -448,6 +456,28 @@ interface Api { }, ): Promise; + /** + * Scoring mode. Every member is optional so a platform that cannot run the + * `viame score` applet simply leaves the mode unavailable. + */ + runScoring?(args: ScoringJobArgs): Promise; + /** Resolve once the scoring job stored on this dataset, launched after this call, ends. */ + watchScoringJob?(datasetId: string): Promise; + /** Runs stored on one dataset, or every run the user can read when omitted. */ + listScoringResults?(datasetId?: string): Promise; + loadScoringResult?(datasetId: string, resultId: string): Promise; + deleteScoringResult?(datasetId: string, resultId: string): Promise; + /** Annotation sets, revisions or on-disk files a source on this dataset can point at. */ + listScoringSources?(datasetId: string): Promise; + /** Datasets that may be named as the other side of a comparison. */ + listScoringDatasets?(): Promise; + /** Open a platform dataset picker; returns null when the user cancels. */ + pickScoringDataset?(excludeIds: string[]): Promise; + /** Save a text export where the user chooses; resolves false when they cancel. */ + saveScoringExport?(args: { filename: string; mime: string; content: string }): Promise; + /** Print the page as it stands (the scoring report view) to a PDF; false when cancelled. */ + exportScoringPdf?(filename: string): Promise; + loadConfig(datasetId: string): Promise; loadDetections(datasetId: string, revision?: number, set?: string): Promise; loadFrameMetadata(datasetId: string): Promise; @@ -758,6 +788,12 @@ export type { TrainingConfigs, MultiCamMedia, MediaImportResponse, + ScoringDatasetSummary, + ScoringJobArgs, + ScoringPair, + ScoringResult, + ScoringResultSummary, + ScoringSourceOptions, }; export type { PercentileStretch, CameraObservations }; diff --git a/client/dive-common/components/Scoring/ScoringClassTable.vue b/client/dive-common/components/Scoring/ScoringClassTable.vue new file mode 100644 index 000000000..eb70b8863 --- /dev/null +++ b/client/dive-common/components/Scoring/ScoringClassTable.vue @@ -0,0 +1,114 @@ + + + + + diff --git a/client/dive-common/components/Scoring/ScoringConfusion.vue b/client/dive-common/components/Scoring/ScoringConfusion.vue new file mode 100644 index 000000000..c5ae172e1 --- /dev/null +++ b/client/dive-common/components/Scoring/ScoringConfusion.vue @@ -0,0 +1,110 @@ + + + + + diff --git a/client/dive-common/components/Scoring/ScoringCurves.vue b/client/dive-common/components/Scoring/ScoringCurves.vue new file mode 100644 index 000000000..cf2f16ab9 --- /dev/null +++ b/client/dive-common/components/Scoring/ScoringCurves.vue @@ -0,0 +1,402 @@ + + + + + diff --git a/client/dive-common/components/Scoring/ScoringErrors.vue b/client/dive-common/components/Scoring/ScoringErrors.vue new file mode 100644 index 000000000..f393a4dc8 --- /dev/null +++ b/client/dive-common/components/Scoring/ScoringErrors.vue @@ -0,0 +1,281 @@ + + + + + diff --git a/client/dive-common/components/Scoring/ScoringPage.vue b/client/dive-common/components/Scoring/ScoringPage.vue new file mode 100644 index 000000000..7fc054ae9 --- /dev/null +++ b/client/dive-common/components/Scoring/ScoringPage.vue @@ -0,0 +1,530 @@ + + + + + + + diff --git a/client/dive-common/components/Scoring/ScoringPairsTable.vue b/client/dive-common/components/Scoring/ScoringPairsTable.vue new file mode 100644 index 000000000..8954a720e --- /dev/null +++ b/client/dive-common/components/Scoring/ScoringPairsTable.vue @@ -0,0 +1,184 @@ + + + + + diff --git a/client/dive-common/components/Scoring/ScoringParamsDialog.vue b/client/dive-common/components/Scoring/ScoringParamsDialog.vue new file mode 100644 index 000000000..ac23a3d5f --- /dev/null +++ b/client/dive-common/components/Scoring/ScoringParamsDialog.vue @@ -0,0 +1,228 @@ + + + + + diff --git a/client/dive-common/components/Scoring/ScoringReport.vue b/client/dive-common/components/Scoring/ScoringReport.vue new file mode 100644 index 000000000..4ab78bead --- /dev/null +++ b/client/dive-common/components/Scoring/ScoringReport.vue @@ -0,0 +1,583 @@ + + + + + diff --git a/client/dive-common/components/Scoring/ScoringSourceDialog.vue b/client/dive-common/components/Scoring/ScoringSourceDialog.vue new file mode 100644 index 000000000..4bb1f8c63 --- /dev/null +++ b/client/dive-common/components/Scoring/ScoringSourceDialog.vue @@ -0,0 +1,255 @@ + + + diff --git a/client/dive-common/components/Scoring/ScoringSourceSelect.vue b/client/dive-common/components/Scoring/ScoringSourceSelect.vue new file mode 100644 index 000000000..fe950f1f8 --- /dev/null +++ b/client/dive-common/components/Scoring/ScoringSourceSelect.vue @@ -0,0 +1,140 @@ + + + + + diff --git a/client/dive-common/components/Scoring/ScoringSummary.vue b/client/dive-common/components/Scoring/ScoringSummary.vue new file mode 100644 index 000000000..24b7ae73d --- /dev/null +++ b/client/dive-common/components/Scoring/ScoringSummary.vue @@ -0,0 +1,212 @@ + + + + + diff --git a/client/dive-common/components/Scoring/ScoringSweep.vue b/client/dive-common/components/Scoring/ScoringSweep.vue new file mode 100644 index 000000000..03081f0ab --- /dev/null +++ b/client/dive-common/components/Scoring/ScoringSweep.vue @@ -0,0 +1,371 @@ + + + + + diff --git a/client/dive-common/components/Scoring/charts/ScoringHeatmap.vue b/client/dive-common/components/Scoring/charts/ScoringHeatmap.vue new file mode 100644 index 000000000..b630048e3 --- /dev/null +++ b/client/dive-common/components/Scoring/charts/ScoringHeatmap.vue @@ -0,0 +1,264 @@ + + + + + diff --git a/client/dive-common/components/Scoring/charts/ScoringLineChart.vue b/client/dive-common/components/Scoring/charts/ScoringLineChart.vue new file mode 100644 index 000000000..003a4b816 --- /dev/null +++ b/client/dive-common/components/Scoring/charts/ScoringLineChart.vue @@ -0,0 +1,561 @@ + + + + + diff --git a/client/dive-common/components/Scoring/charts/chartTypes.ts b/client/dive-common/components/Scoring/charts/chartTypes.ts new file mode 100644 index 000000000..03e103323 --- /dev/null +++ b/client/dive-common/components/Scoring/charts/chartTypes.ts @@ -0,0 +1,24 @@ +export interface ChartPoint { + x: number; + y: number; + /** Extra values shown in the hover tooltip, in insertion order */ + meta?: Record; +} + +export interface ChartSeries { + name: string; + color: string; + points: ChartPoint[]; + dashed?: boolean; +} + +export interface ChartMarker { + x: number; + label: string; + color?: string; +} + +export interface ChartHover { + series: ChartSeries; + point: ChartPoint; +} diff --git a/client/dive-common/scoring/export.spec.ts b/client/dive-common/scoring/export.spec.ts new file mode 100644 index 000000000..a1c691815 --- /dev/null +++ b/client/dive-common/scoring/export.spec.ts @@ -0,0 +1,61 @@ +import { exportFilename, resultToCsv, resultToJson } from './export'; +import { DEFAULT_SCORING_PARAMS } from './metrics'; +import type { ScoringResultFile } from './types'; + +const RESULT: ScoringResultFile = { + version: 1, + id: 'scoring_1.json', + datasetId: 'a', + created: '2026-09-06T14:05:09.000Z', + title: 'Alpha vs Beta', + pairs: [{ computed: { datasetId: 'a', label: 'Alpha' }, truth: { datasetId: 'b', label: 'Beta, GT' } }], + params: { ...DEFAULT_SCORING_PARAMS, labelSynonyms: '' }, + metrics: { + precision: 0.5, + recall: 0.25, + custom_extra: 3, + per_class: { fish: { precision: 1, recall: 0.5, f1_score: 0.6667 } }, + confusion_matrix: { + class_names: ['fish', 'background'], + matrix: [[2, 1], [3, 0]], + normalized_matrix: [[0.67, 0.33], [1, 0]], + per_class_accuracy: { fish: 0.67 }, + }, + sweep: { + interval: 2, + curves: { + overall: { + best: { + idf1: 0.5, idf1_thresh: 0.5, mota: 0.2, mota_thresh: 0, + }, + thresholds: [0, 0.5], + mota: [0.2, 0.1], + }, + }, + }, + }, +}; + +describe('scoring exports', () => { + it('names files by the run timestamp', () => { + expect(exportFilename(RESULT, 'csv')).toMatch(/^scoring_2026\d{4}_\d{6}\.csv$/); + }); + + it('round-trips the run through JSON', () => { + expect(JSON.parse(resultToJson(RESULT))).toEqual(RESULT); + }); + + it('writes labelled summary, per-class, confusion and sweep blocks', () => { + const csv = resultToCsv(RESULT); + expect(csv).toContain('# Run\ntitle,Alpha vs Beta'); + expect(csv).toContain('sequence 1 truth,"Beta, GT"'); + expect(csv).toContain('param iouThreshold,0.5'); + expect(csv).not.toContain('labelSynonyms'); + expect(csv).toContain('precision,Precision,0.5'); + expect(csv).toContain('custom_extra,custom_extra,3'); + expect(csv).toContain('# Per class\nclass,total_gt'); + expect(csv).toContain('\nfish,,,,,,1,0.5,0.6667,'); + expect(csv).toContain('truth \\ computed,fish,background\nfish,2,1\nbackground,3,0'); + expect(csv).toContain('overall,0.5,0.5,0.2,0'); + }); +}); diff --git a/client/dive-common/scoring/export.ts b/client/dive-common/scoring/export.ts new file mode 100644 index 000000000..0d38edd8a --- /dev/null +++ b/client/dive-common/scoring/export.ts @@ -0,0 +1,119 @@ +import type { ScoringResultFile } from './types'; +import { + formatMetric, METRIC_GROUPS, parseScoringMetrics, ScoringMetrics, +} from './metrics'; + +const PER_CLASS_COLUMNS = [ + 'total_gt', 'total_computed', 'true_positives', 'false_positives', 'false_negatives', + 'precision', 'recall', 'f1_score', 'average_precision', 'ap_any', 'ap50', 'ap75', 'ap50_95', +]; + +function csvCell(value: unknown): string { + if (value === null || value === undefined) return ''; + const text = typeof value === 'number' ? String(value) : String(value); + return /[",\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text; +} + +function row(...cells: unknown[]): string { + return cells.map(csvCell).join(','); +} + +function timestampSlug(created: string): string { + const d = new Date(created); + const pad = (n: number) => String(n).padStart(2, '0'); + if (Number.isNaN(d.getTime())) return 'run'; + return `${d.getFullYear()}${pad(d.getMonth() + 1)}${pad(d.getDate())}_${pad(d.getHours())}${pad(d.getMinutes())}${pad(d.getSeconds())}`; +} + +export function exportFilename(result: ScoringResultFile, extension: string): string { + return `scoring_${timestampSlug(result.created)}.${extension}`; +} + +/** The run exactly as stored: parameters, sequences, tool metrics and matches. */ +export function resultToJson(result: ScoringResultFile): string { + return JSON.stringify(result, null, 2); +} + +/** + * Spreadsheet-friendly blocks: the run, every summary metric with its label, + * the per-class table, the confusion matrix and the sweep's best thresholds. + * Blocks are separated by a blank line and introduced by a `#` heading row. + */ +export function resultToCsv(result: ScoringResultFile, parsed?: ScoringMetrics): string { + const metrics = parsed || parseScoringMetrics(result.metrics); + const lines: string[] = []; + + lines.push('# Run'); + lines.push(row('title', result.title)); + lines.push(row('created', result.created)); + result.pairs.forEach((pair, i) => { + lines.push(row(`sequence ${i + 1} computed`, pair.computed.label || pair.computed.datasetId)); + lines.push(row(`sequence ${i + 1} truth`, pair.truth.label || pair.truth.datasetId)); + }); + Object.entries(result.params).forEach(([key, value]) => { + if (key === 'labelSynonyms' && !value) return; + lines.push(row(`param ${key}`, value)); + }); + + lines.push(''); + lines.push('# Summary'); + lines.push(row('metric', 'label', 'value')); + const seen = new Set(); + METRIC_GROUPS.forEach((group) => { + group.metrics.forEach((m) => { + if (m.key in metrics.values) { + seen.add(m.key); + lines.push(row(m.key, m.label, metrics.values[m.key])); + } + }); + }); + Object.entries(metrics.values).forEach(([key, value]) => { + if (!seen.has(key)) lines.push(row(key, key, value)); + }); + + const classNames = Object.keys(metrics.perClass).sort(); + if (classNames.length) { + lines.push(''); + lines.push('# Per class'); + lines.push(row('class', ...PER_CLASS_COLUMNS)); + classNames.forEach((name) => { + lines.push(row(name, ...PER_CLASS_COLUMNS.map((key) => metrics.perClass[name][key]))); + }); + } + + const cm = metrics.confusionMatrix; + if (cm) { + lines.push(''); + lines.push('# Confusion matrix (rows: truth, columns: computed)'); + lines.push(row('truth \\ computed', ...cm.classNames)); + cm.classNames.forEach((name, r) => { + lines.push(row(name, ...(cm.matrix[r] || []))); + }); + } + + if (metrics.sweep) { + lines.push(''); + lines.push('# Sweep best thresholds'); + lines.push(row('curve', 'idf1', 'idf1_threshold', 'mota', 'mota_threshold')); + Object.entries(metrics.sweep.curves).forEach(([name, curve]) => { + lines.push(row(name, curve.best.idf1, curve.best.idf1Thresh, curve.best.mota, curve.best.motaThresh)); + }); + } + + return `${lines.join('\n')}\n`; +} + +/** Browser download of a text file, the fallback when the platform offers no save dialog. */ +export function downloadTextFile(filename: string, content: string, mime: string) { + const blob = new Blob([content], { type: mime }); + const url = URL.createObjectURL(blob); + const anchor = document.createElement('a'); + anchor.href = url; + anchor.download = filename; + document.body.appendChild(anchor); + anchor.click(); + anchor.remove(); + URL.revokeObjectURL(url); +} + +export { formatMetric }; diff --git a/client/dive-common/scoring/metrics.spec.ts b/client/dive-common/scoring/metrics.spec.ts new file mode 100644 index 000000000..9c106b8ce --- /dev/null +++ b/client/dive-common/scoring/metrics.spec.ts @@ -0,0 +1,192 @@ +import { + DEFAULT_SCORING_PARAMS, + headlineMetrics, + parseScoringMatches, + parseScoringMetrics, + perFrameErrorCounts, + scoringCliArgs, + summarizeResult, +} from './metrics'; +import type { RawScoringMatches, ScoringResultFile } from './types'; + +const RAW_METRICS = { + precision: 0.5, + recall: 0.25, + f1_score: 0.3333, + mota: null, + hota: Number.NaN, + true_positives: 3, + config: { + iou_threshold: 0.5, confidence_threshold: 0, match_mode: 'polygon', keypoint_threshold: 0.1, per_class: true, tracking: true, + }, + per_class: { fish: { precision: 1, recall: 0.5, ap50: null } }, + confusion_matrix: { + class_names: ['fish', 'background'], + matrix: [[2, 1], [3, 0]], + normalized_matrix: [[0.6667, 0.3333], [1, 0]], + per_class_accuracy: { fish: 0.6667 }, + }, + pr_curve: { + average_precision: 0.4, + max_f1: 0.5, + best_threshold: 0.3, + points: [{ + recall: 0, precision: 1, confidence: 0.9, f1: 0, tp: 0, fp: 0, fn: 4, + }, { + recall: 0.5, precision: 0.5, confidence: 0.3, f1: 0.5, tp: 2, fp: 2, fn: 2, + }], + }, + roc_curve: { mean_pd: 0.4, max_false_alarms_per_frame: 1, points: [{ false_alarms_per_frame: 0, true_positive_rate: 0, confidence: null }] }, + per_class_pr_curves: { + fish: { + average_precision: 0.4, max_f1: 0.5, best_threshold: 0.3, points: [], + }, + }, + sweep: { + interval: 2, + curves: { + overall: { + best: { + idf1: 0.5, idf1_thresh: 0.5, mota: 0.2, mota_thresh: 0, + }, + thresholds: [0, 0.5], + mota: [0.2, 0.1], + idf1: [0.4, 0.5], + }, + }, + }, +}; + +const RAW_MATCHES: RawScoringMatches = { + columns: ['sequence', 'frame', 'status', 'computed_id', 'gt_id', 'iou', 'confidence', 'computed_class', 'gt_class'], + frame_names: [[0, 0, 'a.png'], [0, 2, 'c.png']], + rows: [ + [0, 0, 'tp', 1, 7, 0.8, 0.9, 'fish', 'fish'], + [0, 0, 'fp', 2, -1, 0, 0.4, 'fish', ''], + [0, 2, 'fn', -1, 8, 0, 0, '', 'fish'], + [0, 2, 'fn', -1, 9, 0, 0, '', 'fish'], + ], +}; + +describe('scoringCliArgs', () => { + const paths = { + computed: '/c.csv', truth: '/t.csv', metricsOut: '/m.json', matchesOut: '/x.json', sweepDir: '/sweep', + }; + + it('emits the sweep, match and curve flags for the defaults', () => { + const args = scoringCliArgs(DEFAULT_SCORING_PARAMS, paths); + expect(args.slice(0, 6)).toEqual(['-c', '/c.csv', '-t', '/t.csv', '-o', '/m.json']); + expect(args).toContain('--json-curves'); + expect(args).toContain('--per-class'); + expect(args).toContain('--sweep-thresholds'); + expect(args).toContain('--output-sweep'); + expect(args).toContain('--output-matches'); + expect(args).not.toContain('--no-tracking'); + expect(args).not.toContain('--labels'); + expect(args[args.indexOf('--match-mode') + 1]).toBe('box'); + }); + + it('drops the sweep and tracking flags when disabled and adds the labels file', () => { + const args = scoringCliArgs({ + ...DEFAULT_SCORING_PARAMS, sweep: false, tracking: false, matchMode: 'polygon', defaultLabel: 'fish', + }, { ...paths, labelsFile: '/labels.txt' }); + expect(args).not.toContain('--sweep-thresholds'); + expect(args).toContain('--no-tracking'); + expect(args[args.indexOf('--match-mode') + 1]).toBe('polygon'); + expect(args[args.indexOf('--labels') + 1]).toBe('/labels.txt'); + expect(args[args.indexOf('--defaultlabel') + 1]).toBe('fish'); + }); +}); + +describe('parseScoringMetrics', () => { + const parsed = parseScoringMetrics(RAW_METRICS); + + it('separates flat values from the structured sections', () => { + expect(parsed.values.precision).toBe(0.5); + expect(parsed.values.mota).toBeNull(); + expect(parsed.values.hota).toBeNull(); + expect(parsed.values.config).toBeUndefined(); + expect(parsed.config?.matchMode).toBe('polygon'); + expect(parsed.perClass.fish.ap50).toBeNull(); + }); + + it('parses the confusion matrix, curves and sweep', () => { + expect(parsed.confusionMatrix?.classNames).toEqual(['fish', 'background']); + expect(parsed.confusionMatrix?.matrix[1][0]).toBe(3); + expect(parsed.prCurve?.points).toHaveLength(2); + expect(parsed.prCurve?.points[1].confidence).toBe(0.3); + expect(parsed.rocCurve?.points[0].confidence).toBeNull(); + expect(Object.keys(parsed.perClassPrCurves)).toEqual(['fish']); + expect(parsed.sweep?.curves.overall.thresholds).toEqual([0, 0.5]); + expect(parsed.sweep?.curves.overall.metrics.mota).toEqual([0.2, 0.1]); + expect(parsed.sweep?.curves.overall.metrics.best).toBeUndefined(); + expect(parsed.sweep?.curves.overall.best.idf1Thresh).toBe(0.5); + }); + + it('tolerates a metrics file with none of the optional sections', () => { + const minimal = parseScoringMetrics({ precision: 1 }); + expect(minimal.confusionMatrix).toBeNull(); + expect(minimal.prCurve).toBeNull(); + expect(minimal.sweep).toBeNull(); + expect(minimal.perClass).toEqual({}); + }); +}); + +describe('parseScoringMatches', () => { + const matches = parseScoringMatches(RAW_MATCHES); + + it('resolves frame names and turns -1 ids into nulls', () => { + expect(matches).toHaveLength(4); + expect(matches[0]).toMatchObject({ + frame: 0, frameName: 'a.png', status: 'tp', computedId: 1, gtId: 7, iou: 0.8, + }); + expect(matches[1].gtId).toBeNull(); + expect(matches[2].computedId).toBeNull(); + expect(matches[2].frameName).toBe('c.png'); + }); + + it('tallies per frame in frame order', () => { + expect(perFrameErrorCounts(matches)).toEqual([ + { + frame: 0, tp: 1, fp: 1, fn: 0, + }, + { + frame: 2, tp: 0, fp: 0, fn: 2, + }, + ]); + }); + + it('returns nothing for a missing payload', () => { + expect(parseScoringMatches(undefined)).toEqual([]); + }); +}); + +describe('summaries', () => { + it('keeps only finite headline numbers', () => { + const headline = headlineMetrics(RAW_METRICS); + expect(headline.precision).toBe(0.5); + expect(headline.mota).toBeNull(); + expect(headline.hota).toBeNull(); + expect('config' in headline).toBe(false); + }); + + it('summarizes a result file without its payloads', () => { + const file: ScoringResultFile = { + version: 1, + id: 'scoring_1.json', + datasetId: 'a', + created: '2026-09-06T00:00:00.000Z', + title: 'a vs b', + pairs: [{ computed: { datasetId: 'a' }, truth: { datasetId: 'b' } }], + params: DEFAULT_SCORING_PARAMS, + metrics: RAW_METRICS, + matches: RAW_MATCHES, + }; + const summary = summarizeResult(file); + expect(summary.headline.precision).toBe(0.5); + expect(summary.datasetId).toBe('a'); + expect(summary.pairs).toHaveLength(1); + expect('metrics' in summary).toBe(false); + expect('matches' in summary).toBe(false); + }); +}); diff --git a/client/dive-common/scoring/metrics.ts b/client/dive-common/scoring/metrics.ts new file mode 100644 index 000000000..6bbca5301 --- /dev/null +++ b/client/dive-common/scoring/metrics.ts @@ -0,0 +1,643 @@ +import type { + RawScoringMatches, + RawScoringMetrics, + ScoringParams, + ScoringResultFile, + ScoringResultSummary, +} from './types'; + +export const DEFAULT_SCORING_PARAMS: ScoringParams = { + iouThreshold: 0.5, + confidenceThreshold: 0.0, + matchMode: 'box', + perClass: true, + topClass: false, + auxConfidence: false, + tracking: true, + keypointThreshold: 0.1, + sweep: true, + sweepInterval: 50, + filterEstimator: 'min', + defaultLabel: '', + labelSynonyms: '', +}; + +export const FILTER_ESTIMATORS: { value: ScoringParams['filterEstimator']; text: string }[] = [ + { value: 'min', text: 'Lower of the IDF1 and MOTA thresholds' }, + { value: 'avg', text: 'Average of the IDF1 and MOTA thresholds' }, + { value: 'avg_minus_1p', text: 'Average minus 0.01' }, + { value: 'idf1', text: 'Threshold maximising IDF1' }, + { value: 'mota', text: 'Threshold maximising MOTA' }, + { value: 'none', text: 'Do not recommend a filter' }, +]; + +export interface ScoringCliPaths { + computed: string; + truth: string; + metricsOut: string; + matchesOut?: string; + sweepDir?: string; + labelsFile?: string; +} + +/** + * The `viame score` argument list for a parameter set. Tokens are returned + * unquoted so each platform can quote for its own shell; paths are passed + * through untouched. + */ +export function scoringCliArgs(params: ScoringParams, paths: ScoringCliPaths): string[] { + const args = [ + '-c', paths.computed, + '-t', paths.truth, + '-o', paths.metricsOut, + '--json-curves', + '--iou', String(params.iouThreshold), + '--conf', String(params.confidenceThreshold), + '--match-mode', params.matchMode, + '--keypoint-threshold', String(params.keypointThreshold), + ]; + if (params.perClass) args.push('--per-class'); + if (params.topClass) args.push('--top-class'); + if (params.auxConfidence) args.push('--aux-confidence'); + if (!params.tracking) args.push('--no-tracking'); + if (params.defaultLabel) args.push('--defaultlabel', params.defaultLabel); + if (paths.labelsFile) args.push('--labels', paths.labelsFile); + if (paths.matchesOut) args.push('--output-matches', paths.matchesOut); + if (params.sweep) { + args.push('--sweep-thresholds', '--sweep-interval', String(params.sweepInterval)); + args.push('--filter-estimator', params.filterEstimator); + if (paths.sweepDir) args.push('--output-sweep', paths.sweepDir); + } + return args; +} + +export const HEADLINE_METRIC_KEYS = [ + 'precision', 'recall', 'f1_score', 'average_precision', 'ap50', 'mean_ap', + 'mota', 'idf1', 'hota', 'mean_iou', 'mean_polygon_iou', 'keypoint_pck', 'length_mape', + 'true_positives', 'false_positives', 'false_negatives', +]; + +function asNumberOrNull(value: unknown): number | null { + return typeof value === 'number' && Number.isFinite(value) ? value : null; +} + +export function headlineMetrics(raw: RawScoringMetrics): Record { + const out: Record = {}; + HEADLINE_METRIC_KEYS.forEach((key) => { + if (key in raw) out[key] = asNumberOrNull(raw[key]); + }); + return out; +} + +export function summarizeResult(file: ScoringResultFile): ScoringResultSummary { + return { + id: file.id, + datasetId: file.datasetId, + created: file.created, + title: file.title, + pairs: file.pairs, + params: file.params, + headline: headlineMetrics(file.metrics), + }; +} + +/** Stable, distinguishable colors for class names outside the annotator's style manager. */ +export const CLASS_PALETTE = [ + '#4c9ac2', '#f4a261', '#2a9d8f', '#e76f51', '#e9c46a', '#a29bfe', '#fd79a8', + '#81ecec', '#ffeaa7', '#55efc4', '#fab1a0', '#74b9ff', '#dfe6e9', '#ff7675', +]; + +// --------------------------------------------------------------------------- +// Structured view of the metrics JSON + +export interface PRCurvePoint { + recall: number; precision: number; confidence: number; f1: number; + tp: number; fp: number; fn: number; +} +export interface PRCurve { + averagePrecision: number | null; + maxF1: number | null; + bestThreshold: number | null; + points: PRCurvePoint[]; +} +export interface ROCCurvePoint { + falseAlarmsPerFrame: number; truePositiveRate: number; confidence: number | null; +} +export interface ROCCurve { + meanPd: number | null; + maxFalseAlarmsPerFrame: number | null; + points: ROCCurvePoint[]; +} +export interface ConfusionMatrix { + classNames: string[]; + matrix: number[][]; + normalized: (number | null)[][]; + perClassAccuracy: Record; +} +export interface SweepCurve { + thresholds: number[]; + metrics: Record; + best: { idf1: number | null; idf1Thresh: number | null; mota: number | null; motaThresh: number | null }; +} +export interface ScoringRunConfig { + iouThreshold: number | null; + confidenceThreshold: number | null; + matchMode: string; + keypointThreshold: number | null; + perClass: boolean; + tracking: boolean; +} +export interface ScoringMetrics { + values: Record; + config: ScoringRunConfig | null; + perClass: Record>; + confusionMatrix: ConfusionMatrix | null; + prCurve: PRCurve | null; + rocCurve: ROCCurve | null; + perClassPrCurves: Record; + sweep: { interval: number; curves: Record } | null; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function parsePrCurve(raw: unknown): PRCurve | null { + if (!isRecord(raw) || !Array.isArray(raw.points)) return null; + return { + averagePrecision: asNumberOrNull(raw.average_precision), + maxF1: asNumberOrNull(raw.max_f1), + bestThreshold: asNumberOrNull(raw.best_threshold), + points: raw.points.filter(isRecord).map((p) => ({ + recall: asNumberOrNull(p.recall) ?? 0, + precision: asNumberOrNull(p.precision) ?? 0, + confidence: asNumberOrNull(p.confidence) ?? 0, + f1: asNumberOrNull(p.f1) ?? 0, + tp: asNumberOrNull(p.tp) ?? 0, + fp: asNumberOrNull(p.fp) ?? 0, + fn: asNumberOrNull(p.fn) ?? 0, + })), + }; +} + +function parseRocCurve(raw: unknown): ROCCurve | null { + if (!isRecord(raw) || !Array.isArray(raw.points)) return null; + return { + meanPd: asNumberOrNull(raw.mean_pd), + maxFalseAlarmsPerFrame: asNumberOrNull(raw.max_false_alarms_per_frame), + points: raw.points.filter(isRecord).map((p) => ({ + falseAlarmsPerFrame: asNumberOrNull(p.false_alarms_per_frame) ?? 0, + truePositiveRate: asNumberOrNull(p.true_positive_rate) ?? 0, + confidence: asNumberOrNull(p.confidence), + })), + }; +} + +function parseNumberMap(raw: unknown): Record { + const out: Record = {}; + if (isRecord(raw)) { + Object.entries(raw).forEach(([k, v]) => { out[k] = asNumberOrNull(v); }); + } + return out; +} + +export function parseScoringMetrics(raw: RawScoringMetrics): ScoringMetrics { + const values: Record = {}; + Object.entries(raw).forEach(([key, value]) => { + if (typeof value === 'number' || value === null) values[key] = asNumberOrNull(value); + }); + + const perClass: Record> = {}; + if (isRecord(raw.per_class)) { + Object.entries(raw.per_class).forEach(([name, metrics]) => { + perClass[name] = parseNumberMap(metrics); + }); + } + + let confusionMatrix: ConfusionMatrix | null = null; + if (isRecord(raw.confusion_matrix) && Array.isArray(raw.confusion_matrix.class_names)) { + const cm = raw.confusion_matrix; + confusionMatrix = { + classNames: (cm.class_names as unknown[]).map(String), + matrix: Array.isArray(cm.matrix) + ? (cm.matrix as unknown[][]).map((row) => row.map((v) => asNumberOrNull(v) ?? 0)) : [], + normalized: Array.isArray(cm.normalized_matrix) + ? (cm.normalized_matrix as unknown[][]).map((row) => row.map(asNumberOrNull)) : [], + perClassAccuracy: parseNumberMap(cm.per_class_accuracy), + }; + } + + const perClassPrCurves: Record = {}; + if (isRecord(raw.per_class_pr_curves)) { + Object.entries(raw.per_class_pr_curves).forEach(([name, curve]) => { + const parsed = parsePrCurve(curve); + if (parsed) perClassPrCurves[name] = parsed; + }); + } + + let sweep: ScoringMetrics['sweep'] = null; + if (isRecord(raw.sweep) && isRecord(raw.sweep.curves)) { + const curves: Record = {}; + Object.entries(raw.sweep.curves).forEach(([name, curve]) => { + if (!isRecord(curve) || !Array.isArray(curve.thresholds)) return; + const metrics: Record = {}; + Object.entries(curve).forEach(([key, series]) => { + if (key !== 'thresholds' && key !== 'best' && Array.isArray(series)) { + metrics[key] = series.map(asNumberOrNull); + } + }); + const best = isRecord(curve.best) ? curve.best : {}; + curves[name] = { + thresholds: (curve.thresholds as unknown[]).map((t) => asNumberOrNull(t) ?? 0), + metrics, + best: { + idf1: asNumberOrNull(best.idf1), + idf1Thresh: asNumberOrNull(best.idf1_thresh), + mota: asNumberOrNull(best.mota), + motaThresh: asNumberOrNull(best.mota_thresh), + }, + }; + }); + sweep = { interval: asNumberOrNull(raw.sweep.interval) ?? 0, curves }; + } + + let config: ScoringRunConfig | null = null; + if (isRecord(raw.config)) { + config = { + iouThreshold: asNumberOrNull(raw.config.iou_threshold), + confidenceThreshold: asNumberOrNull(raw.config.confidence_threshold), + matchMode: String(raw.config.match_mode ?? 'box'), + keypointThreshold: asNumberOrNull(raw.config.keypoint_threshold), + perClass: raw.config.per_class === true, + tracking: raw.config.tracking !== false, + }; + } + + return { + values, + config, + perClass, + confusionMatrix, + prCurve: parsePrCurve(raw.pr_curve), + rocCurve: parseRocCurve(raw.roc_curve), + perClassPrCurves, + sweep, + }; +} + +// --------------------------------------------------------------------------- +// Matches + +export type ScoringMatchStatus = 'tp' | 'fp' | 'fn'; + +export interface ScoringMatch { + sequence: number; + frame: number; + frameName: string; + status: ScoringMatchStatus; + computedId: number | null; + gtId: number | null; + iou: number; + confidence: number; + computedClass: string; + gtClass: string; +} + +export function parseScoringMatches(raw: RawScoringMatches | undefined): ScoringMatch[] { + if (!raw || !Array.isArray(raw.rows)) return []; + const names = new Map(); + (raw.frame_names || []).forEach(([seq, frame, name]) => names.set(`${seq}:${frame}`, name)); + const col = (name: string) => raw.columns.indexOf(name); + const idx = { + sequence: col('sequence'), + frame: col('frame'), + status: col('status'), + computedId: col('computed_id'), + gtId: col('gt_id'), + iou: col('iou'), + confidence: col('confidence'), + computedClass: col('computed_class'), + gtClass: col('gt_class'), + }; + const num = (row: (number | string | null)[], i: number) => asNumberOrNull(row[i]); + const str = (row: (number | string | null)[], i: number) => (i >= 0 && row[i] != null ? String(row[i]) : ''); + return raw.rows.map((row) => { + const sequence = num(row, idx.sequence) ?? 0; + const frame = num(row, idx.frame) ?? 0; + const status = str(row, idx.status) as ScoringMatchStatus; + const computedId = num(row, idx.computedId); + const gtId = num(row, idx.gtId); + return { + sequence, + frame, + frameName: names.get(`${sequence}:${frame}`) ?? '', + status, + computedId: computedId !== null && computedId >= 0 ? computedId : null, + gtId: gtId !== null && gtId >= 0 ? gtId : null, + iou: num(row, idx.iou) ?? 0, + confidence: num(row, idx.confidence) ?? 0, + computedClass: str(row, idx.computedClass), + gtClass: str(row, idx.gtClass), + }; + }); +} + +export interface FrameErrorCount { + frame: number; + tp: number; + fp: number; + fn: number; +} + +/** Per-frame tallies in frame order, one entry per frame that carries any object. */ +export function perFrameErrorCounts(matches: ScoringMatch[]): FrameErrorCount[] { + const byFrame = new Map(); + matches.forEach((m) => { + let entry = byFrame.get(m.frame); + if (!entry) { + entry = { + frame: m.frame, tp: 0, fp: 0, fn: 0, + }; + byFrame.set(m.frame, entry); + } + entry[m.status] += 1; + }); + return Array.from(byFrame.values()).sort((a, b) => a.frame - b.frame); +} + +// --------------------------------------------------------------------------- +// Display definitions + +export interface MetricDefinition { + key: string; + label: string; + description: string; + format: 'ratio' | 'count' | 'pixels' | 'frames' | 'number'; + higherIsBetter?: boolean; +} + +export interface MetricGroup { + name: string; + /** Show the group only when this metric is present and non-zero */ + requires?: string; + metrics: MetricDefinition[]; +} + +export const METRIC_GROUPS: MetricGroup[] = [ + { + name: 'Detection', + metrics: [ + { + key: 'precision', label: 'Precision', description: 'Fraction of computed objects that matched truth', format: 'ratio', higherIsBetter: true, + }, + { + key: 'recall', label: 'Recall', description: 'Fraction of truth objects that were found', format: 'ratio', higherIsBetter: true, + }, + { + key: 'f1_score', label: 'F1', description: 'Harmonic mean of precision and recall', format: 'ratio', higherIsBetter: true, + }, + { + key: 'average_precision', label: 'AP', description: 'Area under the precision-recall curve at the configured IoU', format: 'ratio', higherIsBetter: true, + }, + { + key: 'ap50', label: 'AP@50', description: 'Average precision at IoU 0.5', format: 'ratio', higherIsBetter: true, + }, + { + key: 'ap75', label: 'AP@75', description: 'Average precision at IoU 0.75', format: 'ratio', higherIsBetter: true, + }, + { + key: 'ap50_95', label: 'AP@[.5:.95]', description: 'COCO-style average precision over IoU 0.5 to 0.95', format: 'ratio', higherIsBetter: true, + }, + { + key: 'ap_any', label: 'AP@any', description: 'Average precision counting any overlap as a hit', format: 'ratio', higherIsBetter: true, + }, + { + key: 'mean_ap', label: 'mAP', description: 'Mean of the per-class average precisions', format: 'ratio', higherIsBetter: true, + }, + { + key: 'mcc', label: 'MCC', description: 'Matthews correlation coefficient', format: 'ratio', higherIsBetter: true, + }, + { + key: 'true_positives', label: 'True positives', description: 'Computed objects matched to truth', format: 'count', + }, + { + key: 'false_positives', label: 'False positives', description: 'Computed objects with no matching truth', format: 'count', higherIsBetter: false, + }, + { + key: 'false_negatives', label: 'False negatives', description: 'Truth objects that were missed', format: 'count', higherIsBetter: false, + }, + ], + }, + { + name: 'Localization', + metrics: [ + { + key: 'mean_iou', label: 'Mean IoU', description: 'Mean overlap of matched pairs', format: 'ratio', higherIsBetter: true, + }, + { + key: 'median_iou', label: 'Median IoU', description: 'Median overlap of matched pairs', format: 'ratio', higherIsBetter: true, + }, + { + key: 'mean_center_distance', label: 'Center distance', description: 'Mean distance between matched box centers', format: 'pixels', higherIsBetter: false, + }, + { + key: 'mean_size_error', label: 'Size error', description: 'Mean relative area error of matched boxes', format: 'ratio', higherIsBetter: false, + }, + ], + }, + { + name: 'Segmentation', + requires: 'polygon_pairs', + metrics: [ + { + key: 'polygon_pairs', label: 'Polygon pairs', description: 'Matched pairs with a polygon on both sides', format: 'count', + }, + { + key: 'mean_polygon_iou', label: 'Mean polygon IoU', description: 'Mean mask overlap of those pairs', format: 'ratio', higherIsBetter: true, + }, + { + key: 'median_polygon_iou', label: 'Median polygon IoU', description: 'Median mask overlap of those pairs', format: 'ratio', higherIsBetter: true, + }, + ], + }, + { + name: 'Keypoints', + requires: 'keypoint_pairs', + metrics: [ + { + key: 'keypoint_pairs', label: 'Keypoint pairs', description: 'Matched pairs with head or tail on both sides', format: 'count', + }, + { + key: 'keypoint_pck', label: 'PCK', description: 'Fraction of keypoints within the tolerance', format: 'ratio', higherIsBetter: true, + }, + { + key: 'head_pck', label: 'Head PCK', description: 'Fraction of head points within the tolerance', format: 'ratio', higherIsBetter: true, + }, + { + key: 'tail_pck', label: 'Tail PCK', description: 'Fraction of tail points within the tolerance', format: 'ratio', higherIsBetter: true, + }, + { + key: 'keypoint_mean_error', label: 'Mean error', description: 'Mean keypoint distance from truth', format: 'pixels', higherIsBetter: false, + }, + { + key: 'head_mean_error', label: 'Head error', description: 'Mean head distance from truth', format: 'pixels', higherIsBetter: false, + }, + { + key: 'tail_mean_error', label: 'Tail error', description: 'Mean tail distance from truth', format: 'pixels', higherIsBetter: false, + }, + ], + }, + { + name: 'Length', + requires: 'length_pairs', + metrics: [ + { + key: 'length_pairs', label: 'Length pairs', description: 'Matched pairs with a length on both sides', format: 'count', + }, + { + key: 'length_mae', label: 'MAE', description: 'Mean absolute length error', format: 'number', higherIsBetter: false, + }, + { + key: 'length_mape', label: 'MAPE', description: 'Mean absolute length error relative to truth', format: 'ratio', higherIsBetter: false, + }, + { + key: 'length_rmse', label: 'RMSE', description: 'Root mean square length error', format: 'number', higherIsBetter: false, + }, + { + key: 'length_bias', label: 'Bias', description: 'Mean signed length error, computed minus truth', format: 'number', + }, + ], + }, + { + name: 'Tracking', + requires: 'total_gt_tracks', + metrics: [ + { + key: 'mota', label: 'MOTA', description: 'Multiple object tracking accuracy', format: 'ratio', higherIsBetter: true, + }, + { + key: 'motp', label: 'MOTP', description: 'Mean IoU of matches (higher is better)', format: 'ratio', higherIsBetter: true, + }, + { + key: 'hota', label: 'HOTA', description: 'Higher order tracking accuracy', format: 'ratio', higherIsBetter: true, + }, + { + key: 'deta', label: 'DetA', description: 'HOTA detection accuracy', format: 'ratio', higherIsBetter: true, + }, + { + key: 'assa', label: 'AssA', description: 'HOTA association accuracy', format: 'ratio', higherIsBetter: true, + }, + { + key: 'loca', label: 'LocA', description: 'HOTA localization accuracy', format: 'ratio', higherIsBetter: true, + }, + { + key: 'idf1', label: 'IDF1', description: 'Identity F1', format: 'ratio', higherIsBetter: true, + }, + { + key: 'idp', label: 'IDP', description: 'Identity precision', format: 'ratio', higherIsBetter: true, + }, + { + key: 'idr', label: 'IDR', description: 'Identity recall', format: 'ratio', higherIsBetter: true, + }, + { + key: 'id_switches', label: 'ID switches', description: 'Times a truth track changed computed identity', format: 'count', higherIsBetter: false, + }, + { + key: 'fragmentations', label: 'Fragmentations', description: 'Times a truth track lost and regained coverage', format: 'count', higherIsBetter: false, + }, + { + key: 'mostly_tracked', label: 'Mostly tracked', description: 'Truth tracks covered at least 80% of their span', format: 'count', higherIsBetter: true, + }, + { + key: 'partially_tracked', label: 'Partially tracked', description: 'Truth tracks covered 20% to 80% of their span', format: 'count', + }, + { + key: 'mostly_lost', label: 'Mostly lost', description: 'Truth tracks covered under 20% of their span', format: 'count', higherIsBetter: false, + }, + { + key: 'faf', label: 'False alarms / frame', description: 'False positives per frame', format: 'number', higherIsBetter: false, + }, + { + key: 'avg_track_continuity', label: 'Track continuity', description: 'KWANT continuity of computed tracks', format: 'ratio', higherIsBetter: true, + }, + { + key: 'avg_track_purity', label: 'Track purity', description: 'KWANT purity of computed tracks', format: 'ratio', higherIsBetter: true, + }, + { + key: 'avg_target_continuity', label: 'Target continuity', description: 'KWANT continuity of truth tracks', format: 'ratio', higherIsBetter: true, + }, + { + key: 'avg_target_purity', label: 'Target purity', description: 'KWANT purity of truth tracks', format: 'ratio', higherIsBetter: true, + }, + { + key: 'track_pd', label: 'Track Pd', description: 'Fraction of truth tracks detected', format: 'ratio', higherIsBetter: true, + }, + { + key: 'track_fa', label: 'Track FA', description: 'Computed tracks matching no truth', format: 'count', higherIsBetter: false, + }, + { + key: 'avg_track_length', label: 'Avg track length', description: 'Mean computed track length', format: 'frames', + }, + { + key: 'avg_gt_track_length', label: 'Avg truth length', description: 'Mean truth track length', format: 'frames', + }, + { + key: 'track_completeness', label: 'Completeness', description: 'Mean truth coverage by the best computed track', format: 'ratio', higherIsBetter: true, + }, + ], + }, + { + name: 'Dataset', + metrics: [ + { + key: 'total_frames', label: 'Frames', description: 'Frames carrying any object', format: 'count', + }, + { + key: 'total_gt_objects', label: 'Truth objects', description: 'Truth objects scored', format: 'count', + }, + { + key: 'total_computed', label: 'Computed objects', description: 'Computed objects scored', format: 'count', + }, + { + key: 'total_gt_tracks', label: 'Truth tracks', description: 'Distinct truth track ids', format: 'count', + }, + { + key: 'total_computed_tracks', label: 'Computed tracks', description: 'Distinct computed track ids', format: 'count', + }, + { + key: 'classification_accuracy', label: 'Classification accuracy', description: 'Matched pairs whose classes agree', format: 'ratio', higherIsBetter: true, + }, + ], + }, +]; + +export const METRIC_DEFINITIONS: Record = Object.fromEntries( + METRIC_GROUPS.flatMap((group) => group.metrics.map((m) => [m.key, m])), +); + +export function formatMetric(value: number | null | undefined, format: MetricDefinition['format'] = 'ratio'): string { + if (value === null || value === undefined || !Number.isFinite(value)) return 'n/a'; + switch (format) { + case 'count': + return String(Math.round(value)); + case 'frames': + return `${value.toFixed(1)} f`; + case 'pixels': + return `${value.toFixed(1)} px`; + case 'number': + return value.toFixed(3); + case 'ratio': + default: + return value.toFixed(3); + } +} + +/** Compact label for a source, used in chips and result titles. */ +export function describeSource(source: { datasetId: string; set?: string; revision?: number; file?: string; label?: string }, datasetName?: string): string { + if (source.label) return source.label; + const parts = [datasetName || source.datasetId]; + if (source.set) parts.push(`set ${source.set}`); + if (source.revision !== undefined) parts.push(`rev ${source.revision}`); + if (source.file) parts.push(source.file.split(/[\\/]/).pop() || source.file); + return parts.join(' · '); +} diff --git a/client/dive-common/scoring/types.ts b/client/dive-common/scoring/types.ts new file mode 100644 index 000000000..3a855847b --- /dev/null +++ b/client/dive-common/scoring/types.ts @@ -0,0 +1,144 @@ +/** + * Scoring mode: contract shared by the viewer panel, the web platform and the + * desktop platform. Both platforms run the `viame score` applet on two VIAME + * CSV exports and store its output verbatim inside a {@link ScoringResultFile}. + */ + +export type ScoringMatchMode = 'box' | 'polygon'; + +export type ScoringFilterEstimator = 'none' | 'min' | 'avg' | 'avg_minus_1p' | 'idf1' | 'mota'; + +/** Every `viame score` option the panel exposes. */ +export interface ScoringParams { + /** IoU needed for a computed object to match a truth object (--iou) */ + iouThreshold: number; + /** Computed objects below this confidence are dropped before scoring (--conf) */ + confidenceThreshold: number; + /** Overlap boxes, or polygons where both sides carry one (--match-mode) */ + matchMode: ScoringMatchMode; + /** Also report every class on its own (--per-class) */ + perClass: boolean; + /** Offer each detection only to its best class in per-class scoring (--top-class) */ + topClass: boolean; + /** Rank on the detection confidence column rather than the class score (--aux-confidence) */ + auxConfidence: boolean; + /** Compute MOT, HOTA and KWANT track metrics (omits --no-tracking) */ + tracking: boolean; + /** Keypoint tolerance as a fraction of the truth length (--keypoint-threshold) */ + keypointThreshold: number; + /** Score across a range of confidence thresholds (--sweep-thresholds) */ + sweep: boolean; + /** Number of thresholds in the sweep (--sweep-interval) */ + sweepInterval: number; + /** How swept thresholds become a recommended confidence filter (--filter-estimator) */ + filterEstimator: ScoringFilterEstimator; + /** Class reported for objects that carry none (--defaultlabel) */ + defaultLabel?: string; + /** Class synonym file content, one `canonical: alias, alias` per line (--labels) */ + labelSynonyms?: string; +} + +/** + * One side of a comparison. The dataset supplies the media, so two sources + * naming the same dataset compare two annotation sets of one sequence, and two + * naming different datasets compare separately imported annotations of the + * same footage. + */ +export interface ScoringSource { + datasetId: string; + /** Web annotation set; the default set when empty */ + set?: string; + /** Web revision to score at; the latest when omitted */ + revision?: number; + /** + * Desktop annotation file: an earlier `result_*.json` rotated into the + * project's auxiliary folder, or any annotation file on disk. The dataset's + * current annotations when omitted. + */ + file?: string; + /** Shown in the results view in place of the raw identifiers */ + label?: string; +} + +/** One sequence to score: computed annotations against the truth for the same footage. */ +export interface ScoringPair { + computed: ScoringSource; + truth: ScoringSource; +} + +/** + * A run scores every pair together, so the metrics aggregate across sequences + * the way `viame score` does for a folder of files. The result is stored on + * the first pair's computed dataset. + */ +export interface ScoringJobArgs { + pairs: ScoringPair[]; + params: ScoringParams; + title?: string; +} + +/** Choices a platform can offer for a source on a given dataset. */ +export interface ScoringSourceOptions { + sets: string[]; + revisions: { + revision: number; + description: string; + created: string; + author?: string; + set?: string; + }[]; + files: { + path: string; + name: string; + modified: string; + }[]; + /** The platform can score any annotation file the user browses to */ + allowFilePaths?: boolean; +} + +export interface ScoringDatasetSummary { + id: string; + name: string; + type?: string; +} + +/** The metrics JSON `viame score -o` writes, kept exactly as emitted. */ +export type RawScoringMetrics = Record; + +/** The JSON `viame score --output-matches` writes, kept exactly as emitted. */ +export interface RawScoringMatches { + columns: string[]; + frame_names: [number, number, string][]; + rows: (number | string | null)[][]; +} + +export const SCORING_RESULT_VERSION = 1; + +/** What a platform persists for one run; also what it hands back on load. */ +export interface ScoringResultFile { + version: number; + id: string; + /** Dataset the result is stored on: the first pair's computed dataset */ + datasetId: string; + created: string; + title: string; + /** In the order they were scored; the matches' sequence index refers to this */ + pairs: ScoringPair[]; + params: ScoringParams; + metrics: RawScoringMetrics; + matches?: RawScoringMatches; + summaryText?: string; +} + +/** Listing entry: everything but the bulky payloads, plus headline numbers. */ +export interface ScoringResultSummary { + id: string; + datasetId: string; + created: string; + title: string; + pairs: ScoringPair[]; + params: ScoringParams; + headline: Record; +} + +export type ScoringResult = ScoringResultFile; diff --git a/client/dive-common/use/useScoring.spec.ts b/client/dive-common/use/useScoring.spec.ts new file mode 100644 index 000000000..b3bdc3edc --- /dev/null +++ b/client/dive-common/use/useScoring.spec.ts @@ -0,0 +1,97 @@ +import { createScoringService, ScoringApi } from './useScoring'; +import type { ScoringResultFile, ScoringResultSummary } from '../scoring/types'; +import { DEFAULT_SCORING_PARAMS } from '../scoring/metrics'; + +function makeApi(overrides: Partial = {}): ScoringApi { + const stored: ScoringResultFile = { + version: 1, + id: 'scoring_1.json', + datasetId: 'a', + created: '2026-09-06T00:00:00.000Z', + title: 'a vs b', + pairs: [{ computed: { datasetId: 'a' }, truth: { datasetId: 'b' } }], + params: { ...DEFAULT_SCORING_PARAMS, iouThreshold: 0.7 }, + metrics: { precision: 1 }, + }; + const summary: ScoringResultSummary = { + id: stored.id, + datasetId: 'a', + created: stored.created, + title: stored.title, + pairs: stored.pairs, + params: stored.params, + headline: { precision: 1 }, + }; + return { + runScoring: vi.fn(async () => undefined), + listScoringResults: vi.fn(async () => [summary]), + loadScoringResult: vi.fn(async () => stored), + deleteScoringResult: vi.fn(async () => undefined), + listScoringSources: vi.fn(async (id: string) => ({ + sets: id === 'a' ? ['default', 'groundTruth'] : [], + revisions: [], + files: [], + })), + listScoringDatasets: vi.fn(async () => [{ id: 'a', name: 'Alpha' }, { id: 'b', name: 'Beta' }]), + loadConfig: vi.fn(async () => ({ confidenceFilters: { default: 0.1 } } as never)), + saveConfig: vi.fn(async () => undefined), + openFromDisk: vi.fn(async () => ({ filePaths: [] })), + ...overrides, + }; +} + +describe('createScoringService', () => { + it('defaults the truth side to a ground-truth-named set when one exists', async () => { + const service = createScoringService({ api: makeApi() }); + await service.setDatasets(['a', 'b', 'a']); + expect(service.pairs.value).toHaveLength(2); + expect(service.pairs.value[0].truth).toEqual({ datasetId: 'a', set: 'groundTruth' }); + expect(service.pairs.value[1].truth).toEqual({ datasetId: 'b' }); + }); + + it('refuses to run a pair whose sides are identical', async () => { + const api = makeApi(); + const service = createScoringService({ api }); + await service.setDatasets(['b']); + await service.run(); + expect(api.runScoring).not.toHaveBeenCalled(); + expect(service.error.value).toContain('same annotations'); + }); + + it('launches every pair as one job stored on the first computed dataset', async () => { + const api = makeApi({ watchScoringJob: vi.fn(async () => ({ ok: true })) }); + const service = createScoringService({ api }); + await service.refreshDatasets(); + await service.setDatasets(['a']); + service.setTruth(0, { datasetId: 'b' }); + service.swapPair(0); + expect(service.pairs.value[0].computed.datasetId).toBe('b'); + await service.run(); + const args = (api.runScoring as ReturnType).mock.calls[0][0]; + expect(args.pairs).toHaveLength(1); + expect(args.pairs[0].computed.label).toBe('Beta'); + expect(args.title).toBe('Beta vs Alpha'); + expect(api.watchScoringJob).toHaveBeenCalledWith('b'); + expect(service.selectedResultId.value).toBe('scoring_1.json'); + expect(service.result.value?.title).toBe('a vs b'); + }); + + it('applies confidence filters to every computed dataset of the loaded result', async () => { + const api = makeApi(); + const service = createScoringService({ api }); + await service.refreshResults(); + await service.selectResult('scoring_1.json'); + expect(service.currentFilters.value).toEqual({ default: 0.1 }); + await service.applyConfidenceFilters({ fish: 0.4 }); + expect(api.saveConfig).toHaveBeenCalledWith('a', { confidenceFilters: { default: 0.1, fish: 0.4 } }); + }); + + it('restores a run\'s sequences and parameters into the form', async () => { + const service = createScoringService({ api: makeApi() }); + await service.refreshResults(); + await service.selectResult('scoring_1.json'); + service.useResultSetup(); + expect(service.pairs.value[0].truth.datasetId).toBe('b'); + expect(service.params.iouThreshold).toBe(0.7); + }); +}); diff --git a/client/dive-common/use/useScoring.ts b/client/dive-common/use/useScoring.ts new file mode 100644 index 000000000..15fa4b8b2 --- /dev/null +++ b/client/dive-common/use/useScoring.ts @@ -0,0 +1,449 @@ +import { + computed, inject, provide, reactive, ref, Ref, watch, +} from 'vue'; +import type { Api } from 'dive-common/apispec'; +import type { + ScoringDatasetSummary, + ScoringJobArgs, + ScoringPair, + ScoringParams, + ScoringResult, + ScoringResultSummary, + ScoringSource, + ScoringSourceOptions, +} from 'dive-common/scoring/types'; +import { + CLASS_PALETTE, + DEFAULT_SCORING_PARAMS, + describeSource, + parseScoringMatches, + parseScoringMetrics, + ScoringMatch, + ScoringMetrics, +} from 'dive-common/scoring/metrics'; + +const PARAMS_STORAGE_KEY = 'dive.scoring.params'; +const POLL_INTERVAL_MS = 5000; +const POLL_TIMEOUT_MS = 30 * 60 * 1000; + +export const TRUTH_SET_NAMES = ['groundTruth', 'GT', 'ground_truth', 'Groundtruth', 'GroundTruth', 'gt', 'truth']; + +export type ScoringApi = Pick; + +export interface ScoringServiceDeps { + api: ScoringApi; +} + +export interface ScoringService { + available: Readonly>; + pairs: Ref; + params: ScoringParams; + datasets: Readonly>; + results: Readonly>; + selectedResultId: Readonly>; + result: Readonly>; + metrics: Readonly>; + matches: Readonly>; + /** Confidence filters currently saved on the loaded result's primary computed dataset */ + currentFilters: Readonly>>; + running: Readonly>; + loading: Readonly>; + status: Readonly>; + error: Readonly>; + datasetName(id: string): string; + sourceLabel(source: ScoringSource): string; + classColor(name: string): string; + sourceOptions(datasetId: string): Promise; + refreshDatasets(): Promise; + refreshResults(): Promise; + selectResult(id: string | null): Promise; + deleteResult(id: string): Promise; + addDataset(datasetId: string, summary?: ScoringDatasetSummary): Promise; + setDatasets(datasetIds: string[]): Promise; + removePair(index: number): void; + swapPair(index: number): void; + setComputed(index: number, source: ScoringSource): void; + setTruth(index: number, source: ScoringSource): void; + useResultSetup(): void; + resetParams(): void; + run(): Promise; + applyConfidenceFilters(filters: Record): Promise; + clearError(): void; +} + +function loadStoredParams(): ScoringParams { + try { + const raw = window.localStorage.getItem(PARAMS_STORAGE_KEY); + if (raw) { + const parsed = JSON.parse(raw); + if (parsed && typeof parsed === 'object') { + return { ...DEFAULT_SCORING_PARAMS, ...parsed }; + } + } + } catch { + // Storage may be unavailable; defaults are fine. + } + return { ...DEFAULT_SCORING_PARAMS }; +} + +function storeParams(params: ScoringParams) { + try { + window.localStorage.setItem(PARAMS_STORAGE_KEY, JSON.stringify(params)); + } catch { + // Ignore storage failures. + } +} + +export function sameSource(a: ScoringSource, b: ScoringSource) { + return a.datasetId === b.datasetId + && (a.set || '') === (b.set || '') + && a.revision === b.revision + && (a.file || '') === (b.file || ''); +} + +export function createScoringService(deps: ScoringServiceDeps): ScoringService { + const { api } = deps; + const available = computed(() => typeof api.runScoring === 'function' + && typeof api.listScoringResults === 'function' + && typeof api.loadScoringResult === 'function'); + + const pairs = ref([]); + const params = reactive(loadStoredParams()); + watch(params, () => storeParams({ ...params }), { deep: true }); + + const datasets = ref([]); + const optionsCache = new Map(); + const results = ref([]); + const selectedResultId = ref(null); + const result = ref(null); + const currentFilters = ref>({}); + const running = ref(false); + const loading = ref(false); + const status = ref(null); + const error = ref(null); + + const metrics = computed(() => (result.value ? parseScoringMetrics(result.value.metrics) : null)); + const matches = computed(() => (result.value ? parseScoringMatches(result.value.matches) : [])); + + const classColors = new Map(); + function classColor(name: string) { + let color = classColors.get(name); + if (!color) { + color = CLASS_PALETTE[classColors.size % CLASS_PALETTE.length]; + classColors.set(name, color); + } + return color; + } + + function datasetName(id: string) { + return datasets.value.find((d) => d.id === id)?.name || id; + } + + function sourceLabel(source: ScoringSource) { + return describeSource(source, datasetName(source.datasetId)); + } + + function fail(reason: unknown, fallback: string) { + const message = reason instanceof Error ? reason.message : String(reason || fallback); + error.value = message || fallback; + } + + async function refreshDatasets() { + if (!api.listScoringDatasets) return; + try { + datasets.value = await api.listScoringDatasets(); + } catch (err) { + fail(err, 'Could not list datasets'); + } + } + + async function sourceOptions(datasetId: string): Promise { + const cached = optionsCache.get(datasetId); + if (cached) return cached; + const empty: ScoringSourceOptions = { sets: [], revisions: [], files: [] }; + if (!api.listScoringSources) return empty; + try { + const options = await api.listScoringSources(datasetId); + optionsCache.set(datasetId, options); + return options; + } catch (err) { + fail(err, 'Could not list annotation sources'); + return empty; + } + } + + async function refreshResults() { + if (!api.listScoringResults) return; + loading.value = true; + try { + const list = await api.listScoringResults(); + results.value = [...list].sort((a, b) => b.created.localeCompare(a.created)); + if (selectedResultId.value && !results.value.some((r) => r.id === selectedResultId.value)) { + selectedResultId.value = null; + result.value = null; + } + } catch (err) { + fail(err, 'Could not list scoring results'); + } finally { + loading.value = false; + } + } + + async function loadCurrentFilters() { + currentFilters.value = {}; + const primary = result.value?.pairs[0]?.computed.datasetId; + if (!primary) return; + try { + const config = await api.loadConfig(primary); + currentFilters.value = { ...(config.confidenceFilters || {}) }; + } catch { + // The dataset may be gone; the charts simply omit the current-filter line. + } + } + + async function selectResult(id: string | null) { + selectedResultId.value = id; + if (!id || !api.loadScoringResult) { + result.value = null; + currentFilters.value = {}; + return; + } + const summary = results.value.find((r) => r.id === id); + if (!summary) { + result.value = null; + return; + } + loading.value = true; + try { + result.value = await api.loadScoringResult(summary.datasetId, id); + await loadCurrentFilters(); + } catch (err) { + result.value = null; + fail(err, 'Could not load the scoring result'); + } finally { + loading.value = false; + } + } + + async function deleteResult(id: string) { + if (!api.deleteScoringResult) return; + const summary = results.value.find((r) => r.id === id); + if (!summary) return; + try { + await api.deleteScoringResult(summary.datasetId, id); + if (selectedResultId.value === id) { + selectedResultId.value = null; + result.value = null; + } + await refreshResults(); + } catch (err) { + fail(err, 'Could not delete the scoring result'); + } + } + + /** + * A set named like ground truth is the obvious truth side when the platform + * has sets; otherwise both sides start on the current annotations and the + * user picks a revision, file or other dataset for one of them. + */ + async function defaultPair(datasetId: string): Promise { + const options = await sourceOptions(datasetId); + const gtSet = options.sets.find((s) => TRUTH_SET_NAMES.includes(s)); + return { + computed: { datasetId }, + truth: gtSet ? { datasetId, set: gtSet } : { datasetId }, + }; + } + + function rememberDataset(summary?: ScoringDatasetSummary) { + if (!summary || datasets.value.some((d) => d.id === summary.id)) return; + datasets.value = [...datasets.value, summary]; + } + + async function addDataset(datasetId: string, summary?: ScoringDatasetSummary) { + if (pairs.value.some((p) => p.computed.datasetId === datasetId)) return; + rememberDataset(summary); + pairs.value = [...pairs.value, await defaultPair(datasetId)]; + } + + async function setDatasets(datasetIds: string[]) { + const unique = Array.from(new Set(datasetIds.filter(Boolean))); + pairs.value = await Promise.all(unique.map(defaultPair)); + } + + function removePair(index: number) { + pairs.value = pairs.value.filter((_, i) => i !== index); + } + + function swapPair(index: number) { + pairs.value = pairs.value.map((p, i) => (i === index ? { computed: p.truth, truth: p.computed } : p)); + } + + function setComputed(index: number, source: ScoringSource) { + pairs.value = pairs.value.map((p, i) => (i === index ? { ...p, computed: source } : p)); + } + + function setTruth(index: number, source: ScoringSource) { + pairs.value = pairs.value.map((p, i) => (i === index ? { ...p, truth: source } : p)); + } + + function useResultSetup() { + if (!result.value) return; + pairs.value = result.value.pairs.map((p) => ({ computed: { ...p.computed }, truth: { ...p.truth } })); + Object.assign(params, DEFAULT_SCORING_PARAMS, result.value.params); + } + + function resetParams() { + Object.assign(params, DEFAULT_SCORING_PARAMS); + } + + function clearError() { + error.value = null; + } + + function sleep(ms: number) { + return new Promise((resolve) => { setTimeout(resolve, ms); }); + } + + /** + * Without a job feed the only signal that a run finished is its result + * appearing in the listing, so poll for an entry newer than the launch. + */ + async function pollForResult(startedAt: string): Promise { + const deadline = Date.now() + POLL_TIMEOUT_MS; + while (Date.now() < deadline) { + // eslint-disable-next-line no-await-in-loop + await sleep(POLL_INTERVAL_MS); + // eslint-disable-next-line no-await-in-loop + await refreshResults(); + const fresh = results.value.find((r) => r.created >= startedAt); + if (fresh) return fresh; + } + return null; + } + + function validatePairs(): string | null { + if (pairs.value.length === 0) return 'Add at least one sequence to score.'; + const bad = pairs.value.find((p) => sameSource(p.computed, p.truth)); + if (bad) { + return `${datasetName(bad.computed.datasetId)}: computed and truth point at the same annotations; pick a different set, revision, file or dataset for one of them.`; + } + return null; + } + + async function run() { + if (!api.runScoring || running.value) return; + const problem = validatePairs(); + if (problem) { + error.value = problem; + return; + } + error.value = null; + running.value = true; + status.value = 'Launching scoring job'; + const startedAt = new Date().toISOString(); + const labelled = pairs.value.map((p) => ({ + computed: { ...p.computed, label: sourceLabel(p.computed) }, + truth: { ...p.truth, label: sourceLabel(p.truth) }, + })); + const first = labelled[0]; + const title = labelled.length === 1 + ? `${first.computed.label} vs ${first.truth.label}` + : `${first.computed.label} vs ${first.truth.label} (+${labelled.length - 1} more)`; + const args: ScoringJobArgs = { pairs: labelled, params: { ...params }, title }; + const primary = first.computed.datasetId; + try { + await api.runScoring(args); + status.value = `Scoring ${labelled.length} sequence${labelled.length === 1 ? '' : 's'}`; + let newest: ScoringResultSummary | null = null; + if (api.watchScoringJob) { + const outcome = await api.watchScoringJob(primary); + if (!outcome.ok) { + throw new Error(outcome.message || 'The scoring job failed; see the Jobs tab for its log.'); + } + await refreshResults(); + newest = results.value.find((r) => r.created >= startedAt && r.datasetId === primary) + || results.value[0] || null; + } else { + newest = await pollForResult(startedAt); + if (!newest) throw new Error('Timed out waiting for the scoring result.'); + } + if (newest) await selectResult(newest.id); + status.value = null; + } catch (err) { + status.value = null; + fail(err, 'Scoring failed'); + } finally { + running.value = false; + } + } + + /** Save the filters on every computed dataset of the loaded result. */ + async function applyConfidenceFilters(filters: Record) { + const targets = Array.from(new Set((result.value?.pairs || []).map((p) => p.computed.datasetId))); + try { + await Promise.all(targets.map(async (datasetId) => { + const config = await api.loadConfig(datasetId); + await api.saveConfig(datasetId, { + confidenceFilters: { ...(config.confidenceFilters || {}), ...filters }, + }); + })); + await loadCurrentFilters(); + } catch (err) { + fail(err, 'Could not save the confidence filters'); + } + } + + return { + available, + pairs, + params, + datasets, + results, + selectedResultId, + result, + metrics, + matches, + currentFilters, + running, + loading, + status, + error, + datasetName, + sourceLabel, + classColor, + sourceOptions, + refreshDatasets, + refreshResults, + selectResult, + deleteResult, + addDataset, + setDatasets, + removePair, + swapPair, + setComputed, + setTruth, + useResultSetup, + resetParams, + run, + applyConfidenceFilters, + clearError, + }; +} + +const ScoringSymbol = Symbol('scoring'); + +export function provideScoring(service: ScoringService) { + provide(ScoringSymbol, service); +} + +export function useScoring(): ScoringService { + const service = inject(ScoringSymbol, null); + if (!service) { + throw new Error('Scoring service not provided'); + } + return service; +} diff --git a/client/package.json b/client/package.json index 3603cac89..0baa7f024 100644 --- a/client/package.json +++ b/client/package.json @@ -13,7 +13,7 @@ "dev:electron": "npm run serve:electron", "dev:electron:windows": "npm run serve:electron:windows", "serve": "vite", - "serve:electron": "mkdir -p /tmp/dive-electron && cross-env ELECTRON_RUN_AS_NODE= ELECTRON_DISABLE_SECURITY_WARNINGS=true TMPDIR=/tmp/dive-electron XDG_RUNTIME_DIR=/tmp/dive-electron electron-vite dev --entry=.electron/main/background.js", + "serve:electron": "mkdir -p /tmp/dive-electron && cross-env ELECTRON_RUN_AS_NODE= ELECTRON_DISABLE_SECURITY_WARNINGS=true TMPDIR=/tmp/dive-electron electron-vite dev --entry=.electron/main/background.js", "serve:electron:windows": "cross-env ELECTRON_DISABLE_SECURITY_WARNINGS=true electron-vite dev --entry=.electron/main/background.js", "build:web": "vite build", "build:electron": "electron-vite build && electron-builder --config electron-builder.json", diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index a890d6f59..ae2a29a60 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -14,6 +14,7 @@ import { ExportTrainedPipeline, ConversionArgs, DesktopJob, + RunScoring, } from 'platform/desktop/constants'; import { convertMedia } from 'platform/desktop/backend/native/mediaJobs'; import { closeChildById } from 'platform/desktop/backend/native/processManager'; @@ -132,6 +133,15 @@ export default function register() { ipcMain.on('update-settings', async (_, s: Settings) => { settings.set(s); }); + ipcMain.handle('write-text-file', async (_, args: { path: string; content: string }) => { + await fs.promises.writeFile(args.path, args.content, 'utf-8'); + return args.path; + }); + ipcMain.handle('print-to-pdf', async (event, args: { path: string }) => { + const data = await event.sender.printToPDF({ printBackground: true, pageSize: 'Letter' }); + await fs.promises.writeFile(args.path, data); + return args.path; + }); ipcMain.handle('export-dataset', async (_, args: ExportDatasetArgs) => { const ret = await common.exportDataset(settings.get(), args); return ret; @@ -351,6 +361,29 @@ export default function register() { }; return currentPlatform.train(settings.get(), args, updater); }); + ipcMain.handle('run-scoring', async (event, args: RunScoring) => { + const updater = (update: DesktopJobUpdate) => { + event.sender.send('job-update', update); + }; + return currentPlatform.runScoring(settings.get(), args, updater); + }); + ipcMain.handle('list-scoring-sources', async (_, { datasetId }: { datasetId: string }) => ( + common.listScoringSources(settings.get(), datasetId) + )); + ipcMain.handle('list-scoring-datasets', async () => common.listScoringDatasets(settings.get())); + ipcMain.handle('list-scoring-results', async (_, { datasetId }: { datasetId?: string } = {}) => ( + common.listScoringResults(settings.get(), datasetId) + )); + ipcMain.handle('load-scoring-result', async ( + _, + { datasetId, resultId }: { datasetId: string; resultId: string }, + ) => common.loadScoringResult(settings.get(), datasetId, resultId)); + ipcMain.handle('delete-scoring-result', async ( + _, + { datasetId, resultId }: { datasetId: string; resultId: string }, + ) => { + await common.deleteScoringResult(settings.get(), datasetId, resultId); + }); ipcMain.handle('list-resumable-training', async () => common.findResumableTrainingJobs(settings.get())); ipcMain.handle('discard-resumable-training', async (_event, workingDir: string) => { await common.discardResumableTraining(settings.get(), workingDir); diff --git a/client/platform/desktop/backend/native/common.spec.ts b/client/platform/desktop/backend/native/common.spec.ts index 856fba60d..8b9bfa327 100644 --- a/client/platform/desktop/backend/native/common.spec.ts +++ b/client/platform/desktop/backend/native/common.spec.ts @@ -134,6 +134,61 @@ const console = new Console(process.stdout, process.stderr); const emptyCsvString = '# comment line\n# metadata,fps: 32,"whatever"\n#comment line'; +const scoringTrackFixture = { + version: AnnotationsCurrentVersion, + groups: {}, + tracks: { + 1: { + id: 1, + begin: 0, + end: 0, + attributes: {}, + confidencePairs: [['shark', 0.9]], + features: [{ frame: 0, bounds: [0, 0, 1, 1] }], + }, + }, +}; + +const scoringResultFixture = { + version: 1, + id: 'ignored-on-load', + datasetId: 'projectidScoring', + created: '2026-01-02T03:04:05.000Z', + title: 'current vs old', + pairs: [{ + computed: { datasetId: 'projectidScoring' }, + truth: { datasetId: 'projectidScoring', file: '/home/user/viamedata/DIVE_Projects/projectidScoring/auxiliary/result_old.json' }, + }], + params: { + iouThreshold: 0.5, + confidenceThreshold: 0, + matchMode: 'box', + perClass: true, + topClass: false, + auxConfidence: false, + tracking: true, + keypointThreshold: 0.1, + sweep: false, + sweepInterval: 50, + filterEstimator: 'min', + }, + metrics: { precision: 0.5, recall: 1, per_class: { shark: { precision: 0.5 } } }, +}; + +/** Written before results recorded their storage dataset, and scoring two sequences. */ +const legacyScoringResultFixture = { + version: 1, + id: 'ignored-on-load', + created: '2025-12-31T00:00:00.000Z', + title: 'two sequences', + pairs: [ + { computed: { datasetId: 'projectidScoring2' }, truth: { datasetId: 'projectidScoring' } }, + { computed: { datasetId: 'projectidScoring' }, truth: { datasetId: 'projectidScoring2' } }, + ], + params: scoringResultFixture.params, + metrics: { precision: 0.25 }, +}; + function cocoWithRle(trackId: number, categoryName = 'fish') { return JSON.stringify({ images: [{ id: 1, file_name: 'frame_000001.jpg', frame_index: 0 }], @@ -870,6 +925,46 @@ beforeEach(() => { 'result_whatever.json': JSON.stringify({}), auxiliary: {}, }, + projectidScoring: { + 'dataset.json': JSON.stringify({ + version: 1, + id: 'projectidScoring', + name: 'Scoring Project', + type: 'image-sequence', + fps: 5, + originalBasePath: '/home/user/media/scoring', + originalImageFiles: ['a.png'], + } as JsonConfig), + 'result_current.json': JSON.stringify(scoringTrackFixture), + auxiliary: { + 'scoring_2026-01-02_03-04-05.000.json': JSON.stringify(scoringResultFixture), + 'scoring_broken.json': '{not json', + 'result_old.json': mockfs.file({ + content: JSON.stringify(scoringTrackFixture), + mtime: new Date('2026-01-01T00:00:00Z'), + }), + 'imported_annotations.csv': mockfs.file({ + content: emptyCsvString, + mtime: new Date('2026-01-03T00:00:00Z'), + }), + 'flight_log.csv': '', + }, + }, + projectidScoring2: { + 'dataset.json': JSON.stringify({ + version: 1, + id: 'projectidScoring2', + name: 'Second Scoring Project', + type: 'image-sequence', + fps: 5, + originalBasePath: '/home/user/media/scoring2', + originalImageFiles: ['a.png'], + } as JsonConfig), + 'result_current.json': JSON.stringify(scoringTrackFixture), + auxiliary: { + 'scoring_2025-12-31_00-00-00.000.json': JSON.stringify(legacyScoringResultFixture), + }, + }, projectid5missingMultiCam: { 'meta.json': JSON.stringify({ version: 1, @@ -2980,6 +3075,105 @@ describe('frame metadata import gates', () => { }); }); +describe('scoring results and sources', () => { + const projectDir = '/home/user/viamedata/DIVE_Projects/projectidScoring'; + + it('listScoringResults summarizes readable result files and skips broken ones', async () => { + const warn = vi.spyOn(globalThis.console, 'warn').mockImplementation(() => undefined); + const results = await common.listScoringResults(settings, 'projectidScoring'); + expect(results).toHaveLength(1); + expect(results[0].id).toBe('scoring_2026-01-02_03-04-05.000.json'); + expect(results[0].datasetId).toBe('projectidScoring'); + expect(results[0].title).toBe('current vs old'); + expect(results[0].pairs).toEqual(scoringResultFixture.pairs); + expect(results[0].headline.precision).toBe(0.5); + expect(results[0]).not.toHaveProperty('metrics'); + expect(warn).toHaveBeenCalledTimes(1); + warn.mockRestore(); + }); + + it('listScoringResults without a dataset gathers every project newest first', async () => { + const warn = vi.spyOn(globalThis.console, 'warn').mockImplementation(() => undefined); + const results = await common.listScoringResults(settings); + expect(results.map((r) => [r.id, r.datasetId])).toEqual([ + ['scoring_2026-01-02_03-04-05.000.json', 'projectidScoring'], + ['scoring_2025-12-31_00-00-00.000.json', 'projectidScoring2'], + ]); + expect(results[1].pairs).toHaveLength(2); + expect(results[1].headline.precision).toBe(0.25); + expect(warn).toHaveBeenCalledTimes(1); + warn.mockRestore(); + }); + + it('loadScoringResult returns the file with its basename as id', async () => { + const result = await common.loadScoringResult( + settings, + 'projectidScoring', + 'scoring_2026-01-02_03-04-05.000.json', + ); + expect(result.id).toBe('scoring_2026-01-02_03-04-05.000.json'); + expect(result.metrics.per_class).toEqual({ shark: { precision: 0.5 } }); + expect(result.pairs[0].truth.file).toBe(`${projectDir}/auxiliary/result_old.json`); + }); + + it('deleteScoringResult rejects ids that are not result files inside auxiliary', async () => { + await expect(common.deleteScoringResult(settings, 'projectidScoring', '../result_current.json')) + .rejects.toThrow('not a scoring result id'); + await expect(common.deleteScoringResult(settings, 'projectidScoring', 'scoring_../dataset.json')) + .rejects.toThrow('not a scoring result id'); + await expect(common.deleteScoringResult(settings, 'projectidScoring', 'result_old.json')) + .rejects.toThrow('not a scoring result id'); + expect(fs.existsSync(`${projectDir}/result_current.json`)).toBe(true); + expect(fs.existsSync(`${projectDir}/auxiliary/result_old.json`)).toBe(true); + + await common.deleteScoringResult(settings, 'projectidScoring', 'scoring_2026-01-02_03-04-05.000.json'); + expect(fs.existsSync(`${projectDir}/auxiliary/scoring_2026-01-02_03-04-05.000.json`)).toBe(false); + }); + + it('listScoringSources lists rotated and imported annotation files newest first', async () => { + const options = await common.listScoringSources(settings, 'projectidScoring'); + expect(options.sets).toEqual([]); + expect(options.revisions).toEqual([]); + expect(options.files.map((f) => f.name)).toEqual(['imported_annotations.csv', 'result_old.json']); + expect(options.files[1]).toEqual({ + path: `${projectDir}/auxiliary/result_old.json`, + name: 'result_old.json', + modified: '2026-01-01T00:00:00.000Z', + }); + }); + + it('listScoringDatasets reports every loadable project', async () => { + const datasets = await common.listScoringDatasets(settings); + expect(datasets).toContainEqual({ id: 'projectidScoring', name: 'Scoring Project', type: 'image-sequence' }); + expect(datasets.find((d) => d.id === 'projectid2Bad')).toBeUndefined(); + }); + + it('exportScoringSourceCsv writes current annotations, a rotated json, or copies a csv', async () => { + const current = '/home/user/output/current.csv'; + await common.exportScoringSourceCsv(settings, { datasetId: 'projectidScoring' }, current); + expect(await fs.readFile(current, 'utf-8')).toContain('shark'); + + const rotated = '/home/user/output/rotated.csv'; + await common.exportScoringSourceCsv(settings, { + datasetId: 'projectidScoring', + file: `${projectDir}/auxiliary/result_old.json`, + }, rotated); + expect(await fs.readFile(rotated, 'utf-8')).toContain('shark'); + + const copied = '/home/user/output/copied.csv'; + await common.exportScoringSourceCsv(settings, { + datasetId: 'projectidScoring', + file: `${projectDir}/auxiliary/imported_annotations.csv`, + }, copied); + expect(await fs.readFile(copied, 'utf-8')).toBe(emptyCsvString); + + await expect(common.exportScoringSourceCsv(settings, { + datasetId: 'projectidScoring', + file: `${projectDir}/auxiliary/flight_log.txt`, + }, '/home/user/output/bad.csv')).rejects.toThrow('not a CSV or JSON'); + }); +}); + afterEach(() => { mockfs.restore(); }); diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index cbca22399..1fa16dec5 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -37,6 +37,11 @@ import { METADATA_ATTACHMENT_UNAVAILABLE, isFrameMetadataReadableName, } from 'dive-common/frameMetadata/readability'; import { parentDatasetId, parseCompositeDatasetId } from 'dive-common/compositeDatasetId'; +import type { + ScoringDatasetSummary, ScoringResultFile, ScoringResultSummary, + ScoringSource, ScoringSourceOptions, +} from 'dive-common/scoring/types'; +import { summarizeResult } from 'dive-common/scoring/metrics'; import * as viameSerializers from 'platform/desktop/backend/serializers/viame'; import * as nistSerializers from 'platform/desktop/backend/serializers/nist'; import * as dive from 'platform/desktop/backend/serializers/dive'; @@ -96,6 +101,8 @@ const PortableConfigFileName = 'config.json'; const DiveJobManifestName = 'dive_job_manifest.json'; const PortableConfigFileNameLegacy = 'meta.json'; const CsvFileName = /^.*\.csv$/i; +const ImportedAnnotationFileName = /^imported_.+/i; +const ScoringResultFileName = /^scoring_[\w.-]+\.json$/; const YAMLFileName = /^.*\.ya?ml$/i; const invalidHierarchyMessage = (reason: string) => ( @@ -2785,6 +2792,125 @@ async function exportDataset(settings: Settings, args: ExportDatasetArgs) { }); } +/** + * Annotation files a desktop scoring source can point at: track files rotated + * into auxiliary by earlier saves, and copies of imported annotation files. + */ +async function listScoringSources( + settings: Settings, + datasetId: string, +): Promise { + const projectInfo = await getValidatedProjectDir(settings, datasetId); + const names = await fs.readdir(projectInfo.auxDirAbsPath); + const files = await Promise.all(names + .filter((name) => JsonTrackFileName.test(name) || ImportedAnnotationFileName.test(name)) + .map(async (name) => { + const path = npath.join(projectInfo.auxDirAbsPath, name); + const stat = await fs.stat(path); + return stat.isFile() ? { path, name, modified: stat.mtime.toISOString() } : null; + })); + return { + sets: [], + revisions: [], + files: files + .filter((file): file is NonNullable => file !== null) + .sort((a, b) => b.modified.localeCompare(a.modified)), + allowFilePaths: true, + }; +} + +async function listScoringDatasets(settings: Settings): Promise { + const metas = await autodiscoverData(settings); + return metas.map(({ id, name, type }) => ({ id, name, type })); +} + +async function summarizeScoringResultsIn( + auxDirAbsPath: string, + datasetId: string, +): Promise { + const names = await listNames(auxDirAbsPath); + const summaries: ScoringResultSummary[] = []; + await Promise.all(names.filter((name) => ScoringResultFileName.test(name)).map(async (name) => { + try { + const file = await fs.readJson(npath.join(auxDirAbsPath, name)) as ScoringResultFile; + // Files written before datasetId existed live in the dataset they belong to. + summaries.push(summarizeResult({ ...file, id: name, datasetId: file.datasetId || datasetId })); + } catch (err) { + console.warn(`Skipping unreadable scoring result ${name}:`, err); + } + })); + return summaries; +} + +/** Runs stored on one dataset, or on every project when no dataset is given. */ +async function listScoringResults( + settings: Settings, + datasetId?: string, +): Promise { + let summaries: ScoringResultSummary[]; + if (datasetId) { + const projectInfo = await getValidatedProjectDir(settings, datasetId); + summaries = await summarizeScoringResultsIn(projectInfo.auxDirAbsPath, datasetId); + } else { + const projectIds = await listNames(npath.join(settings.dataPath, ProjectsFolderName)); + summaries = (await Promise.all(projectIds.map((id) => ( + summarizeScoringResultsIn(getProjectDir(settings, id).auxDirAbsPath, id) + )))).flat(); + } + return summaries.sort((a, b) => b.created.localeCompare(a.created)); +} + +async function scoringResultPath(settings: Settings, datasetId: string, resultId: string) { + if (!ScoringResultFileName.test(resultId)) { + throw new Error(`${resultId} is not a scoring result id`); + } + const projectInfo = await getValidatedProjectDir(settings, datasetId); + const path = npath.resolve(projectInfo.auxDirAbsPath, resultId); + if (npath.dirname(path) !== npath.resolve(projectInfo.auxDirAbsPath)) { + throw new Error(`${resultId} is not a scoring result id`); + } + return path; +} + +async function loadScoringResult( + settings: Settings, + datasetId: string, + resultId: string, +): Promise { + const path = await scoringResultPath(settings, datasetId, resultId); + const file = await fs.readJson(path) as ScoringResultFile; + return { ...file, id: npath.basename(path), datasetId: file.datasetId || datasetId }; +} + +async function deleteScoringResult(settings: Settings, datasetId: string, resultId: string) { + await fs.unlink(await scoringResultPath(settings, datasetId, resultId)); +} + +/** + * Write one side of a scoring comparison as VIAME CSV. Every detection is + * kept so the scorer, not the dataset's display filter, applies thresholds. + */ +async function exportScoringSourceCsv(settings: Settings, source: ScoringSource, outPath: string) { + if (source.file && CsvFileName.test(source.file)) { + await fs.copy(source.file, outPath); + return; + } + const projectInfo = await getValidatedProjectDir(settings, source.datasetId); + let trackFile = projectInfo.trackFileAbsPath; + if (source.file) { + if (!JsonFileName.test(source.file)) { + throw new Error(`${source.file} is not a CSV or JSON annotation file`); + } + trackFile = source.file; + } + const meta = await loadJsonConfig(projectInfo.datasetFileAbsPath); + const data = await loadAnnotationFile(trackFile); + await viameSerializers.serializeFile(outPath, data, meta, new Set(), { + excludeBelowThreshold: false, + header: true, + }); +} + async function exportConfiguration(settings: Settings, args: ExportConfigurationArgs) { const projectDirInfo = await getValidatedProjectDir(settings, args.id); const meta = await loadJsonConfig(projectDirInfo.datasetFileAbsPath); @@ -2824,6 +2950,12 @@ export { checkDataset, exportConfiguration, exportDataset, + exportScoringSourceCsv, + listScoringSources, + listScoringDatasets, + listScoringResults, + loadScoringResult, + deleteScoringResult, finalizeMediaImport, getPipelineList, deleteTrainedPipeline, diff --git a/client/platform/desktop/backend/native/linux.ts b/client/platform/desktop/backend/native/linux.ts index 1bc3a077e..e7ca52d67 100644 --- a/client/platform/desktop/backend/native/linux.ts +++ b/client/platform/desktop/backend/native/linux.ts @@ -14,6 +14,7 @@ import { RunTraining, DesktopJobUpdater, ExportTrainedPipeline, + RunScoring, } from 'platform/desktop/constants'; import { observeChild } from 'platform/desktop/backend/native/processManager'; import * as viame from './viame'; @@ -133,6 +134,14 @@ async function train( return viame.train(settings, runTrainingArgs, updater, validateViamePath, getViameConstants(settings)); } +async function runScoring( + settings: Settings, + runScoringArgs: RunScoring, + updater: DesktopJobUpdater, +): Promise { + return viame.runScoring(settings, runScoringArgs, updater, validateViamePath, getViameConstants(settings)); +} + // Based on https://github.com/chrisallenlane/node-nvidia-smi async function nvidiaSmi(): Promise { return new Promise((resolve) => { @@ -168,6 +177,7 @@ export default { runPipeline, exportTrainedPipeline, train, + runScoring, validateViamePath, getViameConstants, getViamePythonExe, diff --git a/client/platform/desktop/backend/native/viame.ts b/client/platform/desktop/backend/native/viame.ts index e248d1c73..9649a1e50 100644 --- a/client/platform/desktop/backend/native/viame.ts +++ b/client/platform/desktop/backend/native/viame.ts @@ -1,6 +1,7 @@ import npath from 'path'; import { spawn } from 'child_process'; import fs from 'fs-extra'; +import moment from 'moment'; import { Settings, DesktopJob, RunPipeline, RunTraining, @@ -8,6 +9,7 @@ import { ExportTrainedPipeline, JsonConfig, JobsOutputFolderName, + RunScoring, } from 'platform/desktop/constants'; import { cleanString } from 'platform/desktop/sharedUtils'; import { serialize } from 'platform/desktop/backend/serializers/viame'; @@ -28,6 +30,9 @@ import { isTranscodePipeline, pipelineCreatesNewDataset, } from 'dive-common/pipelineCreatesDataset'; +import { describeSource, scoringCliArgs } from 'dive-common/scoring/metrics'; +import { SCORING_RESULT_VERSION } from 'dive-common/scoring/types'; +import type { RawScoringMatches, ScoringResultFile } from 'dive-common/scoring/types'; import * as common from './common'; import { jobFileEchoMiddleware, createWorkingDirectory, createCustomWorkingDirectory, splitExt, @@ -1025,9 +1030,168 @@ async function train( return jobBase; } +/** Quote a `viame score` token for the platform shell; bare flags and numbers stay as-is. */ +function shellToken(token: string) { + return /^[\w.-]+$/.test(token) ? token : `"${token}"`; +} + +/** Keep only the last part of the scorer's stdout for the stored summary. */ +const ScoringSummaryTailBytes = 64 * 1024; + +/** + * Run the `viame score` applet over every pair in one invocation and store + * its output as a result file in the first pair's computed dataset. + */ +async function runScoring( + settings: Settings, + args: RunScoring, + updater: DesktopJobUpdater, + validateViamePath: (settings: Settings) => Promise, + viameConstants: ViameConstants, +): Promise { + const isValid = await validateViamePath(settings); + if (isValid !== true) { + throw new Error(isValid); + } + const { pairs, params } = args; + if (pairs.length === 0) { + throw new Error('Scoring needs at least one computed/truth pair'); + } + const storageDatasetId = pairs[0].computed.datasetId; + const projectInfo = await common.getValidatedProjectDir(settings, storageDatasetId); + const jobWorkDir = await createCustomWorkingDirectory(settings, 'Scoring', storageDatasetId); + const joblog = npath.join(jobWorkDir, 'runlog.txt'); + + // Folder mode pairs computed and truth files by basename. + const computedDir = npath.join(jobWorkDir, 'computed'); + const truthDir = npath.join(jobWorkDir, 'truth'); + await fs.ensureDir(computedDir); + await fs.ensureDir(truthDir); + await Promise.all(pairs.map(async (pair, i) => { + const name = `seq_${String(i).padStart(3, '0')}.csv`; + await common.exportScoringSourceCsv(settings, pair.computed, npath.join(computedDir, name)); + await common.exportScoringSourceCsv(settings, pair.truth, npath.join(truthDir, name)); + })); + + let labelsFile: string | undefined; + if (params.labelSynonyms) { + labelsFile = npath.join(jobWorkDir, 'labels.txt'); + await fs.writeFile(labelsFile, params.labelSynonyms); + } + + const metricsOut = npath.join(jobWorkDir, 'metrics.json'); + const matchesOut = npath.join(jobWorkDir, 'matches.json'); + const command = [ + `${viameConstants.setupScriptAbs} &&`, + `"${viameConstants.viameExe}" score`, + ...scoringCliArgs(params, { + computed: computedDir, + truth: truthDir, + metricsOut, + matchesOut, + sweepDir: npath.join(jobWorkDir, 'sweep'), + labelsFile, + }).map(shellToken), + '--input-ext', '.csv', + ]; + + const job = observeChild(spawn(command.join(' '), { + shell: viameConstants.shell, + cwd: jobWorkDir, + })); + if (job.pid === undefined) { + throw new Error('Failed to spawn scoring process'); + } + + let { title } = args; + if (!title) { + title = `${describeSource(pairs[0].computed)} vs ${describeSource(pairs[0].truth)}`; + if (pairs.length > 1) { + title += ` (+${pairs.length - 1} more)`; + } + } + const datasetIds = [...new Set(pairs.flatMap((pair) => [pair.computed.datasetId, pair.truth.datasetId]))]; + const jobBase: DesktopJob = { + key: `scoring_${job.pid}_${jobWorkDir}`, + command: command.join(' '), + jobType: 'scoring', + pid: job.pid, + args, + title: `Scoring ${title}`, + workingDir: jobWorkDir, + datasetIds, + exitCode: job.exitCode, + startTime: new Date(), + }; + const manifestPath = npath.join(jobWorkDir, DiveJobManifestName); + fs.writeFile(manifestPath, JSON.stringify(jobBase, null, 2)); + + updater({ + ...jobBase, + body: [''], + }); + + let stdoutTail = ''; + const echo = jobFileEchoMiddleware(jobBase, updater, joblog); + job.stdout.on('data', (chunk: Buffer) => { + stdoutTail = (stdoutTail + chunk.toString('utf-8')).slice(-ScoringSummaryTailBytes); + echo(chunk); + }); + job.stderr.on('data', echo); + + job.on('exit', async (code) => { + let existingManifest: DesktopJob | undefined; + try { + if (await fs.pathExists(manifestPath)) { + existingManifest = await fs.readJson(manifestPath) as DesktopJob; + } + } catch { + // fall through and record process exit status + } + + let exitCode = code; + const bodyText = ['']; + if (!existingManifest?.cancelledJob && code === 0) { + try { + const created = moment(); + const id = `scoring_${created.format('YYYY-MM-DD_HH-mm-ss.SSS')}.json`; + const result: ScoringResultFile = { + version: SCORING_RESULT_VERSION, + id, + datasetId: storageDatasetId, + created: created.toISOString(), + title, + pairs, + params, + metrics: await fs.readJson(metricsOut), + summaryText: stdoutTail.trim() || undefined, + }; + if (await fs.pathExists(matchesOut)) { + result.matches = await fs.readJson(matchesOut) as RawScoringMatches; + } + await fs.writeJson(npath.join(projectInfo.auxDirAbsPath, id), result); + } catch (err) { + const message = `Failed to record scoring result: ${err instanceof Error ? err.message : String(err)}`; + console.error(err); + await fs.appendFile(joblog, `\n${message}\n`).catch(() => undefined); + exitCode = 1; + bodyText.unshift(message); + } + } + const finalJob = buildTrainingExitManifest(jobBase, exitCode, new Date(), existingManifest); + fs.writeFile(manifestPath, JSON.stringify(finalJob, null, 2)); + updater({ + ...finalJob, + body: bodyText, + }); + }); + return jobBase; +} + export { runPipeline, exportTrainedPipeline, train, + runScoring, DEFAULT_CALIBRATION_KEYS, }; diff --git a/client/platform/desktop/backend/native/windows.ts b/client/platform/desktop/backend/native/windows.ts index 45dc630e0..1ad31243f 100644 --- a/client/platform/desktop/backend/native/windows.ts +++ b/client/platform/desktop/backend/native/windows.ts @@ -13,6 +13,7 @@ import { DesktopJob, RunPipeline, NvidiaSmiReply, RunTraining, DesktopJobUpdater, ExportTrainedPipeline, + RunScoring, } from 'platform/desktop/constants'; import * as viame from './viame'; @@ -126,6 +127,14 @@ async function train( return viame.train(settings, runTrainingArgs, updater, validateFake, getViameConstants(settings)); } +async function runScoring( + settings: Settings, + runScoringArgs: RunScoring, + updater: DesktopJobUpdater, +): Promise { + return viame.runScoring(settings, runScoringArgs, updater, validateFake, getViameConstants(settings)); +} + function checkDefaultNvidiaSmi(resolve: (value: NvidiaSmiReply) => void) { const smi = observeChild(spawn( `"${programFiles}\\NVIDIA Corporation\\NVSMI\\nvidia-smi.exe"`, @@ -194,6 +203,7 @@ export default { runPipeline, exportTrainedPipeline, train, + runScoring, nvidiaSmi, initialize, getViameConstants, diff --git a/client/platform/desktop/constants.ts b/client/platform/desktop/constants.ts index 49c81c007..53caafd73 100644 --- a/client/platform/desktop/constants.ts +++ b/client/platform/desktop/constants.ts @@ -5,6 +5,7 @@ import type { import { Attribute } from 'vue-media-annotator/use/AttributeTypes'; import { AttributeTrackFilter } from 'vue-media-annotator/AttributeTrackFilterControls'; import { ImageEnhancements } from 'vue-media-annotator/use/useImageEnhancements'; +import type { ScoringJobArgs } from 'dive-common/scoring/types'; export const JsonConfigCurrentVersion = 1; export const SettingsCurrentVersion = 1; @@ -189,6 +190,7 @@ export enum JobType { ExportTrainedPipeline, RunPipeline, RunTraining, + RunScoring, } export interface JobArgs { @@ -234,6 +236,10 @@ export interface RunTraining extends JobArgs { resumeWorkingDir?: string; } +export interface RunScoring extends JobArgs, ScoringJobArgs { + type: JobType.RunScoring; +} + export interface ConversionArgs extends JobArgs { type: JobType.Conversion; meta: JsonConfig; @@ -248,7 +254,7 @@ export interface CliTranscodingNotice { mediaCount: number; } -export type Job = ConversionArgs | RunPipeline | RunTraining | ExportTrainedPipeline; +export type Job = ConversionArgs | RunPipeline | RunTraining | ExportTrainedPipeline | RunScoring; export interface DesktopJob { // key unique identifier for this job @@ -256,11 +262,11 @@ export interface DesktopJob { // command that was run command: string; // jobType identify type of job - jobType: 'pipeline' | 'training' | 'conversion' | 'export'; + jobType: 'pipeline' | 'training' | 'conversion' | 'export' | 'scoring'; // title whatever humans should see this job called title: string; // arguments to creation - args: RunPipeline | RunTraining | ExportTrainedPipeline | ConversionArgs; + args: RunPipeline | RunTraining | ExportTrainedPipeline | ConversionArgs | RunScoring; // datasetIds of the involved datasets datasetIds: string[]; // pid of the process spawned diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index 9b9a60c28..1afed360a 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -11,6 +11,7 @@ import type { SegmentationStereoSegmentRequest, SegmentationStereoSegmentResponse, TextQueryRequest, TextQueryResponse, RefineDetectionsRequest, RefineDetectionsResponse, PipelineJobResult, + ScoringDatasetSummary, ScoringJobArgs, ScoringResult, ScoringResultSummary, ScoringSourceOptions, } from 'dive-common/apispec'; import { @@ -26,6 +27,7 @@ import { DesktopMediaImportResponse, ConversionArgs, JobType, DesktopJob, MultiCamBatchScanResult, + RunScoring, } from 'platform/desktop/constants'; import { gpuJobQueue, cpuJobQueue, jobHistory } from './store/jobs'; @@ -176,19 +178,20 @@ async function runPipeline(itemId: string, pipeline: Pipe, pipelineParams?: Pipe } /** - * Resolve when the pipeline job for this dataset finishes. + * Resolve when the first job matching `matches` that started at or after this + * call finishes. * - * The job store is the only place that knows a job ended: a pipeline's own + * The job store is the only place that knows a job ended: a job's own * artifacts cannot say it, because a deterministic re-run writes byte-identical * output and a job that fails writes none at all. Both look exactly like "still * running" to anything watching the output. * * Only jobs starting at or after this call are considered, so an earlier run of - * the same pipe on the same dataset (still in the history) is never mistaken for - * this one. Resolution waits out the queue: `runPipeline` enqueues, so the job - * may not exist for as long as the jobs ahead of it take. + * the same job (still in the history) is never mistaken for this one. + * Resolution waits out the queue: the run functions enqueue, so the job may not + * exist for as long as the jobs ahead of it take. */ -function watchPipelineJob(datasetId: string, pipeline: Pipe): Promise { +function watchJob(matches: (job: DesktopJob) => boolean): Promise { const startedAt = Date.now(); return new Promise((resolve) => { let key: string | null = null; @@ -208,10 +211,7 @@ function watchPipelineJob(datasetId: string, pipeline: Pipe): Promise { const entries = Object.values(jobHistory.value); if (key === null) { - const match = entries.find((entry) => entry.job.jobType === 'pipeline' - && 'pipeline' in entry.job.args - && entry.job.args.pipeline.pipe === pipeline.pipe - && entry.job.datasetIds.includes(datasetId) + const match = entries.find((entry) => matches(entry.job) // A job update carries the start time as an ISO string once it has // crossed the IPC boundary, so normalize before comparing. && new Date(entry.job.startTime).getTime() >= startedAt); @@ -241,6 +241,14 @@ function watchPipelineJob(datasetId: string, pipeline: Pipe): Promise { + return watchJob((job) => job.jobType === 'pipeline' + && 'pipeline' in job.args + && job.args.pipeline.pipe === pipeline.pipe + && job.datasetIds.includes(datasetId)); +} + async function exportTrainedPipeline(path: string, pipeline: Pipe): Promise { const args: ExportTrainedPipeline = { type: JobType.ExportTrainedPipeline, @@ -275,6 +283,41 @@ async function runTraining( gpuJobQueue.enqueue(args); } +/** + * Scoring API + */ + +async function runScoring(args: ScoringJobArgs): Promise { + const spec: RunScoring = { type: JobType.RunScoring, ...args }; + cpuJobQueue.enqueue(spec); +} + +/** Resolve when the scoring job involving this dataset, launched after this call, finishes. */ +function watchScoringJob(datasetId: string): Promise { + return watchJob((job) => job.jobType === 'scoring' && job.datasetIds.includes(datasetId)); +} + +function listScoringSources(datasetId: string): Promise { + return invoke('list-scoring-sources', { datasetId }); +} + +function listScoringDatasets(): Promise { + return invoke('list-scoring-datasets'); +} + +/** Runs stored on one dataset, or every run across all projects when omitted. */ +function listScoringResults(datasetId?: string): Promise { + return invoke('list-scoring-results', { datasetId }); +} + +function loadScoringResult(datasetId: string, resultId: string): Promise { + return invoke('load-scoring-result', { datasetId, resultId }); +} + +function deleteScoringResult(datasetId: string, resultId: string): Promise { + return invoke('delete-scoring-result', { datasetId, resultId }); +} + async function deleteTrainedPipeline(pipeline: Pipe): Promise { return invoke('delete-trained-pipeline', pipeline); } @@ -396,6 +439,29 @@ async function exportDataset(id: string, exclude: boolean, typeFilter: readonly return ''; } +async function saveScoringExport( + { filename, content }: { filename: string; mime: string; content: string }, +): Promise { + const location = await window.diveDesktop.showSaveDialog({ + title: 'Export Scoring Run', + defaultPath: joinPath(await window.diveDesktop.getAppPath('home'), filename), + }); + if (location.canceled || !location.filePath) return false; + await invoke('write-text-file', { path: location.filePath, content }); + return true; +} + +async function exportScoringPdf(filename: string): Promise { + const location = await window.diveDesktop.showSaveDialog({ + title: 'Save Scoring Report', + defaultPath: joinPath(await window.diveDesktop.getAppPath('home'), filename), + filters: [{ name: 'PDF', extensions: ['pdf'] }], + }); + if (location.canceled || !location.filePath) return false; + await invoke('print-to-pdf', { path: location.filePath }); + return true; +} + async function exportConfiguration(id: string): Promise { const location = await window.diveDesktop.showSaveDialog({ title: 'Export Configuration', @@ -836,6 +902,8 @@ function deleteCalibration(datasetId: string): Promise { } export { + saveScoringExport, + exportScoringPdf, /* Standard Specification APIs */ loadConfig, loadDetections, @@ -850,6 +918,13 @@ export { listResumableTrainingJobs, resumeTraining, discardResumableTraining, + runScoring, + watchScoringJob, + listScoringSources, + listScoringDatasets, + listScoringResults, + loadScoringResult, + deleteScoringResult, saveConfig, saveDetections, saveAttributes, diff --git a/client/platform/desktop/frontend/components/JobsQueued.vue b/client/platform/desktop/frontend/components/JobsQueued.vue index 649a1348e..fd0057799 100644 --- a/client/platform/desktop/frontend/components/JobsQueued.vue +++ b/client/platform/desktop/frontend/components/JobsQueued.vue @@ -16,10 +16,15 @@ import { removeJobFromQueue, // removeJobFromQueue, } from 'platform/desktop/frontend/store/jobs'; +import type { ScoringSource } from 'dive-common/scoring/types'; import { datasets } from '../store/dataset'; export default defineComponent({ setup() { + function datasetName(id: string) { + return datasets.value[id]?.name || id; + } + const queuedJobSpecs: Ref = ref([]); function updateQueuedJobSpecs() { queuedJobSpecs.value = []; @@ -39,12 +44,24 @@ export default defineComponent({ return `export trained pipeline: ${jobSpec.path}`; } if (jobSpec.type === JobType.RunTraining) { - const title = `training: ${datasets.value[jobSpec.datasetIds[0]]?.name || jobSpec.datasetIds[0]}`; + const title = `training: ${datasetName(jobSpec.datasetIds[0])}`; if (jobSpec.datasetIds.length > 1) { return `${title} (and ${jobSpec.datasetIds.length - 1} more)`; } return title; } + if (jobSpec.type === JobType.RunScoring) { + const label = (source: ScoringSource) => source.label || datasetName(source.datasetId); + const [first] = jobSpec.pairs; + let { title } = jobSpec; + if (!title && first) { + title = `${label(first.computed)} vs ${label(first.truth)}`; + if (jobSpec.pairs.length > 1) { + title += ` (+${jobSpec.pairs.length - 1} more)`; + } + } + return `scoring: ${title || 'no sequences'}`; + } return 'queued job'; } @@ -58,6 +75,9 @@ export default defineComponent({ if (jobSpec.type === JobType.RunTraining) { return jobSpec.datasetIds; } + if (jobSpec.type === JobType.RunScoring) { + return [...new Set(jobSpec.pairs.flatMap((pair) => [pair.computed.datasetId, pair.truth.datasetId]))]; + } return []; } @@ -69,6 +89,7 @@ export default defineComponent({ JobType, queuedJobSpecs, datasets, + datasetName, getQueuedJobTitle, getJobDatasets, removeJobFromQueue, @@ -114,7 +135,7 @@ export default defineComponent({ class="mr-1" :to="{ name: 'viewer', params: { id: dataset } }" > - {{ datasets[dataset].name }} + {{ datasetName(dataset) }} diff --git a/client/platform/desktop/frontend/components/NavigationBar.vue b/client/platform/desktop/frontend/components/NavigationBar.vue index af89be7cb..f3907c753 100644 --- a/client/platform/desktop/frontend/components/NavigationBar.vue +++ b/client/platform/desktop/frontend/components/NavigationBar.vue @@ -31,6 +31,9 @@ export default defineComponent({ Pipelinemdi-pipe + + Scoringmdi-chart-box-outline + Settingsmdi-cog diff --git a/client/platform/desktop/frontend/components/Recent.vue b/client/platform/desktop/frontend/components/Recent.vue index 602b99fe0..27630ab0b 100644 --- a/client/platform/desktop/frontend/components/Recent.vue +++ b/client/platform/desktop/frontend/components/Recent.vue @@ -261,6 +261,10 @@ export default defineComponent({ function runTrainingOnSelected() { router.push({ name: 'training', query: selectedIdsQuery() }); } + + function scoreSelected() { + router.push({ name: 'scoring', query: selectedIdsQuery() }); + } function getTypeIcon(recent: JsonConfigCache) { if (recent.subType) { if (recent.subType === 'stereo') { @@ -355,6 +359,7 @@ export default defineComponent({ confirmDeleteSelected, runPipelineOnSelected, runTrainingOnSelected, + scoreSelected, isSelected, toggleSelected, toggleSelectAll, @@ -623,6 +628,27 @@ export default defineComponent({ Train a model on the selected datasets + + + Score the selected datasets against ground truth +