Add a scoring page that wraps viame score on desktop and web

client/dive-common/apispec.ts

@@ -13,6 +13,14 @@ import type {
 } from 'vue-media-annotator/alignedView/CameraRegistrationStore';
 import type { CameraRole } from 'dive-common/pipelineCameraOrder';
 import type { PercentileStretch } from 'vue-media-annotator/use/useImageEnhancements';
+import type {
+  ScoringDatasetSummary,
+  ScoringJobArgs,
+  ScoringPair,
+  ScoringResult,
+  ScoringResultSummary,
+  ScoringSourceOptions,
+} from 'dive-common/scoring/types';
 
 type DatasetType = 'image-sequence' | 'video' | 'multi' | 'large-image';
 type MultiTrackRecord = Record<string, TrackData>;
@@ -448,6 +456,28 @@ interface Api {
     },
   ): Promise<unknown>;
 
+  /**
+   * Scoring mode. Every member is optional so a platform that cannot run the
+   * `viame score` applet simply leaves the mode unavailable.
+   */
+  runScoring?(args: ScoringJobArgs): Promise<unknown>;
+  /** Resolve once the scoring job stored on this dataset, launched after this call, ends. */
+  watchScoringJob?(datasetId: string): Promise<PipelineJobResult>;
+  /** Runs stored on one dataset, or every run the user can read when omitted. */
+  listScoringResults?(datasetId?: string): Promise<ScoringResultSummary[]>;
+  loadScoringResult?(datasetId: string, resultId: string): Promise<ScoringResult>;
+  deleteScoringResult?(datasetId: string, resultId: string): Promise<void>;
+  /** Annotation sets, revisions or on-disk files a source on this dataset can point at. */
+  listScoringSources?(datasetId: string): Promise<ScoringSourceOptions>;
+  /** Datasets that may be named as the other side of a comparison. */
+  listScoringDatasets?(): Promise<ScoringDatasetSummary[]>;
+  /** Open a platform dataset picker; returns null when the user cancels. */
+  pickScoringDataset?(excludeIds: string[]): Promise<ScoringDatasetSummary | null>;
+  /** Save a text export where the user chooses; resolves false when they cancel. */
+  saveScoringExport?(args: { filename: string; mime: string; content: string }): Promise<boolean>;
+  /** Print the page as it stands (the scoring report view) to a PDF; false when cancelled. */
+  exportScoringPdf?(filename: string): Promise<boolean>;
+
   loadConfig(datasetId: string): Promise<DatasetConfig>;
   loadDetections(datasetId: string, revision?: number, set?: string): Promise<AnnotationSchemaList>;
   loadFrameMetadata(datasetId: string): Promise<FrameMetadataSourcesResponse>;
@@ -758,6 +788,12 @@ export type {
   TrainingConfigs,
   MultiCamMedia,
   MediaImportResponse,
+  ScoringDatasetSummary,
+  ScoringJobArgs,
+  ScoringPair,
+  ScoringResult,
+  ScoringResultSummary,
+  ScoringSourceOptions,
 };
 
 export type { PercentileStretch, CameraObservations };

client/dive-common/components/Scoring/ScoringClassTable.vue

@@ -0,0 +1,114 @@
+<script lang="ts">
+import { computed, defineComponent } from 'vue';
+import { useScoring } from 'dive-common/use/useScoring';
+import { formatMetric } from 'dive-common/scoring/metrics';
+
+const COLUMNS: { key: string; text: string; format: 'ratio' | 'count' }[] = [
+  { key: 'total_gt', text: 'Truth', format: 'count' },
+  { key: 'total_computed', text: 'Computed', format: 'count' },
+  { key: 'true_positives', text: 'TP', format: 'count' },
+  { key: 'false_positives', text: 'FP', format: 'count' },
+  { key: 'false_negatives', text: 'FN', format: 'count' },
+  { key: 'precision', text: 'Precision', format: 'ratio' },
+  { key: 'recall', text: 'Recall', format: 'ratio' },
+  { key: 'f1_score', text: 'F1', format: 'ratio' },
+  { key: 'average_precision', text: 'AP', format: 'ratio' },
+  { key: 'ap50', text: 'AP@50', format: 'ratio' },
+  { key: 'ap75', text: 'AP@75', format: 'ratio' },
+  { key: 'ap50_95', text: 'AP@[.5:.95]', format: 'ratio' },
+];
+
+export default defineComponent({
+  name: 'ScoringClassTable',
+  setup() {
+    const scoring = useScoring();
+
+    const headers = computed(() => [
+      { text: 'Class', value: 'name', sortable: true },
+      ...COLUMNS.map((c) => ({
+        text: c.text, value: c.key, sortable: true, align: 'end',
+      })),
+    ]);
+
+    const rows = computed(() => {
+      const perClass = scoring.metrics.value?.perClass || {};
+      return Object.entries(perClass).map(([name, metrics]) => ({
+        name,
+        color: scoring.classColor(name),
+        ...metrics,
+      }));
+    });
+
+    function display(value: unknown, key: string) {
+      const col = COLUMNS.find((c) => c.key === key);
+      return formatMetric(typeof value === 'number' ? value : null, col?.format || 'ratio');
+    }
+
+    return {
+      headers,
+      rows,
+      display,
+      COLUMNS,
+    };
+  },
+});
+</script>
+
+<template>
+  <div class="px-2">
+    <div
+      v-if="rows.length === 0"
+      class="text-caption grey--text pa-2"
+    >
+      Per-class metrics were not requested for this run. Enable "Per class" in the scoring parameters.
+    </div>
+    <v-data-table
+      v-else
+      :headers="headers"
+      :items="rows"
+      dense
+      disable-pagination
+      hide-default-footer
+      sort-by="f1_score"
+      sort-desc
+      class="class-table"
+    >
+      <template #[`item.name`]="{ item }">
+        <span
+          class="class-swatch"
+          :style="{ background: item.color }"
+        />
+        {{ item.name }}
+      </template>
+      <template
+        v-for="col in COLUMNS"
+        #[`item.${col.key}`]="{ item }"
+      >
+        <span
+          :key="col.key"
+          class="tabular"
+        >{{ display(item[col.key], col.key) }}</span>
+      </template>
+    </v-data-table>
+  </div>
+</template>
+
+<style lang="scss" scoped>
+.class-swatch {
+  display: inline-block;
+  width: 10px;
+  height: 10px;
+  border-radius: 2px;
+  margin-right: 6px;
+}
+
+.tabular {
+  font-variant-numeric: tabular-nums;
+}
+
+.class-table ::v-deep td,
+.class-table ::v-deep th {
+  font-size: 12px !important;
+  height: 26px !important;
+}
+</style>

client/dive-common/components/Scoring/ScoringConfusion.vue

@@ -0,0 +1,110 @@
+<script lang="ts">
+import { computed, defineComponent, ref } from 'vue';
+import { useScoring } from 'dive-common/use/useScoring';
+import ScoringHeatmap from './charts/ScoringHeatmap.vue';
+
+export default defineComponent({
+  name: 'ScoringConfusion',
+  components: { ScoringHeatmap },
+  setup(_, { emit }) {
+    const scoring = useScoring();
+    const normalized = ref(false);
+    const selected = ref<{ row: number; col: number } | null>(null);
+
+    const matrix = computed(() => scoring.metrics.value?.confusionMatrix || null);
+    const backgroundIndex = computed(() => (matrix.value ? matrix.value.classNames.indexOf('background') : -1));
+    const accuracy = computed(() => scoring.metrics.value?.values.classification_accuracy ?? null);
+
+    function onSelect(cell: { row: number; col: number }) {
+      if (!matrix.value) return;
+      if (selected.value && selected.value.row === cell.row && selected.value.col === cell.col) {
+        selected.value = null;
+        emit('filter', null);
+        return;
+      }
+      selected.value = cell;
+      const truthClass = matrix.value.classNames[cell.row];
+      const computedClass = matrix.value.classNames[cell.col];
+      // Background rows are misses, background columns are false alarms
+      let status: 'tp' | 'fp' | 'fn' | null = null;
+      if (truthClass === 'background') status = 'fp';
+      else if (computedClass === 'background') status = 'fn';
+      else status = 'tp';
+      emit('filter', {
+        status,
+        gtClass: truthClass === 'background' ? null : truthClass,
+        computedClass: computedClass === 'background' ? null : computedClass,
+      });
+    }
+
+    return {
+      matrix,
+      normalized,
+      selected,
+      backgroundIndex,
+      accuracy,
+      onSelect,
+    };
+  },
+});
+</script>
+
+<template>
+  <div class="px-2">
+    <div
+      v-if="!matrix"
+      class="text-caption grey--text pa-2"
+    >
+      No confusion matrix in this result.
+    </div>
+    <div
+      v-else
+      class="d-flex flex-wrap"
+    >
+      <ScoringHeatmap
+        :row-labels="matrix.classNames"
+        :col-labels="matrix.classNames"
+        :counts="matrix.matrix"
+        :fractions="matrix.normalized"
+        :show-fractions="normalized"
+        :background-index="backgroundIndex"
+        :selected="selected"
+        @select="onSelect"
+      />
+      <div class="pl-4 pt-2 confusion-side">
+        <v-switch
+          v-model="normalized"
+          dense
+          hide-details
+          label="Normalize rows"
+          class="mt-0"
+        />
+        <div
+          v-if="accuracy !== null"
+          class="text-body-2 mt-2"
+        >
+          Classification accuracy among matches: <b>{{ (accuracy * 100).toFixed(1) }}%</b>
+        </div>
+        <div class="text-caption grey--text mt-2">
+          Rows are truth classes, columns are computed classes. The background row holds
+          false positives and the background column holds missed objects. Click a cell to
+          browse the objects behind it in the Errors tab.
+        </div>
+        <div class="text-caption mt-2">
+          <div
+            v-for="name in matrix.classNames.filter((n) => n !== 'background')"
+            :key="name"
+          >
+            {{ name }}: {{ matrix.perClassAccuracy[name] === null || matrix.perClassAccuracy[name] === undefined ? 'n/a' : `${(matrix.perClassAccuracy[name] * 100).toFixed(1)}%` }} correct
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+</template>
+
+<style lang="scss" scoped>
+.confusion-side {
+  max-width: 320px;
+}
+</style>