Eval revisited

birdnet_analyzer/evaluation/__init__.py

@@ -11,16 +11,40 @@
 import json
 import logging
 import os
+import typing
 from collections.abc import Sequence
+from typing import Literal, NamedTuple
 
 from birdnet_analyzer.evaluation.assessment.performance_assessor import (
     PerformanceAssessor,
 )
 from birdnet_analyzer.evaluation.preprocessing.data_processor import DataProcessor
 
+if typing.TYPE_CHECKING:
+    import numpy as np
+    import pandas as pd
+
 logger = logging.getLogger(__name__)
 
 
+class EvaluationResult(NamedTuple):
+    """The outcome of :func:`process_data`.
+
+    Besides the metrics and the tensors the assessment ran on, it carries the context
+    a caller needs to explain the numbers: which recordings had predictions but no
+    annotations, and which classes were empty (and therefore excluded from any
+    aggregate score).
+    """
+
+    metrics_df: "pd.DataFrame"
+    pa: PerformanceAssessor
+    predictions: "np.ndarray"
+    labels: "np.ndarray"
+    classes: tuple[str, ...]
+    unmatched_recordings: tuple[str, ...]
+    empty_classes: tuple[str, ...]
+
+
 def process_data(
     annotation_path: str,
     prediction_path: str,
@@ -35,7 +59,9 @@ def process_data(
     metrics_list: tuple[str, ...] = ("accuracy", "precision", "recall"),
     threshold: float = 0.1,
     class_wise: bool = False,
-):
+    averaging: Literal["macro", "micro", "weighted"] = "macro",
+    score_unannotated_as_empty: bool = False,
+) -> EvaluationResult:
     """
     Processes data, computes metrics, and prepares the performance assessment pipeline.
 
@@ -60,10 +86,17 @@ def process_data(
         metrics_list (Tuple[str, ...]): Metrics to compute for performance assessment.
         threshold (float): Confidence threshold for predictions.
         class_wise (bool): Whether to calculate metrics on a per-class basis.
+        averaging (Literal["macro", "micro", "weighted"]): How to aggregate the
+            per-class metrics into the overall score. Ignored when ``class_wise`` is
+            True.
+        score_unannotated_as_empty (bool): Whether recordings that have predictions but
+            no annotation file are kept and scored as all-negative (True) or dropped as
+            unscoreable (False, the default).
 
     Returns:
-        Tuple: Metrics DataFrame, `PerformanceAssessor` object, predictions tensor,
-            labels tensor.
+        EvaluationResult: The metrics DataFrame, the ``PerformanceAssessor``, the
+            prediction and label tensors, and the context (classes, unmatched
+            recordings, empty classes).
     """
     # Load class mapping if provided
     if mapping_path:
@@ -96,6 +129,7 @@ def process_data(
         columns_predictions=columns_predictions,
         columns_annotations=columns_annotations,
         recording_duration=recording_duration,
+        score_unannotated_as_empty=score_unannotated_as_empty,
     )
 
     # Get the available classes and recordings
@@ -126,9 +160,23 @@ def process_data(
     )
 
     # Compute performance metrics
-    metrics_df = pa.calculate_metrics(predictions, labels, per_class_metrics=class_wise)
+    metrics_df = pa.calculate_metrics(
+        predictions,
+        labels,
+        per_class_metrics=class_wise,
+        averaging=averaging,
+        include_support=class_wise,
+    )
 
-    return metrics_df, pa, predictions, labels
+    return EvaluationResult(
+        metrics_df=metrics_df,
+        pa=pa,
+        predictions=predictions,
+        labels=labels,
+        classes=classes,
+        unmatched_recordings=tuple(sorted(processor.unmatched_prediction_files)),
+        empty_classes=pa.empty_classes(labels),
+    )
 
 
 def main():
@@ -193,6 +241,20 @@ def main():
     parser.add_argument(
         "--class_wise", action="store_true", help="Calculate class-wise metrics"
     )
+    parser.add_argument(
+        "--averaging",
+        choices=["macro", "micro", "weighted"],
+        default="macro",
+        help="How to aggregate per-class metrics into the overall score",
+    )
+    parser.add_argument(
+        "--score_unannotated_as_empty",
+        action="store_true",
+        help=(
+            "Keep recordings that have predictions but no annotation file and score "
+            "them as all-negative, instead of dropping them (default)"
+        ),
+    )
     parser.add_argument("--plot_metrics", action="store_true", help="Plot metrics")
     parser.add_argument(
         "--plot_confusion_matrix", action="store_true", help="Plot confusion matrix"
@@ -208,7 +270,7 @@ def main():
     args = parser.parse_args()
 
     # Process data and compute metrics
-    metrics_df, pa, predictions, labels = process_data(
+    result = process_data(
         annotation_path=args.annotation_path,
         prediction_path=args.prediction_path,
         mapping_path=args.mapping_path,
@@ -222,10 +284,22 @@ def main():
         metrics_list=args.metrics,
         threshold=args.threshold,
         class_wise=args.class_wise,
+        averaging=args.averaging,
+        score_unannotated_as_empty=args.score_unannotated_as_empty,
     )
+    pa = result.pa
+    predictions = result.predictions
+    labels = result.labels
 
     # Display the computed metrics
-    logger.info(metrics_df)
+    logger.info(result.metrics_df)
+
+    if result.empty_classes:
+        logger.warning(
+            "Classes with no positive annotations were excluded from the overall "
+            "score: %s",
+            ", ".join(result.empty_classes),
+        )
 
     # Create output directory if needed
     if args.output_dir and not os.path.exists(args.output_dir):

birdnet_analyzer/evaluation/assessment/metrics.py

@@ -28,6 +28,31 @@
 )
 
 
+def _check_inputs(
+    predictions: np.ndarray,
+    labels: np.ndarray,
+    threshold: float | None = None,
+) -> None:
+    """Validate the shared preconditions of every metric function.
+
+    Args:
+        predictions (np.ndarray): Model predictions as probabilities.
+        labels (np.ndarray): True labels.
+        threshold (Optional[float]): If given, must lie in [0, 1]. Pass None for
+            threshold-independent metrics (AP, AUROC).
+
+    Raises:
+        ValueError: If the arrays are empty, mismatched in shape, or the threshold is
+            out of range.
+    """
+    if predictions.size == 0 or labels.size == 0:
+        raise ValueError("Predictions and labels must not be empty.")
+    if predictions.shape != labels.shape:
+        raise ValueError("Predictions and labels must have the same shape.")
+    if threshold is not None and not 0 <= threshold <= 1:
+        raise ValueError(f"Invalid threshold: {threshold}. Must be between 0 and 1.")
+
+
 def calculate_accuracy(
     predictions: np.ndarray,
     labels: np.ndarray,
@@ -57,13 +82,7 @@ def calculate_accuracy(
         ValueError: If inputs are invalid or unsupported task/averaging method is
             specified.
     """
-    # Input validation for predictions, labels, and threshold
-    if predictions.size == 0 or labels.size == 0:
-        raise ValueError("Predictions and labels must not be empty.")
-    if not 0 <= threshold <= 1:
-        raise ValueError(f"Invalid threshold: {threshold}. Must be between 0 and 1.")
-    if predictions.shape != labels.shape:
-        raise ValueError("Predictions and labels must have the same shape.")
+    _check_inputs(predictions, labels, threshold)
 
     # Handle binary and multilabel tasks separately
     if task == "binary":
@@ -148,13 +167,7 @@ def calculate_recall(
     Raises:
         ValueError: If inputs are invalid or unsupported task type is specified.
     """
-    # Validate inputs for size, threshold, and shape
-    if predictions.size == 0 or labels.size == 0:
-        raise ValueError("Predictions and labels must not be empty.")
-    if not 0 <= threshold <= 1:
-        raise ValueError(f"Invalid threshold: {threshold}. Must be between 0 and 1.")
-    if predictions.shape != labels.shape:
-        raise ValueError("Predictions and labels must have the same shape.")
+    _check_inputs(predictions, labels, threshold)
 
     # Adjust averaging method for scikit-learn if none is specified
     averaging = None if averaging_method == "none" else averaging_method
@@ -207,13 +220,7 @@ def calculate_precision(
     Raises:
         ValueError: If inputs are invalid or unsupported task type is specified.
     """
-    # Validate inputs for size, threshold, and shape
-    if predictions.size == 0 or labels.size == 0:
-        raise ValueError("Predictions and labels must not be empty.")
-    if not 0 <= threshold <= 1:
-        raise ValueError(f"Invalid threshold: {threshold}. Must be between 0 and 1.")
-    if predictions.shape != labels.shape:
-        raise ValueError("Predictions and labels must have the same shape.")
+    _check_inputs(predictions, labels, threshold)
 
     # Adjust averaging method for scikit-learn if none is specified
     averaging = None if averaging_method == "none" else averaging_method
@@ -266,13 +273,7 @@ def calculate_f1_score(
     Raises:
         ValueError: If inputs are invalid or unsupported task type is specified.
     """
-    # Validate inputs for size, threshold, and shape
-    if predictions.size == 0 or labels.size == 0:
-        raise ValueError("Predictions and labels must not be empty.")
-    if not 0 <= threshold <= 1:
-        raise ValueError(f"Invalid threshold: {threshold}. Must be between 0 and 1.")
-    if predictions.shape != labels.shape:
-        raise ValueError("Predictions and labels must have the same shape.")
+    _check_inputs(predictions, labels, threshold)
 
     # Adjust averaging method for scikit-learn if none is specified
     averaging = None if averaging_method == "none" else averaging_method
@@ -323,11 +324,7 @@ def calculate_average_precision(
     Raises:
         ValueError: If inputs are invalid or unsupported task type is specified.
     """
-    # Validate inputs for size and shape
-    if predictions.size == 0 or labels.size == 0:
-        raise ValueError("Predictions and labels must not be empty.")
-    if predictions.shape != labels.shape:
-        raise ValueError("Predictions and labels must have the same shape.")
+    _check_inputs(predictions, labels)
 
     # Adjust averaging method for scikit-learn if none is specified
     averaging = None if averaging_method == "none" else averaging_method
@@ -371,11 +368,7 @@ def calculate_auroc(
     Raises:
         ValueError: If inputs are invalid or unsupported task type is specified.
     """
-    # Validate inputs for size and shape
-    if predictions.size == 0 or labels.size == 0:
-        raise ValueError("Predictions and labels must not be empty.")
-    if predictions.shape != labels.shape:
-        raise ValueError("Predictions and labels must have the same shape.")
+    _check_inputs(predictions, labels)
 
     # Adjust averaging method for scikit-learn if none is specified
     averaging = None if averaging_method == "none" else averaging_method