From 33cc73eaa4e4d2c4fe6ee51932331d155bb2ccb7 Mon Sep 17 00:00:00 2001 From: Josef Haupt Date: Mon, 27 Jul 2026 21:57:55 +0200 Subject: [PATCH 1/8] CI model cache --- .github/workflows/ci.yml | 19 +++++++++++++++++++ .gitignore | 3 +++ 2 files changed, 22 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd51b22cd..8b92507e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,14 @@ jobs: runs-on: ${{ matrix.os }} env: IS_GITHUB_RUNNER: "true" + # Fixed, cacheable location for birdnet's model/label downloads. The acoustic + # (~120 MB), geo (~35 MB) and perch-v2 (~380 MB) models all land here, and + # birdnet reads this env var at import time and skips the download when the + # files already exist - so a cache hit avoids re-fetching them from tuc.cloud + # on every job. Placed under the workspace because the `runner` context isn't + # available in job-level env but `github.workspace` is; checkout runs before + # the cache is restored, so it never wipes the restored models. + BIRDNET_APP_DATA: ${{ github.workspace }}/.birdnet-app-data strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] @@ -37,6 +45,17 @@ jobs: with: enable-cache: true cache-dependency-glob: "pyproject.toml" + # Cache the downloaded models across runs. Keyed on the birdnet dependency + # (via pyproject.toml) and the OS only - the model files are identical across + # Python versions, so all matrix jobs on an OS share one cache. birdnet + # re-downloads only missing/updated models, so restoring an older cache is safe. + - name: Cache birdnet models + uses: actions/cache@v4 + with: + path: ${{ env.BIRDNET_APP_DATA }} + key: birdnet-models-${{ runner.os }}-${{ hashFiles('pyproject.toml') }} + restore-keys: | + birdnet-models-${{ runner.os }}- - name: Install dependencies run: uv pip install --system .[embeddings,train,tests,gui-tests] - name: Run tests diff --git a/.gitignore b/.gitignore index 84c8345d7..ba9a9a55f 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,9 @@ uv.lock # Ruff .ruff_cache +# CI model cache location (BIRDNET_APP_DATA in .github/workflows/ci.yml) +.birdnet-app-data + # Models checkpoints From 4224a9569cd4e42b899817bdfab8f24e3cc73087 Mon Sep 17 00:00:00 2001 From: Josef Haupt Date: Tue, 28 Jul 2026 15:02:18 +0200 Subject: [PATCH 2/8] . --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b92507e8..82cbd4d6c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,9 @@ on: concurrency: group: "${{ github.event.pull_request.number }}-${{ github.ref_name }}-${{ github.workflow }}" - cancel-in-progress: true + # Cancel superseded PR runs when new commits are pushed, but let pushes to main + # run to completion so a merge's CI is never cancelled by the next merge. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: running-tests: From b7e4a54f583354133c30c7908ea3a07426b31732 Mon Sep 17 00:00:00 2001 From: Josef Haupt Date: Tue, 28 Jul 2026 16:56:53 +0200 Subject: [PATCH 3/8] . --- .github/workflows/ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82cbd4d6c..3d07dd9b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,10 @@ concurrency: jobs: running-tests: runs-on: ${{ matrix.os }} + # Backstop for a wedged run (e.g. a hung model download or a native deadlock + # that pytest-timeout can't interrupt). Healthy jobs finish in ~2-6 min, plus a + # few for a cold-cache pre-download, so 30 caps a hang's cost with wide margin. + timeout-minutes: 30 env: IS_GITHUB_RUNNER: "true" # Fixed, cacheable location for birdnet's model/label downloads. The acoustic @@ -60,6 +64,15 @@ jobs: birdnet-models-${{ runner.os }}- - name: Install dependencies run: uv pip install --system .[embeddings,train,tests,gui-tests] + # Pre-download the models outside pytest's per-test 120s timeout. On a cold + # cache the acoustic-model download from tuc.cloud can exceed 120s on slower + # (Windows/macOS) runners and trip the timeout mid-download - which also means + # the model cache never captures a complete model, so the next run is cold + # again and the flake is permanent. Fetching them here (no per-test timeout) + # breaks that cycle and lets the cache actually populate. Mirrors the models + # baked into the Docker image. + - name: Pre-download birdnet models + run: python -c "import birdnet; birdnet.load('acoustic', '2.4', 'tf', lang='en_us'); birdnet.load('geo', '2.4', 'tf', lang='en_us')" - name: Run tests run: | python -m pytest From b8968b16887e3074ee8af1ccff06fdbd61c12d85 Mon Sep 17 00:00:00 2001 From: Josef Haupt Date: Wed, 29 Jul 2026 14:01:10 +0200 Subject: [PATCH 4/8] Revisit review tab --- birdnet_analyzer/evaluation/__init__.py | 88 +- .../evaluation/assessment/metrics.py | 69 +- .../assessment/performance_assessor.py | 362 ++++--- .../preprocessing/data_processor.py | 76 +- .../evaluation/preprocessing/utils.py | 76 +- birdnet_analyzer/gui/evaluation.py | 962 +++++++++--------- birdnet_analyzer/lang/de.json | 14 +- birdnet_analyzer/lang/en.json | 14 +- birdnet_analyzer/lang/fi.json | 14 +- birdnet_analyzer/lang/fr.json | 14 +- birdnet_analyzer/lang/id.json | 14 +- birdnet_analyzer/lang/pt-br.json | 14 +- birdnet_analyzer/lang/ru.json | 14 +- birdnet_analyzer/lang/se.json | 14 +- birdnet_analyzer/lang/tlh.json | 14 +- birdnet_analyzer/lang/zh_TW.json | 14 +- .../assessment/test_performance_assessor.py | 79 ++ .../preprocessing/test_data_processor.py | 18 +- tests/evaluation/preprocessing/test_utils.py | 17 +- tests/evaluation/test_process_data.py | 173 ++++ 20 files changed, 1325 insertions(+), 735 deletions(-) create mode 100644 tests/evaluation/test_process_data.py diff --git a/birdnet_analyzer/evaluation/__init__.py b/birdnet_analyzer/evaluation/__init__.py index 55e29bdca..558296a99 100644 --- a/birdnet_analyzer/evaluation/__init__.py +++ b/birdnet_analyzer/evaluation/__init__.py @@ -11,16 +11,40 @@ import json import logging import os +import typing from collections.abc import Sequence +from typing import Literal, NamedTuple from birdnet_analyzer.evaluation.assessment.performance_assessor import ( PerformanceAssessor, ) from birdnet_analyzer.evaluation.preprocessing.data_processor import DataProcessor +if typing.TYPE_CHECKING: + import numpy as np + import pandas as pd + logger = logging.getLogger(__name__) +class EvaluationResult(NamedTuple): + """The outcome of :func:`process_data`. + + Besides the metrics and the tensors the assessment ran on, it carries the context + a caller needs to explain the numbers: which recordings had predictions but no + annotations, and which classes were empty (and therefore excluded from any + aggregate score). + """ + + metrics_df: "pd.DataFrame" + pa: PerformanceAssessor + predictions: "np.ndarray" + labels: "np.ndarray" + classes: tuple[str, ...] + unmatched_recordings: tuple[str, ...] + empty_classes: tuple[str, ...] + + def process_data( annotation_path: str, prediction_path: str, @@ -35,7 +59,9 @@ def process_data( metrics_list: tuple[str, ...] = ("accuracy", "precision", "recall"), threshold: float = 0.1, class_wise: bool = False, -): + averaging: Literal["macro", "micro", "weighted"] = "macro", + score_unannotated_as_empty: bool = False, +) -> EvaluationResult: """ Processes data, computes metrics, and prepares the performance assessment pipeline. @@ -60,10 +86,17 @@ def process_data( metrics_list (Tuple[str, ...]): Metrics to compute for performance assessment. threshold (float): Confidence threshold for predictions. class_wise (bool): Whether to calculate metrics on a per-class basis. + averaging (Literal["macro", "micro", "weighted"]): How to aggregate the + per-class metrics into the overall score. Ignored when ``class_wise`` is + True. + score_unannotated_as_empty (bool): Whether recordings that have predictions but + no annotation file are kept and scored as all-negative (True) or dropped as + unscoreable (False, the default). Returns: - Tuple: Metrics DataFrame, `PerformanceAssessor` object, predictions tensor, - labels tensor. + EvaluationResult: The metrics DataFrame, the ``PerformanceAssessor``, the + prediction and label tensors, and the context (classes, unmatched + recordings, empty classes). """ # Load class mapping if provided if mapping_path: @@ -96,6 +129,7 @@ def process_data( columns_predictions=columns_predictions, columns_annotations=columns_annotations, recording_duration=recording_duration, + score_unannotated_as_empty=score_unannotated_as_empty, ) # Get the available classes and recordings @@ -126,9 +160,23 @@ def process_data( ) # Compute performance metrics - metrics_df = pa.calculate_metrics(predictions, labels, per_class_metrics=class_wise) + metrics_df = pa.calculate_metrics( + predictions, + labels, + per_class_metrics=class_wise, + averaging=averaging, + include_support=class_wise, + ) - return metrics_df, pa, predictions, labels + return EvaluationResult( + metrics_df=metrics_df, + pa=pa, + predictions=predictions, + labels=labels, + classes=classes, + unmatched_recordings=tuple(sorted(processor.unmatched_prediction_files)), + empty_classes=pa.empty_classes(labels), + ) def main(): @@ -193,6 +241,20 @@ def main(): parser.add_argument( "--class_wise", action="store_true", help="Calculate class-wise metrics" ) + parser.add_argument( + "--averaging", + choices=["macro", "micro", "weighted"], + default="macro", + help="How to aggregate per-class metrics into the overall score", + ) + parser.add_argument( + "--score_unannotated_as_empty", + action="store_true", + help=( + "Keep recordings that have predictions but no annotation file and score " + "them as all-negative, instead of dropping them (default)" + ), + ) parser.add_argument("--plot_metrics", action="store_true", help="Plot metrics") parser.add_argument( "--plot_confusion_matrix", action="store_true", help="Plot confusion matrix" @@ -208,7 +270,7 @@ def main(): args = parser.parse_args() # Process data and compute metrics - metrics_df, pa, predictions, labels = process_data( + result = process_data( annotation_path=args.annotation_path, prediction_path=args.prediction_path, mapping_path=args.mapping_path, @@ -222,10 +284,22 @@ def main(): metrics_list=args.metrics, threshold=args.threshold, class_wise=args.class_wise, + averaging=args.averaging, + score_unannotated_as_empty=args.score_unannotated_as_empty, ) + pa = result.pa + predictions = result.predictions + labels = result.labels # Display the computed metrics - logger.info(metrics_df) + logger.info(result.metrics_df) + + if result.empty_classes: + logger.warning( + "Classes with no positive annotations were excluded from the overall " + "score: %s", + ", ".join(result.empty_classes), + ) # Create output directory if needed if args.output_dir and not os.path.exists(args.output_dir): diff --git a/birdnet_analyzer/evaluation/assessment/metrics.py b/birdnet_analyzer/evaluation/assessment/metrics.py index 9b72f1688..3dfb51100 100644 --- a/birdnet_analyzer/evaluation/assessment/metrics.py +++ b/birdnet_analyzer/evaluation/assessment/metrics.py @@ -28,6 +28,31 @@ ) +def _check_inputs( + predictions: np.ndarray, + labels: np.ndarray, + threshold: float | None = None, +) -> None: + """Validate the shared preconditions of every metric function. + + Args: + predictions (np.ndarray): Model predictions as probabilities. + labels (np.ndarray): True labels. + threshold (Optional[float]): If given, must lie in [0, 1]. Pass None for + threshold-independent metrics (AP, AUROC). + + Raises: + ValueError: If the arrays are empty, mismatched in shape, or the threshold is + out of range. + """ + if predictions.size == 0 or labels.size == 0: + raise ValueError("Predictions and labels must not be empty.") + if predictions.shape != labels.shape: + raise ValueError("Predictions and labels must have the same shape.") + if threshold is not None and not 0 <= threshold <= 1: + raise ValueError(f"Invalid threshold: {threshold}. Must be between 0 and 1.") + + def calculate_accuracy( predictions: np.ndarray, labels: np.ndarray, @@ -57,13 +82,7 @@ def calculate_accuracy( ValueError: If inputs are invalid or unsupported task/averaging method is specified. """ - # Input validation for predictions, labels, and threshold - if predictions.size == 0 or labels.size == 0: - raise ValueError("Predictions and labels must not be empty.") - if not 0 <= threshold <= 1: - raise ValueError(f"Invalid threshold: {threshold}. Must be between 0 and 1.") - if predictions.shape != labels.shape: - raise ValueError("Predictions and labels must have the same shape.") + _check_inputs(predictions, labels, threshold) # Handle binary and multilabel tasks separately if task == "binary": @@ -148,13 +167,7 @@ def calculate_recall( Raises: ValueError: If inputs are invalid or unsupported task type is specified. """ - # Validate inputs for size, threshold, and shape - if predictions.size == 0 or labels.size == 0: - raise ValueError("Predictions and labels must not be empty.") - if not 0 <= threshold <= 1: - raise ValueError(f"Invalid threshold: {threshold}. Must be between 0 and 1.") - if predictions.shape != labels.shape: - raise ValueError("Predictions and labels must have the same shape.") + _check_inputs(predictions, labels, threshold) # Adjust averaging method for scikit-learn if none is specified averaging = None if averaging_method == "none" else averaging_method @@ -207,13 +220,7 @@ def calculate_precision( Raises: ValueError: If inputs are invalid or unsupported task type is specified. """ - # Validate inputs for size, threshold, and shape - if predictions.size == 0 or labels.size == 0: - raise ValueError("Predictions and labels must not be empty.") - if not 0 <= threshold <= 1: - raise ValueError(f"Invalid threshold: {threshold}. Must be between 0 and 1.") - if predictions.shape != labels.shape: - raise ValueError("Predictions and labels must have the same shape.") + _check_inputs(predictions, labels, threshold) # Adjust averaging method for scikit-learn if none is specified averaging = None if averaging_method == "none" else averaging_method @@ -266,13 +273,7 @@ def calculate_f1_score( Raises: ValueError: If inputs are invalid or unsupported task type is specified. """ - # Validate inputs for size, threshold, and shape - if predictions.size == 0 or labels.size == 0: - raise ValueError("Predictions and labels must not be empty.") - if not 0 <= threshold <= 1: - raise ValueError(f"Invalid threshold: {threshold}. Must be between 0 and 1.") - if predictions.shape != labels.shape: - raise ValueError("Predictions and labels must have the same shape.") + _check_inputs(predictions, labels, threshold) # Adjust averaging method for scikit-learn if none is specified averaging = None if averaging_method == "none" else averaging_method @@ -323,11 +324,7 @@ def calculate_average_precision( Raises: ValueError: If inputs are invalid or unsupported task type is specified. """ - # Validate inputs for size and shape - if predictions.size == 0 or labels.size == 0: - raise ValueError("Predictions and labels must not be empty.") - if predictions.shape != labels.shape: - raise ValueError("Predictions and labels must have the same shape.") + _check_inputs(predictions, labels) # Adjust averaging method for scikit-learn if none is specified averaging = None if averaging_method == "none" else averaging_method @@ -371,11 +368,7 @@ def calculate_auroc( Raises: ValueError: If inputs are invalid or unsupported task type is specified. """ - # Validate inputs for size and shape - if predictions.size == 0 or labels.size == 0: - raise ValueError("Predictions and labels must not be empty.") - if predictions.shape != labels.shape: - raise ValueError("Predictions and labels must have the same shape.") + _check_inputs(predictions, labels) # Adjust averaging method for scikit-learn if none is specified averaging = None if averaging_method == "none" else averaging_method diff --git a/birdnet_analyzer/evaluation/assessment/performance_assessor.py b/birdnet_analyzer/evaluation/assessment/performance_assessor.py index 6ebd20872..f1b6cae0b 100644 --- a/birdnet_analyzer/evaluation/assessment/performance_assessor.py +++ b/birdnet_analyzer/evaluation/assessment/performance_assessor.py @@ -7,7 +7,7 @@ accuracy, as well as utilities for generating related plots. """ -from typing import Literal +from typing import ClassVar, Literal import numpy as np import pandas as pd @@ -94,124 +94,213 @@ def __init__( # Set default colors for plotting self.colors = ["#3A50B1", "#61A83E", "#D74C4C", "#A13FA1", "#D9A544", "#F3A6E0"] - def calculate_metrics( + # Display labels for each supported metric id, in a stable order. + _METRIC_DISPLAY_NAMES: ClassVar[dict[str, str]] = { + "recall": "Recall", + "precision": "Precision", + "f1": "F1", + "ap": "AP", + "auroc": "AUROC", + "accuracy": "Accuracy", + } + + def _class_names(self) -> list[str]: + """The class labels to use as columns, falling back to ``Class i``.""" + return list(self.classes or [f"Class {i}" for i in range(self.num_classes)]) + + def _compute_metrics_dict( self, predictions: np.ndarray, labels: np.ndarray, - per_class_metrics: bool = False, - ) -> pd.DataFrame: - """ - Calculate multiple performance metrics for the given predictions and labels. + averaging_method, + num_classes: int, + ) -> dict[str, np.ndarray]: + """Compute every requested metric, keyed by its display name. Args: - predictions (np.ndarray): Model predictions as a 2D NumPy array - (probabilities or logits). - labels (np.ndarray): Ground truth labels as a 2D NumPy array. - per_class_metrics (bool): If True, compute metrics for each class - individually. + predictions (np.ndarray): Prediction probabilities. + labels (np.ndarray): Ground truth labels, same shape as predictions. + averaging_method: Averaging passed to the metric functions. ``None`` yields + one value per class; ``"macro"``/``"micro"``/``"weighted"`` aggregate. + num_classes (int): Number of columns in the arrays (needed by accuracy). Returns: - pd.DataFrame: A DataFrame containing the computed metrics. - - Raises: - TypeError: If predictions or labels are not NumPy arrays. - ValueError: If predictions and labels have mismatched dimensions or invalid - shapes. + An ordered mapping of display name to a 1D array of metric values. """ - # Validate that predictions and labels are NumPy arrays - if not isinstance(predictions, np.ndarray): - raise TypeError("predictions must be a NumPy array.") - if not isinstance(labels, np.ndarray): - raise TypeError("labels must be a NumPy array.") + results: dict[str, np.ndarray] = {} - # Ensure predictions and labels have the same shape - if predictions.shape != labels.shape: - raise ValueError("predictions and labels must have the same shape.") - if predictions.ndim != 2: - raise ValueError("predictions and labels must be 2-dimensional arrays.") - if predictions.shape[1] != self.num_classes: - raise ValueError( - f"The number of columns in predictions ({predictions.shape[1]}) " - + f"must match num_classes ({self.num_classes})." - ) - - # Determine the averaging method for metrics - averaging_method = ( - None if per_class_metrics or self.num_classes == 1 else "macro" - ) - - # Dictionary to store the results of each metric - metrics_results = {} - - # Compute each metric in the metrics list for metric_name in self.metrics_list: if metric_name == "recall": - result = metrics.calculate_recall( + value = metrics.calculate_recall( predictions=predictions, labels=labels, task=self.task, threshold=self.threshold, averaging_method=averaging_method, ) - metrics_results["Recall"] = np.atleast_1d(result) elif metric_name == "precision": - result = metrics.calculate_precision( + value = metrics.calculate_precision( predictions=predictions, labels=labels, task=self.task, threshold=self.threshold, averaging_method=averaging_method, ) - metrics_results["Precision"] = np.atleast_1d(result) elif metric_name == "f1": - result = metrics.calculate_f1_score( + value = metrics.calculate_f1_score( predictions=predictions, labels=labels, task=self.task, threshold=self.threshold, averaging_method=averaging_method, ) - metrics_results["F1"] = np.atleast_1d(result) elif metric_name == "ap": - result = metrics.calculate_average_precision( + value = metrics.calculate_average_precision( predictions=predictions, labels=labels, task=self.task, averaging_method=averaging_method, ) - metrics_results["AP"] = np.atleast_1d(result) elif metric_name == "auroc": - result = metrics.calculate_auroc( + value = metrics.calculate_auroc( predictions=predictions, labels=labels, task=self.task, averaging_method=averaging_method, ) - metrics_results["AUROC"] = np.atleast_1d(result) elif metric_name == "accuracy": - result = metrics.calculate_accuracy( + value = metrics.calculate_accuracy( predictions=predictions, labels=labels, task=self.task, - num_classes=self.num_classes, + num_classes=num_classes, threshold=self.threshold, averaging_method=averaging_method, ) - metrics_results["Accuracy"] = np.atleast_1d(result) + else: + continue - # Define column names for the DataFrame - columns = ( - (self.classes or [f"Class {i}" for i in range(self.num_classes)]) - if per_class_metrics - else ["Overall"] + results[self._METRIC_DISPLAY_NAMES[metric_name]] = np.atleast_1d(value) + + return results + + def class_support(self, labels: np.ndarray) -> np.ndarray: + """Number of positive samples per class (the support of each class).""" + return labels.astype(bool).sum(axis=0).astype(int) + + def empty_classes(self, labels: np.ndarray) -> tuple[str, ...]: + """Names of the classes that have no positive annotation in ``labels``. + + These classes are excluded from the aggregate score, because precision, + recall, F1 and AUROC are undefined without a single positive example. + """ + support = self.class_support(labels) + return tuple( + name + for name, count in zip(self._class_names(), support, strict=True) + if count == 0 + ) + + def calculate_metrics( + self, + predictions: np.ndarray, + labels: np.ndarray, + per_class_metrics: bool = False, + averaging: Literal["macro", "micro", "weighted"] = "macro", + drop_empty: bool = True, + include_support: bool = False, + ) -> pd.DataFrame: + """ + Calculate multiple performance metrics for the given predictions and labels. + + Args: + predictions (np.ndarray): Model predictions as a 2D NumPy array + (probabilities or logits). + labels (np.ndarray): Ground truth labels as a 2D NumPy array. + per_class_metrics (bool): If True, compute metrics for each class + individually. If False, return a single aggregate column. + averaging (Literal["macro", "micro", "weighted"]): How to aggregate the + per-class metrics into the single ``Overall`` column. Ignored when + ``per_class_metrics`` is True. + drop_empty (bool): When aggregating, exclude classes with no positive labels + (their metrics are undefined) instead of counting them as zero. Applies + only to the aggregate column. + include_support (bool): When ``per_class_metrics`` is True, append a + ``Support`` row with the number of positive samples per class. + + Returns: + pd.DataFrame: A DataFrame containing the computed metrics. + + Raises: + TypeError: If predictions or labels are not NumPy arrays. + ValueError: If predictions and labels have mismatched dimensions or invalid + shapes. + """ + # Validate that predictions and labels are NumPy arrays + if not isinstance(predictions, np.ndarray): + raise TypeError("predictions must be a NumPy array.") + if not isinstance(labels, np.ndarray): + raise TypeError("labels must be a NumPy array.") + + # Ensure predictions and labels have the same shape + if predictions.shape != labels.shape: + raise ValueError("predictions and labels must have the same shape.") + if predictions.ndim != 2: + raise ValueError("predictions and labels must be 2-dimensional arrays.") + if predictions.shape[1] != self.num_classes: + raise ValueError( + f"The number of columns in predictions ({predictions.shape[1]}) " + + f"must match num_classes ({self.num_classes})." + ) + + if per_class_metrics: + # One value per class: no averaging across classes. + results = self._compute_metrics_dict( + predictions, labels, averaging_method=None, num_classes=self.num_classes + ) + metrics_df = pd.DataFrame.from_dict( + results, orient="index", columns=pd.Index(self._class_names()) + ) + if include_support: + metrics_df.loc["Support"] = self.class_support(labels) + + return metrics_df + + # Aggregate into a single column. Undefined (empty) classes are dropped so they + # cannot silently pull the average down, unless the caller keeps them. + keep = ( + self.class_support(labels) > 0 + if drop_empty + else np.ones(self.num_classes, dtype=bool) + ) + kept = np.flatnonzero(keep) + + if kept.size == 0: + # Nothing scoreable -- report NaN rather than a misleading zero. + results = { + self._METRIC_DISPLAY_NAMES[m]: np.array([np.nan]) + for m in self.metrics_list + } + return pd.DataFrame.from_dict( + results, orient="index", columns=pd.Index(["Overall"]) + ) + + predictions_kept = predictions[:, kept] + labels_kept = labels[:, kept] + # For a binary task the metric functions want no averaging (they use the + # positive-class value). For multilabel we always aggregate across the kept + # classes -- even a single kept column, where ``average=None`` would wrongly + # return one value per binary outcome instead of a single score. + averaging_method = None if self.task == "binary" else averaging + results = self._compute_metrics_dict( + predictions_kept, + labels_kept, + averaging_method=averaging_method, + num_classes=kept.size, ) - # Create a DataFrame to organize metric results - metrics_data = { - key: np.atleast_1d(value) for key, value in metrics_results.items() - } return pd.DataFrame.from_dict( - metrics_data, orient="index", columns=pd.Index(columns) + results, orient="index", columns=pd.Index(["Overall"]) ) def plot_metrics( @@ -268,7 +357,8 @@ def plot_metrics_all_thresholds( Returns: None """ - # Save the original threshold value to restore it later + # Save the original threshold so the sweep never leaks out of this method, + # even if a metric computation or the plotting call raises. original_threshold = self.threshold # Define a range of thresholds for analysis @@ -277,89 +367,83 @@ def plot_metrics_all_thresholds( # Exclude metrics that are not threshold-dependent metrics_to_plot = [m for m in self.metrics_list if m not in ["auroc", "ap"]] - if per_class_metrics: - # Define class names for plotting - class_names = ( - list(self.classes) - if self.classes - else [f"Class {i}" for i in range(self.num_classes)] - ) + try: + if per_class_metrics: + class_names = self._class_names() - # Initialize a dictionary to store metric values per class - metric_values_dict_per_class = { - class_name: {metric: [] for metric in metrics_to_plot} - for class_name in class_names - } + # Initialize a dictionary to store metric values per class + metric_values_dict_per_class = { + class_name: {metric: [] for metric in metrics_to_plot} + for class_name in class_names + } - # Compute metrics for each threshold - for thresh in thresholds: - self.threshold = thresh - metrics_df = self.calculate_metrics( - predictions, labels, per_class_metrics=True - ) - for metric_name in metrics_to_plot: - metric_label = ( - metric_name.capitalize() if metric_name != "f1" else "F1" + # Compute metrics for each threshold + for thresh in thresholds: + self.threshold = thresh + metrics_df = self.calculate_metrics( + predictions, labels, per_class_metrics=True + ) + for metric_name in metrics_to_plot: + metric_label = self._METRIC_DISPLAY_NAMES[metric_name] + for class_name in class_names: + value = metrics_df.loc[metric_label, class_name] + metric_values_dict_per_class[class_name][ + metric_name + ].append(value) + + # Convert lists to NumPy arrays + metric_values_dict_per_class = { + class_name: { + metric: np.array(values) + for metric, values in metrics_dict.items() + } + for class_name, metrics_dict in ( + metric_values_dict_per_class.items() ) - for class_name in class_names: - value = metrics_df.loc[metric_label, class_name] - metric_values_dict_per_class[class_name][metric_name].append( - value - ) - - # Restore the original threshold - self.threshold = original_threshold - - # Convert lists to NumPy arrays - metric_values_dict_per_class = { - class_name: { - metric: np.array(values) for metric, values in metrics_dict.items() } - for class_name, metrics_dict in metric_values_dict_per_class.items() - } - # Plot metrics across thresholds per class - fig = plotting.plot_metrics_across_thresholds_per_class( - thresholds, - metric_values_dict_per_class, - metrics_to_plot, - class_names, - self.colors, - ) - else: - # Initialize a dictionary to store overall metric values - metric_values_dict = {metric_name: [] for metric_name in metrics_to_plot} - - # Compute metrics for each threshold - for thresh in thresholds: - self.threshold = thresh - metrics_df = self.calculate_metrics( - predictions, labels, per_class_metrics=False + # Plot metrics across thresholds per class + fig = plotting.plot_metrics_across_thresholds_per_class( + thresholds, + metric_values_dict_per_class, + metrics_to_plot, + class_names, + self.colors, ) - for metric_name in metrics_to_plot: - metric_label = ( - metric_name.capitalize() if metric_name != "f1" else "F1" + else: + # Initialize a dictionary to store overall metric values + metric_values_dict = { + metric_name: [] for metric_name in metrics_to_plot + } + + # Compute metrics for each threshold + for thresh in thresholds: + self.threshold = thresh + metrics_df = self.calculate_metrics( + predictions, labels, per_class_metrics=False ) - value = metrics_df.loc[metric_label, "Overall"] - metric_values_dict[metric_name].append(value) + for metric_name in metrics_to_plot: + metric_label = self._METRIC_DISPLAY_NAMES[metric_name] + value = metrics_df.loc[metric_label, "Overall"] + metric_values_dict[metric_name].append(value) + + # Convert lists to NumPy arrays + metric_values_dict = { + metric_name: np.array(values) + for metric_name, values in metric_values_dict.items() + } - # Restore the original threshold + # Plot metrics across thresholds + fig = plotting.plot_metrics_across_thresholds( + thresholds, + metric_values_dict, + metrics_to_plot, + self.colors, + ) + finally: + # Always restore the original threshold self.threshold = original_threshold - # Convert lists to NumPy arrays - metric_values_dict = { - metric_name: np.array(values) - for metric_name, values in metric_values_dict.items() - } - - # Plot metrics across thresholds - fig = plotting.plot_metrics_across_thresholds( - thresholds, - metric_values_dict, - metrics_to_plot, - self.colors, - ) - return fig def plot_confusion_matrix( diff --git a/birdnet_analyzer/evaluation/preprocessing/data_processor.py b/birdnet_analyzer/evaluation/preprocessing/data_processor.py index a0eadb829..636bfab49 100644 --- a/birdnet_analyzer/evaluation/preprocessing/data_processor.py +++ b/birdnet_analyzer/evaluation/preprocessing/data_processor.py @@ -61,6 +61,7 @@ def __init__( columns_predictions: dict[str, str] | None = None, columns_annotations: dict[str, str] | None = None, recording_duration: float | None = None, + score_unannotated_as_empty: bool = False, ) -> None: """ Initializes the DataProcessor by loading prediction and annotation data. @@ -86,6 +87,13 @@ class names to standardized class names. mappings for annotation files. recording_duration (Optional[float], optional): User-specified recording duration in seconds. Defaults to None. + score_unannotated_as_empty (bool, optional): How to treat recordings that + have predictions but no matching annotation file. When ``False`` + (default) those recordings are dropped, because without a ground truth + their predictions cannot be scored. When ``True`` they are kept and + treated as if fully annotated with no events, so every prediction on + them counts as a false positive (only correct when the annotations are + exhaustive). Defaults to False. Raises: ValueError: If any parameter is invalid (e.g., negative sample duration). @@ -108,6 +116,7 @@ class names to standardized class names. ) self.recording_duration: float | None = recording_duration + self.score_unannotated_as_empty: bool = score_unannotated_as_empty # Paths and filenames self.prediction_directory_path: str = prediction_directory_path @@ -119,8 +128,8 @@ class names to standardized class names. self.predictions_df: pd.DataFrame = pd.DataFrame() self.annotations_df: pd.DataFrame = pd.DataFrame() - # To track prediction files without matching annotation files - self.umatched_prediction_files: set[str] = set() + # Recordings that have predictions but no matching annotation file. + self.unmatched_prediction_files: set[str] = set() # Placeholder for unique classes across predictions and annotations self.classes: tuple[str, ...] = () @@ -215,7 +224,7 @@ def load_data(self) -> None: ValueError: If file reading fails or data preparation encounters issues. """ - self.umatched_prediction_files = set() # Reset unmatched prediction files + self.unmatched_prediction_files = set() # Reset unmatched prediction files if self.prediction_file_name is None or self.annotation_file_name is None: # Case: No specific files provided; load all files in directories. @@ -247,19 +256,6 @@ def load_data(self) -> None: self.predictions_df[class_col_pred] = self.predictions_df[ class_col_pred ].apply(lambda x: self.class_mapping.get(x, x)) # ty:ignore[unresolved-attribute] - - prediction_filenames = self.predictions_df["recording_filename"].unique() - annotation_filenames = self.annotations_df["recording_filename"].unique() - self.umatched_prediction_files = set(prediction_filenames) - set( - annotation_filenames - ) - if self.umatched_prediction_files: - warnings.warn( - f"Prediction files without matching annotation files: " - f"{', '.join(self.umatched_prediction_files)}. " - "These recordings will not be processed.", - stacklevel=2, - ) else: # Case: Specific files are provided for predictions and annotations. # Ensure filenames correspond to the same recording (heuristic check). @@ -303,6 +299,47 @@ def load_data(self) -> None: class_col_pred ].apply(lambda x: self.class_mapping.get(x, x)) # ty:ignore[unresolved-attribute] + # Reconcile the two recording sets. A recording that only has predictions has no + # ground truth, so its predictions cannot be scored. By default those recordings + # are dropped; only when the caller opts in (exhaustive annotations) are they + # kept and their predictions counted as false positives. This only applies when + # matching whole directories -- an explicitly named single file pair is always a + # deliberate 1:1 comparison. + directory_mode = ( + self.prediction_file_name is None or self.annotation_file_name is None + ) + if directory_mode and "recording_filename" in self.predictions_df.columns: + prediction_recordings = set( + self.predictions_df["recording_filename"].unique() + ) + annotation_recordings = set( + self.annotations_df["recording_filename"].unique() + ) + self.unmatched_prediction_files = ( + prediction_recordings - annotation_recordings + ) + + if self.unmatched_prediction_files: + names = ", ".join(sorted(map(str, self.unmatched_prediction_files))) + if self.score_unannotated_as_empty: + warnings.warn( + f"Prediction files without matching annotation file: {names}. " + "Their annotations are assumed empty, so every prediction on " + "them counts as a false positive.", + stacklevel=2, + ) + else: + warnings.warn( + f"Prediction files without matching annotation file: {names}. " + "These recordings are dropped and not scored.", + stacklevel=2, + ) + self.predictions_df = self.predictions_df[ + self.predictions_df["recording_filename"].isin( + annotation_recordings + ) + ] + # Consolidate all unique classes from predictions and annotations class_col_pred = self.get_column_name("Class", prediction=True) class_col_annot = self.get_column_name("Class", prediction=False) @@ -552,6 +589,13 @@ def update_samples_with_predictions( confidence scores for those samples, retaining the maximum confidence value if multiple predictions overlap. + Note: + `min_overlap` is applied as a margin on both sides of the sample window, not + as a guaranteed intersection length: a prediction shorter than `min_overlap` + that falls entirely inside a window still marks it. This keeps short events + from being dropped, but means the effective overlap requirement is looser + than `min_overlap` seconds for events shorter than the sample duration. + Args: pred_df (pd.DataFrame): DataFrame containing prediction information. samples_df (pd.DataFrame): DataFrame of samples to be updated with diff --git a/birdnet_analyzer/evaluation/preprocessing/utils.py b/birdnet_analyzer/evaluation/preprocessing/utils.py index b2d0f9e03..1b491b830 100644 --- a/birdnet_analyzer/evaluation/preprocessing/utils.py +++ b/birdnet_analyzer/evaluation/preprocessing/utils.py @@ -2,7 +2,7 @@ Utility Functions for Data Processing Tasks This module provides helper functions to handle common data processing tasks, such as: -- Extracting recording filenames from file paths or filenames. +- Extracting recording keys from file paths or selection-table filenames. - Reading and concatenating text files from a specified directory. It is designed to work seamlessly with pandas and file system operations. @@ -12,41 +12,79 @@ import pandas as pd +# Suffixes BirdNET appends to a recording name when it writes a selection/result table, +# e.g. ``soundscape.BirdNET.selection.table.txt``. Stripped so the table matches the +# annotation file (and audio file) it belongs to. +_TABLE_SUFFIXES = ( + ".BirdNET.selection.table", + ".BirdNET.results", + ".BirdNET", +) -def extract_recording_filename(path_column: pd.Series) -> pd.Series: +# Audio extensions that may still cling to a name after the table suffix is removed, +# e.g. ``soundscape.wav.BirdNET.selection.table.txt`` -> ``soundscape.wav`` -> key. +_AUDIO_EXTENSIONS = (".wav", ".flac", ".mp3", ".ogg", ".m4a", ".aac", ".wave") + + +def recording_key(name) -> str: + """Derives the recording identifier a prediction/annotation entry belongs to. + + This is the single source of truth for matching prediction files to annotation + files. It takes the base name (dropping any directory and the final extension), + then removes a BirdNET table suffix and a residual audio extension, so that a + Raven selection table, a plain annotation file, and the ``Begin File`` column of + the same recording all collapse to the same key. + + Unlike the previous ``split(".")[0]`` approach, a dot inside the recording name + (dates such as ``2023.05.01_dawn``) is preserved, so distinct recordings never + collide. + + Args: + name: A file path, file name, or recording reference. Non-string values + (e.g. ``NaN``) are returned unchanged. + + Returns: + The recording key, or the input unchanged if it is not a string. """ - Extract the recording filename from a path column. + if not isinstance(name, str): + return name - This function processes a pandas Series containing file paths and extracts the base - filename (without the extension) for each path. + key = os.path.splitext(os.path.basename(name))[0] + + for suffix in _TABLE_SUFFIXES: + if key.endswith(suffix): + key = key[: -len(suffix)] + break + + root, ext = os.path.splitext(key) + if ext.lower() in _AUDIO_EXTENSIONS: + key = root + + return key + + +def extract_recording_filename(path_column: pd.Series) -> pd.Series: + """Extract the recording key from a column of file paths (e.g. ``Begin File``). Args: path_column (pd.Series): A pandas Series containing file paths. Returns: - pd.Series: A pandas Series containing the extracted recording filenames. + pd.Series: The recording key for each entry. """ - # Apply a lambda function to extract the base filename without extension - return path_column.apply( - lambda x: os.path.splitext(os.path.basename(x))[0] if isinstance(x, str) else x - ) # ty:ignore[invalid-return-type] + return path_column.apply(recording_key) def extract_recording_filename_from_filename(filename_series: pd.Series) -> pd.Series: - """ - Extract the recording filename from a filename Series. - - This function processes a pandas Series containing filenames and extracts the base - filename (without the extension) for each. + """Extract the recording key from a column of selection-table file names. Args: - filename_series (pd.Series): A pandas Series containing filenames. + filename_series (pd.Series): A pandas Series containing file names. Returns: - pd.Series: A pandas Series containing the extracted recording filenames. + pd.Series: The recording key for each entry. """ - # Apply a lambda function to split filenames and remove the extension - return filename_series.apply(lambda x: x.split(".")[0] if isinstance(x, str) else x) # ty:ignore[invalid-return-type] + return filename_series.apply(recording_key) def read_and_concatenate_files_in_directory(directory_path: str) -> pd.DataFrame: diff --git a/birdnet_analyzer/gui/evaluation.py b/birdnet_analyzer/gui/evaluation.py index 5d0d14d63..52bdc7f26 100644 --- a/birdnet_analyzer/gui/evaluation.py +++ b/birdnet_analyzer/gui/evaluation.py @@ -2,9 +2,6 @@ import json import logging -import os -import shutil -import tempfile import typing import gradio as gr @@ -22,9 +19,12 @@ logger = logging.getLogger(__name__) +# Averaging methods offered for the overall score, with their stable ids. +AVERAGING_IDS = ("macro", "micro", "weighted") + class ProcessorState(typing.NamedTuple): - """State of the DataProcessor.""" + """State of the DataProcessor together with the directories it read.""" processor: DataProcessor annotation_dir: str @@ -62,6 +62,22 @@ def build_evaluation_tab() -> gu.TAB_BUILDER_RESULT: "Confidence": loc.localize("eval-tab-column-confidence-label"), } + annotation_column_order = [ + "Start Time", + "End Time", + "Class", + "Recording", + "Duration", + ] + prediction_column_order = [ + "Start Time", + "End Time", + "Class", + "Confidence", + "Recording", + "Duration", + ] + def download_class_mapping_template(): try: template_mapping = { @@ -90,7 +106,7 @@ def download_class_mapping_template(): ) from e def download_results_table( - pa: PerformanceAssessor, predictions, labels, class_wise_value + pa: PerformanceAssessor, predictions, labels, class_wise_value, averaging_value ): if pa is None or predictions is None or labels is None: raise gr.Error( @@ -106,7 +122,11 @@ def download_results_table( if file_location: metrics_df = pa.calculate_metrics( - predictions, labels, per_class_metrics=class_wise_value + predictions, + labels, + per_class_metrics=class_wise_value, + averaging=averaging_value, + include_support=class_wise_value, ) if file_location.split(".")[-1].lower() == "tsv": @@ -146,7 +166,7 @@ def download_data_table(processor_state: ProcessorState): f"{loc.localize('eval-tab-error-saving-data-table')} {e}" ) from e - def get_columns_from_uploaded_files(files): + def get_columns_from_files(files): columns = set() if files: @@ -163,99 +183,61 @@ def get_columns_from_uploaded_files(files): return sorted(columns) - def save_uploaded_files(files): - if not files: - return None - - temp_dir = tempfile.mkdtemp() - - for file_obj in files: - dest_path = os.path.join(temp_dir, os.path.basename(file_obj)) - shutil.copy(file_obj, dest_path) - - return temp_dir - - # Single initialize_processor that can reuse given directories. - def initialize_processor( - annotation_files, - prediction_files, + def build_processor( + annotation_dir, + prediction_dir, mapping_file_obj, sample_duration_value, min_overlap_value, - recording_duration, - ann_start_time, - ann_end_time, - ann_class, - ann_recording, - ann_duration, - pred_start_time, - pred_end_time, - pred_class, - pred_confidence, - pred_recording, - pred_duration, - annotation_dir=None, - prediction_dir=None, + recording_duration_value: str, + score_unannotated_value: bool, + ann_cols: dict[str, str], + pred_cols: dict[str, str], ): + """Builds a DataProcessor straight from the selected directories. + + The selection dialog already hands back real directories, so nothing is copied: + the processor reads the folders in place. Returns ``None`` if either directory + is missing. + """ from birdnet_analyzer.evaluation.preprocessing.data_processor import ( DataProcessor, ) - if not annotation_files or not prediction_files: - return [], [], None, None, None - - if annotation_dir is None: - annotation_dir = save_uploaded_files(annotation_files) - - if prediction_dir is None: - prediction_dir = save_uploaded_files(prediction_files) - - # Fallback for annotation columns. - ann_start_time = ann_start_time or annotation_default_columns["Start Time"] - ann_end_time = ann_end_time or annotation_default_columns["End Time"] - ann_class = ann_class or annotation_default_columns["Class"] - ann_recording = ann_recording or annotation_default_columns["Recording"] - ann_duration = ann_duration or annotation_default_columns["Duration"] + if not annotation_dir or not prediction_dir: + return None - # Fallback for prediction columns. - pred_start_time = pred_start_time or prediction_default_columns["Start Time"] - pred_end_time = pred_end_time or prediction_default_columns["End Time"] - pred_class = pred_class or prediction_default_columns["Class"] - pred_confidence = pred_confidence or prediction_default_columns["Confidence"] - pred_recording = pred_recording or prediction_default_columns["Recording"] - pred_duration = pred_duration or prediction_default_columns["Duration"] + try: + rec_dur = ( + float(recording_duration_value.strip()) + if recording_duration_value + else None + ) + except (ValueError, TypeError): + rec_dur = None + # Fill any blank column choice with its default. cols_ann = { - "Start Time": ann_start_time, - "End Time": ann_end_time, - "Class": ann_class, - "Recording": ann_recording, - "Duration": ann_duration, + key: ann_cols.get(key) or annotation_default_columns[key] + for key in annotation_column_order } cols_pred = { - "Start Time": pred_start_time, - "End Time": pred_end_time, - "Class": pred_class, - "Confidence": pred_confidence, - "Recording": pred_recording, - "Duration": pred_duration, + key: pred_cols.get(key) or prediction_default_columns[key] + for key in prediction_column_order } - # Handle mapping file: if it has a temp_files attribute use that, otherwise - # assume it's a filepath. if mapping_file_obj and hasattr(mapping_file_obj, "temp_files"): - mapping_path = list(mapping_file_obj.temp_files)[0] + mapping_path = next(iter(mapping_file_obj.temp_files)) else: mapping_path = mapping_file_obj or None + class_mapping = None if mapping_path: with open(mapping_path) as f: class_mapping = json.load(f) - else: - class_mapping = None try: - proc = DataProcessor( + processor = DataProcessor( prediction_directory_path=prediction_dir, prediction_file_name=None, annotation_directory_path=annotation_dir, @@ -265,115 +247,22 @@ def initialize_processor( min_overlap=min_overlap_value, columns_predictions=cols_pred, columns_annotations=cols_ann, - recording_duration=recording_duration, + recording_duration=rec_dur, + score_unannotated_as_empty=score_unannotated_value, ) - avail_classes = list(proc.classes) # Ensure it's a list - avail_recordings = proc.samples_df["filename"].unique().tolist() - - if len(proc.umatched_prediction_files) > 0: - gr.Warning( - f"{loc.localize('eval-tab-warning-unmatched-predictions')}: " - f"{', '.join(proc.umatched_prediction_files)}. " - f"{loc.localize('eval-tab-warning-unmatched-predictions-info')}" - ) - - return avail_classes, avail_recordings, proc, annotation_dir, prediction_dir except KeyError as e: logger.error(f"Column missing in files: {e}", exc_info=e) raise gr.Error( - f"{loc.localize('eval-tab-error-missing-col')}: " - + str(e) - + f". {loc.localize('eval-tab-error-missing-col-info')}" + f"{loc.localize('eval-tab-error-missing-col')}: {e}. " + f"{loc.localize('eval-tab-error-missing-col-info')}" ) from e except Exception as e: logger.error(f"Error initializing processor: {e}", exc_info=e) - raise gr.Error( - f"{loc.localize('eval-tab-error-init-processor')}:" + str(e) + f"{loc.localize('eval-tab-error-init-processor')}: {e}" ) from e - # update_selections is triggered when files or mapping file change. - # It creates the temporary directories once and stores them along with the - # processor. - # It now also receives the current selection values so that user selections are - # preserved. - def update_selections( - annotation_files, - prediction_files, - mapping_file_obj, - sample_duration_value, - min_overlap_value, - recording_duration_value: str, - ann_start_time, - ann_end_time, - ann_class, - ann_recording, - ann_duration, - pred_start_time, - pred_end_time, - pred_class, - pred_confidence, - pred_recording, - pred_duration, - current_classes, - current_recordings, - ): - try: - rec_dur = ( - float(recording_duration_value.strip()) - if recording_duration_value - else None - ) - except (ValueError, TypeError): - rec_dur = None - - # Create temporary directories once. - annotation_dir = save_uploaded_files(annotation_files) - prediction_dir = save_uploaded_files(prediction_files) - avail_classes, avail_recordings, proc, annotation_dir, prediction_dir = ( - initialize_processor( - annotation_files, - prediction_files, - mapping_file_obj, - sample_duration_value, - min_overlap_value, - rec_dur, - ann_start_time, - ann_end_time, - ann_class, - ann_recording, - ann_duration, - pred_start_time, - pred_end_time, - pred_class, - pred_confidence, - pred_recording, - pred_duration, - annotation_dir, - prediction_dir, - ) - ) - # Build a state dictionary to store the processor and the directories. - state = ProcessorState(proc, annotation_dir, prediction_dir) - # If no current selection exists, default to all available classes/recordings; - # otherwise, preserve any selections that are still valid. - new_classes = ( - avail_classes - if not current_classes - else [c for c in current_classes if c in avail_classes] or avail_classes - ) - new_recordings = ( - avail_recordings - if not current_recordings - else [r for r in current_recordings if r in avail_recordings] - or avail_recordings - ) - - return ( - gr.update(choices=avail_classes, value=new_classes), - gr.update(choices=avail_recordings, value=new_recordings), - state, - ) + return ProcessorState(processor, annotation_dir, prediction_dir) with gr.Tab(loc.localize("eval-tab-title")): processor_state = gr.State() @@ -382,6 +271,8 @@ def update_selections( labels_state = gr.State() annotation_files_state = gr.State() prediction_files_state = gr.State() + annotation_dir_state = gr.State() + prediction_dir_state = gr.State() plot_name_state = gr.State() gu.info_box( @@ -392,68 +283,66 @@ def update_selections( def get_selection_tables(directory): from pathlib import Path - directory = Path(directory) - - return list(directory.glob("*.txt")) - - # Update column dropdowns when files are uploaded. - def update_annotation_columns(uploaded_files): - cols = get_columns_from_uploaded_files(uploaded_files) - cols = ["", *cols] - updates = [] - - for label in ["Start Time", "End Time", "Class", "Recording", "Duration"]: - default_val = annotation_default_columns.get(label) - val = default_val if default_val in cols else None - updates.append(gr.update(choices=cols, value=val)) + return list(Path(directory).glob("*.txt")) - return updates + def update_annotation_columns(files): + cols = ["", *get_columns_from_files(files)] - def update_prediction_columns(uploaded_files): - cols = get_columns_from_uploaded_files(uploaded_files) - cols = ["", *cols] - updates = [] + return [ + gr.update( + choices=cols, + value=annotation_default_columns[label] + if annotation_default_columns[label] in cols + else None, + ) + for label in annotation_column_order + ] - for label in [ - "Start Time", - "End Time", - "Class", - "Confidence", - "Recording", - "Duration", - ]: - default_val = prediction_default_columns.get(label) - val = default_val if default_val in cols else None - updates.append(gr.update(choices=cols, value=val)) + def update_prediction_columns(files): + cols = ["", *get_columns_from_files(files)] - return updates + return [ + gr.update( + choices=cols, + value=prediction_default_columns[label] + if prediction_default_columns[label] in cols + else None, + ) + for label in prediction_column_order + ] - def get_selection_func(state_key, on_select): - def select_directory_on_empty(): + def get_selection_func(state_key, on_select, column_labels): + def select_directory(current_files, current_dir): folder = gu.select_folder(state_key=state_key) - if folder: - files = get_selection_tables(folder) - files_to_display = ( - [*files[:100], [f"{len(files) - 100} more..."]] - if len(files) > 100 - else files - ) + if not folder: + # Keep everything as it was when the dialog is cancelled. return [ - files, - folder, - files_to_display, - gr.update(visible=True), - *on_select(files), + current_files, + current_dir, + gr.update(), + gr.update(), + gr.update(), + *[gr.update() for _ in column_labels], ] + files = get_selection_tables(folder) + files_to_display = ( + [*files[:100], [f"{len(files) - 100} more..."]] + if len(files) > 100 + else files + ) + return [ - "", - "", - [[loc.localize("eval-tab-no-files-found")]], + files, + folder, + folder, + files_to_display, + gr.update(visible=True), + *on_select(files), ] - return select_directory_on_empty + return select_directory with gr.Group(), gr.Row(equal_height=True): annotation_select_directory_btn = gr.Button( @@ -477,6 +366,7 @@ def select_directory_on_empty(): headers=[ loc.localize("eval-tab-selections-column-file-header"), ], + buttons=[], ) with gr.Group(), gr.Row(equal_height=True): @@ -501,6 +391,7 @@ def select_directory_on_empty(): headers=[ loc.localize("eval-tab-selections-column-file-header"), ], + buttons=[], ) # ----------------------- Annotations Columns Box ----------------------- @@ -511,12 +402,10 @@ def select_directory_on_empty(): ), gr.Row(), ): - annotation_columns: dict[str, gr.Dropdown] = {} - - for col in ["Start Time", "End Time", "Class", "Recording", "Duration"]: - annotation_columns[col] = gr.Dropdown( - choices=[], label=localized_column_labels[col] - ) + annotation_columns: dict[str, gr.Dropdown] = { + col: gr.Dropdown(choices=[], label=localized_column_labels[col]) + for col in annotation_column_order + } # ----------------------- Predictions Columns Box ----------------------- with ( @@ -526,19 +415,10 @@ def select_directory_on_empty(): ), gr.Row(), ): - prediction_columns: dict[str, gr.Dropdown] = {} - - for col in [ - "Start Time", - "End Time", - "Class", - "Confidence", - "Recording", - "Duration", - ]: - prediction_columns[col] = gr.Dropdown( - choices=[], label=localized_column_labels[col] - ) + prediction_columns: dict[str, gr.Dropdown] = { + col: gr.Dropdown(choices=[], label=localized_column_labels[col]) + for col in prediction_column_order + } # ----------------------- Class Mapping Box ----------------------- with gr.Group(visible=False) as mapping_group: @@ -600,49 +480,57 @@ def select_directory_on_empty(): gr.Accordion( loc.localize("eval-tab-parameters-accordion-label"), open=False ), - gr.Row(), ): - sample_duration = state.persist( - "sample_duration_number", - gr.Number, - value=3, - label=loc.localize("eval-tab-sample-duration-number-label"), - precision=0, - info=loc.localize("eval-tab-sample-duration-number-info"), - ) - recording_duration = state.persist( - "recording_duration_textbox", - gr.Textbox, - value="", - label=loc.localize("eval-tab-recording-duration-textbox-label"), - placeholder=loc.localize( - "eval-tab-recording-duration-textbox-placeholder" - ), - info=loc.localize("eval-tab-recording-duration-textbox-info"), - ) - min_overlap = state.persist( - "min_overlap_number", - gr.Number, - value=0.5, - label=loc.localize("eval-tab-min-overlap-number-label"), - info=loc.localize("eval-tab-min-overlap-number-info"), - ) - threshold = state.persist( - "threshold_slider", - gr.Slider, - minimum=0.01, - maximum=0.99, - value=0.1, - label=loc.localize("eval-tab-threshold-number-label"), - info=loc.localize("eval-tab-threshold-number-info"), - ) - class_wise = state.persist( - "class_wise_checkbox", - gr.Checkbox, - label=loc.localize("eval-tab-classwise-checkbox-label"), - value=False, - info=loc.localize("eval-tab-classwise-checkbox-info"), - ) + with gr.Row(): + sample_duration = state.persist( + "sample_duration_number", + gr.Number, + value=3, + label=loc.localize("eval-tab-sample-duration-number-label"), + precision=0, + info=loc.localize("eval-tab-sample-duration-number-info"), + ) + recording_duration = state.persist( + "recording_duration_textbox", + gr.Textbox, + value="", + label=loc.localize("eval-tab-recording-duration-textbox-label"), + placeholder=loc.localize( + "eval-tab-recording-duration-textbox-placeholder" + ), + info=loc.localize("eval-tab-recording-duration-textbox-info"), + ) + min_overlap = state.persist( + "min_overlap_number", + gr.Number, + value=0.5, + label=loc.localize("eval-tab-min-overlap-number-label"), + info=loc.localize("eval-tab-min-overlap-number-info"), + ) + threshold = state.persist( + "threshold_slider", + gr.Slider, + minimum=0.01, + maximum=0.99, + value=0.1, + label=loc.localize("eval-tab-threshold-number-label"), + info=loc.localize("eval-tab-threshold-number-info"), + ) + with gr.Row(): + class_wise = state.persist( + "class_wise_checkbox", + gr.Checkbox, + label=loc.localize("eval-tab-classwise-checkbox-label"), + value=False, + info=loc.localize("eval-tab-classwise-checkbox-info"), + ) + score_unannotated = state.persist( + "score_unannotated_checkbox", + gr.Checkbox, + label=loc.localize("eval-tab-score-unannotated-checkbox-label"), + value=False, + info=loc.localize("eval-tab-score-unannotated-checkbox-info"), + ) # ----------------------- Metrics Box ----------------------- with ( @@ -656,36 +544,44 @@ def select_directory_on_empty(): "auroc": ( loc.localize("eval-tab-metric-auroc-label"), loc.localize("eval-tab-auroc-checkbox-info"), + True, ), "precision": ( loc.localize("eval-tab-metric-precision-label"), loc.localize("eval-tab-precision-checkbox-info"), + True, ), "recall": ( loc.localize("eval-tab-metric-recall-label"), loc.localize("eval-tab-recall-checkbox-info"), + True, ), "f1": ( loc.localize("eval-tab-metric-f1-score-label"), loc.localize("eval-tab-f1-score-checkbox-info"), + True, ), "ap": ( loc.localize("eval-tab-metric-ap-label"), loc.localize("eval-tab-ap-checkbox-info"), + True, ), + # Accuracy is dominated by true negatives in soundscape data, so it is + # available but off by default. "accuracy": ( loc.localize("eval-tab-metric-accuracy-label"), loc.localize("eval-tab-accuracy-checkbox-info"), + False, ), } metrics_checkboxes = {} - for metric_id, (metric_name, description) in metric_info.items(): + for metric_id, (metric_name, description, default) in metric_info.items(): metrics_checkboxes[metric_id] = state.persist( f"{metric_id}_checkbox", gr.Checkbox, label=metric_name, - value=True, + value=default, info=description, ) @@ -696,7 +592,31 @@ def select_directory_on_empty(): variant="primary", ) - with gr.Column(visible=False) as action_col: + # ----------------------- Results ----------------------- + with gr.Column(visible=False) as results_col: + gr.Markdown(f"### {loc.localize('eval-tab-per-class-metrics-label')}") + per_class_table = gr.Dataframe( + show_label=False, type="pandas", interactive=False, buttons=[] + ) + + gr.Markdown(f"### {loc.localize('eval-tab-overall-metrics-label')}") + averaging_radio = state.persist( + "averaging_radio", + gr.Radio, + choices=[ + (loc.localize(f"eval-tab-averaging-{avg}-label"), avg) + for avg in AVERAGING_IDS + ], + value="macro", + label=loc.localize("eval-tab-averaging-radio-label"), + info=loc.localize("eval-tab-averaging-radio-info"), + ) + overall_table = gr.Dataframe( + show_label=False, type="pandas", interactive=False, buttons=[] + ) + + notes_markdown = gr.Markdown(visible=False) + with gr.Row(): plot_metrics_button = gr.Button( loc.localize("eval-tab-plot-metrics-button-label") @@ -718,12 +638,15 @@ def select_directory_on_empty(): download_results_button.click( fn=download_results_table, - inputs=[pa_state, predictions_state, labels_state, class_wise], + inputs=[ + pa_state, + predictions_state, + labels_state, + class_wise, + averaging_radio, + ], ) download_data_button.click(fn=download_data_table, inputs=[processor_state]) - metric_table = gr.Dataframe( - show_label=False, type="pandas", visible=False, interactive=False - ) with gr.Group(visible=False) as plot_group: plot_output = gr.Plot(show_label=False) @@ -732,189 +655,318 @@ def select_directory_on_empty(): size="sm", ) - # Update available selections (classes and recordings) and the processor state - # when files or mapping file change. - # Also pass the current selection values so that user selections are preserved. - for comp in ( - list(annotation_columns.values()) - + list(prediction_columns.values()) - + [mapping_file] + # ------------------------------------------------------------------ + # Building / refreshing the processor and the class/recording choices + # ------------------------------------------------------------------ + def refresh_processor( + annotation_dir, + prediction_dir, + mapping_file_obj, + sample_duration_value, + min_overlap_value, + recording_duration_value, + score_unannotated_value, + current_classes, + current_recordings, + *column_values, ): + n_ann = len(annotation_column_order) + ann_cols = dict( + zip(annotation_column_order, column_values[:n_ann], strict=True) + ) + pred_cols = dict( + zip(prediction_column_order, column_values[n_ann:], strict=True) + ) + + proc_state = build_processor( + annotation_dir, + prediction_dir, + mapping_file_obj, + sample_duration_value, + min_overlap_value, + recording_duration_value, + score_unannotated_value, + ann_cols, + pred_cols, + ) + + if proc_state is None: + return gr.update(), gr.update(), None + + processor = proc_state.processor + avail_classes = list(processor.classes) + avail_recordings = processor.samples_df["filename"].unique().tolist() + + # Keep any still-valid user selection, otherwise select everything. + kept_classes = [c for c in (current_classes or []) if c in avail_classes] + new_classes = kept_classes or avail_classes + kept_recordings = [ + r for r in (current_recordings or []) if r in avail_recordings + ] + new_recordings = kept_recordings or avail_recordings + + return ( + gr.update(choices=avail_classes, value=new_classes), + gr.update(choices=avail_recordings, value=new_recordings), + proc_state, + ) + + refresh_inputs = [ + annotation_dir_state, + prediction_dir_state, + mapping_file, + sample_duration, + min_overlap, + recording_duration, + score_unannotated, + select_classes_checkboxgroup, + select_recordings_checkboxgroup, + *annotation_columns.values(), + *prediction_columns.values(), + ] + refresh_outputs = [ + select_classes_checkboxgroup, + select_recordings_checkboxgroup, + processor_state, + ] + + # Rebuild when a mapping/column/parameter the processor depends on changes. The + # column dropdowns use ``.input`` so programmatically setting their defaults on + # folder selection does not trigger a rebuild storm. + change_triggers = ( + mapping_file, + sample_duration, + min_overlap, + recording_duration, + score_unannotated, + ) + for comp in change_triggers: comp.change( - fn=update_selections, - inputs=[ - annotation_files_state, - prediction_files_state, - mapping_file, - sample_duration, - min_overlap, - recording_duration, - annotation_columns["Start Time"], - annotation_columns["End Time"], - annotation_columns["Class"], - annotation_columns["Recording"], - annotation_columns["Duration"], - prediction_columns["Start Time"], - prediction_columns["End Time"], - prediction_columns["Class"], - prediction_columns["Confidence"], - prediction_columns["Recording"], - prediction_columns["Duration"], - select_classes_checkboxgroup, - select_recordings_checkboxgroup, - ], - outputs=[ - select_classes_checkboxgroup, - select_recordings_checkboxgroup, - processor_state, - ], + refresh_processor, inputs=refresh_inputs, outputs=refresh_outputs + ) + for comp in (*annotation_columns.values(), *prediction_columns.values()): + comp.input( + refresh_processor, inputs=refresh_inputs, outputs=refresh_outputs + ) + + # ------------------------------------------------------------------ + # Metric calculation and rendering + # ------------------------------------------------------------------ + def _build_assessor( + proc_state, + selected_classes, + selected_recordings, + threshold_value, + metric_ids, + ): + from birdnet_analyzer.evaluation.assessment.performance_assessor import ( + PerformanceAssessor, ) - # calculate_metrics now uses the stored temporary directories from - # processor_state. - # The function now accepts selected_classes and selected_recordings as inputs. + processor = proc_state.processor + predictions, labels, classes = processor.get_filtered_tensors( + selected_classes, selected_recordings + ) + num_classes = len(classes) + task = "binary" if num_classes == 1 else "multilabel" + pa = PerformanceAssessor( + num_classes=num_classes, + threshold=threshold_value, + classes=classes, + task=task, + metrics_list=metric_ids, + ) + + return pa, predictions, labels + + def _as_display_table(metrics_df: pd.DataFrame) -> pd.DataFrame: + # Metrics are the index and classes the columns; transpose so each row is a + # class (or the overall score) and each metric a column, which reads better. + return metrics_df.T.reset_index(names=[""]).round(3) + + def _build_notes(proc_state, empty_classes, score_unannotated_value): + lines = [] + unmatched = sorted(proc_state.processor.unmatched_prediction_files) + + if unmatched: + key = ( + "eval-tab-note-unmatched-empty" + if score_unannotated_value + else "eval-tab-note-unmatched-dropped" + ) + lines.append(f"⚠️ {loc.localize(key)} {', '.join(unmatched)}") + + if empty_classes: + lines.append( + f"📊 {loc.localize('eval-tab-note-empty-classes')} " + f"{', '.join(empty_classes)}" + ) + + return "\n\n".join(lines) + def calculate_metrics( - mapping_file_obj, - sample_duration_value, - min_overlap_value, - recording_duration_value: str, - ann_start_time, - ann_end_time, - ann_class, - ann_recording, - ann_duration, - pred_start_time, - pred_end_time, - pred_class, - pred_confidence, - pred_recording, - pred_duration, + proc_state: ProcessorState, threshold_value, class_wise_value, + averaging_value, + score_unannotated_value, selected_classes_list, selected_recordings_list, - proc_state: ProcessorState, *metrics_checkbox_values, ): - from birdnet_analyzer.evaluation import process_data + if proc_state is None: + raise gr.Error( + loc.localize("eval-tab-error-init-processor"), + print_exception=False, + ) - metrics = tuple( + metric_ids = tuple( metric_id for value, metric_id in zip( metrics_checkbox_values, metrics_checkboxes, strict=True ) if value ) + if not metric_ids: + metric_ids = ("precision", "recall", "f1") - # Fall back to available classes from processor state if none selected. - if not selected_classes_list and proc_state and proc_state.processor: + if not selected_classes_list: selected_classes_list = list(proc_state.processor.classes) - if not selected_classes_list: raise gr.Error(loc.localize("eval-tab-error-no-class-selected")) try: - rec_dur = ( - float(recording_duration_value.strip()) - if recording_duration_value - else None + pa, predictions, labels = _build_assessor( + proc_state, + selected_classes_list, + selected_recordings_list, + threshold_value, + metric_ids, ) - except (ValueError, TypeError) as e: - raise gr.Error( - loc.localize("eval-tab-error-no-valid-recording-duration") - ) from e - - if mapping_file_obj and hasattr(mapping_file_obj, "temp_files"): - mapping_path = list(mapping_file_obj.temp_files)[0] - else: - mapping_path = mapping_file_obj or None - try: - metrics_df, pa, preds, labs = process_data( - annotation_path=proc_state.annotation_dir, - prediction_path=proc_state.prediction_dir, - mapping_path=mapping_path, - sample_duration=sample_duration_value, - min_overlap=min_overlap_value, - recording_duration=rec_dur, - columns_annotations={ - "Start Time": ann_start_time, - "End Time": ann_end_time, - "Class": ann_class, - "Recording": ann_recording, - "Duration": ann_duration, - }, - columns_predictions={ - "Start Time": pred_start_time, - "End Time": pred_end_time, - "Class": pred_class, - "Confidence": pred_confidence, - "Recording": pred_recording, - "Duration": pred_duration, - }, - selected_classes=selected_classes_list, - selected_recordings=selected_recordings_list, - metrics_list=metrics, - threshold=threshold_value, - class_wise=class_wise_value, + per_class_df = pa.calculate_metrics( + predictions, labels, per_class_metrics=True, include_support=True ) - - table = metrics_df.T.reset_index(names=[""]) - - return ( - gr.update(value=table, visible=True), - gr.update(visible=True), - pa, - preds, - labs, - gr.update(), - gr.update(), - proc_state, + overall_df = pa.calculate_metrics( + predictions, labels, averaging=averaging_value ) + empty_classes = pa.empty_classes(labels) + notes = _build_notes(proc_state, empty_classes, score_unannotated_value) + except gr.Error: + raise except Exception as e: logger.error(f"Error processing data: {e}", exc_info=e) raise gr.Error( f"{loc.localize('eval-tab-error-during-processing')}: {e}" ) from e - # Updated calculate_button click now passes the selected classes and recordings. + return ( + gr.update(visible=True), + gr.update(value=_as_display_table(per_class_df)), + gr.update(value=_as_display_table(overall_df)), + gr.update(value=notes, visible=bool(notes)), + pa, + predictions, + labels, + ) + calculate_button.click( calculate_metrics, inputs=[ - mapping_file, - sample_duration, - min_overlap, - recording_duration, - annotation_columns["Start Time"], - annotation_columns["End Time"], - annotation_columns["Class"], - annotation_columns["Recording"], - annotation_columns["Duration"], - prediction_columns["Start Time"], - prediction_columns["End Time"], - prediction_columns["Class"], - prediction_columns["Confidence"], - prediction_columns["Recording"], - prediction_columns["Duration"], + processor_state, threshold, class_wise, + averaging_radio, + score_unannotated, select_classes_checkboxgroup, select_recordings_checkboxgroup, - processor_state, - *list(metrics_checkboxes.values()), + *metrics_checkboxes.values(), ], outputs=[ - metric_table, - action_col, + results_col, + per_class_table, + overall_table, + notes_markdown, pa_state, predictions_state, labels_state, - select_classes_checkboxgroup, - select_recordings_checkboxgroup, - processor_state, ], ) + def recompute_overall( + pa: PerformanceAssessor, predictions, labels, averaging_value + ): + if pa is None or predictions is None or labels is None: + return gr.update() + + overall_df = pa.calculate_metrics( + predictions, labels, averaging=averaging_value + ) + + return gr.update(value=_as_display_table(overall_df)) + + averaging_radio.input( + recompute_overall, + inputs=[pa_state, predictions_state, labels_state, averaging_radio], + outputs=[overall_table], + ) + + annotation_select_directory_btn.click( + get_selection_func( + "eval-annotations-dir", + update_annotation_columns, + annotation_column_order, + ), + inputs=[annotation_files_state, annotation_dir_state], + outputs=[ + annotation_files_state, + annotation_dir_state, + annotation_selected_textbox, + annotation_directory_input, + annotation_group, + *[annotation_columns[label] for label in annotation_column_order], + ], + show_progress="full", + ).then(refresh_processor, inputs=refresh_inputs, outputs=refresh_outputs) + + prediction_select_directory_btn.click( + get_selection_func( + "eval-predictions-dir", + update_prediction_columns, + prediction_column_order, + ), + inputs=[prediction_files_state, prediction_dir_state], + outputs=[ + prediction_files_state, + prediction_dir_state, + prediction_selected_textbox, + prediction_directory_input, + prediction_group, + *[prediction_columns[label] for label in prediction_column_order], + ], + show_progress="full", + ).then(refresh_processor, inputs=refresh_inputs, outputs=refresh_outputs) + + def toggle_after_selection(annotation_dir, prediction_dir): + visible = bool(annotation_dir and prediction_dir) + + return [gr.update(visible=visible)] * 2 + + annotation_directory_input.change( + toggle_after_selection, + inputs=[annotation_dir_state, prediction_dir_state], + outputs=[mapping_group, class_recording_group], + ) + + prediction_directory_input.change( + toggle_after_selection, + inputs=[annotation_dir_state, prediction_dir_state], + outputs=[mapping_group, class_recording_group], + ) + + # ------------------------------------------------------------------ + # Plots + # ------------------------------------------------------------------ def plot_metrics( pa: PerformanceAssessor, predictions, labels, class_wise_value ): @@ -967,64 +1019,6 @@ def plot_confusion_matrix(pa: PerformanceAssessor, predictions, labels): outputs=[plot_group, plot_output, plot_name_state], ) - annotation_select_directory_btn.click( - get_selection_func("eval-annotations-dir", update_annotation_columns), - outputs=[ - annotation_files_state, - annotation_selected_textbox, - annotation_directory_input, - annotation_group, - ] - + [ - annotation_columns[label] - for label in [ - "Start Time", - "End Time", - "Class", - "Recording", - "Duration", - ] - ], - show_progress="full", - ) - - prediction_select_directory_btn.click( - get_selection_func("eval-predictions-dir", update_prediction_columns), - outputs=[ - prediction_files_state, - prediction_selected_textbox, - prediction_directory_input, - prediction_group, - ] - + [ - prediction_columns[label] - for label in [ - "Start Time", - "End Time", - "Class", - "Confidence", - "Recording", - "Duration", - ] - ], - show_progress="full", - ) - - def toggle_after_selection(annotation_files, prediction_files): - return [gr.update(visible=annotation_files and prediction_files)] * 2 - - annotation_directory_input.change( - toggle_after_selection, - inputs=[annotation_files_state, prediction_files_state], - outputs=[mapping_group, class_recording_group], - ) - - prediction_directory_input.change( - toggle_after_selection, - inputs=[annotation_files_state, prediction_files_state], - outputs=[mapping_group, class_recording_group], - ) - def plot_metrics_all_thresholds( pa: PerformanceAssessor, predictions, labels, class_wise_value ): diff --git a/birdnet_analyzer/lang/de.json b/birdnet_analyzer/lang/de.json index 5ccaf6b6c..0dc0d34d5 100644 --- a/birdnet_analyzer/lang/de.json +++ b/birdnet_analyzer/lang/de.json @@ -65,6 +65,11 @@ "eval-tab-annotation-selection-textbox-placeholder": "Kein Annotationsverzeichnis ausgewählt", "eval-tab-ap-checkbox-info": "Average Precision fasst die Precision-Recall-Kurve zusammen, indem sie die Precision über alle Recall-Werte mittelt.", "eval-tab-auroc-checkbox-info": "AUROC misst, ob das Modell eine zufällige positive Probe höher bewertet als eine zufällige negative Probe.", + "eval-tab-averaging-macro-label": "Makro (ungewichteter Mittelwert)", + "eval-tab-averaging-micro-label": "Mikro (zusammengefasst)", + "eval-tab-averaging-radio-info": "Wie die klassenweisen Werte zum Gesamtwert zusammengefasst werden.", + "eval-tab-averaging-radio-label": "Mittelung", + "eval-tab-averaging-weighted-label": "Gewichtet (nach Häufigkeit)", "eval-tab-calculate-metrics-button-label": "Metriken berechnen", "eval-tab-class-mapping-accordion-label": "Klassen-Zuordnung (optional)", "eval-tab-classwise-checkbox-info": "Berechnen Sie Metriken für jede Klasse separat.", @@ -107,7 +112,12 @@ "eval-tab-min-overlap-number-info": "Erforderliche Überlappung, um eine Anmerkung einer Probe zuzuordnen.", "eval-tab-min-overlap-number-label": "Minimale Überlappung (s)", "eval-tab-no-files-found": "Keine Dateien gefunden", + "eval-tab-note-empty-classes": "Klassen ohne positive Annotationen werden als „n/v“ angezeigt und aus dem Gesamtwert ausgeschlossen:", + "eval-tab-note-unmatched-dropped": "Vorhersagedateien ohne passende Annotationsdatei wurden verworfen und nicht bewertet:", + "eval-tab-note-unmatched-empty": "Vorhersagedateien ohne passende Annotationsdatei werden als vollständig negativ bewertet (jede Vorhersage zählt als Falsch-Positiv):", + "eval-tab-overall-metrics-label": "Gesamtwert", "eval-tab-parameters-accordion-label": "Parameter", + "eval-tab-per-class-metrics-label": "Metriken pro Klasse", "eval-tab-plot-confusion-matrix-button-label": "Konfusionsmatrix darstellen", "eval-tab-plot-metrics-all-thresholds-button-label": "Metriken für alle Schwellenwerte darstellen", "eval-tab-plot-metrics-button-label": "Metriken darstellen", @@ -122,6 +132,8 @@ "eval-tab-result-table-download-button-label": "Ergebnistabelle herunterladen", "eval-tab-sample-duration-number-info": "Länge der Probe (in Sekunden).", "eval-tab-sample-duration-number-label": "Probendauer (s)", + "eval-tab-score-unannotated-checkbox-info": "Aufnahmen mit Vorhersagen, aber ohne Annotationsdatei werden so behandelt, als enthielten sie keine Ereignisse, sodass jede Vorhersage darauf als Falsch-Positiv zählt. Standardmäßig aus; dann werden sie verworfen.", + "eval-tab-score-unannotated-checkbox-label": "Nicht annotierte Aufnahmen als leer bewerten", "eval-tab-select-classes-checkboxgroup-info": "Wählen Sie Klassen für die Auswertung aus.", "eval-tab-select-classes-checkboxgroup-label": "Klassen auswählen", "eval-tab-select-classes-recordings-accordion-label": "Klassen und Aufnahmen auswählen", @@ -133,8 +145,6 @@ "eval-tab-title": "Auswertung", "eval-tab-upload-mapping-file-label": "Zuordnungsdatei hochladen", "eval-tab-warning-error-reading-file": "Fehler beim Lesen der Datei", - "eval-tab-warning-unmatched-predictions": "Folgende Vorhersagedateien haben keine entsprechende Annotationsdatei", - "eval-tab-warning-unmatched-predictions-info": "für diese Dateien werden die Annotationen als leer angenommen", "footer-help": "Dokumentation und Support finden Sie unter", "footer-support": "Unterstützen Sie unsere Arbeit", "inference-settings-accordion-label": "Inferenzeinstellungen", diff --git a/birdnet_analyzer/lang/en.json b/birdnet_analyzer/lang/en.json index 3f5bb22fb..0c11b05c7 100644 --- a/birdnet_analyzer/lang/en.json +++ b/birdnet_analyzer/lang/en.json @@ -65,6 +65,11 @@ "eval-tab-annotation-selection-textbox-placeholder": "No annotation directory selected", "eval-tab-ap-checkbox-info": "Average Precision summarizes the precision-recall curve by averaging precision across all recall levels.", "eval-tab-auroc-checkbox-info": "AUROC measures the likelihood that the model ranks a random positive case higher than a random negative case.", + "eval-tab-averaging-macro-label": "Macro (unweighted mean)", + "eval-tab-averaging-micro-label": "Micro (pooled)", + "eval-tab-averaging-radio-info": "How the per-class scores are combined into the overall score.", + "eval-tab-averaging-radio-label": "Averaging", + "eval-tab-averaging-weighted-label": "Weighted (by support)", "eval-tab-calculate-metrics-button-label": "Calculate metrics", "eval-tab-class-mapping-accordion-label": "Class mapping (optional)", "eval-tab-classwise-checkbox-info": "Calculate metrics for each class separately.", @@ -107,7 +112,12 @@ "eval-tab-min-overlap-number-info": "Overlap needed to assign an annotation to a sample.", "eval-tab-min-overlap-number-label": "Minimum overlap (s)", "eval-tab-no-files-found": "No files found", + "eval-tab-note-empty-classes": "Classes without positive annotations are shown as n/a and excluded from the overall score:", + "eval-tab-note-unmatched-dropped": "Prediction files without a matching annotation file were dropped and not scored:", + "eval-tab-note-unmatched-empty": "Prediction files without a matching annotation file are scored as all-negative (every prediction counts as a false positive):", + "eval-tab-overall-metrics-label": "Overall score", "eval-tab-parameters-accordion-label": "Parameters", + "eval-tab-per-class-metrics-label": "Per-class metrics", "eval-tab-plot-confusion-matrix-button-label": "Plot confusion matrix", "eval-tab-plot-metrics-all-thresholds-button-label": "Plot metrics for all thresholds", "eval-tab-plot-metrics-button-label": "Plot metrics", @@ -122,6 +132,8 @@ "eval-tab-result-table-download-button-label": "Download results table", "eval-tab-sample-duration-number-info": "Audio sample length (in seconds).", "eval-tab-sample-duration-number-label": "Sample duration (s)", + "eval-tab-score-unannotated-checkbox-info": "Treat recordings that have predictions but no annotation file as if they contained no events, so every prediction on them counts as a false positive. Off by default, they are dropped instead.", + "eval-tab-score-unannotated-checkbox-label": "Score un-annotated recordings as empty", "eval-tab-select-classes-checkboxgroup-info": "Select classes to include in the evaluation.", "eval-tab-select-classes-checkboxgroup-label": "Select classes", "eval-tab-select-classes-recordings-accordion-label": "Select classes and recordings", @@ -133,8 +145,6 @@ "eval-tab-title": "Evaluation", "eval-tab-upload-mapping-file-label": "Upload mapping file", "eval-tab-warning-error-reading-file": "Error reading file", - "eval-tab-warning-unmatched-predictions": "Following prediction files do not have a corresponding annotation file", - "eval-tab-warning-unmatched-predictions-info": "for these files, the annotations will assumed to be empty", "footer-help": "For docs and support visit", "footer-support": "Support our work", "inference-settings-accordion-label": "Inference settings", diff --git a/birdnet_analyzer/lang/fi.json b/birdnet_analyzer/lang/fi.json index 5bcb1599d..13a5a698e 100644 --- a/birdnet_analyzer/lang/fi.json +++ b/birdnet_analyzer/lang/fi.json @@ -65,6 +65,11 @@ "eval-tab-annotation-selection-textbox-placeholder": "Annotaatiohakemistoa ei valittu", "eval-tab-ap-checkbox-info": "Keskimääräinen tarkkuus tiivistää tarkkuus-takaisinsoitto-käyrän.", "eval-tab-auroc-checkbox-info": "AUROC mittaa, kuinka hyvin malli erottaa positiiviset ja negatiiviset tapaukset.", + "eval-tab-averaging-macro-label": "Makro (painottamaton keskiarvo)", + "eval-tab-averaging-micro-label": "Mikro (yhdistetty)", + "eval-tab-averaging-radio-info": "Miten luokkakohtaiset tulokset yhdistetään kokonaistulokseksi.", + "eval-tab-averaging-radio-label": "Keskiarvotus", + "eval-tab-averaging-weighted-label": "Painotettu (esiintymien mukaan)", "eval-tab-calculate-metrics-button-label": "Laske mittarit", "eval-tab-class-mapping-accordion-label": "Luokkakartoitus (valinnainen)", "eval-tab-classwise-checkbox-info": "Laske mittarit erikseen kullekin luokalle.", @@ -107,7 +112,12 @@ "eval-tab-min-overlap-number-info": "Vaadittu päällekkäisyys merkinnän ja näytteen välillä.", "eval-tab-min-overlap-number-label": "Vähimmäispäällekkäisyys (s)", "eval-tab-no-files-found": "Ei tiedostoja löytynyt", + "eval-tab-note-empty-classes": "Luokat, joilla ei ole positiivisia annotaatioita, näytetään merkinnällä ”ei saat.” ja jätetään pois kokonaistuloksesta:", + "eval-tab-note-unmatched-dropped": "Ennustetiedostot, joilla ei ole vastaavaa annotaatiotiedostoa, hylättiin eikä niitä arvioitu:", + "eval-tab-note-unmatched-empty": "Ennustetiedostot, joilla ei ole vastaavaa annotaatiotiedostoa, arvioidaan täysin negatiivisina (jokainen ennuste lasketaan vääräksi positiiviseksi):", + "eval-tab-overall-metrics-label": "Kokonaistulos", "eval-tab-parameters-accordion-label": "Parametrit", + "eval-tab-per-class-metrics-label": "Luokkakohtaiset mittarit", "eval-tab-plot-confusion-matrix-button-label": "Piirrä sekaannusmatriisi", "eval-tab-plot-metrics-all-thresholds-button-label": "Piirrä mittarit kaikille kynnysarvoille", "eval-tab-plot-metrics-button-label": "Piirrä mittarit", @@ -122,6 +132,8 @@ "eval-tab-result-table-download-button-label": "Lataa tulostaulukko", "eval-tab-sample-duration-number-info": "Ääninäytteen pituus sekunneissa.", "eval-tab-sample-duration-number-label": "Näytteen kesto (s)", + "eval-tab-score-unannotated-checkbox-info": "Käsittele tallenteet, joilla on ennusteita mutta ei annotaatiotiedostoa, ikään kuin niissä ei olisi tapahtumia, jolloin jokainen niiden ennuste lasketaan vääräksi positiiviseksi. Oletuksena pois päältä, jolloin ne hylätään.", + "eval-tab-score-unannotated-checkbox-label": "Arvioi merkitsemättömät tallenteet tyhjinä", "eval-tab-select-classes-checkboxgroup-info": "Valitse arvioitavat luokat.", "eval-tab-select-classes-checkboxgroup-label": "Valitse luokat", "eval-tab-select-classes-recordings-accordion-label": "Valitse luokat ja äänitteet", @@ -133,8 +145,6 @@ "eval-tab-title": "Arviointi", "eval-tab-upload-mapping-file-label": "Lataa kartoitustiedosto", "eval-tab-warning-error-reading-file": "Virhe luettaessa tiedostoa", - "eval-tab-warning-unmatched-predictions": "Seuraavilla ennustetiedostoilla ei ole vastaavaa annotaatiotiedostoa", - "eval-tab-warning-unmatched-predictions-info": "näiden tiedostojen annotaatioiden oletetaan olevan tyhjiä", "footer-help": "Dokumentaatio ja tuki osoitteessa", "footer-support": "Tue työtämme", "inference-settings-accordion-label": "Päättelyasetukset", diff --git a/birdnet_analyzer/lang/fr.json b/birdnet_analyzer/lang/fr.json index 45e4d2b9b..3720adc2a 100644 --- a/birdnet_analyzer/lang/fr.json +++ b/birdnet_analyzer/lang/fr.json @@ -65,6 +65,11 @@ "eval-tab-annotation-selection-textbox-placeholder": "Aucun répertoire d'annotations sélectionné", "eval-tab-ap-checkbox-info": "La Précision Moyenne résume la courbe précision-rappel en faisant la moyenne de la précision à tous les niveaux de rappel.", "eval-tab-auroc-checkbox-info": "L'AUROC mesure la probabilité que le modèle classe un cas positif aléatoire plus haut qu'un cas négatif aléatoire.", + "eval-tab-averaging-macro-label": "Macro (moyenne non pondérée)", + "eval-tab-averaging-micro-label": "Micro (agrégé)", + "eval-tab-averaging-radio-info": "Comment les scores par classe sont combinés en un score global.", + "eval-tab-averaging-radio-label": "Moyennage", + "eval-tab-averaging-weighted-label": "Pondéré (par effectif)", "eval-tab-calculate-metrics-button-label": "Calculer les métriques", "eval-tab-class-mapping-accordion-label": "Correspondance des classes (optionnelle)", "eval-tab-classwise-checkbox-info": "Calculer les métriques pour chaque classe séparément.", @@ -107,7 +112,12 @@ "eval-tab-min-overlap-number-info": "Chevauchement nécessaire pour attribuer une annotation à un échantillon.", "eval-tab-min-overlap-number-label": "Chevauchement minimum (s)", "eval-tab-no-files-found": "Aucun fichier trouvé", + "eval-tab-note-empty-classes": "Les classes sans annotation positive sont affichées comme « n/a » et exclues du score global :", + "eval-tab-note-unmatched-dropped": "Les fichiers de prédiction sans fichier d'annotation correspondant ont été ignorés et non évalués :", + "eval-tab-note-unmatched-empty": "Les fichiers de prédiction sans fichier d'annotation correspondant sont évalués comme entièrement négatifs (chaque prédiction compte comme un faux positif) :", + "eval-tab-overall-metrics-label": "Score global", "eval-tab-parameters-accordion-label": "Paramètres", + "eval-tab-per-class-metrics-label": "Métriques par classe", "eval-tab-plot-confusion-matrix-button-label": "Tracer la matrice de confusion", "eval-tab-plot-metrics-all-thresholds-button-label": "Tracer les métriques pour tous les seuils", "eval-tab-plot-metrics-button-label": "Tracer les métriques", @@ -122,6 +132,8 @@ "eval-tab-result-table-download-button-label": "Télécharger la table de résultats", "eval-tab-sample-duration-number-info": "Durée de l'échantillon audio (en secondes).", "eval-tab-sample-duration-number-label": "Durée de l'échantillon (s)", + "eval-tab-score-unannotated-checkbox-info": "Traiter les enregistrements ayant des prédictions mais aucun fichier d'annotation comme s'ils ne contenaient aucun événement, de sorte que chaque prédiction compte comme un faux positif. Désactivé par défaut, ils sont alors ignorés.", + "eval-tab-score-unannotated-checkbox-label": "Évaluer les enregistrements non annotés comme vides", "eval-tab-select-classes-checkboxgroup-info": "Sélectionnez les classes à inclure dans l'évaluation.", "eval-tab-select-classes-checkboxgroup-label": "Sélectionner les classes", "eval-tab-select-classes-recordings-accordion-label": "Sélectionner les classes et enregistrements", @@ -133,8 +145,6 @@ "eval-tab-title": "Évaluation", "eval-tab-upload-mapping-file-label": "Téléverser le fichier de correspondance", "eval-tab-warning-error-reading-file": "Erreur lors de la lecture du fichier", - "eval-tab-warning-unmatched-predictions": "Les fichiers de prédiction suivants n'ont pas de fichier d'annotation correspondant", - "eval-tab-warning-unmatched-predictions-info": "pour ces fichiers, les annotations seront supposées vides", "footer-help": "Pour de la documentation, visiter", "footer-support": "Soutenez notre travail", "inference-settings-accordion-label": "Paramètres d'inférence", diff --git a/birdnet_analyzer/lang/id.json b/birdnet_analyzer/lang/id.json index e95fd24ef..1945794fc 100644 --- a/birdnet_analyzer/lang/id.json +++ b/birdnet_analyzer/lang/id.json @@ -65,6 +65,11 @@ "eval-tab-annotation-selection-textbox-placeholder": "Tidak ada direktori anotasi yang dipilih", "eval-tab-ap-checkbox-info": "Average Precision merangkum kurva precision-recall dengan merata-ratakan precision di semua level recall.", "eval-tab-auroc-checkbox-info": "AUROC mengukur kemungkinan model memberi peringkat lebih tinggi pada kasus positif acak dibanding kasus negatif acak.", + "eval-tab-averaging-macro-label": "Makro (rata-rata tak berbobot)", + "eval-tab-averaging-micro-label": "Mikro (digabung)", + "eval-tab-averaging-radio-info": "Bagaimana skor per kelas digabungkan menjadi skor keseluruhan.", + "eval-tab-averaging-radio-label": "Perataan", + "eval-tab-averaging-weighted-label": "Berbobot (menurut dukungan)", "eval-tab-calculate-metrics-button-label": "Hitung metrik", "eval-tab-class-mapping-accordion-label": "Pemetaan kelas (opsional)", "eval-tab-classwise-checkbox-info": "Hitung metrik untuk setiap kelas secara terpisah.", @@ -107,7 +112,12 @@ "eval-tab-min-overlap-number-info": "Tumpang tindih yang diperlukan untuk menetapkan anotasi ke sampel.", "eval-tab-min-overlap-number-label": "Tumpang tindih minimum (detik)", "eval-tab-no-files-found": "Tidak ada file yang ditemukan", + "eval-tab-note-empty-classes": "Kelas tanpa anotasi positif ditampilkan sebagai \"t/a\" dan dikecualikan dari skor keseluruhan:", + "eval-tab-note-unmatched-dropped": "Berkas prediksi tanpa berkas anotasi yang cocok telah dibuang dan tidak dinilai:", + "eval-tab-note-unmatched-empty": "Berkas prediksi tanpa berkas anotasi yang cocok dinilai sebagai seluruhnya negatif (setiap prediksi dihitung sebagai positif palsu):", + "eval-tab-overall-metrics-label": "Skor keseluruhan", "eval-tab-parameters-accordion-label": "Parameter", + "eval-tab-per-class-metrics-label": "Metrik per kelas", "eval-tab-plot-confusion-matrix-button-label": "Buat confusion matrix", "eval-tab-plot-metrics-all-thresholds-button-label": "Buat metrik untuk semua threshold", "eval-tab-plot-metrics-button-label": "Buat metrik", @@ -122,6 +132,8 @@ "eval-tab-result-table-download-button-label": "Unduh tabel hasil", "eval-tab-sample-duration-number-info": "Panjang sampel audio (dalam detik).", "eval-tab-sample-duration-number-label": "Durasi sampel (detik)", + "eval-tab-score-unannotated-checkbox-info": "Perlakukan rekaman yang memiliki prediksi tetapi tanpa berkas anotasi seolah-olah tidak berisi peristiwa, sehingga setiap prediksi padanya dihitung sebagai positif palsu. Nonaktif secara bawaan, rekaman tersebut dibuang.", + "eval-tab-score-unannotated-checkbox-label": "Nilai rekaman tanpa anotasi sebagai kosong", "eval-tab-select-classes-checkboxgroup-info": "Pilih kelas untuk disertakan dalam evaluasi.", "eval-tab-select-classes-checkboxgroup-label": "Pilih kelas", "eval-tab-select-classes-recordings-accordion-label": "Pilih kelas dan rekaman", @@ -133,8 +145,6 @@ "eval-tab-title": "Evaluasi", "eval-tab-upload-mapping-file-label": "Unggah file pemetaan", "eval-tab-warning-error-reading-file": "Error saat membaca file", - "eval-tab-warning-unmatched-predictions": "File prediksi berikut tidak memiliki file anotasi yang sesuai", - "eval-tab-warning-unmatched-predictions-info": "untuk file-file ini, anotasi akan dianggap kosong", "footer-help": "Untuk dokumen dan dukungan kunjungi", "footer-support": "Dukung Pekerjaan Kami", "inference-settings-accordion-label": "Pengaturan inferensi", diff --git a/birdnet_analyzer/lang/pt-br.json b/birdnet_analyzer/lang/pt-br.json index 3d48d0c20..16d583a39 100644 --- a/birdnet_analyzer/lang/pt-br.json +++ b/birdnet_analyzer/lang/pt-br.json @@ -65,6 +65,11 @@ "eval-tab-annotation-selection-textbox-placeholder": "Nenhum diretório de anotações selecionado", "eval-tab-ap-checkbox-info": "A Precisão Média resume a curva precisão-recall calculando a média da precisão em todos os níveis de recall.", "eval-tab-auroc-checkbox-info": "AUROC mede a probabilidade de o modelo classificar um caso positivo aleatório acima de um caso negativo aleatório.", + "eval-tab-averaging-macro-label": "Macro (média não ponderada)", + "eval-tab-averaging-micro-label": "Micro (agrupado)", + "eval-tab-averaging-radio-info": "Como as pontuações por classe são combinadas na pontuação geral.", + "eval-tab-averaging-radio-label": "Média", + "eval-tab-averaging-weighted-label": "Ponderada (por suporte)", "eval-tab-calculate-metrics-button-label": "Calcular métricas", "eval-tab-class-mapping-accordion-label": "Mapeamento de classes (opcional)", "eval-tab-classwise-checkbox-info": "Calcular métricas para cada classe separadamente.", @@ -107,7 +112,12 @@ "eval-tab-min-overlap-number-info": "Sobreposição necessária para atribuir uma anotação a uma amostra.", "eval-tab-min-overlap-number-label": "Sobreposição mínima (s)", "eval-tab-no-files-found": "Nenhum arquivo encontrado", + "eval-tab-note-empty-classes": "Classes sem anotações positivas são exibidas como \"n/d\" e excluídas da pontuação geral:", + "eval-tab-note-unmatched-dropped": "Arquivos de previsão sem arquivo de anotação correspondente foram descartados e não avaliados:", + "eval-tab-note-unmatched-empty": "Arquivos de previsão sem arquivo de anotação correspondente são avaliados como totalmente negativos (cada previsão conta como falso positivo):", + "eval-tab-overall-metrics-label": "Pontuação geral", "eval-tab-parameters-accordion-label": "Parâmetros", + "eval-tab-per-class-metrics-label": "Métricas por classe", "eval-tab-plot-confusion-matrix-button-label": "Plotar matriz de confusão", "eval-tab-plot-metrics-all-thresholds-button-label": "Plotar métricas para todos os limiares", "eval-tab-plot-metrics-button-label": "Plotar métricas", @@ -122,6 +132,8 @@ "eval-tab-result-table-download-button-label": "Baixar tabela de resultados", "eval-tab-sample-duration-number-info": "Duração da amostra de áudio (em segundos).", "eval-tab-sample-duration-number-label": "Duração da amostra (s)", + "eval-tab-score-unannotated-checkbox-info": "Tratar gravações que têm previsões, mas nenhum arquivo de anotação, como se não contivessem eventos, de modo que cada previsão nelas conte como falso positivo. Desativado por padrão, elas são descartadas.", + "eval-tab-score-unannotated-checkbox-label": "Avaliar gravações não anotadas como vazias", "eval-tab-select-classes-checkboxgroup-info": "Selecione as classes a serem incluídas na avaliação.", "eval-tab-select-classes-checkboxgroup-label": "Selecionar classes", "eval-tab-select-classes-recordings-accordion-label": "Selecionar classes e gravações", @@ -133,8 +145,6 @@ "eval-tab-title": "Avaliação", "eval-tab-upload-mapping-file-label": "Carregar arquivo de mapeamento", "eval-tab-warning-error-reading-file": "Erro ao ler arquivo", - "eval-tab-warning-unmatched-predictions": "Os seguintes arquivos de previsão não possuem arquivo de anotação correspondente", - "eval-tab-warning-unmatched-predictions-info": "para esses arquivos, as anotações serão consideradas vazias", "footer-help": "Para documentos e suporte visite", "footer-support": "Apoie nosso trabalho", "inference-settings-accordion-label": "Configurações de inferência", diff --git a/birdnet_analyzer/lang/ru.json b/birdnet_analyzer/lang/ru.json index cf73464da..0a4cfde75 100644 --- a/birdnet_analyzer/lang/ru.json +++ b/birdnet_analyzer/lang/ru.json @@ -65,6 +65,11 @@ "eval-tab-annotation-selection-textbox-placeholder": "Каталог аннотаций не выбран", "eval-tab-ap-checkbox-info": "Средняя точность суммирует кривую точности-полноты, усредняя точность по всем уровням полноты.", "eval-tab-auroc-checkbox-info": "AUROC измеряет вероятность того, что модель оценит случайный положительный случай выше случайного отрицательного.", + "eval-tab-averaging-macro-label": "Макро (невзвешенное среднее)", + "eval-tab-averaging-micro-label": "Микро (объединённо)", + "eval-tab-averaging-radio-info": "Как поклассовые оценки объединяются в общую оценку.", + "eval-tab-averaging-radio-label": "Усреднение", + "eval-tab-averaging-weighted-label": "Взвешенное (по числу примеров)", "eval-tab-calculate-metrics-button-label": "Рассчитать метрики", "eval-tab-class-mapping-accordion-label": "Сопоставление классов (опционально)", "eval-tab-classwise-checkbox-info": "Рассчитать метрики для каждого класса отдельно.", @@ -107,7 +112,12 @@ "eval-tab-min-overlap-number-info": "Необходимое перекрытие для присвоения аннотации образцу.", "eval-tab-min-overlap-number-label": "Минимальное перекрытие (сек)", "eval-tab-no-files-found": "Файлы не найдены", + "eval-tab-note-empty-classes": "Классы без положительных аннотаций отображаются как «н/д» и исключаются из общей оценки:", + "eval-tab-note-unmatched-dropped": "Файлы предсказаний без соответствующего файла аннотаций были отброшены и не оценивались:", + "eval-tab-note-unmatched-empty": "Файлы предсказаний без соответствующего файла аннотаций оцениваются как полностью отрицательные (каждое предсказание считается ложноположительным):", + "eval-tab-overall-metrics-label": "Общая оценка", "eval-tab-parameters-accordion-label": "Параметры", + "eval-tab-per-class-metrics-label": "Метрики по классам", "eval-tab-plot-confusion-matrix-button-label": "Построить матрицу ошибок", "eval-tab-plot-metrics-all-thresholds-button-label": "Построить метрики для всех порогов", "eval-tab-plot-metrics-button-label": "Построить метрики", @@ -122,6 +132,8 @@ "eval-tab-result-table-download-button-label": "Скачать таблицу результатов", "eval-tab-sample-duration-number-info": "Длительность аудиообразца (в секундах).", "eval-tab-sample-duration-number-label": "Длительность образца (сек)", + "eval-tab-score-unannotated-checkbox-info": "Обрабатывать записи, для которых есть предсказания, но нет файла аннотаций, как не содержащие событий, так что каждое предсказание на них считается ложноположительным. По умолчанию выключено — такие записи отбрасываются.", + "eval-tab-score-unannotated-checkbox-label": "Считать неаннотированные записи пустыми", "eval-tab-select-classes-checkboxgroup-info": "Выберите классы для включения в оценку.", "eval-tab-select-classes-checkboxgroup-label": "Выбрать классы", "eval-tab-select-classes-recordings-accordion-label": "Выбор классов и записей", @@ -133,8 +145,6 @@ "eval-tab-title": "Оценка", "eval-tab-upload-mapping-file-label": "Загрузить файл сопоставления", "eval-tab-warning-error-reading-file": "Ошибка чтения файла", - "eval-tab-warning-unmatched-predictions": "Следующие файлы прогнозов не имеют соответствующего файла аннотаций", - "eval-tab-warning-unmatched-predictions-info": "для этих файлов аннотации будут считаться пустыми", "footer-help": "Для получения документации и поддержки посетите", "footer-support": "Поддержите нашу работу", "inference-settings-accordion-label": "Настройки вывода", diff --git a/birdnet_analyzer/lang/se.json b/birdnet_analyzer/lang/se.json index 6273e5e56..46fe18f9f 100644 --- a/birdnet_analyzer/lang/se.json +++ b/birdnet_analyzer/lang/se.json @@ -65,6 +65,11 @@ "eval-tab-annotation-selection-textbox-placeholder": "Ingen anteckningskatalog vald", "eval-tab-ap-checkbox-info": "Genomsnittlig precision sammanfattar precision-återkallningskurvan genom att beräkna medelvärdet av precisionen över alla återkallningsnivåer.", "eval-tab-auroc-checkbox-info": "AUROC mäter sannolikheten att modellen rankar ett slumpmässigt positivt fall högre än ett slumpmässigt negativt fall.", + "eval-tab-averaging-macro-label": "Makro (oviktat medelvärde)", + "eval-tab-averaging-micro-label": "Mikro (sammanslaget)", + "eval-tab-averaging-radio-info": "Hur poängen per klass kombineras till den totala poängen.", + "eval-tab-averaging-radio-label": "Medelvärdesberäkning", + "eval-tab-averaging-weighted-label": "Viktat (efter antal)", "eval-tab-calculate-metrics-button-label": "Beräkna mått", "eval-tab-class-mapping-accordion-label": "Klassmappning (valfritt)", "eval-tab-classwise-checkbox-info": "Beräkna mått för varje klass separat.", @@ -107,7 +112,12 @@ "eval-tab-min-overlap-number-info": "Överlappning som krävs för att tilldela en anteckning till ett prov.", "eval-tab-min-overlap-number-label": "Minsta överlappning (s)", "eval-tab-no-files-found": "Inga filer hittades", + "eval-tab-note-empty-classes": "Klasser utan positiva annoteringar visas som ”ej till.” och utesluts från den totala poängen:", + "eval-tab-note-unmatched-dropped": "Prediktionsfiler utan matchande annoteringsfil kastades bort och bedömdes inte:", + "eval-tab-note-unmatched-empty": "Prediktionsfiler utan matchande annoteringsfil bedöms som helt negativa (varje prediktion räknas som falskt positiv):", + "eval-tab-overall-metrics-label": "Total poäng", "eval-tab-parameters-accordion-label": "Parametrar", + "eval-tab-per-class-metrics-label": "Mått per klass", "eval-tab-plot-confusion-matrix-button-label": "Plotta förvirringsmatris", "eval-tab-plot-metrics-all-thresholds-button-label": "Plotta mått för alla tröskelvärden", "eval-tab-plot-metrics-button-label": "Plotta mått", @@ -122,6 +132,8 @@ "eval-tab-result-table-download-button-label": "Ladda ner resultattabell", "eval-tab-sample-duration-number-info": "Ljudprovslängd (i sekunder).", "eval-tab-sample-duration-number-label": "Provlängd (s)", + "eval-tab-score-unannotated-checkbox-info": "Behandla inspelningar som har prediktioner men ingen annoteringsfil som om de inte innehåller några händelser, så att varje prediktion på dem räknas som falskt positiv. Avstängt som standard, då kastas de bort.", + "eval-tab-score-unannotated-checkbox-label": "Bedöm oannoterade inspelningar som tomma", "eval-tab-select-classes-checkboxgroup-info": "Välj klasser att inkludera i utvärderingen.", "eval-tab-select-classes-checkboxgroup-label": "Välj klasser", "eval-tab-select-classes-recordings-accordion-label": "Välj klasser och inspelningar", @@ -133,8 +145,6 @@ "eval-tab-title": "Utvärdering", "eval-tab-upload-mapping-file-label": "Ladda upp mappningsfil", "eval-tab-warning-error-reading-file": "Fel vid läsning av fil", - "eval-tab-warning-unmatched-predictions": "Följande prediktionsfiler har ingen motsvarande annotationsfil", - "eval-tab-warning-unmatched-predictions-info": "för dessa filer kommer annotationerna antas vara tomma", "footer-help": "För dokumentation och support besök", "footer-support": "Stöd vårt arbete", "inference-settings-accordion-label": "Inferensinställningar", diff --git a/birdnet_analyzer/lang/tlh.json b/birdnet_analyzer/lang/tlh.json index 544b00408..537c6826a 100644 --- a/birdnet_analyzer/lang/tlh.json +++ b/birdnet_analyzer/lang/tlh.json @@ -65,6 +65,11 @@ "eval-tab-annotation-selection-textbox-placeholder": "QIj Daq wIvlu'be'", "eval-tab-ap-checkbox-info": "qel patlh vIt.", "eval-tab-auroc-checkbox-info": "qel patlh vIt.", + "eval-tab-averaging-macro-label": "Macro (unweighted mean)", + "eval-tab-averaging-micro-label": "Micro (pooled)", + "eval-tab-averaging-radio-info": "How the per-class scores are combined into the overall score.", + "eval-tab-averaging-radio-label": "Averaging", + "eval-tab-averaging-weighted-label": "Weighted (by support)", "eval-tab-calculate-metrics-button-label": "qelmey yInob", "eval-tab-class-mapping-accordion-label": "Segh map", "eval-tab-classwise-checkbox-info": "Segh Dar qelmey.", @@ -107,7 +112,12 @@ "eval-tab-min-overlap-number-info": "wavmey qeq taS ngeD.", "eval-tab-min-overlap-number-label": "quS taS ngeD (s)", "eval-tab-no-files-found": "wavmey tu'Ha'", + "eval-tab-note-empty-classes": "Classes without positive annotations are shown as n/a and excluded from the overall score:", + "eval-tab-note-unmatched-dropped": "Prediction files without a matching annotation file were dropped and not scored:", + "eval-tab-note-unmatched-empty": "Prediction files without a matching annotation file are scored as all-negative (every prediction counts as a false positive):", + "eval-tab-overall-metrics-label": "Overall score", "eval-tab-parameters-accordion-label": "chutmey", + "eval-tab-per-class-metrics-label": "Per-class metrics", "eval-tab-plot-confusion-matrix-button-label": "qel vItlh", "eval-tab-plot-metrics-all-thresholds-button-label": "Hoch qelmey vItlh", "eval-tab-plot-metrics-button-label": "qelmey vItlh", @@ -122,6 +132,8 @@ "eval-tab-result-table-download-button-label": "qelmey yInob", "eval-tab-sample-duration-number-info": "wav tup.", "eval-tab-sample-duration-number-label": "wav tup (s)", + "eval-tab-score-unannotated-checkbox-info": "Treat recordings that have predictions but no annotation file as if they contained no events, so every prediction on them counts as a false positive. Off by default, they are dropped instead.", + "eval-tab-score-unannotated-checkbox-label": "Score un-annotated recordings as empty", "eval-tab-select-classes-checkboxgroup-info": "Seghmey wIv.", "eval-tab-select-classes-checkboxgroup-label": "Seghmey wIv", "eval-tab-select-classes-recordings-accordion-label": "Seghmey je wavmey wIv", @@ -133,8 +145,6 @@ "eval-tab-title": "qel", "eval-tab-upload-mapping-file-label": "map wav yIchel", "eval-tab-warning-error-reading-file": "wav qelHa'", - "eval-tab-warning-unmatched-predictions": "nota' wav ghajbe' nob wav vaj", - "eval-tab-warning-unmatched-predictions-info": "wav vaj chISHa' nota' lupoQ", "footer-help": "qaSmo' chutmey chu' yIchoH 'e' QaH", "footer-support": "Qapma' yIQan", "inference-settings-accordion-label": "poj chut mutlh", diff --git a/birdnet_analyzer/lang/zh_TW.json b/birdnet_analyzer/lang/zh_TW.json index c7d6e81ac..5c53198cb 100644 --- a/birdnet_analyzer/lang/zh_TW.json +++ b/birdnet_analyzer/lang/zh_TW.json @@ -65,6 +65,11 @@ "eval-tab-annotation-selection-textbox-placeholder": "未選擇標註目錄", "eval-tab-ap-checkbox-info": "平均精確度通過平均所有召回級別的精確度來總結精確度-召回曲線。", "eval-tab-auroc-checkbox-info": "AUROC 測量模型將隨機正例排名高於隨機負例的可能性。", + "eval-tab-averaging-macro-label": "宏平均（未加權平均）", + "eval-tab-averaging-micro-label": "微平均（彙總）", + "eval-tab-averaging-radio-info": "如何將各類別的分數合併為整體分數。", + "eval-tab-averaging-radio-label": "平均方式", + "eval-tab-averaging-weighted-label": "加權（依樣本數）", "eval-tab-calculate-metrics-button-label": "計算指標", "eval-tab-class-mapping-accordion-label": "類別映射 (可選)", "eval-tab-classwise-checkbox-info": "為每個類別單獨計算指標。", @@ -107,7 +112,12 @@ "eval-tab-min-overlap-number-info": "將註釋分配給樣本所需的重疊。", "eval-tab-min-overlap-number-label": "最小重疊 (秒)", "eval-tab-no-files-found": "未找到文件", + "eval-tab-note-empty-classes": "沒有正例標註的類別會顯示為「不適用」並排除於整體分數之外：", + "eval-tab-note-unmatched-dropped": "沒有對應標註檔的預測檔已被略過且未評估：", + "eval-tab-note-unmatched-empty": "沒有對應標註檔的預測檔會被評估為全部為負（每個預測都計為誤報）：", + "eval-tab-overall-metrics-label": "整體分數", "eval-tab-parameters-accordion-label": "參數", + "eval-tab-per-class-metrics-label": "各類別指標", "eval-tab-plot-confusion-matrix-button-label": "繪製混淆矩陣", "eval-tab-plot-metrics-all-thresholds-button-label": "繪製所有閾值的指標", "eval-tab-plot-metrics-button-label": "繪製指標", @@ -122,6 +132,8 @@ "eval-tab-result-table-download-button-label": "下載結果表", "eval-tab-sample-duration-number-info": "音頻樣本長度 (秒)。", "eval-tab-sample-duration-number-label": "樣本持續時間 (秒)", + "eval-tab-score-unannotated-checkbox-info": "將有預測但沒有標註檔的錄音視為不含任何事件，因此其上的每個預測都會計為誤報。預設為關閉，此時這些錄音會被略過。", + "eval-tab-score-unannotated-checkbox-label": "將未標註的錄音視為空白進行評估", "eval-tab-select-classes-checkboxgroup-info": "選擇要包含在評估中的類別。", "eval-tab-select-classes-checkboxgroup-label": "選擇類別", "eval-tab-select-classes-recordings-accordion-label": "選擇類別和錄音", @@ -133,8 +145,6 @@ "eval-tab-title": "評估", "eval-tab-upload-mapping-file-label": "上傳映射文件", "eval-tab-warning-error-reading-file": "讀取文件出錯", - "eval-tab-warning-unmatched-predictions": "以下預測文件沒有對應的標註文件", - "eval-tab-warning-unmatched-predictions-info": "對於這些文件，標註將被假定為空", "footer-help": "參考檔案以及尋求協助", "footer-support": "支持我們的工作", "inference-settings-accordion-label": "參數設定", diff --git a/tests/evaluation/assessment/test_performance_assessor.py b/tests/evaluation/assessment/test_performance_assessor.py index 3a9251d03..3ca5784e5 100644 --- a/tests/evaluation/assessment/test_performance_assessor.py +++ b/tests/evaluation/assessment/test_performance_assessor.py @@ -290,6 +290,85 @@ def test_calculate_metrics_with_no_classes(self): assert list(metrics_df.columns) == expected_columns +class TestPerformanceAssessorAggregation: + """Tests for empty-class handling, averaging, and support.""" + + def setup_method(self): + # Class 0: 2 positives, both predicted (recall 1). Class 1: never positive + # (empty). Class 2: 1 positive, predicted (recall 1). + self.classes = ("A", "Empty", "C") + self.labels = np.zeros((10, 3), dtype=int) + self.labels[:2, 0] = 1 + self.labels[0, 2] = 1 + self.predictions = np.zeros((10, 3)) + self.predictions[:2, 0] = 0.9 + self.predictions[:3, 1] = 0.9 # false positives on the empty class + self.predictions[0, 2] = 0.9 + + def _assessor(self): + return PerformanceAssessor( + num_classes=3, + threshold=0.1, + classes=self.classes, + metrics_list=("precision", "recall"), + ) + + def test_empty_classes_detected(self): + pa = self._assessor() + assert pa.empty_classes(self.labels) == ("Empty",) + + def test_class_support(self): + pa = self._assessor() + assert list(pa.class_support(self.labels)) == [2, 0, 1] + + def test_empty_class_excluded_from_overall(self): + pa = self._assessor() + overall = pa.calculate_metrics(self.predictions, self.labels) + # A and C are perfect; the empty class is excluded so precision stays 1. + assert overall.loc["Precision", "Overall"] == pytest.approx(1.0) + + def test_drop_empty_false_includes_empty_class(self): + pa = self._assessor() + overall = pa.calculate_metrics( + self.predictions, self.labels, drop_empty=False + ) + # The empty class contributes precision 0, pulling the macro mean below 1. + assert overall.loc["Precision", "Overall"] < 1.0 + + def test_per_class_includes_support_row(self): + pa = self._assessor() + per_class = pa.calculate_metrics( + self.predictions, self.labels, per_class_metrics=True, include_support=True + ) + assert "Support" in per_class.index + assert list(per_class.loc["Support"]) == [2, 0, 1] + + def test_all_empty_labels_give_nan(self): + pa = self._assessor() + overall = pa.calculate_metrics(self.predictions, np.zeros((10, 3), dtype=int)) + assert np.isnan(overall.loc["Precision", "Overall"]) + + def test_micro_and_macro_differ(self): + # Class A: 4 positives, 1 recalled (recall 1/4). Class C: 1 positive, recalled. + labels = np.zeros((10, 3), dtype=int) + labels[:4, 0] = 1 + labels[0, 2] = 1 + predictions = np.zeros((10, 3)) + predictions[0, 0] = 0.9 + predictions[0, 2] = 0.9 + pa = PerformanceAssessor( + num_classes=3, + threshold=0.1, + classes=self.classes, + metrics_list=("recall",), + ) + macro = pa.calculate_metrics(predictions, labels, averaging="macro") + micro = pa.calculate_metrics(predictions, labels, averaging="micro") + # macro = mean(1/4, 1) = 0.625; micro = 2/5 = 0.4. + assert macro.loc["Recall", "Overall"] == pytest.approx(0.625) + assert micro.loc["Recall", "Overall"] == pytest.approx(0.4) + + class TestPerformanceAssessorPlotMetrics: """ Test suite for the PerformanceAssessor plot_metrics method. diff --git a/tests/evaluation/preprocessing/test_data_processor.py b/tests/evaluation/preprocessing/test_data_processor.py index 30f149e14..868af6eeb 100644 --- a/tests/evaluation/preprocessing/test_data_processor.py +++ b/tests/evaluation/preprocessing/test_data_processor.py @@ -360,12 +360,13 @@ def test_load_data_with_none_filenames(self, mock_read_concat): "source_file": ["file1.txt", "file2.txt"], } ) + # Same recordings as the predictions so nothing is dropped as unmatched. mock_annotations_df = pd.DataFrame( { "Class": ["A", "C"], "Start Time": [0.5, 1.5], "End Time": [1.5, 2.5], - "source_file": ["file3.txt", "file4.txt"], + "source_file": ["file1.txt", "file2.txt"], } ) mock_read_concat.side_effect = [mock_predictions_df, mock_annotations_df] @@ -2115,8 +2116,11 @@ def setup_method(self): def teardown_method(self): """Stop patching.""" - self.patcher_pred.stop() + # Stop in reverse (LIFO) order. Both patchers target the same function, so + # stopping the first one first would leave the module attribute pointing at + # the first patcher's mock instead of the real function. self.patcher_annot.stop() + self.patcher_pred.stop() def test_empty_samples_df(self): """Test when samples_df is empty.""" @@ -2286,8 +2290,11 @@ def setup_method(self): def teardown_method(self): """Stop patching.""" - self.patcher_pred.stop() + # Stop in reverse (LIFO) order. Both patchers target the same function, so + # stopping the first one first would leave the module attribute pointing at + # the first patcher's mock instead of the real function. self.patcher_annot.stop() + self.patcher_pred.stop() def test_default_mapping_prediction(self): """Test default mapping for predictions.""" @@ -2384,8 +2391,11 @@ def setup_method(self): def teardown_method(self): """Stop patching.""" - self.patcher_pred.stop() + # Stop in reverse (LIFO) order. Both patchers target the same function, so + # stopping the first one first would leave the module attribute pointing at + # the first patcher's mock instead of the real function. self.patcher_annot.stop() + self.patcher_pred.stop() def test_empty_samples_df(self): """Test when samples_df is empty.""" diff --git a/tests/evaluation/preprocessing/test_utils.py b/tests/evaluation/preprocessing/test_utils.py index ede7e196e..9c9bcb6ec 100644 --- a/tests/evaluation/preprocessing/test_utils.py +++ b/tests/evaluation/preprocessing/test_utils.py @@ -157,11 +157,11 @@ def test_extract_recording_filename_from_filename_multiple_dots(): Test extract_recording_filename_from_filename with filenames containing multiple dots. - Ensures that the function extracts the base filename correctly when multiple dots - are present. + Only the final extension is stripped, so dots inside the recording name (e.g. + dates) are preserved. This mirrors extract_recording_filename so both agree. """ input_series = pd.Series(["file.name.ext", "another.file.name.ext"]) - expected_output = pd.Series(["file", "another"]) + expected_output = pd.Series(["file.name", "another.file.name"]) output_series = extract_recording_filename_from_filename(input_series) pd.testing.assert_series_equal(output_series, expected_output) @@ -221,11 +221,11 @@ def test_extract_recording_filename_from_filename_starting_dot(): """ Test extract_recording_filename_from_filename with filenames starting with a dot. - Ensures that the function correctly handles hidden files or filenames that start - with a dot. + A leading dot marks a hidden file rather than an extension, so the name is kept + (only a trailing extension is removed), consistent with os.path.splitext. """ input_series = pd.Series([".hiddenfile", ".anotherhiddenfile.txt"]) - expected_output = pd.Series(["", ""]) + expected_output = pd.Series([".hiddenfile", ".anotherhiddenfile"]) output_series = extract_recording_filename_from_filename(input_series) pd.testing.assert_series_equal(output_series, expected_output) @@ -246,10 +246,11 @@ def test_extract_recording_filename_from_filename_only_dots(): """ Test extract_recording_filename_from_filename with filenames that contain only dots. - Ensures that the function handles filenames made entirely of dots. + Ensures that the function handles filenames made entirely of dots. os.path.splitext + treats these as extensionless, so they are returned unchanged. """ input_series = pd.Series(["...", ".."]) - expected_output = pd.Series(["", ""]) + expected_output = pd.Series(["...", ".."]) output_series = extract_recording_filename_from_filename(input_series) pd.testing.assert_series_equal(output_series, expected_output) diff --git a/tests/evaluation/test_process_data.py b/tests/evaluation/test_process_data.py new file mode 100644 index 000000000..78ff2746b --- /dev/null +++ b/tests/evaluation/test_process_data.py @@ -0,0 +1,173 @@ +"""End-to-end tests for the evaluation pipeline (``process_data`` + ``DataProcessor``). + +These exercise the behaviour that used to produce "weird results": prediction files +without a matching annotation, recording names containing dots, and empty classes +dragging down the aggregate score. +""" + +import pytest + +from birdnet_analyzer.evaluation import process_data +from birdnet_analyzer.evaluation.preprocessing.data_processor import DataProcessor + +ANN_HEADER = ["Start Time", "End Time", "Class"] +PRED_HEADER = ["Start Time", "End Time", "Class", "Confidence"] + + +def _write_table(path, header, rows): + lines = ["\t".join(header)] + lines += ["\t".join(str(v) for v in row) for row in rows] + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def _dirs(tmp_path): + ann = tmp_path / "annotations" + pred = tmp_path / "predictions" + ann.mkdir() + pred.mkdir() + + return ann, pred + + +def test_unmatched_prediction_file_dropped_by_default(tmp_path): + """A recording with predictions but no annotation is dropped, not scored.""" + ann, pred = _dirs(tmp_path) + _write_table(ann / "rec1.txt", ANN_HEADER, [[0, 3, "X"]]) + _write_table(pred / "rec1.txt", PRED_HEADER, [[0, 3, "X", 0.9]]) + # rec2 has no annotation file at all. + _write_table( + pred / "rec2.txt", PRED_HEADER, [[0, 3, "X", 0.8], [3, 6, "X", 0.7]] + ) + + result = process_data( + annotation_path=str(ann), + prediction_path=str(pred), + recording_duration=30, + metrics_list=("precision", "recall"), + ) + + assert result.unmatched_recordings == ("rec2",) + # Only rec1 (30 s / 3 s = 10 samples) is scored. + assert result.predictions.shape[0] == 10 + # rec2's predictions are not counted as false positives. + assert result.metrics_df.loc["Precision", "Overall"] == 1.0 + + +def test_unmatched_prediction_file_scored_as_empty_when_opted_in(tmp_path): + """Opting in keeps unmatched recordings and scores their predictions as FPs.""" + ann, pred = _dirs(tmp_path) + _write_table(ann / "rec1.txt", ANN_HEADER, [[0, 3, "X"]]) + _write_table(pred / "rec1.txt", PRED_HEADER, [[0, 3, "X", 0.9]]) + _write_table( + pred / "rec2.txt", PRED_HEADER, [[0, 3, "X", 0.8], [3, 6, "X", 0.7]] + ) + + result = process_data( + annotation_path=str(ann), + prediction_path=str(pred), + recording_duration=30, + metrics_list=("precision", "recall"), + score_unannotated_as_empty=True, + ) + + assert result.unmatched_recordings == ("rec2",) + # Both recordings are scored: 20 samples. + assert result.predictions.shape[0] == 20 + # rec2's two predictions become false positives -> precision 1 / (1 + 2). + assert result.metrics_df.loc["Precision", "Overall"] == 1 / 3 + + +def test_dotted_recording_names_do_not_collide(tmp_path): + """Date-style names with dots stay distinct instead of merging on the first dot.""" + ann, pred = _dirs(tmp_path) + for name in ("2023.05.01", "2023.06.01"): + _write_table(ann / f"{name}.txt", ANN_HEADER, [[0, 3, "X"]]) + _write_table(pred / f"{name}.txt", PRED_HEADER, [[0, 3, "X", 0.9]]) + + processor = DataProcessor( + prediction_directory_path=str(pred), + annotation_directory_path=str(ann), + recording_duration=30, + ) + + assert set(processor.samples_df["filename"].unique()) == { + "2023.05.01", + "2023.06.01", + } + assert processor.unmatched_prediction_files == set() + + +def test_empty_class_excluded_from_overall(tmp_path): + """A class with predictions but no annotations is left out of the average.""" + ann, pred = _dirs(tmp_path) + _write_table(ann / "rec1.txt", ANN_HEADER, [[0, 3, "X"]]) + # Y is predicted but never annotated -> an empty class. + _write_table( + pred / "rec1.txt", PRED_HEADER, [[0, 3, "X", 0.9], [0, 3, "Y", 0.9]] + ) + + result = process_data( + annotation_path=str(ann), + prediction_path=str(pred), + recording_duration=30, + metrics_list=("precision", "recall", "f1"), + ) + + assert result.empty_classes == ("Y",) + # X is perfect; Y (empty) does not pull the overall precision down. + assert result.metrics_df.loc["Precision", "Overall"] == 1.0 + + +def test_support_counts_reported_per_class(tmp_path): + """Per-class output carries a Support row with the positive-sample counts.""" + ann, pred = _dirs(tmp_path) + _write_table(ann / "rec1.txt", ANN_HEADER, [[0, 3, "X"], [3, 6, "X"]]) + _write_table(pred / "rec1.txt", PRED_HEADER, [[0, 3, "X", 0.9]]) + + result = process_data( + annotation_path=str(ann), + prediction_path=str(pred), + recording_duration=30, + metrics_list=("precision",), + class_wise=True, + ) + + assert "Support" in result.metrics_df.index + assert result.metrics_df.loc["Support", "X"] == 2 + + +def test_averaging_methods_differ_end_to_end(tmp_path): + """macro / micro / weighted give different overall numbers on imbalanced data.""" + ann, pred = _dirs(tmp_path) + # Class A: three annotated windows, model gets one of them (recall 1/3). + # Class B: one annotated window, model gets it (recall 1/1). + _write_table( + ann / "rec1.txt", + ANN_HEADER, + [[0, 3, "A"], [3, 6, "A"], [6, 9, "A"], [0, 3, "B"]], + ) + _write_table( + pred / "rec1.txt", + PRED_HEADER, + [[0, 3, "A", 0.9], [0, 3, "B", 0.9]], + ) + + def overall_recall(averaging): + return process_data( + annotation_path=str(ann), + prediction_path=str(pred), + recording_duration=30, + metrics_list=("recall",), + averaging=averaging, + ).metrics_df.loc["Recall", "Overall"] + + macro = overall_recall("macro") + micro = overall_recall("micro") + weighted = overall_recall("weighted") + + # macro = mean(1/3, 1) = 2/3; micro pools TP/FN = 2/4 = 0.5; weighted by support + # = (3*(1/3) + 1*1) / 4 = 0.5. + assert macro == pytest.approx(2 / 3, rel=1e-6) + assert micro == pytest.approx(0.5, rel=1e-6) + assert weighted == pytest.approx(0.5, rel=1e-6) + assert macro != micro From b503dace5cdbfaea33001efc2b9397af865dc6f7 Mon Sep 17 00:00:00 2001 From: Josef Haupt Date: Wed, 29 Jul 2026 18:25:33 +0200 Subject: [PATCH 5/8] copilot comments --- .../evaluation/preprocessing/utils.py | 2 +- birdnet_analyzer/gui/evaluation.py | 24 ++++++++++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/birdnet_analyzer/evaluation/preprocessing/utils.py b/birdnet_analyzer/evaluation/preprocessing/utils.py index 1b491b830..aac1c7108 100644 --- a/birdnet_analyzer/evaluation/preprocessing/utils.py +++ b/birdnet_analyzer/evaluation/preprocessing/utils.py @@ -26,7 +26,7 @@ _AUDIO_EXTENSIONS = (".wav", ".flac", ".mp3", ".ogg", ".m4a", ".aac", ".wave") -def recording_key(name) -> str: +def recording_key(name): """Derives the recording identifier a prediction/annotation entry belongs to. This is the single source of truth for matching prediction files to annotation diff --git a/birdnet_analyzer/gui/evaluation.py b/birdnet_analyzer/gui/evaluation.py index 52bdc7f26..e15251edc 100644 --- a/birdnet_analyzer/gui/evaluation.py +++ b/birdnet_analyzer/gui/evaluation.py @@ -327,17 +327,29 @@ def select_directory(current_files, current_dir): ] files = get_selection_tables(folder) - files_to_display = ( - [*files[:100], [f"{len(files) - 100} more..."]] - if len(files) > 100 - else files - ) + + if not files: + # Folder has no selection tables: tell the user and leave the tab + # unarmed (no directory stored, column box hidden). + return [ + [], + "", + folder, + [[loc.localize("eval-tab-no-files-found")]], + gr.update(visible=False), + *on_select([]), + ] + + # gr.Matrix expects 2D data: one row per file in the single column. + rows = [[f.name] for f in files[:100]] + if len(files) > 100: + rows.append([f"{len(files) - 100} more..."]) return [ files, folder, folder, - files_to_display, + rows, gr.update(visible=True), *on_select(files), ] From 7f6e1533078e8697150c4eb3eb3737303b512de2 Mon Sep 17 00:00:00 2001 From: Josef Haupt Date: Wed, 29 Jul 2026 18:41:16 +0200 Subject: [PATCH 6/8] simple test case --- tests/evaluation/test_process_data.py | 52 +++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/evaluation/test_process_data.py b/tests/evaluation/test_process_data.py index 78ff2746b..a9b412120 100644 --- a/tests/evaluation/test_process_data.py +++ b/tests/evaluation/test_process_data.py @@ -171,3 +171,55 @@ def overall_recall(averaging): assert micro == pytest.approx(0.5, rel=1e-6) assert weighted == pytest.approx(0.5, rel=1e-6) assert macro != micro + + +def test_raven_annotation_matches_birdnet_prediction_table(tmp_path): + """Real-world Raven/BirdNET naming and columns still line up. + + Mirrors a real dataset: the annotation is a Raven table whose name contains dots + (``.Table.1.selections``) and whose class lives in a ``Call type`` column, matched + via its ``Begin File`` column. The prediction is a BirdNET selection table whose + name carries the ``.BirdNET.selection.table`` suffix and which only has a + ``Begin Path`` column (no ``Begin File``), so it is matched via its file name. + Both must collapse to the same recording key. + """ + ann, pred = _dirs(tmp_path) + audio = "S19_20180214_080003_resampled.wav" + _write_table( + ann / "S19_20180214_080003.Table.1.selections.txt", + ["Begin Time (s)", "End Time (s)", "Begin File", "Call type"], + [[0, 3, audio, "gibbon.female"], [6, 9, audio, "noise"]], + ) + _write_table( + pred / "S19_20180214_080003_resampled.BirdNET.selection.table.txt", + ["Begin Time (s)", "End Time (s)", "Common Name", "Confidence", "Begin Path"], + [[0, 3, "gibbon.female", 0.9, f"/data/{audio}"]], + ) + + result = process_data( + annotation_path=str(ann), + prediction_path=str(pred), + recording_duration=9, + columns_annotations={ + "Start Time": "Begin Time (s)", + "End Time": "End Time (s)", + "Class": "Call type", + "Recording": "Begin File", + }, + columns_predictions={ + "Start Time": "Begin Time (s)", + "End Time": "End Time (s)", + "Class": "Common Name", + "Confidence": "Confidence", + }, + metrics_list=("precision", "recall"), + class_wise=True, + ) + + # The dotted annotation name and the BirdNET suffix resolve to the same recording. + assert result.unmatched_recordings == () + assert set(result.classes) == {"gibbon.female", "noise"} + # "noise" is annotated (so not empty) but never predicted. + assert result.empty_classes == () + assert result.metrics_df.loc["Precision", "gibbon.female"] == 1.0 + assert result.metrics_df.loc["Recall", "noise"] == 0.0 From c4b6ad98adc3e0706376f2309e6e337c6d56a203 Mon Sep 17 00:00:00 2001 From: Josef Haupt Date: Wed, 29 Jul 2026 19:47:12 +0200 Subject: [PATCH 7/8] always display table --- birdnet_analyzer/gui/evaluation.py | 291 ++++++++++++++--------------- birdnet_analyzer/lang/de.json | 9 +- birdnet_analyzer/lang/en.json | 9 +- birdnet_analyzer/lang/fi.json | 9 +- birdnet_analyzer/lang/fr.json | 9 +- birdnet_analyzer/lang/id.json | 9 +- birdnet_analyzer/lang/pt-br.json | 9 +- birdnet_analyzer/lang/ru.json | 9 +- birdnet_analyzer/lang/se.json | 9 +- birdnet_analyzer/lang/tlh.json | 9 +- birdnet_analyzer/lang/zh_TW.json | 9 +- 11 files changed, 178 insertions(+), 203 deletions(-) diff --git a/birdnet_analyzer/gui/evaluation.py b/birdnet_analyzer/gui/evaluation.py index e15251edc..fb48e1cab 100644 --- a/birdnet_analyzer/gui/evaluation.py +++ b/birdnet_analyzer/gui/evaluation.py @@ -273,7 +273,6 @@ def build_processor( prediction_files_state = gr.State() annotation_dir_state = gr.State() prediction_dir_state = gr.State() - plot_name_state = gr.State() gu.info_box( description=loc.localize("eval-tab-info-text"), @@ -548,7 +547,6 @@ def select_directory(current_files, current_dir): with ( gr.Group(), gr.Accordion(loc.localize("eval-tab-metrics-accordian-label"), open=False), - gr.Row(), ): # The labels are translated, so the metrics are keyed by the id the # PerformanceAssessor expects, which does not change with the GUI language. @@ -588,30 +586,20 @@ def select_directory(current_files, current_dir): } metrics_checkboxes = {} - for metric_id, (metric_name, description, default) in metric_info.items(): - metrics_checkboxes[metric_id] = state.persist( - f"{metric_id}_checkbox", - gr.Checkbox, - label=metric_name, - value=default, - info=description, - ) - - # ----------------------- Actions Box ----------------------- - - calculate_button = gr.Button( - loc.localize("eval-tab-calculate-metrics-button-label"), - variant="primary", - ) - - # ----------------------- Results ----------------------- - with gr.Column(visible=False) as results_col: - gr.Markdown(f"### {loc.localize('eval-tab-per-class-metrics-label')}") - per_class_table = gr.Dataframe( - show_label=False, type="pandas", interactive=False, buttons=[] - ) + with gr.Row(): + for metric_id, ( + metric_name, + description, + default, + ) in metric_info.items(): + metrics_checkboxes[metric_id] = state.persist( + f"{metric_id}_checkbox", + gr.Checkbox, + label=metric_name, + value=default, + info=description, + ) - gr.Markdown(f"### {loc.localize('eval-tab-overall-metrics-label')}") averaging_radio = state.persist( "averaging_radio", gr.Radio, @@ -623,22 +611,41 @@ def select_directory(current_files, current_dir): label=loc.localize("eval-tab-averaging-radio-label"), info=loc.localize("eval-tab-averaging-radio-info"), ) - overall_table = gr.Dataframe( + + # ----------------------- Actions Box ----------------------- + + calculate_button = gr.Button( + loc.localize("eval-tab-calculate-metrics-button-label"), + variant="primary", + ) + + # ----------------------- Results ----------------------- + with gr.Column(visible=False) as results_col: + # One row per class plus a final aggregate row ("Overall"). + metrics_table = gr.Dataframe( show_label=False, type="pandas", interactive=False, buttons=[] ) notes_markdown = gr.Markdown(visible=False) - with gr.Row(): - plot_metrics_button = gr.Button( - loc.localize("eval-tab-plot-metrics-button-label") - ) - plot_confusion_button = gr.Button( - loc.localize("eval-tab-plot-confusion-matrix-button-label") - ) - plot_metrics_all_thresholds_button = gr.Button( - loc.localize("eval-tab-plot-metrics-all-thresholds-button-label") - ) + # The plots are generated with the metrics and always shown, one + # switchable tab per plot. + with gr.Tabs(): + with gr.Tab(loc.localize("eval-tab-plot-tab-metrics-label")): + metrics_plot = gr.Plot(show_label=False) + metrics_plot_dl_btn = gr.Button( + loc.localize("eval-tab-download-plot-button-label"), size="sm" + ) + with gr.Tab(loc.localize("eval-tab-plot-tab-confusion-matrix-label")): + confusion_plot = gr.Plot(show_label=False) + confusion_plot_dl_btn = gr.Button( + loc.localize("eval-tab-download-plot-button-label"), size="sm" + ) + with gr.Tab(loc.localize("eval-tab-plot-tab-thresholds-label")): + thresholds_plot = gr.Plot(show_label=False) + thresholds_plot_dl_btn = gr.Button( + loc.localize("eval-tab-download-plot-button-label"), size="sm" + ) with gr.Row(): download_results_button = gr.DownloadButton( @@ -660,11 +667,22 @@ def select_directory(current_files, current_dir): ) download_data_button.click(fn=download_data_table, inputs=[processor_state]) - with gr.Group(visible=False) as plot_group: - plot_output = gr.Plot(show_label=False) - plot_output_dl_btn = gr.Button( - loc.localize("eval-tab-download-plot-button-label"), - size="sm", + def download_plot_guarded(plot, plot_name): + if plot is None: + raise gr.Error( + loc.localize("eval-tab-error-calc-metrics-first"), + print_exception=False, + ) + + gu.download_plot(plot, plot_name) + + for dl_btn, plot_comp, plot_name in ( + (metrics_plot_dl_btn, metrics_plot, "metrics"), + (confusion_plot_dl_btn, confusion_plot, "confusion_matrix"), + (thresholds_plot_dl_btn, thresholds_plot, "metrics_all_thresholds"), + ): + dl_btn.click( + download_plot_guarded, inputs=[plot_comp, gr.State(plot_name)] ) # ------------------------------------------------------------------ @@ -791,10 +809,69 @@ def _build_assessor( return pa, predictions, labels - def _as_display_table(metrics_df: pd.DataFrame) -> pd.DataFrame: - # Metrics are the index and classes the columns; transpose so each row is a - # class (or the overall score) and each metric a column, which reads better. - return metrics_df.T.reset_index(names=[""]).round(3) + def _build_metrics_table(pa, predictions, labels, averaging_value): + """One table with a row per class and a final aggregate "Overall" row. + + The aggregate is recomputed with the selected averaging; its support is the + total number of positive samples across classes. + """ + per_class_df = pa.calculate_metrics( + predictions, labels, per_class_metrics=True, include_support=True + ) + overall_df = pa.calculate_metrics( + predictions, labels, averaging=averaging_value + ) + overall_df.loc["Support"] = int(pa.class_support(labels).sum()) + overall_df.columns = pd.Index( + [loc.localize("eval-tab-overall-row-label")] + ) + + merged = pd.concat([per_class_df, overall_df], axis=1) + # One row per class/aggregate, one column per metric. + table = merged.T.reset_index(names=[""]).round(3) + table["Support"] = table["Support"].astype(int) + + return table + + def _build_figures(pa, predictions, labels, class_wise_value): + """Builds the three result plots, tolerating individual failures. + + A plot that cannot be drawn (e.g. a degenerate confusion matrix) must not + take down the whole calculation, so each failure is reported as a warning + and its pane is simply left empty. + """ + import matplotlib.pyplot as plt + + def safe(build, error_key): + try: + fig = build() + plt.close(fig) + + return fig + except Exception as e: + logger.error(f"Error building plot: {e}", exc_info=e) + gr.Warning(f"{loc.localize(error_key)}: {e}") + + return None + + return ( + safe( + lambda: pa.plot_metrics( + predictions, labels, per_class_metrics=class_wise_value + ), + "eval-tab-error-plotting-metrics", + ), + safe( + lambda: pa.plot_confusion_matrix(predictions, labels), + "eval-tab-error-plotting-confusion-matrix", + ), + safe( + lambda: pa.plot_metrics_all_thresholds( + predictions, labels, per_class_metrics=class_wise_value + ), + "eval-tab-error-plotting-metrics-all-thresholds", + ), + ) def _build_notes(proc_state, empty_classes, score_unannotated_value): lines = [] @@ -856,12 +933,7 @@ def calculate_metrics( metric_ids, ) - per_class_df = pa.calculate_metrics( - predictions, labels, per_class_metrics=True, include_support=True - ) - overall_df = pa.calculate_metrics( - predictions, labels, averaging=averaging_value - ) + table = _build_metrics_table(pa, predictions, labels, averaging_value) empty_classes = pa.empty_classes(labels) notes = _build_notes(proc_state, empty_classes, score_unannotated_value) except gr.Error: @@ -872,11 +944,17 @@ def calculate_metrics( f"{loc.localize('eval-tab-error-during-processing')}: {e}" ) from e + fig_metrics, fig_confusion, fig_thresholds = _build_figures( + pa, predictions, labels, class_wise_value + ) + return ( gr.update(visible=True), - gr.update(value=_as_display_table(per_class_df)), - gr.update(value=_as_display_table(overall_df)), + gr.update(value=table), gr.update(value=notes, visible=bool(notes)), + fig_metrics, + fig_confusion, + fig_thresholds, pa, predictions, labels, @@ -896,9 +974,11 @@ def calculate_metrics( ], outputs=[ results_col, - per_class_table, - overall_table, + metrics_table, notes_markdown, + metrics_plot, + confusion_plot, + thresholds_plot, pa_state, predictions_state, labels_state, @@ -911,16 +991,14 @@ def recompute_overall( if pa is None or predictions is None or labels is None: return gr.update() - overall_df = pa.calculate_metrics( - predictions, labels, averaging=averaging_value + return gr.update( + value=_build_metrics_table(pa, predictions, labels, averaging_value) ) - return gr.update(value=_as_display_table(overall_df)) - averaging_radio.input( recompute_overall, inputs=[pa_state, predictions_state, labels_state, averaging_radio], - outputs=[overall_table], + outputs=[metrics_table], ) annotation_select_directory_btn.click( @@ -976,98 +1054,5 @@ def toggle_after_selection(annotation_dir, prediction_dir): outputs=[mapping_group, class_recording_group], ) - # ------------------------------------------------------------------ - # Plots - # ------------------------------------------------------------------ - def plot_metrics( - pa: PerformanceAssessor, predictions, labels, class_wise_value - ): - import matplotlib.pyplot as plt - - if pa is None or predictions is None or labels is None: - raise gr.Error( - loc.localize("eval-tab-error-calc-metrics-first"), - print_exception=False, - ) - try: - fig = pa.plot_metrics( - predictions, labels, per_class_metrics=class_wise_value - ) - plt.close(fig) - - return gr.update(visible=True), gr.update(value=fig), "metrics" - except Exception as e: - raise gr.Error( - f"{loc.localize('eval-tab-error-plotting-metrics')}: {e}" - ) from e - - plot_metrics_button.click( - plot_metrics, - inputs=[pa_state, predictions_state, labels_state, class_wise], - outputs=[plot_group, plot_output, plot_name_state], - ) - - def plot_confusion_matrix(pa: PerformanceAssessor, predictions, labels): - import matplotlib.pyplot as plt - - if pa is None or predictions is None or labels is None: - raise gr.Error( - loc.localize("eval-tab-error-calc-metrics-first"), - print_exception=False, - ) - try: - fig = pa.plot_confusion_matrix(predictions, labels) - plt.close(fig) - - return gr.update(visible=True), fig, "confusion_matrix" - except Exception as e: - raise gr.Error( - f"{loc.localize('eval-tab-error-plotting-confusion-matrix')}: {e}" - ) from e - - plot_confusion_button.click( - plot_confusion_matrix, - inputs=[pa_state, predictions_state, labels_state], - outputs=[plot_group, plot_output, plot_name_state], - ) - - def plot_metrics_all_thresholds( - pa: PerformanceAssessor, predictions, labels, class_wise_value - ): - import matplotlib.pyplot as plt - - if pa is None or predictions is None or labels is None: - raise gr.Error( - loc.localize("eval-tab-error-calc-metrics-first"), - print_exception=False, - ) - try: - fig = pa.plot_metrics_all_thresholds( - predictions, labels, per_class_metrics=class_wise_value - ) - plt.close(fig) - - return ( - gr.update(visible=True), - gr.update(value=fig), - "metrics_all_thresholds", - ) - except Exception as e: - raise gr.Error( - f"{loc.localize('eval-tab-error-plotting-metrics-all-thresholds')}:" - f" {e}" - ) from e - - plot_metrics_all_thresholds_button.click( - plot_metrics_all_thresholds, - inputs=[pa_state, predictions_state, labels_state, class_wise], - outputs=[plot_group, plot_output, plot_name_state], - ) - - plot_output_dl_btn.click( - gu.download_plot, inputs=[plot_output, plot_name_state] - ) - - if __name__ == "__main__": gu.open_window(build_evaluation_tab) diff --git a/birdnet_analyzer/lang/de.json b/birdnet_analyzer/lang/de.json index 0dc0d34d5..3a728e5c1 100644 --- a/birdnet_analyzer/lang/de.json +++ b/birdnet_analyzer/lang/de.json @@ -115,12 +115,11 @@ "eval-tab-note-empty-classes": "Klassen ohne positive Annotationen werden als „n/v“ angezeigt und aus dem Gesamtwert ausgeschlossen:", "eval-tab-note-unmatched-dropped": "Vorhersagedateien ohne passende Annotationsdatei wurden verworfen und nicht bewertet:", "eval-tab-note-unmatched-empty": "Vorhersagedateien ohne passende Annotationsdatei werden als vollständig negativ bewertet (jede Vorhersage zählt als Falsch-Positiv):", - "eval-tab-overall-metrics-label": "Gesamtwert", + "eval-tab-overall-row-label": "Gesamt", "eval-tab-parameters-accordion-label": "Parameter", - "eval-tab-per-class-metrics-label": "Metriken pro Klasse", - "eval-tab-plot-confusion-matrix-button-label": "Konfusionsmatrix darstellen", - "eval-tab-plot-metrics-all-thresholds-button-label": "Metriken für alle Schwellenwerte darstellen", - "eval-tab-plot-metrics-button-label": "Metriken darstellen", + "eval-tab-plot-tab-confusion-matrix-label": "Konfusionsmatrix", + "eval-tab-plot-tab-metrics-label": "Metriken", + "eval-tab-plot-tab-thresholds-label": "Metriken über alle Schwellenwerte", "eval-tab-precision-checkbox-info": "Precision misst, wie oft positive Vorhersagen tatsächlich korrekt sind.", "eval-tab-prediction-col-accordion-label": "Vorhersagespalten", "eval-tab-prediction-selection-button-label": "Verzeichnis mit Vorhersage auswählen", diff --git a/birdnet_analyzer/lang/en.json b/birdnet_analyzer/lang/en.json index 0c11b05c7..9d150d614 100644 --- a/birdnet_analyzer/lang/en.json +++ b/birdnet_analyzer/lang/en.json @@ -115,12 +115,11 @@ "eval-tab-note-empty-classes": "Classes without positive annotations are shown as n/a and excluded from the overall score:", "eval-tab-note-unmatched-dropped": "Prediction files without a matching annotation file were dropped and not scored:", "eval-tab-note-unmatched-empty": "Prediction files without a matching annotation file are scored as all-negative (every prediction counts as a false positive):", - "eval-tab-overall-metrics-label": "Overall score", + "eval-tab-overall-row-label": "Overall", "eval-tab-parameters-accordion-label": "Parameters", - "eval-tab-per-class-metrics-label": "Per-class metrics", - "eval-tab-plot-confusion-matrix-button-label": "Plot confusion matrix", - "eval-tab-plot-metrics-all-thresholds-button-label": "Plot metrics for all thresholds", - "eval-tab-plot-metrics-button-label": "Plot metrics", + "eval-tab-plot-tab-confusion-matrix-label": "Confusion matrix", + "eval-tab-plot-tab-metrics-label": "Metrics", + "eval-tab-plot-tab-thresholds-label": "Metrics across thresholds", "eval-tab-precision-checkbox-info": "Precision measures how often the models positive predictions are actually correct.", "eval-tab-prediction-col-accordion-label": "Prediction columns", "eval-tab-prediction-selection-button-label": "Select prediction directory", diff --git a/birdnet_analyzer/lang/fi.json b/birdnet_analyzer/lang/fi.json index 13a5a698e..7263e41c9 100644 --- a/birdnet_analyzer/lang/fi.json +++ b/birdnet_analyzer/lang/fi.json @@ -115,12 +115,11 @@ "eval-tab-note-empty-classes": "Luokat, joilla ei ole positiivisia annotaatioita, näytetään merkinnällä ”ei saat.” ja jätetään pois kokonaistuloksesta:", "eval-tab-note-unmatched-dropped": "Ennustetiedostot, joilla ei ole vastaavaa annotaatiotiedostoa, hylättiin eikä niitä arvioitu:", "eval-tab-note-unmatched-empty": "Ennustetiedostot, joilla ei ole vastaavaa annotaatiotiedostoa, arvioidaan täysin negatiivisina (jokainen ennuste lasketaan vääräksi positiiviseksi):", - "eval-tab-overall-metrics-label": "Kokonaistulos", + "eval-tab-overall-row-label": "Yhteensä", "eval-tab-parameters-accordion-label": "Parametrit", - "eval-tab-per-class-metrics-label": "Luokkakohtaiset mittarit", - "eval-tab-plot-confusion-matrix-button-label": "Piirrä sekaannusmatriisi", - "eval-tab-plot-metrics-all-thresholds-button-label": "Piirrä mittarit kaikille kynnysarvoille", - "eval-tab-plot-metrics-button-label": "Piirrä mittarit", + "eval-tab-plot-tab-confusion-matrix-label": "Sekaannusmatriisi", + "eval-tab-plot-tab-metrics-label": "Mittarit", + "eval-tab-plot-tab-thresholds-label": "Mittarit eri kynnysarvoilla", "eval-tab-precision-checkbox-info": "Tarkkuus mittaa, kuinka usein positiiviset ennusteet ovat oikeita.", "eval-tab-prediction-col-accordion-label": "Ennustesarakkeet", "eval-tab-prediction-selection-button-label": "Valitse ennustekansio", diff --git a/birdnet_analyzer/lang/fr.json b/birdnet_analyzer/lang/fr.json index 3720adc2a..90565fe2e 100644 --- a/birdnet_analyzer/lang/fr.json +++ b/birdnet_analyzer/lang/fr.json @@ -115,12 +115,11 @@ "eval-tab-note-empty-classes": "Les classes sans annotation positive sont affichées comme « n/a » et exclues du score global :", "eval-tab-note-unmatched-dropped": "Les fichiers de prédiction sans fichier d'annotation correspondant ont été ignorés et non évalués :", "eval-tab-note-unmatched-empty": "Les fichiers de prédiction sans fichier d'annotation correspondant sont évalués comme entièrement négatifs (chaque prédiction compte comme un faux positif) :", - "eval-tab-overall-metrics-label": "Score global", + "eval-tab-overall-row-label": "Global", "eval-tab-parameters-accordion-label": "Paramètres", - "eval-tab-per-class-metrics-label": "Métriques par classe", - "eval-tab-plot-confusion-matrix-button-label": "Tracer la matrice de confusion", - "eval-tab-plot-metrics-all-thresholds-button-label": "Tracer les métriques pour tous les seuils", - "eval-tab-plot-metrics-button-label": "Tracer les métriques", + "eval-tab-plot-tab-confusion-matrix-label": "Matrice de confusion", + "eval-tab-plot-tab-metrics-label": "Métriques", + "eval-tab-plot-tab-thresholds-label": "Métriques selon les seuils", "eval-tab-precision-checkbox-info": "La précision mesure la fréquence à laquelle les prédictions positives du modèle sont réellement correctes.", "eval-tab-prediction-col-accordion-label": "Colonnes de prédiction", "eval-tab-prediction-selection-button-label": "Sélectionner le répertoire de prédiction", diff --git a/birdnet_analyzer/lang/id.json b/birdnet_analyzer/lang/id.json index 1945794fc..3befaf2fd 100644 --- a/birdnet_analyzer/lang/id.json +++ b/birdnet_analyzer/lang/id.json @@ -115,12 +115,11 @@ "eval-tab-note-empty-classes": "Kelas tanpa anotasi positif ditampilkan sebagai \"t/a\" dan dikecualikan dari skor keseluruhan:", "eval-tab-note-unmatched-dropped": "Berkas prediksi tanpa berkas anotasi yang cocok telah dibuang dan tidak dinilai:", "eval-tab-note-unmatched-empty": "Berkas prediksi tanpa berkas anotasi yang cocok dinilai sebagai seluruhnya negatif (setiap prediksi dihitung sebagai positif palsu):", - "eval-tab-overall-metrics-label": "Skor keseluruhan", + "eval-tab-overall-row-label": "Keseluruhan", "eval-tab-parameters-accordion-label": "Parameter", - "eval-tab-per-class-metrics-label": "Metrik per kelas", - "eval-tab-plot-confusion-matrix-button-label": "Buat confusion matrix", - "eval-tab-plot-metrics-all-thresholds-button-label": "Buat metrik untuk semua threshold", - "eval-tab-plot-metrics-button-label": "Buat metrik", + "eval-tab-plot-tab-confusion-matrix-label": "Matriks kebingungan", + "eval-tab-plot-tab-metrics-label": "Metrik", + "eval-tab-plot-tab-thresholds-label": "Metrik lintas ambang", "eval-tab-precision-checkbox-info": "Precision mengukur seberapa sering prediksi positif model benar.", "eval-tab-prediction-col-accordion-label": "Kolom prediksi", "eval-tab-prediction-selection-button-label": "Pilih direktori prediksi", diff --git a/birdnet_analyzer/lang/pt-br.json b/birdnet_analyzer/lang/pt-br.json index 16d583a39..da99a3afc 100644 --- a/birdnet_analyzer/lang/pt-br.json +++ b/birdnet_analyzer/lang/pt-br.json @@ -115,12 +115,11 @@ "eval-tab-note-empty-classes": "Classes sem anotações positivas são exibidas como \"n/d\" e excluídas da pontuação geral:", "eval-tab-note-unmatched-dropped": "Arquivos de previsão sem arquivo de anotação correspondente foram descartados e não avaliados:", "eval-tab-note-unmatched-empty": "Arquivos de previsão sem arquivo de anotação correspondente são avaliados como totalmente negativos (cada previsão conta como falso positivo):", - "eval-tab-overall-metrics-label": "Pontuação geral", + "eval-tab-overall-row-label": "Geral", "eval-tab-parameters-accordion-label": "Parâmetros", - "eval-tab-per-class-metrics-label": "Métricas por classe", - "eval-tab-plot-confusion-matrix-button-label": "Plotar matriz de confusão", - "eval-tab-plot-metrics-all-thresholds-button-label": "Plotar métricas para todos os limiares", - "eval-tab-plot-metrics-button-label": "Plotar métricas", + "eval-tab-plot-tab-confusion-matrix-label": "Matriz de confusão", + "eval-tab-plot-tab-metrics-label": "Métricas", + "eval-tab-plot-tab-thresholds-label": "Métricas por limiar", "eval-tab-precision-checkbox-info": "A precisão mede com que frequência as previsões positivas do modelo estão corretas.", "eval-tab-prediction-col-accordion-label": "Colunas de previsão", "eval-tab-prediction-selection-button-label": "Selecionar diretório de previsões", diff --git a/birdnet_analyzer/lang/ru.json b/birdnet_analyzer/lang/ru.json index 0a4cfde75..6b075832b 100644 --- a/birdnet_analyzer/lang/ru.json +++ b/birdnet_analyzer/lang/ru.json @@ -115,12 +115,11 @@ "eval-tab-note-empty-classes": "Классы без положительных аннотаций отображаются как «н/д» и исключаются из общей оценки:", "eval-tab-note-unmatched-dropped": "Файлы предсказаний без соответствующего файла аннотаций были отброшены и не оценивались:", "eval-tab-note-unmatched-empty": "Файлы предсказаний без соответствующего файла аннотаций оцениваются как полностью отрицательные (каждое предсказание считается ложноположительным):", - "eval-tab-overall-metrics-label": "Общая оценка", + "eval-tab-overall-row-label": "Итог", "eval-tab-parameters-accordion-label": "Параметры", - "eval-tab-per-class-metrics-label": "Метрики по классам", - "eval-tab-plot-confusion-matrix-button-label": "Построить матрицу ошибок", - "eval-tab-plot-metrics-all-thresholds-button-label": "Построить метрики для всех порогов", - "eval-tab-plot-metrics-button-label": "Построить метрики", + "eval-tab-plot-tab-confusion-matrix-label": "Матрица ошибок", + "eval-tab-plot-tab-metrics-label": "Метрики", + "eval-tab-plot-tab-thresholds-label": "Метрики по порогам", "eval-tab-precision-checkbox-info": "Точность измеряет, как часто положительные прогнозы модели действительно верны.", "eval-tab-prediction-col-accordion-label": "Колонки прогнозов", "eval-tab-prediction-selection-button-label": "Выбрать каталог прогнозов", diff --git a/birdnet_analyzer/lang/se.json b/birdnet_analyzer/lang/se.json index 46fe18f9f..72de74996 100644 --- a/birdnet_analyzer/lang/se.json +++ b/birdnet_analyzer/lang/se.json @@ -115,12 +115,11 @@ "eval-tab-note-empty-classes": "Klasser utan positiva annoteringar visas som ”ej till.” och utesluts från den totala poängen:", "eval-tab-note-unmatched-dropped": "Prediktionsfiler utan matchande annoteringsfil kastades bort och bedömdes inte:", "eval-tab-note-unmatched-empty": "Prediktionsfiler utan matchande annoteringsfil bedöms som helt negativa (varje prediktion räknas som falskt positiv):", - "eval-tab-overall-metrics-label": "Total poäng", + "eval-tab-overall-row-label": "Totalt", "eval-tab-parameters-accordion-label": "Parametrar", - "eval-tab-per-class-metrics-label": "Mått per klass", - "eval-tab-plot-confusion-matrix-button-label": "Plotta förvirringsmatris", - "eval-tab-plot-metrics-all-thresholds-button-label": "Plotta mått för alla tröskelvärden", - "eval-tab-plot-metrics-button-label": "Plotta mått", + "eval-tab-plot-tab-confusion-matrix-label": "Förväxlingsmatris", + "eval-tab-plot-tab-metrics-label": "Mått", + "eval-tab-plot-tab-thresholds-label": "Mått över tröskelvärden", "eval-tab-precision-checkbox-info": "Precision mäter hur ofta modellens positiva förutsägelser faktiskt är korrekta.", "eval-tab-prediction-col-accordion-label": "Förutsägelsekolumner", "eval-tab-prediction-selection-button-label": "Välj förutsägelsekatalog", diff --git a/birdnet_analyzer/lang/tlh.json b/birdnet_analyzer/lang/tlh.json index 537c6826a..2d8ed7ac3 100644 --- a/birdnet_analyzer/lang/tlh.json +++ b/birdnet_analyzer/lang/tlh.json @@ -115,12 +115,11 @@ "eval-tab-note-empty-classes": "Classes without positive annotations are shown as n/a and excluded from the overall score:", "eval-tab-note-unmatched-dropped": "Prediction files without a matching annotation file were dropped and not scored:", "eval-tab-note-unmatched-empty": "Prediction files without a matching annotation file are scored as all-negative (every prediction counts as a false positive):", - "eval-tab-overall-metrics-label": "Overall score", + "eval-tab-overall-row-label": "Overall", "eval-tab-parameters-accordion-label": "chutmey", - "eval-tab-per-class-metrics-label": "Per-class metrics", - "eval-tab-plot-confusion-matrix-button-label": "qel vItlh", - "eval-tab-plot-metrics-all-thresholds-button-label": "Hoch qelmey vItlh", - "eval-tab-plot-metrics-button-label": "qelmey vItlh", + "eval-tab-plot-tab-confusion-matrix-label": "Confusion matrix", + "eval-tab-plot-tab-metrics-label": "Metrics", + "eval-tab-plot-tab-thresholds-label": "Metrics across thresholds", "eval-tab-precision-checkbox-info": "qel patlh vIt.", "eval-tab-prediction-col-accordion-label": "Qelmey", "eval-tab-prediction-selection-button-label": "Qel Daq yIwIv", diff --git a/birdnet_analyzer/lang/zh_TW.json b/birdnet_analyzer/lang/zh_TW.json index 5c53198cb..45f1ac5e0 100644 --- a/birdnet_analyzer/lang/zh_TW.json +++ b/birdnet_analyzer/lang/zh_TW.json @@ -115,12 +115,11 @@ "eval-tab-note-empty-classes": "沒有正例標註的類別會顯示為「不適用」並排除於整體分數之外：", "eval-tab-note-unmatched-dropped": "沒有對應標註檔的預測檔已被略過且未評估：", "eval-tab-note-unmatched-empty": "沒有對應標註檔的預測檔會被評估為全部為負（每個預測都計為誤報）：", - "eval-tab-overall-metrics-label": "整體分數", + "eval-tab-overall-row-label": "整體", "eval-tab-parameters-accordion-label": "參數", - "eval-tab-per-class-metrics-label": "各類別指標", - "eval-tab-plot-confusion-matrix-button-label": "繪製混淆矩陣", - "eval-tab-plot-metrics-all-thresholds-button-label": "繪製所有閾值的指標", - "eval-tab-plot-metrics-button-label": "繪製指標", + "eval-tab-plot-tab-confusion-matrix-label": "混淆矩陣", + "eval-tab-plot-tab-metrics-label": "指標", + "eval-tab-plot-tab-thresholds-label": "跨閾值指標", "eval-tab-precision-checkbox-info": "精確度測量模型的正面預測實際正確的頻率。", "eval-tab-prediction-col-accordion-label": "預測欄位", "eval-tab-prediction-selection-button-label": "選擇預測目錄", From ebac2f5e85e7af43037fe6dac9566b868e6bcc28 Mon Sep 17 00:00:00 2001 From: Josef Haupt Date: Wed, 29 Jul 2026 20:37:05 +0200 Subject: [PATCH 8/8] updated result view --- birdnet_analyzer/gui/assets/gui.css | 29 ++++++++++++- birdnet_analyzer/gui/evaluation.py | 65 +++++++++++++++++++++-------- 2 files changed, 75 insertions(+), 19 deletions(-) diff --git a/birdnet_analyzer/gui/assets/gui.css b/birdnet_analyzer/gui/assets/gui.css index e1c25a019..77fa5f1b9 100644 --- a/birdnet_analyzer/gui/assets/gui.css +++ b/birdnet_analyzer/gui/assets/gui.css @@ -71,4 +71,31 @@ footer { .info-accordion button span { font-weight: bolder; -} \ No newline at end of file +} + +/* The "Overall" footer row of the evaluation metrics table. It is a separate + one-row table glued under the class table (the virtualized Dataframe cannot + pin a row), so it stays visible while the class rows scroll. The strong top + border marks it as the aggregate of the rows above. */ + +/* Both tables resolve their percentage column widths against their own viewport + width. Reserve the scrollbar gutter on both, so a scrollbar appearing on only + the class table cannot make the footer columns drift out of alignment. */ +.eval-metrics-table .virtual-table-viewport, +.eval-overall-footer .virtual-table-viewport { + scrollbar-gutter: stable; +} + +.eval-overall-footer { + border-top: 2px solid var(--body-text-color) !important; + border-top-left-radius: 0 !important; + border-top-right-radius: 0 !important; +} + +.eval-overall-footer .header-table { + display: none !important; +} + +.eval-overall-footer .virtual-row { + font-weight: var(--weight-semibold); +} diff --git a/birdnet_analyzer/gui/evaluation.py b/birdnet_analyzer/gui/evaluation.py index fb48e1cab..e800b5f11 100644 --- a/birdnet_analyzer/gui/evaluation.py +++ b/birdnet_analyzer/gui/evaluation.py @@ -621,10 +621,25 @@ def select_directory(current_files, current_dir): # ----------------------- Results ----------------------- with gr.Column(visible=False) as results_col: - # One row per class plus a final aggregate row ("Overall"). - metrics_table = gr.Dataframe( - show_label=False, type="pandas", interactive=False, buttons=[] - ) + # One row per class, with the aggregate in a separate one-row footer + # table that stays visible while the class rows scroll. (The virtualized + # gradio Dataframe cannot pin a row, so the footer is its own component; + # shared column widths keep the two aligned.) + with gr.Group(): + metrics_table = gr.Dataframe( + show_label=False, + type="pandas", + interactive=False, + buttons=[], + elem_classes="eval-metrics-table", + ) + overall_table = gr.Dataframe( + show_label=False, + type="pandas", + interactive=False, + buttons=[], + elem_classes="eval-overall-footer", + ) notes_markdown = gr.Markdown(visible=False) @@ -809,11 +824,12 @@ def _build_assessor( return pa, predictions, labels - def _build_metrics_table(pa, predictions, labels, averaging_value): - """One table with a row per class and a final aggregate "Overall" row. + def _build_metrics_tables(pa, predictions, labels, averaging_value): + """The per-class table and the aggregate "Overall" footer row. - The aggregate is recomputed with the selected averaging; its support is the - total number of positive samples across classes. + The aggregate is computed with the selected averaging; its support is the + total number of positive samples across classes. Both tables share the + same column layout, so the returned widths keep them aligned. """ per_class_df = pa.calculate_metrics( predictions, labels, per_class_metrics=True, include_support=True @@ -826,12 +842,18 @@ def _build_metrics_table(pa, predictions, labels, averaging_value): [loc.localize("eval-tab-overall-row-label")] ) - merged = pd.concat([per_class_df, overall_df], axis=1) - # One row per class/aggregate, one column per metric. - table = merged.T.reset_index(names=[""]).round(3) - table["Support"] = table["Support"].astype(int) + def display(df): + # One row per class/aggregate, one column per metric. + table = df.T.reset_index(names=[""]).round(3) + table["Support"] = table["Support"].astype(int) + + return table - return table + # The name column gets a fixed share, the metric columns split the rest. + n_metric_cols = len(per_class_df.index) + widths = ["22%"] + [f"{round(78 / n_metric_cols, 2)}%"] * n_metric_cols + + return display(per_class_df), display(overall_df), widths def _build_figures(pa, predictions, labels, class_wise_value): """Builds the three result plots, tolerating individual failures. @@ -933,7 +955,9 @@ def calculate_metrics( metric_ids, ) - table = _build_metrics_table(pa, predictions, labels, averaging_value) + class_table, overall_row, widths = _build_metrics_tables( + pa, predictions, labels, averaging_value + ) empty_classes = pa.empty_classes(labels) notes = _build_notes(proc_state, empty_classes, score_unannotated_value) except gr.Error: @@ -950,7 +974,8 @@ def calculate_metrics( return ( gr.update(visible=True), - gr.update(value=table), + gr.update(value=class_table, column_widths=widths), + gr.update(value=overall_row, column_widths=widths), gr.update(value=notes, visible=bool(notes)), fig_metrics, fig_confusion, @@ -975,6 +1000,7 @@ def calculate_metrics( outputs=[ results_col, metrics_table, + overall_table, notes_markdown, metrics_plot, confusion_plot, @@ -991,14 +1017,17 @@ def recompute_overall( if pa is None or predictions is None or labels is None: return gr.update() - return gr.update( - value=_build_metrics_table(pa, predictions, labels, averaging_value) + # Only the aggregate depends on the averaging method. + _, overall_row, widths = _build_metrics_tables( + pa, predictions, labels, averaging_value ) + return gr.update(value=overall_row, column_widths=widths) + averaging_radio.input( recompute_overall, inputs=[pa_state, predictions_state, labels_state, averaging_radio], - outputs=[metrics_table], + outputs=[overall_table], ) annotation_select_directory_btn.click(