diff --git a/birdnet_analyzer/model.py b/birdnet_analyzer/model.py index 0f2bac7b3..d73413f5a 100644 --- a/birdnet_analyzer/model.py +++ b/birdnet_analyzer/model.py @@ -151,11 +151,10 @@ def random_split(x, y, rng: Generator, val_ratio=0.2): A tuple of (x_train, y_train, x_val, y_val). """ num_classes = y.shape[1] - x_train, y_train, x_val, y_val = [], [], [], [] + train_indices, val_indices = [], [] for i in range(num_classes): positive_indices = np.where(y[:, i] == 1)[0] - negative_indices = np.where(y[:, i] == -1)[0] num_samples = len(positive_indices) num_samples_train = max(1, int(num_samples * (1 - val_ratio))) @@ -163,18 +162,18 @@ def random_split(x, y, rng: Generator, val_ratio=0.2): rng.shuffle(positive_indices) - train_indices = positive_indices[:num_samples_train] - val_indices = positive_indices[ + class_train_indices = positive_indices[:num_samples_train] + class_val_indices = positive_indices[ num_samples_train : num_samples_train + num_samples_val ] - x_train.append(x[train_indices]) - y_train.append(y[train_indices]) - x_val.append(x[val_indices]) - y_val.append(y[val_indices]) + train_indices.append(class_train_indices) + val_indices.append(class_val_indices) - x_train.append(x[negative_indices]) - y_train.append(y[negative_indices]) + # Negative samples are not class-specific in single-label training. Appending + # them in the loop above duplicates every negative sample once per class. + negative_indices = np.unique(np.where(y == -1)[0]) + train_indices.append(negative_indices) non_event_indices = np.where(np.sum(y[:, :], axis=1) == 0)[0] num_samples = len(non_event_indices) @@ -183,36 +182,21 @@ def random_split(x, y, rng: Generator, val_ratio=0.2): rng.shuffle(non_event_indices) - train_indices = non_event_indices[:num_samples_train] - val_indices = non_event_indices[ + non_event_train_indices = non_event_indices[:num_samples_train] + non_event_val_indices = non_event_indices[ num_samples_train : num_samples_train + num_samples_val ] - x_train.append(x[train_indices]) - y_train.append(y[train_indices]) - x_val.append(x[val_indices]) - y_val.append(y[val_indices]) + train_indices.append(non_event_train_indices) + val_indices.append(non_event_val_indices) - x_train = np.concatenate(x_train) - y_train = np.concatenate(y_train) - x_val = np.concatenate(x_val) - y_val = np.concatenate(y_val) + train_indices = np.concatenate(train_indices) + val_indices = np.concatenate(val_indices) - indices = np.arange(len(x_train)) + rng.shuffle(train_indices) + rng.shuffle(val_indices) - rng.shuffle(indices) - - x_train = x_train[indices] - y_train = y_train[indices] - - indices = np.arange(len(x_val)) - - rng.shuffle(indices) - - x_val = x_val[indices] - y_val = y_val[indices] - - return x_train, y_train, x_val, y_val + return x[train_indices], y[train_indices], x[val_indices], y[val_indices] def random_multilabel_split(x, y, rng: Generator, val_ratio=0.2): @@ -230,15 +214,16 @@ def random_multilabel_split(x, y, rng: Generator, val_ratio=0.2): A tuple of (x_train, y_train, x_val, y_val). """ - class_combinations = np.unique(y, axis=0) - x_train, y_train, x_val, y_val = [], [], [], [] + class_combinations, combination_ids = np.unique( + y, axis=0, return_inverse=True + ) + train_indices, val_indices = [], [] - for class_combination in class_combinations: - indices = np.where((y == class_combination).all(axis=1))[0] + for combination_id, class_combination in enumerate(class_combinations): + indices = np.flatnonzero(combination_ids == combination_id) if -1 in class_combination: - x_train.append(x[indices]) - y_train.append(y[indices]) + train_indices.append(indices) else: num_samples = len(indices) num_samples_train = max(1, int(num_samples * (1 - val_ratio))) @@ -246,32 +231,20 @@ def random_multilabel_split(x, y, rng: Generator, val_ratio=0.2): rng.shuffle(indices) - train_indices = indices[:num_samples_train] - val_indices = indices[ + combination_train_indices = indices[:num_samples_train] + combination_val_indices = indices[ num_samples_train : num_samples_train + num_samples_val ] - x_train.append(x[train_indices]) - y_train.append(y[train_indices]) - x_val.append(x[val_indices]) - y_val.append(y[val_indices]) + train_indices.append(combination_train_indices) + val_indices.append(combination_val_indices) - x_train = np.concatenate(x_train) - y_train = np.concatenate(y_train) - x_val = np.concatenate(x_val) - y_val = np.concatenate(y_val) + train_indices = np.concatenate(train_indices) + val_indices = np.concatenate(val_indices) + rng.shuffle(train_indices) + rng.shuffle(val_indices) - indices = np.arange(len(x_train)) - rng.shuffle(indices) - x_train = x_train[indices] - y_train = y_train[indices] - - indices = np.arange(len(x_val)) - rng.shuffle(indices) - x_val = x_val[indices] - y_val = y_val[indices] - - return x_train, y_train, x_val, y_val + return x[train_indices], y[train_indices], x[val_indices], y[val_indices] def upsample_core( @@ -303,33 +276,33 @@ def upsample_core( x_temp = [] if is_binary: - minority_label = 1 if y.sum(axis=0) < len(y) - y.sum(axis=0) else 0 + positive_count = y.sum(axis=0) + minority_label = 1 if positive_count < len(y) - positive_count else 0 + source_indices = np.flatnonzero(y == minority_label) + missing_samples = min_samples - len(source_indices) - while np.where(y == minority_label)[0].shape[0] + len(y_temp) < min_samples: - random_index = rng.choice(np.where(y == minority_label)[0], size=size) - x_app, y_app = apply(x, y, random_index) + for _ in range(max(0, missing_samples)): + random_index = rng.choice(source_indices, size=size) + x_app, y_app = apply(x, y, random_index, source_indices) y_temp.append(y_app) x_temp.append(x_app) else: for i in range(y.shape[1]): - class_x_temp = [] - class_y_temp = [] + source_indices = np.flatnonzero(y[:, i] == 1) + missing_samples = min_samples - len(source_indices) - while y[:, i].sum() + len(class_y_temp) < min_samples: - try: - random_index = rng.choice(np.where(y[:, i] == 1)[0], size=size) - except ValueError as e: - raise get_empty_class_exception()(index=i) from e + if missing_samples <= 0: + continue - # Apply - x_app, y_app = apply(x, y, random_index) - class_y_temp.append(y_app) - class_x_temp.append(x_app) + if not len(source_indices): + raise get_empty_class_exception()(index=i) - if len(class_y_temp) > 0: - x_temp.extend(class_x_temp) - y_temp.extend(class_y_temp) + for _ in range(missing_samples): + random_index = rng.choice(source_indices, size=size) + x_app, y_app = apply(x, y, random_index, source_indices) + y_temp.append(y_app) + x_temp.append(x_app) return x_temp, y_temp @@ -366,43 +339,60 @@ def upsampling( x_temp = [] y_temp = [] - if mode == "repeat": - - def applyRepeat(x, y, random_index): - return x[random_index[0]], y[random_index[0]] - - x_temp, y_temp = upsample_core( - x, y, min_samples, rng, applyRepeat, is_binary, size=1 - ) - - elif mode == "mean": - - def applyMean(x, y, random_indices): - mean = np.mean(x[random_indices], axis=0) + if mode in {"repeat", "mean", "linear"}: + if is_binary: + positive_count = y.sum(axis=0) + minority_label = 1 if positive_count < len(y) - positive_count else 0 + source_groups = [(None, np.flatnonzero(y == minority_label))] + else: + source_groups = [ + (class_index, np.flatnonzero(y[:, class_index] == 1)) + for class_index in range(y.shape[1]) + ] - return mean, y[random_indices[0]] + sample_size = 1 if mode == "repeat" else 2 + for class_index, source_indices in source_groups: + missing_samples = min_samples - len(source_indices) + if missing_samples <= 0: + continue - x_temp, y_temp = upsample_core(x, y, min_samples, rng, applyMean, is_binary) - elif mode == "linear": + if not len(source_indices): + if class_index is None: + raise ValueError("The minority class is empty.") + raise get_empty_class_exception()(index=class_index) - def applyLinearCombination(x, y, random_indices): - alpha = rng.uniform(0, 1) - new_sample = ( - alpha * x[random_indices[0]] + (1 - alpha) * x[random_indices[1]] + sampled_indices = rng.choice( + source_indices, size=(missing_samples, sample_size) ) + x_sources = x[sampled_indices] - return new_sample, y[random_indices[0]] + if mode == "repeat": + x_temp.append(x_sources[:, 0]) + elif mode == "mean": + x_temp.append(np.mean(x_sources, axis=1)) + else: + alpha = rng.uniform(0, 1, size=(missing_samples, 1)) + x_temp.append( + alpha * x_sources[:, 0] + (1 - alpha) * x_sources[:, 1] + ) - x_temp, y_temp = upsample_core( - x, y, min_samples, rng, applyLinearCombination, is_binary - ) + y_temp.append(y[sampled_indices[:, 0]]) elif mode == "smote": - def applySmote(x, y, random_index, k=5): - distances = np.sqrt(np.sum((x - x[random_index[0]]) ** 2, axis=1)) - indices = np.argsort(distances)[1 : k + 1] - random_neighbor = rng.choice(indices) + def applySmote(x, y, random_index, source_indices, k=5): + source_index = random_index[0] + neighbor_indices = source_indices[source_indices != source_index] + if not len(neighbor_indices): + return x[source_index], y[source_index] + + differences = x[neighbor_indices] - x[source_index] + distances = np.einsum("ij,ij->i", differences, differences) + nearest_count = min(k, len(neighbor_indices)) + nearest_indices = neighbor_indices[ + np.argpartition(distances, nearest_count - 1)[:nearest_count] + ] + random_neighbor = rng.choice(nearest_indices) diff = x[random_neighbor] - x[random_index[0]] weight = rng.uniform(0, 1) new_sample = x[random_index[0]] + weight * diff @@ -414,16 +404,8 @@ def applySmote(x, y, random_index, k=5): ) if len(x_temp) > 0: - x = np.vstack((x, np.array(x_temp))) - y = np.vstack((y, np.array(y_temp))) - - indices = np.arange(len(x)) - rng.shuffle(indices) - x = x[indices] - y = y[indices] - - del x_temp - del y_temp + x = np.vstack((x, *x_temp)) + y = np.vstack((y, *y_temp)) return x, y @@ -540,10 +522,6 @@ def on_epoch_end(self, epoch, logs=None): self.on_epoch_end_fn(epoch, logs) rng = np.random.default_rng(RANDOM_SEED) - idx = np.arange(x_train.shape[0]) - rng.shuffle(idx) - x_train = x_train[idx] - y_train = y_train[idx] if val_split > 0: if not is_multi_label: @@ -554,6 +532,11 @@ def on_epoch_end(self, epoch, logs=None): x_train, y_train, x_val, y_val = random_multilabel_split( x_train, y_train, rng, val_split ) + else: + idx = np.arange(x_train.shape[0]) + rng.shuffle(idx) + x_train = x_train[idx] + y_train = y_train[idx] if upsampling_ratio > 0: x_train, y_train = upsampling( @@ -634,6 +617,7 @@ def _focal_loss(y_true, y_pred): batch_size=batch_size, validation_data=(x_val, y_val), callbacks=callbacks, + shuffle=True, ) os.environ["CUDA_VISIBLE_DEVICES"] = setting_cache diff --git a/birdnet_analyzer/train/utils.py b/birdnet_analyzer/train/utils.py index 0201aa7ff..2c547ac61 100644 --- a/birdnet_analyzer/train/utils.py +++ b/birdnet_analyzer/train/utils.py @@ -158,6 +158,29 @@ def _read_and_crop_file( return sig_splits, labels +def _check_input_folders(audio_input: str, train_folders: list[str]): + """Reject training folders without supported audio files before model setup.""" + empty_folders = [] + + for folder in train_folders: + folder_path = os.path.join(audio_input, folder) + has_audio_file = any( + entry.is_file() + and not entry.name.startswith(".") + and entry.name.rsplit(".", 1)[-1].lower() in ALLOWED_FILETYPES + for entry in os.scandir(folder_path) + ) + + if not has_audio_file: + empty_folders.append(folder) + + if empty_folders: + raise ValueError( + "The following training data folders do not contain any supported audio " + f"files: {', '.join(empty_folders)}" + ) + + def _load_training_data( audio_input: str, test_data: str | None = None, @@ -248,6 +271,8 @@ def _load_training_data( "validation-only-repeat-upsampling-for-multi-label", ) + _check_input_folders(audio_input, train_folders) + x_train, y_train, x_test, y_test = [], [], [], [] model = load("acoustic", "2.4", "tf") model_sr = int(model.get_sample_rate()) diff --git a/tests/train/test_model.py b/tests/train/test_model.py new file mode 100644 index 000000000..8bb940eb9 --- /dev/null +++ b/tests/train/test_model.py @@ -0,0 +1,101 @@ +import numpy as np +import pytest + +from birdnet_analyzer.model import ( + get_empty_class_exception, + random_multilabel_split, + random_split, + upsampling, +) + + +def test_random_split_adds_negative_samples_only_once(): + x = np.arange(10).reshape(-1, 1) + y = np.array( + [ + [1, 0, 0], + [1, 0, 0], + [0, 1, 0], + [0, 1, 0], + [0, 0, 1], + [0, 0, 1], + [-1, -1, -1], + [-1, -1, -1], + [-1, -1, -1], + [-1, -1, -1], + ] + ) + + x_train, _, x_val, _ = random_split( + x, y, np.random.default_rng(42), val_ratio=0.5 + ) + + assert len(x_train) + len(x_val) == len(x) + assert len(np.unique(x_train)) == len(x_train) + assert not np.intersect1d(x_train, x_val).size + + +def test_random_multilabel_split_keeps_negative_combinations_in_training(): + x = np.arange(10).reshape(-1, 1) + y = np.array( + [ + [1, 0, 0], + [1, 0, 0], + [0, 1, 0], + [0, 1, 0], + [1, 1, 0], + [1, 1, 0], + [0, -1, 0], + [0, -1, 0], + [0, 0, 0], + [0, 0, 0], + ] + ) + + x_train, _, x_val, y_val = random_multilabel_split( + x, y, np.random.default_rng(42), val_ratio=0.5 + ) + + assert len(x_train) + len(x_val) == len(x) + assert not np.intersect1d(x_train, x_val).size + assert {6, 7}.issubset(set(x_train[:, 0])) + assert not np.any(y_val == -1) + + +@pytest.mark.parametrize("mode", ["repeat", "mean", "linear"]) +def test_upsampling_balances_each_class_to_requested_ratio(mode): + x = np.arange(12, dtype="float32").reshape(6, 2) + y = np.array([[1, 0], [1, 0], [1, 0], [1, 0], [1, 0], [0, 1]]) + + x_upsampled, y_upsampled = upsampling( + x, y, np.random.default_rng(42), is_binary=False, ratio=1.0, mode=mode + ) + + assert len(x_upsampled) == 10 + assert np.array_equal(y_upsampled.sum(axis=0), [5, 5]) + + +@pytest.mark.parametrize("mode", ["repeat", "mean", "linear"]) +def test_upsampling_rejects_empty_classes(mode): + x = np.arange(8, dtype="float32").reshape(4, 2) + y = np.array([[1, 0], [1, 0], [1, 0], [1, 0]]) + + with pytest.raises(get_empty_class_exception()) as error: + upsampling( + x, y, np.random.default_rng(42), is_binary=False, ratio=1.0, mode=mode + ) + + assert error.value.index == 1 + + +def test_smote_only_interpolates_between_samples_of_the_same_class(): + x = np.array([[0.0], [1.0], [100.0], [101.0], [102.0]]) + y = np.array([[1, 0], [1, 0], [0, 1], [0, 1], [0, 1]]) + + x_upsampled, y_upsampled = upsampling( + x, y, np.random.default_rng(42), is_binary=False, ratio=1.0, mode="smote" + ) + + assert len(x_upsampled) == 6 + assert np.array_equal(y_upsampled.sum(axis=0), [3, 3]) + assert 0 <= x_upsampled[-1, 0] <= 1 diff --git a/tests/train/test_train_loading.py b/tests/train/test_train_loading.py index 865e2c83e..5c6b1d9bb 100644 --- a/tests/train/test_train_loading.py +++ b/tests/train/test_train_loading.py @@ -14,13 +14,14 @@ import os import tempfile +from unittest.mock import patch import numpy as np import pytest import soundfile as sf from birdnet_analyzer import model_utils -from birdnet_analyzer.train.utils import _read_and_crop_file +from birdnet_analyzer.train.utils import _load_training_data, _read_and_crop_file SR = 48000 SIG_LENGTH = 3.0 @@ -81,6 +82,22 @@ def test_read_and_crop_bad_file_returns_empty(): assert labels == [] +def test_load_training_data_lists_empty_folders_before_loading_model(tmp_path): + (tmp_path / "blackbird").mkdir() + (tmp_path / "bluebird").mkdir() + robin_dir = tmp_path / "robin" + robin_dir.mkdir() + (robin_dir / "recording.wav").touch() + + with patch("birdnet_analyzer.train.utils.load") as mock_load, pytest.raises( + ValueError, match=r"blackbird.*bluebird" + ) as error: + _load_training_data(str(tmp_path)) + + assert "robin" not in str(error.value) + mock_load.assert_not_called() + + @pytest.fixture def short_wav(): """A 0.4 s wav -> shorter than the 1 s min_len, so split_signal yields nothing."""