From ad49c54b5429be5eae22fa25a83d19c2783ebdf6 Mon Sep 17 00:00:00 2001 From: AtomicGlance Date: Fri, 4 Sep 2026 09:33:39 +0330 Subject: [PATCH 1/4] Add subject ID discovery to Dandiset APIs --- dandi/dandiapi.py | 14 ++++++++++++++ dandi/dandiset.py | 17 +++++++++++++++++ dandi/tests/test_dandiapi.py | 23 +++++++++++++++++++++++ dandi/tests/test_dandiset.py | 14 ++++++++++++++ docs/source/modref/dandiapi.rst | 10 ++++++++++ docs/source/modref/dandiset.rst | 21 +++++++++++++++++++++ docs/source/modref/index.rst | 1 + 7 files changed, 100 insertions(+) create mode 100644 docs/source/modref/dandiset.rst diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index ccf17239b..251b7401d 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -1426,6 +1426,20 @@ def get_assets(self, order: str | None = None) -> Iterator[RemoteAsset]: f"No such version: {self.version_id!r} of Dandiset {self.identifier}" ) + def get_subject_ids(self) -> list[str]: + """Return sorted subject identifiers from top-level ``sub-*`` asset paths. + + The Archive API does not return directory objects, so asset records + are streamed in path order and only their paths are inspected. Asset + payloads and metadata are not downloaded. + """ + subject_ids = set[str]() + for asset in self.get_assets(order="path"): + parts = PurePosixPath(asset.path).parts + if len(parts) > 1 and parts[0].startswith("sub-") and len(parts[0]) > 4: + subject_ids.add(parts[0][4:]) + return sorted(subject_ids) + def get_asset(self, asset_id: str) -> RemoteAsset: """ Fetch the asset in this version of the Dandiset with the given asset diff --git a/dandi/dandiset.py b/dandi/dandiset.py index 22cbce892..a844c1f3f 100644 --- a/dandi/dandiset.py +++ b/dandi/dandiset.py @@ -168,6 +168,23 @@ def assets(self, allow_all: bool = False) -> AssetView: data[PurePosixPath(df.path)] = df return AssetView(data) + def get_subject_ids(self) -> list[str]: + """Return sorted subject identifiers from top-level ``sub-*`` directories. + + Only immediate children of the local Dandiset are inspected; files + inside subject directories are not read. + """ + return sorted( + { + path.name[4:] + for path in self.path_obj.iterdir() + if path.is_dir() + and not path.is_symlink() + and path.name.startswith("sub-") + and len(path.name) > 4 + } + ) + def metadata_file(self) -> DandisetMetadataFile: df = dandi_file(self._metadata_file_obj, dandiset_path=self.path) assert isinstance(df, DandisetMetadataFile) diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index 8da28852e..58b59ad86 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -8,6 +8,7 @@ import random import re from shutil import rmtree +from types import SimpleNamespace from typing import Any import anys @@ -996,6 +997,28 @@ def _get_assets_with_path_prefix(prefix: str, **kw: Any) -> list[str]: ] +def test_remote_get_subject_ids(mocker: MockerFixture) -> None: + client = mocker.Mock(_instance_id="test") + dandiset = RemoteDandiset(client, "000001", version=DRAFT) + get_assets = mocker.patch.object( + dandiset, + "get_assets", + return_value=iter( + [ + SimpleNamespace(path="sub-mouse2/session/file.nwb"), + SimpleNamespace(path="sub-mouse1/file.nwb"), + SimpleNamespace(path="sub-mouse2/other/file.nwb"), + SimpleNamespace(path="sub-root.nwb"), + SimpleNamespace(path="other/file.nwb"), + SimpleNamespace(path="sub-/file.nwb"), + ] + ), + ) + + assert dandiset.get_subject_ids() == ["mouse1", "mouse2"] + get_assets.assert_called_once_with(order="path") + + def test_get_assets_by_glob(text_dandiset: SampleDandiset) -> None: assert sorted( asset.path for asset in text_dandiset.dandiset.get_assets_by_glob("*a*.txt") diff --git a/dandi/tests/test_dandiset.py b/dandi/tests/test_dandiset.py index 39653345a..8a0bf6ae2 100644 --- a/dandi/tests/test_dandiset.py +++ b/dandi/tests/test_dandiset.py @@ -1,3 +1,5 @@ +from pathlib import Path + from ..dandiset import Dandiset @@ -6,3 +8,15 @@ def test_get_dandiset_record() -> None: # Should have only header with "DO NOT EDIT" assert out.startswith("# DO NOT EDIT") assert "000000" in out + + +def test_get_subject_ids(tmp_path: Path) -> None: + (tmp_path / "dandiset.yaml").write_text("identifier: '000001'\n") + (tmp_path / "sub-mouse2").mkdir() + (tmp_path / "sub-mouse2" / "record.nwb").touch() + (tmp_path / "sub-mouse1").mkdir() + (tmp_path / "sub-").mkdir() + (tmp_path / "sub-root.nwb").touch() + (tmp_path / "subjects").mkdir() + + assert Dandiset(tmp_path).get_subject_ids() == ["mouse1", "mouse2"] diff --git a/docs/source/modref/dandiapi.rst b/docs/source/modref/dandiapi.rst index 8212711d0..46d6a17a0 100644 --- a/docs/source/modref/dandiapi.rst +++ b/docs/source/modref/dandiapi.rst @@ -36,6 +36,16 @@ be passed to functions of pynwb etc. You can see more usages of DANDI API to assist with data streaming at `PyNWB: Streaming NWB files `_. +To discover the subject labels represented by a remote Dandiset, use +``RemoteDandiset.get_subject_ids()``. It streams asset paths ordered by path +and does not download asset payloads or metadata: + +.. code-block:: python + + with DandiAPIClient() as client: + dandiset = client.get_dandiset("000001") + print(dandiset.get_subject_ids()) + Client ------ diff --git a/docs/source/modref/dandiset.rst b/docs/source/modref/dandiset.rst new file mode 100644 index 000000000..ea0e323bd --- /dev/null +++ b/docs/source/modref/dandiset.rst @@ -0,0 +1,21 @@ +.. module:: dandi.dandiset + +``dandi.dandiset`` +================== + +This module provides the local Dandiset API. A local Dandiset can report the +subject labels represented by its top-level ``sub-*`` directories without +opening or scanning the files within them: + +.. code-block:: python + + from dandi.dandiset import Dandiset + + dandiset = Dandiset("/data/my-dandiset") + print(dandiset.get_subject_ids()) + +.. autoclass:: Dandiset() + :members: + +.. autoclass:: AssetView() + :members: diff --git a/docs/source/modref/index.rst b/docs/source/modref/index.rst index 373b48502..ce215299c 100644 --- a/docs/source/modref/index.rst +++ b/docs/source/modref/index.rst @@ -32,6 +32,7 @@ Object-oriented interfaces to manipulate Dandisets and assets on a DANDI instanc .. toctree:: dandiarchive + dandiset Low-level user interfaces ========================= From 763770f84e8578de50fdf061d1c08a1322457109 Mon Sep 17 00:00:00 2001 From: AtomicGlance Date: Fri, 4 Sep 2026 16:36:38 +0330 Subject: [PATCH 2/4] Harden subject ID discovery contract --- dandi/consts.py | 6 ++++++ dandi/dandiapi.py | 12 ++++++++--- dandi/dandiset.py | 35 ++++++++++++++++++++------------- dandi/organize.py | 8 +++++--- dandi/tests/test_dandiapi.py | 22 +++++++++++++++++++++ dandi/tests/test_dandiset.py | 25 ++++++++++++++++++++++- dandi/utils.py | 16 ++++++++++++++- docs/source/modref/dandiapi.rst | 4 +++- docs/source/modref/dandiset.rst | 6 ++++-- 9 files changed, 109 insertions(+), 25 deletions(-) diff --git a/dandi/consts.py b/dandi/consts.py index baca5465c..1d6f618cf 100644 --- a/dandi/consts.py +++ b/dandi/consts.py @@ -14,6 +14,12 @@ from enum import Enum, StrEnum import os +# Labels used in DANDI-organized subject and session path components. Keep +# these expressions here so path discovery and path validation use the same +# syntax. +DANDI_LABEL_REGEX = r"[^_*\\/<>:|\"'?%@;.]+" +DANDI_SUBJECT_FOLDER_REGEX = rf"sub-{DANDI_LABEL_REGEX}" + #: A list of metadata fields which dandi extracts from .nwb files. #: Additional fields (such as ``number_of_*``) might be added by #: `get_metadata()` diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 251b7401d..510e8a12a 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -63,6 +63,7 @@ is_interactive, is_page2_url, joinurl, + parse_dandi_subject_dirname, ) if TYPE_CHECKING: @@ -1431,13 +1432,18 @@ def get_subject_ids(self) -> list[str]: The Archive API does not return directory objects, so asset records are streamed in path order and only their paths are inspected. Asset - payloads and metadata are not downloaded. + payloads and metadata are not downloaded. Runtime is linear in the + number of assets in the Dandiset. + + .. versionadded:: 0.79.0 """ subject_ids = set[str]() for asset in self.get_assets(order="path"): parts = PurePosixPath(asset.path).parts - if len(parts) > 1 and parts[0].startswith("sub-") and len(parts[0]) > 4: - subject_ids.add(parts[0][4:]) + if len(parts) > 1: + subject_id = parse_dandi_subject_dirname(parts[0]) + if subject_id is not None: + subject_ids.add(subject_id) return sorted(subject_ids) def get_asset(self, asset_id: str) -> RemoteAsset: diff --git a/dandi/dandiset.py b/dandi/dandiset.py index a844c1f3f..d3b378b9b 100644 --- a/dandi/dandiset.py +++ b/dandi/dandiset.py @@ -12,7 +12,13 @@ from . import get_logger from .consts import dandiset_metadata_file from .files import DandisetMetadataFile, LocalAsset, dandi_file, find_dandi_files -from .utils import find_parent_directory_containing, under_paths, yaml_dump, yaml_load +from .utils import ( + find_parent_directory_containing, + parse_dandi_subject_dirname, + under_paths, + yaml_dump, + yaml_load, +) if TYPE_CHECKING: from typing_extensions import Self @@ -169,21 +175,22 @@ def assets(self, allow_all: bool = False) -> AssetView: return AssetView(data) def get_subject_ids(self) -> list[str]: - """Return sorted subject identifiers from top-level ``sub-*`` directories. + """Return sorted IDs from populated top-level subject directories. - Only immediate children of the local Dandiset are inspected; files - inside subject directories are not read. + Directory entries are inspected, but file contents are not read. + Empty and symlinked subject directories are ignored so the result + describes subjects that can also be represented by remote assets. + + .. versionadded:: 0.79.0 """ - return sorted( - { - path.name[4:] - for path in self.path_obj.iterdir() - if path.is_dir() - and not path.is_symlink() - and path.name.startswith("sub-") - and len(path.name) > 4 - } - ) + subject_ids = set[str]() + for path in self.path_obj.iterdir(): + if not path.is_dir() or path.is_symlink(): + continue + subject_id = parse_dandi_subject_dirname(path.name) + if subject_id is not None and any(p.is_file() for p in path.rglob("*")): + subject_ids.add(subject_id) + return sorted(subject_ids) def metadata_file(self) -> DandisetMetadataFile: df = dandi_file(self._metadata_file_obj, dandiset_path=self.path) diff --git a/dandi/organize.py b/dandi/organize.py index 896702f8a..b15cb6141 100644 --- a/dandi/organize.py +++ b/dandi/organize.py @@ -27,7 +27,11 @@ import ruamel.yaml from . import get_logger -from .consts import dandi_layout_fields +from .consts import ( + DANDI_LABEL_REGEX as LABELREGEX, + DANDI_SUBJECT_FOLDER_REGEX as ORGANIZED_FOLDER_REGEX, + dandi_layout_fields, +) from .dandiset import Dandiset from .exceptions import OrganizeImpossibleError from .utils import ( @@ -1149,7 +1153,6 @@ def msg_(msg, n, cond=None): ) -LABELREGEX = r"[^_*\\/<>:|\"'?%@;.]+" ORGANIZED_FILENAME_REGEX = ( rf"sub-{LABELREGEX}" rf"(_ses-{LABELREGEX})?" @@ -1157,7 +1160,6 @@ def msg_(msg, n, cond=None): r"(_[a-z]+(\+[a-z]+)*)?" r"\.nwb" ) -ORGANIZED_FOLDER_REGEX = rf"sub-{LABELREGEX}" def validate_organized_path( diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index 58b59ad86..9ecddd387 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -1011,6 +1011,7 @@ def test_remote_get_subject_ids(mocker: MockerFixture) -> None: SimpleNamespace(path="sub-root.nwb"), SimpleNamespace(path="other/file.nwb"), SimpleNamespace(path="sub-/file.nwb"), + SimpleNamespace(path="sub-bad_name/file.nwb"), ] ), ) @@ -1019,6 +1020,27 @@ def test_remote_get_subject_ids(mocker: MockerFixture) -> None: get_assets.assert_called_once_with(order="path") +def test_remote_get_subject_ids_empty(mocker: MockerFixture) -> None: + client = mocker.Mock(_instance_id="test") + dandiset = RemoteDandiset(client, "000001", version=DRAFT) + mocker.patch.object(dandiset, "get_assets", return_value=iter(())) + + assert dandiset.get_subject_ids() == [] + + +def test_remote_get_subject_ids_propagates_api_errors( + mocker: MockerFixture, +) -> None: + client = mocker.Mock(_instance_id="test") + dandiset = RemoteDandiset(client, "000001", version=DRAFT) + mocker.patch.object( + dandiset, "get_assets", side_effect=RuntimeError("pagination failed") + ) + + with pytest.raises(RuntimeError, match="pagination failed"): + dandiset.get_subject_ids() + + def test_get_assets_by_glob(text_dandiset: SampleDandiset) -> None: assert sorted( asset.path for asset in text_dandiset.dandiset.get_assets_by_glob("*a*.txt") diff --git a/dandi/tests/test_dandiset.py b/dandi/tests/test_dandiset.py index 8a0bf6ae2..6edff90d0 100644 --- a/dandi/tests/test_dandiset.py +++ b/dandi/tests/test_dandiset.py @@ -1,5 +1,7 @@ from pathlib import Path +from pytest_mock import MockerFixture + from ..dandiset import Dandiset @@ -10,13 +12,34 @@ def test_get_dandiset_record() -> None: assert "000000" in out -def test_get_subject_ids(tmp_path: Path) -> None: +def test_get_subject_ids(tmp_path: Path, mocker: MockerFixture) -> None: (tmp_path / "dandiset.yaml").write_text("identifier: '000001'\n") (tmp_path / "sub-mouse2").mkdir() (tmp_path / "sub-mouse2" / "record.nwb").touch() (tmp_path / "sub-mouse1").mkdir() + (tmp_path / "sub-mouse1" / "session").mkdir() + (tmp_path / "sub-mouse1" / "session" / "record.nwb").touch() (tmp_path / "sub-").mkdir() + (tmp_path / "sub-bad_name").mkdir() + (tmp_path / "sub-bad_name" / "record.nwb").touch() + (tmp_path / "sub-empty").mkdir() + linked = tmp_path / "sub-linked" + linked.mkdir() + (linked / "record.nwb").touch() (tmp_path / "sub-root.nwb").touch() (tmp_path / "subjects").mkdir() + original_is_symlink = Path.is_symlink + mocker.patch.object( + Path, + "is_symlink", + autospec=True, + side_effect=lambda path: path == linked or original_is_symlink(path), + ) assert Dandiset(tmp_path).get_subject_ids() == ["mouse1", "mouse2"] + + +def test_get_subject_ids_empty(tmp_path: Path) -> None: + (tmp_path / "dandiset.yaml").write_text("identifier: '000001'\n") + + assert Dandiset(tmp_path).get_subject_ids() == [] diff --git a/dandi/utils.py b/dandi/utils.py index cd6ea7afd..c002011c2 100644 --- a/dandi/utils.py +++ b/dandi/utils.py @@ -35,7 +35,12 @@ from yarl import URL from . import __version__, get_logger -from .consts import DandiInstance, known_instances, known_instances_rev +from .consts import ( + DANDI_SUBJECT_FOLDER_REGEX, + DandiInstance, + known_instances, + known_instances_rev, +) from .exceptions import BadCliVersionError, CliVersionTooOldError AnyPath = Union[str, Path] @@ -43,6 +48,15 @@ lgr = get_logger() + +def parse_dandi_subject_dirname(name: str) -> str | None: + """Return the ID encoded by a valid DANDI ``sub-*`` directory name.""" + + if re.fullmatch(DANDI_SUBJECT_FOLDER_REGEX, name) is None: + return None + return name[4:] + + _sys_excepthook = sys.excepthook # Just in case we ever need original one # diff --git a/docs/source/modref/dandiapi.rst b/docs/source/modref/dandiapi.rst index 46d6a17a0..5431ed382 100644 --- a/docs/source/modref/dandiapi.rst +++ b/docs/source/modref/dandiapi.rst @@ -38,7 +38,9 @@ You can see more usages of DANDI API to assist with data streaming at To discover the subject labels represented by a remote Dandiset, use ``RemoteDandiset.get_subject_ids()``. It streams asset paths ordered by path -and does not download asset payloads or metadata: +and does not download asset payloads or metadata. This is an ``O(number of +assets)`` operation because the Archive does not currently provide distinct +top-level path prefixes: .. code-block:: python diff --git a/docs/source/modref/dandiset.rst b/docs/source/modref/dandiset.rst index ea0e323bd..35e41a3e7 100644 --- a/docs/source/modref/dandiset.rst +++ b/docs/source/modref/dandiset.rst @@ -4,8 +4,10 @@ ================== This module provides the local Dandiset API. A local Dandiset can report the -subject labels represented by its top-level ``sub-*`` directories without -opening or scanning the files within them: +subject labels represented by populated, valid top-level ``sub-*`` directories. +It walks directory entries to establish that a subject contains a file, but it +does not open file contents. Empty and symlinked subject directories are +ignored: .. code-block:: python From a5b2950b16722d9b192ede1fd7dab699e424c44e Mon Sep 17 00:00:00 2001 From: AtomicGlance Date: Fri, 4 Sep 2026 16:55:57 +0330 Subject: [PATCH 3/4] Keep subject parsing change focused --- dandi/consts.py | 6 ------ dandi/organize.py | 8 +++----- dandi/utils.py | 11 ++++------- 3 files changed, 7 insertions(+), 18 deletions(-) diff --git a/dandi/consts.py b/dandi/consts.py index 1d6f618cf..baca5465c 100644 --- a/dandi/consts.py +++ b/dandi/consts.py @@ -14,12 +14,6 @@ from enum import Enum, StrEnum import os -# Labels used in DANDI-organized subject and session path components. Keep -# these expressions here so path discovery and path validation use the same -# syntax. -DANDI_LABEL_REGEX = r"[^_*\\/<>:|\"'?%@;.]+" -DANDI_SUBJECT_FOLDER_REGEX = rf"sub-{DANDI_LABEL_REGEX}" - #: A list of metadata fields which dandi extracts from .nwb files. #: Additional fields (such as ``number_of_*``) might be added by #: `get_metadata()` diff --git a/dandi/organize.py b/dandi/organize.py index b15cb6141..896702f8a 100644 --- a/dandi/organize.py +++ b/dandi/organize.py @@ -27,11 +27,7 @@ import ruamel.yaml from . import get_logger -from .consts import ( - DANDI_LABEL_REGEX as LABELREGEX, - DANDI_SUBJECT_FOLDER_REGEX as ORGANIZED_FOLDER_REGEX, - dandi_layout_fields, -) +from .consts import dandi_layout_fields from .dandiset import Dandiset from .exceptions import OrganizeImpossibleError from .utils import ( @@ -1153,6 +1149,7 @@ def msg_(msg, n, cond=None): ) +LABELREGEX = r"[^_*\\/<>:|\"'?%@;.]+" ORGANIZED_FILENAME_REGEX = ( rf"sub-{LABELREGEX}" rf"(_ses-{LABELREGEX})?" @@ -1160,6 +1157,7 @@ def msg_(msg, n, cond=None): r"(_[a-z]+(\+[a-z]+)*)?" r"\.nwb" ) +ORGANIZED_FOLDER_REGEX = rf"sub-{LABELREGEX}" def validate_organized_path( diff --git a/dandi/utils.py b/dandi/utils.py index c002011c2..f6bcf6027 100644 --- a/dandi/utils.py +++ b/dandi/utils.py @@ -35,12 +35,7 @@ from yarl import URL from . import __version__, get_logger -from .consts import ( - DANDI_SUBJECT_FOLDER_REGEX, - DandiInstance, - known_instances, - known_instances_rev, -) +from .consts import DandiInstance, known_instances, known_instances_rev from .exceptions import BadCliVersionError, CliVersionTooOldError AnyPath = Union[str, Path] @@ -52,7 +47,9 @@ def parse_dandi_subject_dirname(name: str) -> str | None: """Return the ID encoded by a valid DANDI ``sub-*`` directory name.""" - if re.fullmatch(DANDI_SUBJECT_FOLDER_REGEX, name) is None: + # Match the label syntax used by ``dandi organize`` without importing that + # module, which imports this utility module itself. + if re.fullmatch(r"sub-[^_*\\/<>:|\"'?%@;.]+", name) is None: return None return name[4:] From c68257f19f36a40fea0ec441b6bdecdd2471fb66 Mon Sep 17 00:00:00 2001 From: AtomicGlance Date: Fri, 4 Sep 2026 20:13:13 +0330 Subject: [PATCH 4/4] Use path endpoint for remote subject discovery --- dandi/dandiapi.py | 28 +++++++++++--------- dandi/tests/test_dandiapi.py | 45 ++++++++++++++++++--------------- docs/source/modref/dandiapi.rst | 6 ++--- 3 files changed, 42 insertions(+), 37 deletions(-) diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index 510e8a12a..24be6a024 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -1430,21 +1430,25 @@ def get_assets(self, order: str | None = None) -> Iterator[RemoteAsset]: def get_subject_ids(self) -> list[str]: """Return sorted subject identifiers from top-level ``sub-*`` asset paths. - The Archive API does not return directory objects, so asset records - are streamed in path order and only their paths are inspected. Asset - payloads and metadata are not downloaded. Runtime is linear in the - number of assets in the Dandiset. + The Archive's path endpoint returns the immediate children of the + Dandiset root, so discovery does not require listing every asset. + Asset payloads and metadata are not downloaded. .. versionadded:: 0.79.0 """ - subject_ids = set[str]() - for asset in self.get_assets(order="path"): - parts = PurePosixPath(asset.path).parts - if len(parts) > 1: - subject_id = parse_dandi_subject_dirname(parts[0]) - if subject_id is not None: - subject_ids.add(subject_id) - return sorted(subject_ids) + try: + paths = self.client.paginate(f"{self.version_api_path}assets/paths/") + return sorted( + subject_id + for item in paths + if item["asset"] is None + if (subject_id := parse_dandi_subject_dirname(item["path"])) + is not None + ) + except HTTP404Error: + raise NotFoundError( + f"No such version: {self.version_id!r} of Dandiset {self.identifier}" + ) def get_asset(self, asset_id: str) -> RemoteAsset: """ diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index 9ecddd387..292519f2b 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -8,7 +8,6 @@ import random import re from shutil import rmtree -from types import SimpleNamespace from typing import Any import anys @@ -41,7 +40,7 @@ VersionStatus, ) from ..download import download -from ..exceptions import NotFoundError, SchemaVersionError +from ..exceptions import HTTP404Error, NotFoundError, SchemaVersionError from ..files import GenericAsset, dandi_file from ..utils import list_paths @@ -1000,30 +999,27 @@ def _get_assets_with_path_prefix(prefix: str, **kw: Any) -> list[str]: def test_remote_get_subject_ids(mocker: MockerFixture) -> None: client = mocker.Mock(_instance_id="test") dandiset = RemoteDandiset(client, "000001", version=DRAFT) - get_assets = mocker.patch.object( - dandiset, - "get_assets", - return_value=iter( - [ - SimpleNamespace(path="sub-mouse2/session/file.nwb"), - SimpleNamespace(path="sub-mouse1/file.nwb"), - SimpleNamespace(path="sub-mouse2/other/file.nwb"), - SimpleNamespace(path="sub-root.nwb"), - SimpleNamespace(path="other/file.nwb"), - SimpleNamespace(path="sub-/file.nwb"), - SimpleNamespace(path="sub-bad_name/file.nwb"), - ] - ), + client.paginate.return_value = iter( + [ + {"path": "sub-mouse2", "asset": None}, + {"path": "sub-mouse1", "asset": None}, + {"path": "sub-root", "asset": {"asset_id": "a"}}, + {"path": "other", "asset": None}, + {"path": "sub-", "asset": None}, + {"path": "sub-bad_name", "asset": None}, + ] ) assert dandiset.get_subject_ids() == ["mouse1", "mouse2"] - get_assets.assert_called_once_with(order="path") + client.paginate.assert_called_once_with( + "/dandisets/000001/versions/draft/assets/paths/" + ) def test_remote_get_subject_ids_empty(mocker: MockerFixture) -> None: client = mocker.Mock(_instance_id="test") dandiset = RemoteDandiset(client, "000001", version=DRAFT) - mocker.patch.object(dandiset, "get_assets", return_value=iter(())) + client.paginate.return_value = iter(()) assert dandiset.get_subject_ids() == [] @@ -1033,14 +1029,21 @@ def test_remote_get_subject_ids_propagates_api_errors( ) -> None: client = mocker.Mock(_instance_id="test") dandiset = RemoteDandiset(client, "000001", version=DRAFT) - mocker.patch.object( - dandiset, "get_assets", side_effect=RuntimeError("pagination failed") - ) + client.paginate.side_effect = RuntimeError("pagination failed") with pytest.raises(RuntimeError, match="pagination failed"): dandiset.get_subject_ids() +def test_remote_get_subject_ids_missing_version(mocker: MockerFixture) -> None: + client = mocker.Mock(_instance_id="test") + dandiset = RemoteDandiset(client, "000001", version=DRAFT) + client.paginate.side_effect = HTTP404Error("not found") + + with pytest.raises(NotFoundError, match="No such version: 'draft'"): + dandiset.get_subject_ids() + + def test_get_assets_by_glob(text_dandiset: SampleDandiset) -> None: assert sorted( asset.path for asset in text_dandiset.dandiset.get_assets_by_glob("*a*.txt") diff --git a/docs/source/modref/dandiapi.rst b/docs/source/modref/dandiapi.rst index 5431ed382..5ee1e0c7b 100644 --- a/docs/source/modref/dandiapi.rst +++ b/docs/source/modref/dandiapi.rst @@ -37,10 +37,8 @@ You can see more usages of DANDI API to assist with data streaming at `PyNWB: Streaming NWB files `_. To discover the subject labels represented by a remote Dandiset, use -``RemoteDandiset.get_subject_ids()``. It streams asset paths ordered by path -and does not download asset payloads or metadata. This is an ``O(number of -assets)`` operation because the Archive does not currently provide distinct -top-level path prefixes: +``RemoteDandiset.get_subject_ids()``. It queries the Archive's efficient +top-level path endpoint and does not download asset payloads or metadata: .. code-block:: python