diff --git a/dandi/dandiapi.py b/dandi/dandiapi.py index ccf17239b..24be6a024 100644 --- a/dandi/dandiapi.py +++ b/dandi/dandiapi.py @@ -63,6 +63,7 @@ is_interactive, is_page2_url, joinurl, + parse_dandi_subject_dirname, ) if TYPE_CHECKING: @@ -1426,6 +1427,29 @@ def get_assets(self, order: str | None = None) -> Iterator[RemoteAsset]: f"No such version: {self.version_id!r} of Dandiset {self.identifier}" ) + def get_subject_ids(self) -> list[str]: + """Return sorted subject identifiers from top-level ``sub-*`` asset paths. + + The Archive's path endpoint returns the immediate children of the + Dandiset root, so discovery does not require listing every asset. + Asset payloads and metadata are not downloaded. + + .. versionadded:: 0.79.0 + """ + try: + paths = self.client.paginate(f"{self.version_api_path}assets/paths/") + return sorted( + subject_id + for item in paths + if item["asset"] is None + if (subject_id := parse_dandi_subject_dirname(item["path"])) + is not None + ) + except HTTP404Error: + raise NotFoundError( + f"No such version: {self.version_id!r} of Dandiset {self.identifier}" + ) + def get_asset(self, asset_id: str) -> RemoteAsset: """ Fetch the asset in this version of the Dandiset with the given asset diff --git a/dandi/dandiset.py b/dandi/dandiset.py index 22cbce892..d3b378b9b 100644 --- a/dandi/dandiset.py +++ b/dandi/dandiset.py @@ -12,7 +12,13 @@ from . import get_logger from .consts import dandiset_metadata_file from .files import DandisetMetadataFile, LocalAsset, dandi_file, find_dandi_files -from .utils import find_parent_directory_containing, under_paths, yaml_dump, yaml_load +from .utils import ( + find_parent_directory_containing, + parse_dandi_subject_dirname, + under_paths, + yaml_dump, + yaml_load, +) if TYPE_CHECKING: from typing_extensions import Self @@ -168,6 +174,24 @@ def assets(self, allow_all: bool = False) -> AssetView: data[PurePosixPath(df.path)] = df return AssetView(data) + def get_subject_ids(self) -> list[str]: + """Return sorted IDs from populated top-level subject directories. + + Directory entries are inspected, but file contents are not read. + Empty and symlinked subject directories are ignored so the result + describes subjects that can also be represented by remote assets. + + .. versionadded:: 0.79.0 + """ + subject_ids = set[str]() + for path in self.path_obj.iterdir(): + if not path.is_dir() or path.is_symlink(): + continue + subject_id = parse_dandi_subject_dirname(path.name) + if subject_id is not None and any(p.is_file() for p in path.rglob("*")): + subject_ids.add(subject_id) + return sorted(subject_ids) + def metadata_file(self) -> DandisetMetadataFile: df = dandi_file(self._metadata_file_obj, dandiset_path=self.path) assert isinstance(df, DandisetMetadataFile) diff --git a/dandi/tests/test_dandiapi.py b/dandi/tests/test_dandiapi.py index 8da28852e..292519f2b 100644 --- a/dandi/tests/test_dandiapi.py +++ b/dandi/tests/test_dandiapi.py @@ -40,7 +40,7 @@ VersionStatus, ) from ..download import download -from ..exceptions import NotFoundError, SchemaVersionError +from ..exceptions import HTTP404Error, NotFoundError, SchemaVersionError from ..files import GenericAsset, dandi_file from ..utils import list_paths @@ -996,6 +996,54 @@ def _get_assets_with_path_prefix(prefix: str, **kw: Any) -> list[str]: ] +def test_remote_get_subject_ids(mocker: MockerFixture) -> None: + client = mocker.Mock(_instance_id="test") + dandiset = RemoteDandiset(client, "000001", version=DRAFT) + client.paginate.return_value = iter( + [ + {"path": "sub-mouse2", "asset": None}, + {"path": "sub-mouse1", "asset": None}, + {"path": "sub-root", "asset": {"asset_id": "a"}}, + {"path": "other", "asset": None}, + {"path": "sub-", "asset": None}, + {"path": "sub-bad_name", "asset": None}, + ] + ) + + assert dandiset.get_subject_ids() == ["mouse1", "mouse2"] + client.paginate.assert_called_once_with( + "/dandisets/000001/versions/draft/assets/paths/" + ) + + +def test_remote_get_subject_ids_empty(mocker: MockerFixture) -> None: + client = mocker.Mock(_instance_id="test") + dandiset = RemoteDandiset(client, "000001", version=DRAFT) + client.paginate.return_value = iter(()) + + assert dandiset.get_subject_ids() == [] + + +def test_remote_get_subject_ids_propagates_api_errors( + mocker: MockerFixture, +) -> None: + client = mocker.Mock(_instance_id="test") + dandiset = RemoteDandiset(client, "000001", version=DRAFT) + client.paginate.side_effect = RuntimeError("pagination failed") + + with pytest.raises(RuntimeError, match="pagination failed"): + dandiset.get_subject_ids() + + +def test_remote_get_subject_ids_missing_version(mocker: MockerFixture) -> None: + client = mocker.Mock(_instance_id="test") + dandiset = RemoteDandiset(client, "000001", version=DRAFT) + client.paginate.side_effect = HTTP404Error("not found") + + with pytest.raises(NotFoundError, match="No such version: 'draft'"): + dandiset.get_subject_ids() + + def test_get_assets_by_glob(text_dandiset: SampleDandiset) -> None: assert sorted( asset.path for asset in text_dandiset.dandiset.get_assets_by_glob("*a*.txt") diff --git a/dandi/tests/test_dandiset.py b/dandi/tests/test_dandiset.py index 39653345a..6edff90d0 100644 --- a/dandi/tests/test_dandiset.py +++ b/dandi/tests/test_dandiset.py @@ -1,3 +1,7 @@ +from pathlib import Path + +from pytest_mock import MockerFixture + from ..dandiset import Dandiset @@ -6,3 +10,36 @@ def test_get_dandiset_record() -> None: # Should have only header with "DO NOT EDIT" assert out.startswith("# DO NOT EDIT") assert "000000" in out + + +def test_get_subject_ids(tmp_path: Path, mocker: MockerFixture) -> None: + (tmp_path / "dandiset.yaml").write_text("identifier: '000001'\n") + (tmp_path / "sub-mouse2").mkdir() + (tmp_path / "sub-mouse2" / "record.nwb").touch() + (tmp_path / "sub-mouse1").mkdir() + (tmp_path / "sub-mouse1" / "session").mkdir() + (tmp_path / "sub-mouse1" / "session" / "record.nwb").touch() + (tmp_path / "sub-").mkdir() + (tmp_path / "sub-bad_name").mkdir() + (tmp_path / "sub-bad_name" / "record.nwb").touch() + (tmp_path / "sub-empty").mkdir() + linked = tmp_path / "sub-linked" + linked.mkdir() + (linked / "record.nwb").touch() + (tmp_path / "sub-root.nwb").touch() + (tmp_path / "subjects").mkdir() + original_is_symlink = Path.is_symlink + mocker.patch.object( + Path, + "is_symlink", + autospec=True, + side_effect=lambda path: path == linked or original_is_symlink(path), + ) + + assert Dandiset(tmp_path).get_subject_ids() == ["mouse1", "mouse2"] + + +def test_get_subject_ids_empty(tmp_path: Path) -> None: + (tmp_path / "dandiset.yaml").write_text("identifier: '000001'\n") + + assert Dandiset(tmp_path).get_subject_ids() == [] diff --git a/dandi/utils.py b/dandi/utils.py index cd6ea7afd..f6bcf6027 100644 --- a/dandi/utils.py +++ b/dandi/utils.py @@ -43,6 +43,17 @@ lgr = get_logger() + +def parse_dandi_subject_dirname(name: str) -> str | None: + """Return the ID encoded by a valid DANDI ``sub-*`` directory name.""" + + # Match the label syntax used by ``dandi organize`` without importing that + # module, which imports this utility module itself. + if re.fullmatch(r"sub-[^_*\\/<>:|\"'?%@;.]+", name) is None: + return None + return name[4:] + + _sys_excepthook = sys.excepthook # Just in case we ever need original one # diff --git a/docs/source/modref/dandiapi.rst b/docs/source/modref/dandiapi.rst index 8212711d0..5ee1e0c7b 100644 --- a/docs/source/modref/dandiapi.rst +++ b/docs/source/modref/dandiapi.rst @@ -36,6 +36,16 @@ be passed to functions of pynwb etc. You can see more usages of DANDI API to assist with data streaming at `PyNWB: Streaming NWB files `_. +To discover the subject labels represented by a remote Dandiset, use +``RemoteDandiset.get_subject_ids()``. It queries the Archive's efficient +top-level path endpoint and does not download asset payloads or metadata: + +.. code-block:: python + + with DandiAPIClient() as client: + dandiset = client.get_dandiset("000001") + print(dandiset.get_subject_ids()) + Client ------ diff --git a/docs/source/modref/dandiset.rst b/docs/source/modref/dandiset.rst new file mode 100644 index 000000000..35e41a3e7 --- /dev/null +++ b/docs/source/modref/dandiset.rst @@ -0,0 +1,23 @@ +.. module:: dandi.dandiset + +``dandi.dandiset`` +================== + +This module provides the local Dandiset API. A local Dandiset can report the +subject labels represented by populated, valid top-level ``sub-*`` directories. +It walks directory entries to establish that a subject contains a file, but it +does not open file contents. Empty and symlinked subject directories are +ignored: + +.. code-block:: python + + from dandi.dandiset import Dandiset + + dandiset = Dandiset("/data/my-dandiset") + print(dandiset.get_subject_ids()) + +.. autoclass:: Dandiset() + :members: + +.. autoclass:: AssetView() + :members: diff --git a/docs/source/modref/index.rst b/docs/source/modref/index.rst index 373b48502..ce215299c 100644 --- a/docs/source/modref/index.rst +++ b/docs/source/modref/index.rst @@ -32,6 +32,7 @@ Object-oriented interfaces to manipulate Dandisets and assets on a DANDI instanc .. toctree:: dandiarchive + dandiset Low-level user interfaces =========================