Add subject ID discovery to Dandiset APIs

dandi/dandiapi.py

@@ -63,6 +63,7 @@
     is_interactive,
     is_page2_url,
     joinurl,
+    parse_dandi_subject_dirname,
 )
 
 if TYPE_CHECKING:
@@ -1426,6 +1427,29 @@ def get_assets(self, order: str | None = None) -> Iterator[RemoteAsset]:
                 f"No such version: {self.version_id!r} of Dandiset {self.identifier}"
             )
 
+    def get_subject_ids(self) -> list[str]:
+        """Return sorted subject identifiers from top-level ``sub-*`` asset paths.
+
+        The Archive's path endpoint returns the immediate children of the
+        Dandiset root, so discovery does not require listing every asset.
+        Asset payloads and metadata are not downloaded.
+
+        .. versionadded:: 0.79.0
+        """
+        try:
+            paths = self.client.paginate(f"{self.version_api_path}assets/paths/")
+            return sorted(
+                subject_id
+                for item in paths
+                if item["asset"] is None
+                if (subject_id := parse_dandi_subject_dirname(item["path"]))
+                is not None
+            )
+        except HTTP404Error:
+            raise NotFoundError(
+                f"No such version: {self.version_id!r} of Dandiset {self.identifier}"
+            )
+
     def get_asset(self, asset_id: str) -> RemoteAsset:
         """
         Fetch the asset in this version of the Dandiset with the given asset

dandi/dandiset.py

@@ -12,7 +12,13 @@
 from . import get_logger
 from .consts import dandiset_metadata_file
 from .files import DandisetMetadataFile, LocalAsset, dandi_file, find_dandi_files
-from .utils import find_parent_directory_containing, under_paths, yaml_dump, yaml_load
+from .utils import (
+    find_parent_directory_containing,
+    parse_dandi_subject_dirname,
+    under_paths,
+    yaml_dump,
+    yaml_load,
+)
 
 if TYPE_CHECKING:
     from typing_extensions import Self
@@ -168,6 +174,24 @@ def assets(self, allow_all: bool = False) -> AssetView:
             data[PurePosixPath(df.path)] = df
         return AssetView(data)
 
+    def get_subject_ids(self) -> list[str]:
+        """Return sorted IDs from populated top-level subject directories.
+
+        Directory entries are inspected, but file contents are not read.
+        Empty and symlinked subject directories are ignored so the result
+        describes subjects that can also be represented by remote assets.
+
+        .. versionadded:: 0.79.0
+        """
+        subject_ids = set[str]()
+        for path in self.path_obj.iterdir():
+            if not path.is_dir() or path.is_symlink():
+                continue
+            subject_id = parse_dandi_subject_dirname(path.name)
+            if subject_id is not None and any(p.is_file() for p in path.rglob("*")):
+                subject_ids.add(subject_id)
+        return sorted(subject_ids)
+
     def metadata_file(self) -> DandisetMetadataFile:
         df = dandi_file(self._metadata_file_obj, dandiset_path=self.path)
         assert isinstance(df, DandisetMetadataFile)

dandi/tests/test_dandiapi.py

@@ -40,7 +40,7 @@
     VersionStatus,
 )
 from ..download import download
-from ..exceptions import NotFoundError, SchemaVersionError
+from ..exceptions import HTTP404Error, NotFoundError, SchemaVersionError
 from ..files import GenericAsset, dandi_file
 from ..utils import list_paths
 
@@ -996,6 +996,54 @@ def _get_assets_with_path_prefix(prefix: str, **kw: Any) -> list[str]:
     ]
 
 
+def test_remote_get_subject_ids(mocker: MockerFixture) -> None:
+    client = mocker.Mock(_instance_id="test")
+    dandiset = RemoteDandiset(client, "000001", version=DRAFT)
+    client.paginate.return_value = iter(
+        [
+            {"path": "sub-mouse2", "asset": None},
+            {"path": "sub-mouse1", "asset": None},
+            {"path": "sub-root", "asset": {"asset_id": "a"}},
+            {"path": "other", "asset": None},
+            {"path": "sub-", "asset": None},
+            {"path": "sub-bad_name", "asset": None},
+        ]
+    )
+
+    assert dandiset.get_subject_ids() == ["mouse1", "mouse2"]
+    client.paginate.assert_called_once_with(
+        "/dandisets/000001/versions/draft/assets/paths/"
+    )
+
+
+def test_remote_get_subject_ids_empty(mocker: MockerFixture) -> None:
+    client = mocker.Mock(_instance_id="test")
+    dandiset = RemoteDandiset(client, "000001", version=DRAFT)
+    client.paginate.return_value = iter(())
+
+    assert dandiset.get_subject_ids() == []
+
+
+def test_remote_get_subject_ids_propagates_api_errors(
+    mocker: MockerFixture,
+) -> None:
+    client = mocker.Mock(_instance_id="test")
+    dandiset = RemoteDandiset(client, "000001", version=DRAFT)
+    client.paginate.side_effect = RuntimeError("pagination failed")
+
+    with pytest.raises(RuntimeError, match="pagination failed"):
+        dandiset.get_subject_ids()
+
+
+def test_remote_get_subject_ids_missing_version(mocker: MockerFixture) -> None:
+    client = mocker.Mock(_instance_id="test")
+    dandiset = RemoteDandiset(client, "000001", version=DRAFT)
+    client.paginate.side_effect = HTTP404Error("not found")
+
+    with pytest.raises(NotFoundError, match="No such version: 'draft'"):
+        dandiset.get_subject_ids()
+
+
 def test_get_assets_by_glob(text_dandiset: SampleDandiset) -> None:
     assert sorted(
         asset.path for asset in text_dandiset.dandiset.get_assets_by_glob("*a*.txt")

dandi/tests/test_dandiset.py

@@ -1,3 +1,7 @@
+from pathlib import Path
+
+from pytest_mock import MockerFixture
+
 from ..dandiset import Dandiset
 
 
@@ -6,3 +10,36 @@ def test_get_dandiset_record() -> None:
     # Should have only header with "DO NOT EDIT"
     assert out.startswith("# DO NOT EDIT")
     assert "000000" in out
+
+
+def test_get_subject_ids(tmp_path: Path, mocker: MockerFixture) -> None:
+    (tmp_path / "dandiset.yaml").write_text("identifier: '000001'\n")
+    (tmp_path / "sub-mouse2").mkdir()
+    (tmp_path / "sub-mouse2" / "record.nwb").touch()
+    (tmp_path / "sub-mouse1").mkdir()
+    (tmp_path / "sub-mouse1" / "session").mkdir()
+    (tmp_path / "sub-mouse1" / "session" / "record.nwb").touch()
+    (tmp_path / "sub-").mkdir()
+    (tmp_path / "sub-bad_name").mkdir()
+    (tmp_path / "sub-bad_name" / "record.nwb").touch()
+    (tmp_path / "sub-empty").mkdir()
+    linked = tmp_path / "sub-linked"
+    linked.mkdir()
+    (linked / "record.nwb").touch()
+    (tmp_path / "sub-root.nwb").touch()
+    (tmp_path / "subjects").mkdir()
+    original_is_symlink = Path.is_symlink
+    mocker.patch.object(
+        Path,
+        "is_symlink",
+        autospec=True,
+        side_effect=lambda path: path == linked or original_is_symlink(path),
+    )
+
+    assert Dandiset(tmp_path).get_subject_ids() == ["mouse1", "mouse2"]
+
+
+def test_get_subject_ids_empty(tmp_path: Path) -> None:
+    (tmp_path / "dandiset.yaml").write_text("identifier: '000001'\n")
+
+    assert Dandiset(tmp_path).get_subject_ids() == []

dandi/utils.py

@@ -43,6 +43,17 @@
 
 lgr = get_logger()
 
+
+def parse_dandi_subject_dirname(name: str) -> str | None:
+    """Return the ID encoded by a valid DANDI ``sub-*`` directory name."""
+
+    # Match the label syntax used by ``dandi organize`` without importing that
+    # module, which imports this utility module itself.
+    if re.fullmatch(r"sub-[^_*\\/<>:|\"'?%@;.]+", name) is None:
+        return None
+    return name[4:]
+
+
 _sys_excepthook = sys.excepthook  # Just in case we ever need original one
 
 #

docs/source/modref/dandiapi.rst

@@ -36,6 +36,16 @@ be passed to functions of pynwb etc.
 You can see more usages of DANDI API to assist with data streaming at
 `PyNWB: Streaming NWB files <https://pynwb.readthedocs.io/en/stable/tutorials/advanced_io/streaming.html>`_.
 
+To discover the subject labels represented by a remote Dandiset, use
+``RemoteDandiset.get_subject_ids()``.  It queries the Archive's efficient
+top-level path endpoint and does not download asset payloads or metadata:
+
+.. code-block:: python
+
+    with DandiAPIClient() as client:
+        dandiset = client.get_dandiset("000001")
+        print(dandiset.get_subject_ids())
+
 Client
 ------
 

docs/source/modref/dandiset.rst

@@ -0,0 +1,23 @@
+.. module:: dandi.dandiset
+
+``dandi.dandiset``
+==================
+
+This module provides the local Dandiset API.  A local Dandiset can report the
+subject labels represented by populated, valid top-level ``sub-*`` directories.
+It walks directory entries to establish that a subject contains a file, but it
+does not open file contents. Empty and symlinked subject directories are
+ignored:
+
+.. code-block:: python
+
+    from dandi.dandiset import Dandiset
+
+    dandiset = Dandiset("/data/my-dandiset")
+    print(dandiset.get_subject_ids())
+
+.. autoclass:: Dandiset()
+    :members:
+
+.. autoclass:: AssetView()
+    :members:

docs/source/modref/index.rst

@@ -32,6 +32,7 @@ Object-oriented interfaces to manipulate Dandisets and assets on a DANDI instanc
 .. toctree::
 
    dandiarchive
+   dandiset
 
 Low-level user interfaces
 =========================