From e23ef370278ed7ab28b74291d2d74d4f521d9e9f Mon Sep 17 00:00:00 2001 From: "Peter C. Petersen" Date: Wed, 14 Oct 2026 23:32:03 +0200 Subject: [PATCH 1/6] Add project export and improve pagination Add direct and background project exports with job polling, download verification, and clear errors. Update load_all pagination to use page/per_page and merge sideloaded records. Document the export API and add tests. --- README.md | 51 ++++++ brainstem_api_tools/__init__.py | 2 +- brainstem_api_tools/brainstem_api_client.py | 182 +++++++++++++++++--- tests/test_client.py | 165 +++++++++++++++++- 4 files changed, 372 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 8ada93d..2600ae9 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,57 @@ client.save('session', id='', data={'description': 'updated'}) client.delete('session', id='') ``` +## Exporting a project + +`export_project()` returns a whole project as one JSON-serializable dict. It is the same file as the JSON download on the BrainSTEM website (**Project → Download → JSON**). Use it for backups, NWB conversion, external integrations or analysis pipelines. You need contributor (change) access to the project. + +```python +import json + +export = client.export_project('') + +print(export['export_info']['format_version'], export['export_info']['counts']) +for record in export['records']['stem.session']: + print(record['pk'], record['fields']['name']) + +with open('project_backup.json', 'w') as f: + json.dump(export, f) +``` + +Large projects can take longer than the server's 30-second request limit. For those, pass `background=True`: the server builds the export as a job, the client waits for it and downloads the result, and you also get an email when it is ready. + +```python +export = client.export_project('', background=True) +``` + +The export has this structure: + +```text +{ + "export_info": { + "format_version": "2.0", + "exported_at": "", + "project_id": "", + "lab_resources": "", + "record_count": 2344, + "counts": {"stem.project": 1, "stem.subject": 21, ...} + }, + "records": { + "stem.project": [{"model": "stem.project", "pk": "", "fields": {...}}], + "stem.subject": [...], + "stem.session": [...], + ... + } +} +``` + +- `records` is keyed by model label (`app_label.model_name`). Every label is always present, with an empty list when the project has no such records. +- Links to other records are primary keys. Many-to-many links are sorted lists of primary keys. +- Lab resources (setups, data storages, equipment, protocols, ...) are included when you can read them and the project uses them. Two people exporting the same project can therefore get different lab resources. +- A minor format version (2.1, 2.2, ...) only adds labels and keys, so ignore any you don't recognise. + +Problems raise `ExportError` with a readable message, for example when you lack access, a background job fails, or the server doesn't support exports yet. + ## Command-line Interface After installation a `brainstem` command is available in your shell. diff --git a/brainstem_api_tools/__init__.py b/brainstem_api_tools/__init__.py index 7b19a80..da50ef7 100644 --- a/brainstem_api_tools/__init__.py +++ b/brainstem_api_tools/__init__.py @@ -1,3 +1,3 @@ -from .brainstem_api_client import BrainstemClient, ModelType, PortalType, AuthenticationError +from .brainstem_api_client import BrainstemClient, ModelType, PortalType, AuthenticationError, ExportError __version__ = "2.0.0" diff --git a/brainstem_api_tools/brainstem_api_client.py b/brainstem_api_tools/brainstem_api_client.py index da55b1f..821b713 100644 --- a/brainstem_api_tools/brainstem_api_client.py +++ b/brainstem_api_tools/brainstem_api_client.py @@ -1,3 +1,6 @@ +import gzip +import hashlib +import json import os import stat import time @@ -115,6 +118,10 @@ class AuthenticationError(Exception): pass +class ExportError(Exception): + """A project export could not be produced or downloaded.""" + + # --------------------------------------------------------------------------- # Client # --------------------------------------------------------------------------- @@ -131,6 +138,7 @@ def __init__( url: str = None, ) -> None: base = (url.rstrip("/") + "/") if url else self.BASE_URL + self._base_url = base self._address = base + "api/" self._session = requests.Session() @@ -291,43 +299,69 @@ def load(self, if id is not None: raise ValueError("load_all=True cannot be used together with id.") - # --- auto-paginate and merge all pages --- + # The API paginates with page/per_page; translate limit/offset. page_size = limit or 100 - params["limit"] = page_size - params.setdefault("offset", offset or 0) + if offset and offset % page_size: + raise ValueError( + "With load_all=True, offset must be a multiple of limit (the page size)." + ) + params.pop("limit", None) + params.pop("offset", None) + return self._load_all_pages(url, params, page_size, + start_page=(offset or 0) // page_size + 1) + + def _get_json(self, url: str, params: dict = None) -> dict: + """GET *url* and return the decoded JSON, raising on HTTP errors.""" + resp = self._session.get(url, params=params, timeout=self.DEFAULT_TIMEOUT) + if resp.status_code == 401: + raise AuthenticationError( + "API token is invalid or expired. Run `brainstem login` to re-authenticate." + ) + resp.raise_for_status() + return resp.json() + + def _load_all_pages(self, url: str, params: dict, + page_size: int = 100, start_page: int = 1) -> dict: + """Follow pagination and merge every page into a single dict. + All list-valued keys are concatenated across pages: the primary + records as well as any sideloaded related records (de-duplicated by + ``id``). Non-list keys are taken from the first page. + """ + params = dict(params, per_page=page_size) + page = start_page combined: dict = {} - records_key: str = None + seen_ids: dict = {} while True: - resp = self._session.get(url, params=params, timeout=self.DEFAULT_TIMEOUT) - if resp.status_code == 401: - raise AuthenticationError( - "API token is invalid or expired. Run `brainstem login` to re-authenticate." - ) - resp.raise_for_status() - data = resp.json() + data = self._get_json(url, dict(params, page=page)) + lists = {k: v for k, v in data.items() if isinstance(v, list)} - if records_key is None: - # Detect the list key (e.g. 'sessions', 'projects') - records_key = next( - (k for k, v in data.items() if isinstance(v, list)), None - ) - if records_key is None: + if not combined: + if not lists: raise ValueError( "load_all=True requires a paginated list response, but the API " "returned no list-valued key. Use load_all=False for single-object endpoints." ) - combined = {k: v for k, v in data.items() if k != records_key} - combined[records_key] = [] - - combined[records_key].extend(data.get(records_key, [])) - total = data.get("count", len(combined[records_key])) - - if len(combined[records_key]) >= total: + combined = {k: v for k, v in data.items() if k not in lists} + + for key, records in lists.items(): + bucket = combined.setdefault(key, []) + ids = seen_ids.setdefault(key, set()) + for record in records: + rid = record.get("id") if isinstance(record, dict) else None + if rid is not None: + if rid in ids: + continue + ids.add(rid) + bucket.append(record) + + total_pages = (data.get("meta") or {}).get("total_pages") + if total_pages is None and "count" in data: + total_pages = -(-data["count"] // page_size) # ceil division + if total_pages is None or page >= total_pages or not any(lists.values()): break - - params["offset"] = params.get("offset", 0) + page_size + page += 1 return combined @@ -382,6 +416,102 @@ def delete(self, url = self._build_url(portal, app, model, id) return self._session.delete(url, timeout=self.DEFAULT_TIMEOUT) + # ------------------------------------------------------------------ + # Export + # ------------------------------------------------------------------ + + EXPORT_TIMEOUT: int = 90 # seconds; the server builds the whole project per request + + def export_project(self, project_id: str, background: bool = False, + poll_interval: float = 5.0, timeout: float = 3600) -> dict: + """Export a project as the server's versioned JSON export. + + Returns the same payload as the JSON download on the BrainSTEM website + (format documented in the server's ``docs/project-export-format.md``):: + + { + "export_info": {"format_version", "exported_at", "project_id", + "lab_resources", "record_count", "counts"}, + "records": {"stem.project": [...], "stem.subject": [...], ...} + } + + Each record has the shape ``{"model", "pk", "fields"}``. Exporting + requires contributor (change) access to the project. + + Parameters + ---------- + project_id : UUID of the project to export. + background : When ``True``, queue the export as a server-side job, + wait for it to finish and download the result. Use this + for large projects, which can exceed the server's + 30-second request limit. The server also emails you + when the job finishes. + poll_interval : Seconds between job status checks (background only). + timeout : Maximum seconds to wait for the job (background only). + """ + if not project_id: + raise ValueError("'project_id' is required to export a project.") + url = self._build_url("private", "stem", "project", project_id, "export/") + + if not background: + # Bypass the session's automatic retries: a timed-out export would be + # rebuilt up to three more times, each counting against the export rate limit. + resp = requests.get(url, headers=self._session.headers, timeout=self.EXPORT_TIMEOUT) + if resp.status_code in (503, 504): + raise ExportError( + "The export did not finish within the server's request time limit. " + "Use export_project(..., background=True) for large projects." + ) + self._check_export_response(resp) + return resp.json() + + resp = self._session.post(url, timeout=self.DEFAULT_TIMEOUT) + self._check_export_response(resp) + job = resp.json()["export_job"] + job_url = self._build_url("private", "stem", "project", project_id, + f"export-jobs/{job['id']}/") + + deadline = time.monotonic() + timeout + while job["status"] in ("pending", "running"): + if time.monotonic() > deadline: + raise ExportError( + f"Export job {job['id']} did not finish within {timeout} seconds " + f"(status: {job['status']}, {job.get('progress_percent', 0)}%)." + ) + time.sleep(poll_interval) + job = self._get_json(job_url)["export_job"] + + if job["status"] != "completed": + raise ExportError(job.get("error_message") or f"Export job ended as '{job['status']}'.") + + resp = self._session.get(self._base_url + job["download_url"].lstrip("/"), + timeout=self.EXPORT_TIMEOUT) + self._check_export_response(resp) + content = resp.content + if content[:2] == b"\x1f\x8b": # gzip magic number + expected = job.get("file_sha256") + if expected and hashlib.sha256(content).hexdigest() != expected: + raise ExportError(f"Export job {job['id']}: downloaded file failed its checksum.") + content = gzip.decompress(content) + return json.loads(content) + + def _check_export_response(self, resp: Response) -> None: + """Turn export endpoint errors into clear exceptions.""" + if resp.ok: + return + if resp.status_code == 401: + raise AuthenticationError( + "API token is invalid or expired. Run `brainstem login` to re-authenticate." + ) + is_json = "json" in resp.headers.get("Content-Type", "") + if resp.status_code == 404 and not is_json: + raise ExportError("This BrainSTEM server does not support project export yet.") + if resp.status_code == 403: + raise ExportError("Exporting a project requires contributor (change) access to it.") + if resp.status_code in (409, 410, 429) and is_json: + raise ExportError(resp.json().get("detail", resp.reason)) + resp.raise_for_status() + def __enter__(self): return self diff --git a/tests/test_client.py b/tests/test_client.py index 3851c2e..e3d16a8 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -4,12 +4,18 @@ network access is required. """ +import gzip +import hashlib +import json + import pytest +import requests from unittest.mock import MagicMock, patch, PropertyMock from brainstem_api_tools.brainstem_api_client import ( BrainstemClient, AuthenticationError, + ExportError, _MODEL_TO_APP, _resolve_model, _resolve_portal, @@ -322,10 +328,43 @@ def test_two_pages_merged(self): {"sessions": [{"id": "1"}, {"id": "2"}], "count": 3}, {"sessions": [{"id": "3"}], "count": 3}, ]) - result = self.client.load("session", load_all=True) + result = self.client.load("session", limit=2, load_all=True) assert len(result["sessions"]) == 3 assert [r["id"] for r in result["sessions"]] == ["1", "2", "3"] + def test_meta_total_pages_drives_paging(self): + meta = {"per_page": 100, "total_results": 3, "total_pages": 2} + self._mock_pages([ + {"sessions": [{"id": "1"}, {"id": "2"}], "meta": {**meta, "page": 1}}, + {"sessions": [{"id": "3"}], "meta": {**meta, "page": 2}}, + ]) + result = self.client.load("session", load_all=True) + assert [r["id"] for r in result["sessions"]] == ["1", "2", "3"] + pages = [c[1]["params"]["page"] for c in self.client._session.get.call_args_list] + assert pages == [1, 2] + params = self.client._session.get.call_args[1]["params"] + assert params["per_page"] == 100 + assert "limit" not in params and "offset" not in params + + def test_sideloaded_lists_merged_and_deduplicated(self): + meta = {"total_pages": 2} + self._mock_pages([ + {"data_acquisitions": [{"id": "da1"}], "sessions": [{"id": "1"}], "meta": meta}, + {"data_acquisitions": [{"id": "da1"}, {"id": "da2"}], "sessions": [{"id": "2"}], "meta": meta}, + ]) + result = self.client.load("session", include=["dataacquisition"], load_all=True) + assert [r["id"] for r in result["sessions"]] == ["1", "2"] + assert [r["id"] for r in result["data_acquisitions"]] == ["da1", "da2"] + + def test_offset_translated_to_page(self): + self._mock_pages([{"sessions": [{"id": "3"}], "meta": {"total_pages": 2}}]) + self.client.load("session", limit=2, offset=2, load_all=True) + assert self.client._session.get.call_args[1]["params"]["page"] == 2 + + def test_offset_not_multiple_of_limit_raises(self): + with pytest.raises(ValueError, match="multiple of limit"): + self.client.load("session", limit=10, offset=5, load_all=True) + def test_load_all_false_returns_response(self): r = MagicMock() self.client._session.get = MagicMock(return_value=r) @@ -333,6 +372,130 @@ def test_load_all_false_returns_response(self): assert result is r +# --------------------------------------------------------------------------- +# export_project() +# --------------------------------------------------------------------------- + +PROJECT_ID = "11111111-0000-0000-0000-000000000000" +JOB_ID = "22222222-0000-0000-0000-000000000000" +EXPORT_URL = f"https://www.brainstem.org/api/private/stem/project/{PROJECT_ID}/export/" +PAYLOAD = { + "export_info": {"format_version": "2.0", "project_id": PROJECT_ID, "record_count": 1, + "counts": {"stem.project": 1}}, + "records": {"stem.project": [{"model": "stem.project", "pk": PROJECT_ID, "fields": {}}]}, +} + + +def export_response(status_code=200, json_body=None, content=None, content_type="application/json"): + resp = mock_response(status_code, json_body) + resp.headers = {"Content-Type": content_type} + resp.reason = "reason" + if content is not None: + resp.content = content + if status_code >= 400: + resp.raise_for_status.side_effect = requests.HTTPError(str(status_code)) + return resp + + +def job(status, **extra): + return {"export_job": {"id": JOB_ID, "status": status, "progress_percent": 0, **extra}} + + +class TestExportProject: + def setup_method(self): + self.client = make_client() + + def test_direct_export_returns_payload(self): + with patch("requests.get", return_value=export_response(200, PAYLOAD)) as get: + result = self.client.export_project(PROJECT_ID) + assert result == PAYLOAD + assert get.call_args[0][0] == EXPORT_URL + assert get.call_args[1]["headers"]["Authorization"] == f"Bearer {TOKEN}" + + def test_direct_export_bypasses_session_retries(self): + self.client._session.get = MagicMock() + with patch("requests.get", return_value=export_response(200, PAYLOAD)): + self.client.export_project(PROJECT_ID) + self.client._session.get.assert_not_called() + + def test_direct_timeout_suggests_background(self): + with patch("requests.get", return_value=export_response(503, content_type="text/html")): + with pytest.raises(ExportError, match="background=True"): + self.client.export_project(PROJECT_ID) + + def test_server_without_endpoint(self): + with patch("requests.get", return_value=export_response(404, content_type="text/html")): + with pytest.raises(ExportError, match="does not support project export"): + self.client.export_project(PROJECT_ID) + + def test_unknown_project_raises_http_error(self): + with patch("requests.get", return_value=export_response(404, {"detail": "Not found."})): + with pytest.raises(requests.HTTPError): + self.client.export_project(PROJECT_ID) + + def test_forbidden(self): + with patch("requests.get", return_value=export_response(403, {"detail": "x"})): + with pytest.raises(ExportError, match="contributor"): + self.client.export_project(PROJECT_ID) + + def test_401_raises_authentication_error(self): + with patch("requests.get", return_value=export_response(401, {"detail": "x"})): + with pytest.raises(AuthenticationError): + self.client.export_project(PROJECT_ID) + + def test_throttled_shows_server_detail(self): + with patch("requests.get", return_value=export_response(429, {"detail": "Request was throttled."})): + with pytest.raises(ExportError, match="throttled"): + self.client.export_project(PROJECT_ID) + + def test_missing_id_raises(self): + with pytest.raises(ValueError, match="project_id"): + self.client.export_project(None) + + def _run_background(self, final_job, download): + self.client._session.post = MagicMock(return_value=export_response(202, job("pending"))) + self.client._session.get = MagicMock(side_effect=[ + export_response(200, job("running")), + export_response(200, final_job), + download, + ]) + with patch("time.sleep"): + return self.client.export_project(PROJECT_ID, background=True) + + def test_background_export_polls_downloads_and_unpacks(self): + gz = gzip.compress(json.dumps(PAYLOAD).encode()) + download_path = f"/api/private/stem/project/{PROJECT_ID}/export-jobs/{JOB_ID}/download/" + result = self._run_background( + job("completed", download_url=download_path, file_sha256=hashlib.sha256(gz).hexdigest()), + export_response(200, content=gz, content_type="application/gzip"), + ) + assert result == PAYLOAD + assert self.client._session.post.call_args[0][0] == EXPORT_URL + urls = [c[0][0] for c in self.client._session.get.call_args_list] + assert urls[0].endswith(f"/project/{PROJECT_ID}/export-jobs/{JOB_ID}/") + assert urls[2] == "https://www.brainstem.org" + download_path + + def test_background_checksum_mismatch(self): + gz = gzip.compress(json.dumps(PAYLOAD).encode()) + with pytest.raises(ExportError, match="checksum"): + self._run_background( + job("completed", download_url="/x/", file_sha256="0" * 64), + export_response(200, content=gz, content_type="application/gzip"), + ) + + def test_background_failed_job_raises_server_message(self): + with pytest.raises(ExportError, match="no longer export"): + self._run_background( + job("failed", error_message="The requester can no longer export this project."), None, + ) + + def test_background_timeout(self): + self.client._session.post = MagicMock(return_value=export_response(202, job("pending"))) + self.client._session.get = MagicMock(return_value=export_response(200, job("running"))) + with patch("time.sleep"), pytest.raises(ExportError, match=JOB_ID): + self.client.export_project(PROJECT_ID, background=True, timeout=-1) + + # --------------------------------------------------------------------------- # Convenience loaders # --------------------------------------------------------------------------- From 5b0a3169081f8ad3eb54009b52a518f6755198ea Mon Sep 17 00:00:00 2001 From: "Peter C. Petersen" Date: Thu, 15 Oct 2026 12:22:55 +0200 Subject: [PATCH 2/6] Add session exports and clarify export errors Share direct and background export logic between projects and sessions. Clarify missing-record and permission errors, and document read access requirements and the export format 2.1 metadata. --- README.md | 46 +++++++++++---- brainstem_api_tools/brainstem_api_client.py | 65 +++++++++++++++------ tests/test_client.py | 65 +++++++++++++++++++-- 3 files changed, 145 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 2600ae9..a24cf07 100644 --- a/README.md +++ b/README.md @@ -103,9 +103,9 @@ client.save('session', id='', data={'description': 'updated'}) client.delete('session', id='') ``` -## Exporting a project +## Exporting projects and sessions -`export_project()` returns a whole project as one JSON-serializable dict. It is the same file as the JSON download on the BrainSTEM website (**Project → Download → JSON**). Use it for backups, NWB conversion, external integrations or analysis pipelines. You need contributor (change) access to the project. +`export_project()` returns a whole project as one JSON-serializable dict. It is the same file as the JSON download on the BrainSTEM website (**Project → Download → JSON**). Use it for backups, NWB conversion, external integrations or analysis pipelines. You need read access to the project. ```python import json @@ -120,21 +120,47 @@ with open('project_backup.json', 'w') as f: json.dump(export, f) ``` -Large projects can take longer than the server's 30-second request limit. For those, pass `background=True`: the server builds the export as a job, the client waits for it and downloads the result, and you also get an email when it is ready. +### Session exports + +`export_session()` exports one session and everything it touches: the content of one NWB file. You need read access to one of the session's projects. + +```python +export = client.export_session('') +``` + +The file has the same format as a project export, with `export_info['scope'] == 'session'`. It contains: + +- the session with its data acquisitions, manipulations, behaviors and epochs; +- the subjects and specimen sets linked to the session, with their full procedure and log history; +- the session's projects you can read (`export_info['project_ids']`; a session can belong to several); +- the lab resources, catalog entries and users these records reference. + +A subject's procedure history is complete, including procedures after the session, so filter by date where that matters. + +### Large exports + +Large exports can take longer than the server's 30-second request limit. For those, pass `background=True`: the server builds the export as a job, the client waits for it and downloads the result, and you also get an email when it is ready. ```python export = client.export_project('', background=True) +export = client.export_session('', background=True) ``` -The export has this structure: +### Format + +Both exports have this structure: ```text { "export_info": { - "format_version": "2.0", + "format_version": "2.1", "exported_at": "", - "project_id": "", + "scope": "project", # or "session" + "project_id": "", # null for a session export + "project_ids": ["", ...], + "session_id": null, # the session's uuid for a session export "lab_resources": "", + "restricted": {"personal_attributes.datastorage": [{"pk": "", "name": "Lab server"}], ...}, "record_count": 2344, "counts": {"stem.project": 1, "stem.subject": 21, ...} }, @@ -147,12 +173,12 @@ The export has this structure: } ``` -- `records` is keyed by model label (`app_label.model_name`). Every label is always present, with an empty list when the project has no such records. -- Links to other records are primary keys. Many-to-many links are sorted lists of primary keys. -- Lab resources (setups, data storages, equipment, protocols, ...) are included when you can read them and the project uses them. Two people exporting the same project can therefore get different lab resources. +- `records` is keyed by model label (`app_label.model_name`). Every label is always present, with an empty list when there are no such records. +- Links to other records are primary keys. Many-to-many links are sorted lists of primary keys. Referenced catalog entries (species, brain regions, suppliers, ...) and users (names only) are included in the file. +- Lab resources (setups, data storages, equipment, protocols, ...) are included when you can read them and the exported records use them. Two people exporting the same project can therefore get different lab resources. Referenced lab resources you cannot read are listed by name in `export_info['restricted']`. - A minor format version (2.1, 2.2, ...) only adds labels and keys, so ignore any you don't recognise. -Problems raise `ExportError` with a readable message, for example when you lack access, a background job fails, or the server doesn't support exports yet. +Problems raise `ExportError` with a readable message, for example when the project or session doesn't exist or you can't read it, a background job fails, or the server doesn't support exports yet. ## Command-line Interface diff --git a/brainstem_api_tools/brainstem_api_client.py b/brainstem_api_tools/brainstem_api_client.py index 821b713..6d12632 100644 --- a/brainstem_api_tools/brainstem_api_client.py +++ b/brainstem_api_tools/brainstem_api_client.py @@ -420,7 +420,7 @@ def delete(self, # Export # ------------------------------------------------------------------ - EXPORT_TIMEOUT: int = 90 # seconds; the server builds the whole project per request + EXPORT_TIMEOUT: int = 90 # seconds; the server builds the whole export per request def export_project(self, project_id: str, background: bool = False, poll_interval: float = 5.0, timeout: float = 3600) -> dict: @@ -430,13 +430,15 @@ def export_project(self, project_id: str, background: bool = False, (format documented in the server's ``docs/project-export-format.md``):: { - "export_info": {"format_version", "exported_at", "project_id", - "lab_resources", "record_count", "counts"}, + "export_info": {"format_version", "exported_at", "scope", + "project_id", "project_ids", "session_id", + "lab_resources", "restricted", "record_count", + "counts"}, "records": {"stem.project": [...], "stem.subject": [...], ...} } Each record has the shape ``{"model", "pk", "fields"}``. Exporting - requires contributor (change) access to the project. + requires read access to the project. Parameters ---------- @@ -449,9 +451,35 @@ def export_project(self, project_id: str, background: bool = False, poll_interval : Seconds between job status checks (background only). timeout : Maximum seconds to wait for the job (background only). """ - if not project_id: - raise ValueError("'project_id' is required to export a project.") - url = self._build_url("private", "stem", "project", project_id, "export/") + return self._export("project", project_id, background, poll_interval, timeout) + + def export_session(self, session_id: str, background: bool = False, + poll_interval: float = 5.0, timeout: float = 3600) -> dict: + """Export one session and everything it touches: the content of one NWB file. + + Returns the same format as ``export_project`` (``export_info.scope`` is + ``"session"``), holding the session with its data acquisitions, + manipulations, behaviors and epochs; the subjects and specimen sets + linked to it with their full procedure and log history; the session's + projects; and the lab resources, catalog entries and users these + reference. Only records you can read are included, in any of the + session's projects. + + A subject's procedure history is complete, including procedures after + the session, so filter by date where that matters. Exporting requires + read access to one of the session's projects. + + Parameters are the same as for ``export_project``, with + ``session_id`` the UUID of the session to export. + """ + return self._export("session", session_id, background, poll_interval, timeout) + + def _export(self, model: str, record_id: str, background: bool, + poll_interval: float, timeout: float) -> dict: + """Run a direct or background export of a project or session record.""" + if not record_id: + raise ValueError(f"'{model}_id' is required to export a {model}.") + url = self._build_url("private", "stem", model, record_id, "export/") if not background: # Bypass the session's automatic retries: a timed-out export would be @@ -460,15 +488,15 @@ def export_project(self, project_id: str, background: bool = False, if resp.status_code in (503, 504): raise ExportError( "The export did not finish within the server's request time limit. " - "Use export_project(..., background=True) for large projects." + f"Use export_{model}(..., background=True) for large exports." ) - self._check_export_response(resp) + self._check_export_response(resp, model) return resp.json() resp = self._session.post(url, timeout=self.DEFAULT_TIMEOUT) - self._check_export_response(resp) + self._check_export_response(resp, model) job = resp.json()["export_job"] - job_url = self._build_url("private", "stem", "project", project_id, + job_url = self._build_url("private", "stem", model, record_id, f"export-jobs/{job['id']}/") deadline = time.monotonic() + timeout @@ -486,7 +514,7 @@ def export_project(self, project_id: str, background: bool = False, resp = self._session.get(self._base_url + job["download_url"].lstrip("/"), timeout=self.EXPORT_TIMEOUT) - self._check_export_response(resp) + self._check_export_response(resp, model) content = resp.content if content[:2] == b"\x1f\x8b": # gzip magic number expected = job.get("file_sha256") @@ -495,7 +523,7 @@ def export_project(self, project_id: str, background: bool = False, content = gzip.decompress(content) return json.loads(content) - def _check_export_response(self, resp: Response) -> None: + def _check_export_response(self, resp: Response, model: str) -> None: """Turn export endpoint errors into clear exceptions.""" if resp.ok: return @@ -504,11 +532,14 @@ def _check_export_response(self, resp: Response) -> None: "API token is invalid or expired. Run `brainstem login` to re-authenticate." ) is_json = "json" in resp.headers.get("Content-Type", "") - if resp.status_code == 404 and not is_json: - raise ExportError("This BrainSTEM server does not support project export yet.") + if resp.status_code == 404: + if not is_json: + raise ExportError(f"This BrainSTEM server does not support {model} export yet.") + # The API answers 404, not 403, when the record exists but you cannot read it. + raise ExportError(f"The {model} was not found, or you do not have read access to it.") if resp.status_code == 403: - raise ExportError("Exporting a project requires contributor (change) access to it.") - if resp.status_code in (409, 410, 429) and is_json: + raise ExportError(f"You do not have permission to export this {model}.") + if resp.status_code in (400, 409, 410, 429) and is_json: raise ExportError(resp.json().get("detail", resp.reason)) resp.raise_for_status() diff --git a/tests/test_client.py b/tests/test_client.py index e3d16a8..ae04d99 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -373,7 +373,7 @@ def test_load_all_false_returns_response(self): # --------------------------------------------------------------------------- -# export_project() +# export_project() / export_session() # --------------------------------------------------------------------------- PROJECT_ID = "11111111-0000-0000-0000-000000000000" @@ -428,14 +428,19 @@ def test_server_without_endpoint(self): with pytest.raises(ExportError, match="does not support project export"): self.client.export_project(PROJECT_ID) - def test_unknown_project_raises_http_error(self): + def test_unknown_or_unreadable_project(self): with patch("requests.get", return_value=export_response(404, {"detail": "Not found."})): - with pytest.raises(requests.HTTPError): + with pytest.raises(ExportError, match="project was not found, or you do not have read access"): self.client.export_project(PROJECT_ID) def test_forbidden(self): with patch("requests.get", return_value=export_response(403, {"detail": "x"})): - with pytest.raises(ExportError, match="contributor"): + with pytest.raises(ExportError, match="permission to export this project"): + self.client.export_project(PROJECT_ID) + + def test_unexpected_status_raises_http_error(self): + with patch("requests.get", return_value=export_response(500, {"detail": "x"})): + with pytest.raises(requests.HTTPError): self.client.export_project(PROJECT_ID) def test_401_raises_authentication_error(self): @@ -496,6 +501,58 @@ def test_background_timeout(self): self.client.export_project(PROJECT_ID, background=True, timeout=-1) +SESSION_ID = "33333333-0000-0000-0000-000000000000" +SESSION_EXPORT_URL = f"https://www.brainstem.org/api/private/stem/session/{SESSION_ID}/export/" +SESSION_PAYLOAD = { + "export_info": {"format_version": "2.1", "scope": "session", "session_id": SESSION_ID, + "project_id": None, "project_ids": [PROJECT_ID]}, + "records": {"stem.session": [{"model": "stem.session", "pk": SESSION_ID, "fields": {}}]}, +} + + +class TestExportSession: + def setup_method(self): + self.client = make_client() + + def test_direct_export_uses_session_endpoint(self): + with patch("requests.get", return_value=export_response(200, SESSION_PAYLOAD)) as get: + result = self.client.export_session(SESSION_ID) + assert result == SESSION_PAYLOAD + assert get.call_args[0][0] == SESSION_EXPORT_URL + + def test_background_export_polls_under_the_session(self): + gz = gzip.compress(json.dumps(SESSION_PAYLOAD).encode()) + download_path = f"/api/private/stem/session/{SESSION_ID}/export-jobs/{JOB_ID}/download/" + self.client._session.post = MagicMock(return_value=export_response(202, job("pending"))) + self.client._session.get = MagicMock(side_effect=[ + export_response(200, job("completed", download_url=download_path, + file_sha256=hashlib.sha256(gz).hexdigest())), + export_response(200, content=gz, content_type="application/gzip"), + ]) + with patch("time.sleep"): + result = self.client.export_session(SESSION_ID, background=True) + assert result == SESSION_PAYLOAD + assert self.client._session.post.call_args[0][0] == SESSION_EXPORT_URL + urls = [c[0][0] for c in self.client._session.get.call_args_list] + assert urls[0].endswith(f"/session/{SESSION_ID}/export-jobs/{JOB_ID}/") + assert urls[1] == "https://www.brainstem.org" + download_path + + def test_errors_name_the_session(self): + with patch("requests.get", return_value=export_response(404, {"detail": "Not found."})): + with pytest.raises(ExportError, match="session was not found"): + self.client.export_session(SESSION_ID) + with patch("requests.get", return_value=export_response(404, content_type="text/html")): + with pytest.raises(ExportError, match="does not support session export"): + self.client.export_session(SESSION_ID) + with patch("requests.get", return_value=export_response(503, content_type="text/html")): + with pytest.raises(ExportError, match=r"export_session\(\.\.\., background=True\)"): + self.client.export_session(SESSION_ID) + + def test_missing_id_raises(self): + with pytest.raises(ValueError, match="session_id"): + self.client.export_session("") + + # --------------------------------------------------------------------------- # Convenience loaders # --------------------------------------------------------------------------- From 844f5b82784c389ec78a01a071603b354416653d Mon Sep 17 00:00:00 2001 From: "Peter C. Petersen" Date: Thu, 15 Oct 2026 16:19:38 +0200 Subject: [PATCH 3/6] Align client and CLI with current BrainSTEM API Fix pagination and relation filters, expand model routing for the specimen release, and add anatomical region helpers with deprecated aliases. Support anonymous access and environment-based configuration, add CLI page/all options, and stop printing login tokens. Update documentation and regression coverage. --- README.md | 81 +++- brainstem_api_tools/brainstem_api_client.py | 389 ++++++++++++++------ brainstem_api_tools/cli.py | 85 ++++- tests/test_client.py | 386 ++++++++++++++++++- 4 files changed, 774 insertions(+), 167 deletions(-) diff --git a/README.md b/README.md index a24cf07..f55d988 100644 --- a/README.md +++ b/README.md @@ -30,11 +30,22 @@ For headless environments (no browser available): client = BrainstemClient(headless=True) # prints a URL + code to enter manually ``` +To read public data without logging in (no token is sent; only `portal='public'` works): +```python +client = BrainstemClient(anonymous=True) +client.load("project", portal="public", load_all=True) +``` + To connect to a different server (e.g. a local development instance): ```python client = BrainstemClient(url="http://127.0.0.1:8000/") ``` +The client and the CLI also read two environment variables when no `token` / `url` is given: + +- `BRAINSTEM_API_TOKEN`: API token, used before the cached token. +- `BRAINSTEM_API_URL`: server base URL (default `https://www.brainstem.org/`). + > **Security note:** The cached token file is stored with owner-read-only permissions (`0600`). Treat it like a password and do not commit it to version control. ## Getting Started @@ -46,7 +57,7 @@ The tutorial demonstrates how to: - **Authenticate:** Load the client and authenticate via your browser. - **Load Data:** Load sessions and filter data using flexible options. -- **Paginate:** Retrieve large datasets using `limit` / `offset` or `load_all=True`. +- **Paginate:** Retrieve large datasets page by page (`limit` / `offset`) or all at once (`load_all=True`). - **Convenience Loaders:** Use `load_session()`, `load_subject()`, etc. for common queries. - **Update Entries:** Modify existing records and update them in the database. - **Create Entries:** Submit new data entries with required fields. @@ -71,26 +82,32 @@ response = client.load( include=['projects'], ) -# Paginate results (manual) +# Paginate results (manual). The API pages with page/per_page: limit is sent +# as per_page (1 to 500, default 100) and offset is converted to a page number, +# so offset must be a multiple of limit. Here: page 2 of 20 records. page2 = client.load('session', limit=20, offset=20).json() +print(page2['meta']) # {'page': 2, 'per_page': 20, 'total_results': ..., 'total_pages': ...} -# Auto-paginate — fetches every page and returns a merged dict +# Auto-paginate — fetches every page and returns a merged dict. Records of +# sideloaded relations (include=[...]) are merged per key as well. all_sessions = client.load('session', load_all=True) -# Convenience loaders — sensible defaults with named filter kwargs -# load_session embeds dataacquisition, behaviors, manipulations, epochs +# Convenience loaders — sensible defaults with named filter kwargs. Their +# filters send the bare field (filter{projects}=), which every server accepts. +# load_session sideloads dataacquisition, behaviors, manipulations, epochs sessions = client.load_session(name='Rat', load_all=True) -# load_subject embeds procedures and subjectlogs +# load_subject sideloads procedures and subjectlogs subjects = client.load_subject(sex='M', projects='', load_all=True) -# load_project embeds sessions, subjects, collections, cohorts +# load_project sideloads sessions, subjects, collections, cohorts projects = client.load_project(name='MyProject') # load_behavior / load_dataacquisition / load_manipulation scope by session UUID behaviors = client.load_behavior(session='', load_all=True) -# load_procedure scopes by subject UUID +# load_procedure / load_subjectlog scope by subject UUID; load_procedurelog by +# procedure, load_equipment by setup and load_strain by species UUID. procedures = client.load_procedure(subject='', load_all=True) # Create a record @@ -99,10 +116,48 @@ client.save('session', data={'name': 'New session', 'projects': [' # Update a record client.save('session', id='', data={'description': 'updated'}) +# Sending a list field replaces the whole list. Lists read with include (and +# the convenience loaders' default includes) hold only records you may read, +# so read the record without include before changing one: +session = client.load('session', id='').json()['session'] +client.save('session', id='', + data={'projects': session['projects'] + ['']}) + # Delete a record client.delete('session', id='') ``` +### Specimens, interfaces and materials + +Specimen sets and specimens (`stem`), experimental interfaces, material usages and specimen procedures/logs (`modules`), and anatomical regions (`taxonomies`) load like any other model: + +```python +specimen_sets = client.load('specimenset', filters={'projects': ''}, load_all=True) +interfaces = client.load('experimentalinterface', load_all=True) +``` + +### Species, strains and brain regions + +```python +# Search species and strains by common name as well as scientific name +mice = client.load_species(popular_name='mouse').json() +strains = client.load_strain(popular_name='black 6').json() + +# Brain regions are anatomical regions; find one by acronym, then list its subregions +ca1 = client.load_anatomicalregion(acronym='CA1').json()['anatomical_regions'][0] +subregions = client.load_anatomicalregion_descendants(ca1['id']).json()['anatomical_regions'] +for region in subregions: + print(region['depth'], region['acronym'], region['name']) +``` + +`brainregion` and `load_brainregion()` are deprecated names for `anatomicalregion` and `load_anatomicalregion()`. They still work, with a `FutureWarning`, and load anatomical regions. + +### Known limitations + +- Specimen-era models (specimen sets, specimens, experimental interfaces, material usages, anatomical regions, ...), `popular_name` and region descendants need a server with the specimen release. Older servers answer 404 for those models and 400 for a `popular_name` filter. +- `offset` must be a multiple of the page size (`limit`, default 100), and a page holds at most 500 records. Use `load_all=True` to get everything. +- Lists read with `include` show only the related records you may read; see the note on `save` above. + ## Exporting projects and sessions `export_project()` returns a whole project as one JSON-serializable dict. It is the same file as the JSON download on the BrainSTEM website (**Project → Download → JSON**). Use it for backups, NWB conversion, external integrations or analysis pipelines. You need read access to the project. @@ -210,8 +265,12 @@ brainstem load session --filters name.icontains=Rat --sort -name --include proje # Load a single record by UUID brainstem load session --id -# Manual pagination -brainstem load session --limit 20 --offset 20 +# Manual pagination (page 2, 20 records per page) +brainstem load session --per-page 20 --page 2 +brainstem load session --limit 20 --offset 20 # equivalent + +# Fetch every page and print one merged JSON object +brainstem load session --all # Public portal brainstem load project --portal public @@ -231,7 +290,7 @@ brainstem save session --id --data '{"description":"updated"}' brainstem delete session --id ``` -All subcommands accept `--token`, `--headless`, and `--url` to override defaults. +All subcommands accept `--token`, `--headless`, and `--url` to override defaults (`--token` and `--url` default to `$BRAINSTEM_API_TOKEN` and `$BRAINSTEM_API_URL`). `brainstem login` never prints the token; it reports where it is cached. ## Contributing Contributions are welcome! Feel free to open issues or submit pull requests on GitHub. diff --git a/brainstem_api_tools/brainstem_api_client.py b/brainstem_api_tools/brainstem_api_client.py index 6d12632..7b6c913 100644 --- a/brainstem_api_tools/brainstem_api_client.py +++ b/brainstem_api_tools/brainstem_api_client.py @@ -4,6 +4,7 @@ import os import stat import time +import warnings import webbrowser from enum import Enum from pathlib import Path @@ -21,68 +22,134 @@ _MODEL_TO_APP: dict = { # stem + "breeding": "stem", + "cohort": "stem", + "cohortsubject": "stem", + "collection": "stem", + "collectionsession": "stem", + "epoch": "stem", "project": "stem", - "subject": "stem", + "projectgroupmembershipinvitation": "stem", + "projectmembershipinvitation": "stem", "session": "stem", - "collection": "stem", - "cohort": "stem", - "breeding": "stem", - "project_membership_invitation": "stem", - "project_group_membership_invitation": "stem", + "sessiondatastorage": "stem", + "specimen": "stem", + "specimenset": "stem", + "subject": "stem", # modules - "procedure": "modules", "behavior": "modules", + "consumablestock": "modules", "dataacquisition": "modules", - "manipulation": "modules", + "dataacquisitioninterface": "modules", + "dataacquisitionprocedure": "modules", + "dataacquisitiontemplate": "modules", + "episodepersonnel": "modules", "equipment": "modules", - "consumablestock": "modules", + "experimentalinterface": "modules", + "experimentalinterfacetemplate": "modules", + "manipulation": "modules", + "manipulationinterface": "modules", + "manipulationtemplate": "modules", + "materialusage": "modules", + "materialusagetemplate": "modules", + "procedure": "modules", + "procedureepisode": "modules", + "procedureepisodetemplate": "modules", "procedurelog": "modules", + "proceduretemplate": "modules", + "specimenlog": "modules", + "specimenprocedure": "modules", + "specimenproceduretemplate": "modules", "subjectlog": "modules", # personal_attributes "behavioralassay": "personal_attributes", + "behavioralassaygroupmembershipinvitation": "personal_attributes", + "behavioralassaymembershipinvitation": "personal_attributes", "datastorage": "personal_attributes", + "datastoragegroupmembershipinvitation": "personal_attributes", + "datastoragemembershipinvitation": "personal_attributes", "inventory": "personal_attributes", + "inventorygroupmembershipinvitation": "personal_attributes", + "inventorymembershipinvitation": "personal_attributes", "license": "personal_attributes", + "licensegroupmembershipinvitation": "personal_attributes", + "licensemembershipinvitation": "personal_attributes", "protocol": "personal_attributes", + "protocolgroupmembershipinvitation": "personal_attributes", + "protocolmembershipinvitation": "personal_attributes", "setup": "personal_attributes", + "setupgroupmembershipinvitation": "personal_attributes", + "setupmembershipinvitation": "personal_attributes", # resources "consumable": "resources", + "consumableapproval": "resources", "hardwaredevice": "resources", + "hardwaredeviceapproval": "resources", "supplier": "resources", + "supplierapproval": "resources", # taxonomies + "anatomicalreference": "taxonomies", + "anatomicalregion": "taxonomies", "behavioralcategory": "taxonomies", + "behavioralcategoryapproval": "taxonomies", "behavioralparadigm": "taxonomies", - "brainregion": "taxonomies", + "behavioralparadigmapproval": "taxonomies", + "coordinatetransform": "taxonomies", "regulatoryauthority": "taxonomies", + "regulatoryauthorityapproval": "taxonomies", "setuptype": "taxonomies", + "setuptypeapproval": "taxonomies", "species": "taxonomies", + "speciesapproval": "taxonomies", "strain": "taxonomies", "strainapproval": "taxonomies", # dissemination "journal": "dissemination", "journalapproval": "dissemination", "publication": "dissemination", + "publicationapproval": "dissemination", # users + "admingroup": "users", "group": "users", - "group_membership_invitation": "users", - "group_membership_request": "users", + "groupmembershipinvitation": "users", + "groupmembershiprequest": "users", "laboratory": "users", + "pendinginvitation": "users", "user": "users", } _TOKEN_FILE = Path.home() / ".config" / "brainstem" / "token" +# Deprecated model names, resolved to the server's current name with a warning: +# the underscored names earlier versions of this client used (they never matched +# a server route), and "brainregion", renamed "anatomicalregion". +_MODEL_ALIASES: dict = { + "brainregion": "anatomicalregion", + "project_membership_invitation": "projectmembershipinvitation", + "project_group_membership_invitation": "projectgroupmembershipinvitation", + "group_membership_invitation": "groupmembershipinvitation", + "group_membership_request": "groupmembershiprequest", +} + # Derived from _MODEL_TO_APP so there is exactly one source of truth. -ModelType = Enum("ModelType", {k: k for k in _MODEL_TO_APP}) # type: ignore[misc] +ModelType = Enum( # type: ignore[misc] + "ModelType", {k: k for k in [*_MODEL_TO_APP, *_MODEL_ALIASES]} +) _VALID_PORTALS = {"private", "public", "super"} def _resolve_model(model) -> str: - """Accept a plain string or a ModelType member and return the string value.""" + """Accept a plain string or a ModelType member and return the server's resource name.""" if isinstance(model, ModelType): - return model.value + model = model.value model = str(model) + if model in _MODEL_ALIASES: + warnings.warn( + f"Model name '{model}' is deprecated; use '{_MODEL_ALIASES[model]}'.", + FutureWarning, stacklevel=3, + ) + return _MODEL_ALIASES[model] if model not in _MODEL_TO_APP: raise ValueError( f"Unknown model '{model}'. Valid models are: " @@ -130,13 +197,32 @@ class BrainstemClient: BASE_URL = "https://www.brainstem.org/" DEFAULT_TIMEOUT: int = 30 # seconds; applied to all HTTP calls + MAX_PAGE_SIZE: int = 500 # the API returns at most this many records per page def __init__( self, token: str = None, headless: bool = False, url: str = None, + anonymous: bool = False, ) -> None: + """Create a client. + + Parameters + ---------- + token : API token. Defaults to ``$BRAINSTEM_API_TOKEN``, then the + cached token, then the browser device authorization flow. + headless : Print the verification URL and code instead of opening a browser. + url : Base URL of the server. Defaults to ``$BRAINSTEM_API_URL``, + then ``https://www.brainstem.org/``. + anonymous : Send no token and never start a login. Only the public + portal (``portal='public'``) is readable. + """ + if url is None: + url = os.environ.get("BRAINSTEM_API_URL") + if token is None: + token = os.environ.get("BRAINSTEM_API_TOKEN") + base = (url.rstrip("/") + "/") if url else self.BASE_URL self._base_url = base self._address = base + "api/" @@ -147,6 +233,10 @@ def __init__( self._session.mount("https://", HTTPAdapter(max_retries=_retry)) self._session.mount("http://", HTTPAdapter(max_retries=_retry)) + if anonymous: + self._token = None + return + if token: self._token = token else: @@ -268,11 +358,20 @@ def load(self, Supports ``.icontains``, ``.startswith``, ``.endswith``, ``.gt``, ``.gte``, ``.lt``, ``.lte``. sort : List of fields to sort by. Prefix with ``'-'`` for descending. - include : Related models to embed, e.g. ``['dataacquisition', 'behaviors']``. - limit : Max records per page (API maximum: 100). - offset : Number of records to skip (for pagination). + include : Related fields to sideload, e.g. ``['dataacquisition', 'behaviors']``. + Their records are returned as extra lists next to the main + one, and only records you may read are included. See ``save`` + before writing back a list field read this way. + limit : Records per page, 1 to 500 (default 100). Sent to the API as + ``per_page``. With *load_all*, a larger value is reduced to + 500, which returns the same records in more pages. + offset : Number of records to skip. The API pages with ``page`` and + ``per_page``, so *offset* must be a multiple of the page size + (*limit*, or 100 if *limit* is not set); it is sent as + ``page = offset // page_size + 1``. load_all : When ``True``, automatically follow pagination and return a - combined ``dict`` with all records merged under the model key. + combined ``dict`` with all records merged under the model key, + starting at the page selected by *offset*. When ``False`` (default), returns the raw ``Response``. """ model = _resolve_model(model) @@ -288,27 +387,34 @@ def load(self, params.setdefault("sort[]", []).append(field) for rel in (include or []): params.setdefault("include[]", []).append(f"{rel}.*") - if limit is not None: - params["limit"] = limit - if offset is not None: - params["offset"] = offset - if not load_all: - return self._session.get(url, params=params, timeout=self.DEFAULT_TIMEOUT) - - if id is not None: + if load_all and id is not None: raise ValueError("load_all=True cannot be used together with id.") - # The API paginates with page/per_page; translate limit/offset. - page_size = limit or 100 - if offset and offset % page_size: + # The API paginates with page/per_page (limit/offset are ignored by + # the server), so translate limit/offset into those parameters. + page_size = 100 if limit is None else limit + if load_all: + page_size = min(page_size, self.MAX_PAGE_SIZE) + if not id and not 1 <= page_size <= self.MAX_PAGE_SIZE: + raise ValueError( + f"limit ({limit}) must be between 1 and {self.MAX_PAGE_SIZE}, " + "the API's maximum page size." + ) + if not id and offset and offset % page_size: raise ValueError( - "With load_all=True, offset must be a multiple of limit (the page size)." + f"offset ({offset}) must be a multiple of limit (the page size, " + f"{page_size}): the API pages with page/per_page." ) - params.pop("limit", None) - params.pop("offset", None) - return self._load_all_pages(url, params, page_size, - start_page=(offset or 0) // page_size + 1) + start_page = (offset or 0) // page_size + 1 + + if load_all: + return self._load_all_pages(url, params, page_size, start_page=start_page) + + if not id and (limit is not None or offset is not None): + params["per_page"] = page_size + params["page"] = start_page + return self._session.get(url, params=params, timeout=self.DEFAULT_TIMEOUT) def _get_json(self, url: str, params: dict = None) -> dict: """GET *url* and return the decoded JSON, raising on HTTP errors.""" @@ -376,6 +482,14 @@ def save(self, When *id* is provided the record is **updated** (PATCH). When *id* is omitted a **new record** is created (POST). + .. warning:: + Sending a list field (``projects``, ``subjects``, ``behaviors``, ...) + replaces the whole list. A list read with ``include`` (including the + convenience loaders' default includes) holds only the records you may + read, so writing it back drops links to records you cannot see. Read + the record without ``include`` first, e.g. + ``client.load('session', id=uid).json()``, and change that list. + Parameters ---------- model : Model name string or ``ModelType`` member, e.g. ``'session'``. @@ -592,13 +706,13 @@ def load_project(self, portal="private", id: str = None, name : Filter by project name (case-insensitive contains). sessions : Filter by session UUID. subjects : Filter by subject UUID. - tags : Filter by tag. + tags : Filter by numeric tag id (the server does not filter by tag name). """ return self._convenience_load( "project", default_include=["sessions", "subjects", "collections", "cohorts"], - filter_map={"name": "name.icontains", "sessions": "sessions.id", - "subjects": "subjects.id", "tags": "tags"}, + filter_map={"name": "name.icontains", "sessions": "sessions", + "subjects": "subjects", "tags": "tags"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, name=name, sessions=sessions, subjects=subjects, tags=tags, @@ -618,13 +732,13 @@ def load_subject(self, portal="private", id: str = None, projects : Filter by project UUID. strain : Filter by strain UUID. sex : Filter by sex (``'M'``, ``'F'``, or ``'U'``). - tags : Filter by tag. + tags : Filter by numeric tag id (the server does not filter by tag name). """ return self._convenience_load( "subject", default_include=["procedures", "subjectlogs"], - filter_map={"name": "name.icontains", "projects": "projects.id", - "strain": "strain.id", "sex": "sex", "tags": "tags"}, + filter_map={"name": "name.icontains", "projects": "projects", + "strain": "strain", "sex": "sex", "tags": "tags"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, name=name, projects=projects, strain=strain, sex=sex, tags=tags, @@ -643,13 +757,13 @@ def load_session(self, portal="private", id: str = None, name : Filter by session name (case-insensitive contains). projects : Filter by project UUID. datastorage : Filter by data storage UUID. - tags : Filter by tag. + tags : Filter by numeric tag id (the server does not filter by tag name). """ return self._convenience_load( "session", default_include=["dataacquisition", "behaviors", "manipulations", "epochs"], - filter_map={"name": "name.icontains", "projects": "projects.id", - "datastorage": "datastorage.id", "tags": "tags"}, + filter_map={"name": "name.icontains", "projects": "projects", + "datastorage": "datastorage", "tags": "tags"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, name=name, projects=projects, datastorage=datastorage, tags=tags, @@ -665,7 +779,7 @@ def load_collection(self, portal="private", id: str = None, Parameters ---------- name : Filter by collection name (case-insensitive contains). - tags : Filter by tag. + tags : Filter by numeric tag id (the server does not filter by tag name). """ return self._convenience_load( "collection", @@ -686,7 +800,7 @@ def load_cohort(self, portal="private", id: str = None, Parameters ---------- name : Filter by cohort name (case-insensitive contains). - tags : Filter by tag. + tags : Filter by numeric tag id (the server does not filter by tag name). """ return self._convenience_load( "cohort", @@ -698,7 +812,7 @@ def load_cohort(self, portal="private", id: str = None, ) def load_behavior(self, portal="private", id: str = None, - session: str = None, tags: str = None, + session: str = None, filters: dict = None, sort: list = None, include: list = None, limit: int = None, offset: int = None, load_all: bool = False): @@ -707,19 +821,18 @@ def load_behavior(self, portal="private", id: str = None, Parameters ---------- session : Filter by session UUID. - tags : Filter by tag. """ return self._convenience_load( "behavior", default_include=[], - filter_map={"session": "session.id", "tags": "tags"}, + filter_map={"session": "session"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, - session=session, tags=tags, + session=session, ) def load_dataacquisition(self, portal="private", id: str = None, - session: str = None, tags: str = None, + session: str = None, filters: dict = None, sort: list = None, include: list = None, limit: int = None, offset: int = None, load_all: bool = False): @@ -728,19 +841,18 @@ def load_dataacquisition(self, portal="private", id: str = None, Parameters ---------- session : Filter by session UUID. - tags : Filter by tag. """ return self._convenience_load( "dataacquisition", default_include=[], - filter_map={"session": "session.id", "tags": "tags"}, + filter_map={"session": "session"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, - session=session, tags=tags, + session=session, ) def load_manipulation(self, portal="private", id: str = None, - session: str = None, tags: str = None, + session: str = None, filters: dict = None, sort: list = None, include: list = None, limit: int = None, offset: int = None, load_all: bool = False): @@ -749,19 +861,18 @@ def load_manipulation(self, portal="private", id: str = None, Parameters ---------- session : Filter by session UUID. - tags : Filter by tag. """ return self._convenience_load( "manipulation", default_include=[], - filter_map={"session": "session.id", "tags": "tags"}, + filter_map={"session": "session"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, - session=session, tags=tags, + session=session, ) def load_procedure(self, portal="private", id: str = None, - subject: str = None, tags: str = None, + subject: str = None, filters: dict = None, sort: list = None, include: list = None, limit: int = None, offset: int = None, load_all: bool = False): @@ -770,19 +881,18 @@ def load_procedure(self, portal="private", id: str = None, Parameters ---------- subject : Filter by subject UUID. - tags : Filter by tag. """ return self._convenience_load( "procedure", default_include=[], - filter_map={"subject": "subject.id", "tags": "tags"}, + filter_map={"subject": "subject"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, - subject=subject, tags=tags, + subject=subject, ) def load_procedurelog(self, portal="private", id: str = None, - procedure: str = None, tags: str = None, + procedure: str = None, filters: dict = None, sort: list = None, include: list = None, limit: int = None, offset: int = None, load_all: bool = False): @@ -791,19 +901,18 @@ def load_procedurelog(self, portal="private", id: str = None, Parameters ---------- procedure : Filter by procedure UUID. - tags : Filter by tag. """ return self._convenience_load( "procedurelog", default_include=[], - filter_map={"procedure": "procedure.id", "tags": "tags"}, + filter_map={"procedure": "procedure"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, - procedure=procedure, tags=tags, + procedure=procedure, ) def load_subjectlog(self, portal="private", id: str = None, - subject: str = None, tags: str = None, + subject: str = None, filters: dict = None, sort: list = None, include: list = None, limit: int = None, offset: int = None, load_all: bool = False): @@ -812,19 +921,18 @@ def load_subjectlog(self, portal="private", id: str = None, Parameters ---------- subject : Filter by subject UUID. - tags : Filter by tag. """ return self._convenience_load( "subjectlog", default_include=[], - filter_map={"subject": "subject.id", "tags": "tags"}, + filter_map={"subject": "subject"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, - subject=subject, tags=tags, + subject=subject, ) def load_equipment(self, portal="private", id: str = None, - name: str = None, setup: str = None, tags: str = None, + name: str = None, setup: str = None, filters: dict = None, sort: list = None, include: list = None, limit: int = None, offset: int = None, load_all: bool = False): @@ -834,19 +942,18 @@ def load_equipment(self, portal="private", id: str = None, ---------- name : Filter by equipment name (case-insensitive contains). setup : Filter by setup UUID. - tags : Filter by tag. """ return self._convenience_load( "equipment", default_include=[], - filter_map={"name": "name.icontains", "setup": "setup.id", "tags": "tags"}, + filter_map={"name": "name.icontains", "setup": "setup"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, - name=name, setup=setup, tags=tags, + name=name, setup=setup, ) def load_consumablestock(self, portal="private", id: str = None, - tags: str = None, + inventory: str = None, filters: dict = None, sort: list = None, include: list = None, limit: int = None, offset: int = None, load_all: bool = False): @@ -854,19 +961,19 @@ def load_consumablestock(self, portal="private", id: str = None, Parameters ---------- - tags : Filter by tag. + inventory : Filter by inventory UUID. """ return self._convenience_load( "consumablestock", default_include=[], - filter_map={"tags": "tags"}, + filter_map={"inventory": "inventory"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, - tags=tags, + inventory=inventory, ) def load_behavioralassay(self, portal="private", id: str = None, - name: str = None, tags: str = None, + name: str = None, filters: dict = None, sort: list = None, include: list = None, limit: int = None, offset: int = None, load_all: bool = False): @@ -875,19 +982,18 @@ def load_behavioralassay(self, portal="private", id: str = None, Parameters ---------- name : Filter by name (case-insensitive contains). - tags : Filter by tag. """ return self._convenience_load( "behavioralassay", default_include=[], - filter_map={"name": "name.icontains", "tags": "tags"}, + filter_map={"name": "name.icontains"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, - name=name, tags=tags, + name=name, ) def load_datastorage(self, portal="private", id: str = None, - name: str = None, tags: str = None, + name: str = None, filters: dict = None, sort: list = None, include: list = None, limit: int = None, offset: int = None, load_all: bool = False): @@ -896,19 +1002,18 @@ def load_datastorage(self, portal="private", id: str = None, Parameters ---------- name : Filter by name (case-insensitive contains). - tags : Filter by tag. """ return self._convenience_load( "datastorage", default_include=[], - filter_map={"name": "name.icontains", "tags": "tags"}, + filter_map={"name": "name.icontains"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, - name=name, tags=tags, + name=name, ) def load_setup(self, portal="private", id: str = None, - name: str = None, tags: str = None, + name: str = None, filters: dict = None, sort: list = None, include: list = None, limit: int = None, offset: int = None, load_all: bool = False): @@ -917,19 +1022,18 @@ def load_setup(self, portal="private", id: str = None, Parameters ---------- name : Filter by name (case-insensitive contains). - tags : Filter by tag. """ return self._convenience_load( "setup", default_include=[], - filter_map={"name": "name.icontains", "tags": "tags"}, + filter_map={"name": "name.icontains"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, - name=name, tags=tags, + name=name, ) def load_hardwaredevice(self, portal="private", id: str = None, - name: str = None, tags: str = None, + name: str = None, filters: dict = None, sort: list = None, include: list = None, limit: int = None, offset: int = None, load_all: bool = False): @@ -938,83 +1042,128 @@ def load_hardwaredevice(self, portal="private", id: str = None, Parameters ---------- name : Filter by name (case-insensitive contains). - tags : Filter by tag. """ return self._convenience_load( "hardwaredevice", default_include=[], - filter_map={"name": "name.icontains", "tags": "tags"}, + filter_map={"name": "name.icontains"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, - name=name, tags=tags, + name=name, ) def load_brainregion(self, portal="private", id: str = None, - name: str = None, tags: str = None, + name: str = None, filters: dict = None, sort: list = None, include: list = None, limit: int = None, offset: int = None, load_all: bool = False): - """Load brain region record(s). + """Deprecated: use ``load_anatomicalregion``, which this calls. + + Brain regions are ``anatomicalregion`` records; the response holds them + under ``anatomical_regions``. Parameters ---------- name : Filter by name (case-insensitive contains). - tags : Filter by tag. + """ + warnings.warn( + "load_brainregion() is deprecated; use load_anatomicalregion().", + FutureWarning, stacklevel=2, + ) + return self.load_anatomicalregion( + portal=portal, id=id, name=name, filters=filters, sort=sort, + include=include, limit=limit, offset=offset, load_all=load_all, + ) + + def load_anatomicalregion(self, portal="private", id: str = None, + name: str = None, acronym: str = None, + reference: str = None, + filters: dict = None, sort: list = None, + include: list = None, limit: int = None, + offset: int = None, load_all: bool = False): + """Load anatomical (brain) region record(s). + + Parameters + ---------- + name : Filter by name (case-insensitive contains). + acronym : Filter by acronym (case-insensitive exact match), e.g. ``'CA1'``. + reference : Filter by anatomical reference (atlas) UUID. """ return self._convenience_load( - "brainregion", + "anatomicalregion", default_include=[], - filter_map={"name": "name.icontains", "tags": "tags"}, + filter_map={"name": "name.icontains", "acronym": "acronym.iexact", + "reference": "reference"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, - name=name, tags=tags, + name=name, acronym=acronym, reference=reference, ) + def load_anatomicalregion_descendants(self, id: str, portal="private") -> Response: + """Load every region below anatomical region *id* in its atlas. + + Returns the raw ``Response``; its JSON is + ``{"anatomical_regions": [{"id", "name", "acronym", "parent_acronym", + "is_obsolete", "depth"}, ...]}``, ordered by depth (1 = direct child). + Atlases without a hierarchy return an empty list. + """ + if not id: + raise ValueError("'id' is required to load the descendants of a region.") + url = self._build_url(_resolve_portal(portal), "taxonomies", "anatomicalregion", + id, "descendants/") + return self._session.get(url, timeout=self.DEFAULT_TIMEOUT) + def load_species(self, portal="private", id: str = None, - name: str = None, tags: str = None, + name: str = None, filters: dict = None, sort: list = None, include: list = None, limit: int = None, - offset: int = None, load_all: bool = False): + offset: int = None, load_all: bool = False, + popular_name: str = None): """Load species record(s). Parameters ---------- - name : Filter by name (case-insensitive contains). - tags : Filter by tag. + name : Filter by scientific name (case-insensitive contains). + popular_name : Filter by common name (case-insensitive contains), e.g. + ``'mouse'``. Needs a server with the specimen release. """ return self._convenience_load( "species", default_include=[], - filter_map={"name": "name.icontains", "tags": "tags"}, + filter_map={"name": "name.icontains", + "popular_name": "popular_name.icontains"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, - name=name, tags=tags, + name=name, popular_name=popular_name, ) def load_strain(self, portal="private", id: str = None, - name: str = None, species: str = None, tags: str = None, + name: str = None, species: str = None, filters: dict = None, sort: list = None, include: list = None, limit: int = None, - offset: int = None, load_all: bool = False): + offset: int = None, load_all: bool = False, + popular_name: str = None): """Load strain record(s). Parameters ---------- - name : Filter by name (case-insensitive contains). - species : Filter by species UUID. - tags : Filter by tag. + name : Filter by name (case-insensitive contains). + species : Filter by species UUID. + popular_name : Filter by common name (case-insensitive contains). + Needs a server with the specimen release. """ return self._convenience_load( "strain", default_include=[], - filter_map={"name": "name.icontains", "species": "species.id", "tags": "tags"}, + filter_map={"name": "name.icontains", "species": "species", + "popular_name": "popular_name.icontains"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, - name=name, species=species, tags=tags, + name=name, species=species, popular_name=popular_name, ) def load_publication(self, portal="private", id: str = None, - name: str = None, tags: str = None, + name: str = None, filters: dict = None, sort: list = None, include: list = None, limit: int = None, offset: int = None, load_all: bool = False): @@ -1023,19 +1172,18 @@ def load_publication(self, portal="private", id: str = None, Parameters ---------- name : Filter by name (case-insensitive contains). - tags : Filter by tag. """ return self._convenience_load( "publication", default_include=[], - filter_map={"name": "name.icontains", "tags": "tags"}, + filter_map={"name": "name.icontains"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, - name=name, tags=tags, + name=name, ) def load_laboratory(self, portal="private", id: str = None, - name: str = None, tags: str = None, + name: str = None, filters: dict = None, sort: list = None, include: list = None, limit: int = None, offset: int = None, load_all: bool = False): @@ -1044,13 +1192,12 @@ def load_laboratory(self, portal="private", id: str = None, Parameters ---------- name : Filter by name (case-insensitive contains). - tags : Filter by tag. """ return self._convenience_load( "laboratory", default_include=[], - filter_map={"name": "name.icontains", "tags": "tags"}, + filter_map={"name": "name.icontains"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, - name=name, tags=tags, + name=name, ) diff --git a/brainstem_api_tools/cli.py b/brainstem_api_tools/cli.py index 043b5d1..6eb4d8c 100644 --- a/brainstem_api_tools/cli.py +++ b/brainstem_api_tools/cli.py @@ -10,6 +10,8 @@ brainstem load session --portal public --filters name.icontains=rat --sort -name brainstem load session --id brainstem load session --limit 20 --offset 40 + brainstem load session --per-page 20 --page 3 + brainstem load session --all brainstem save session --data '{"name":"New","projects":[""]}' brainstem save session --id --data '{"description":"updated"}' brainstem delete session --id @@ -20,32 +22,50 @@ import os import sys -from .brainstem_api_client import BrainstemClient, _MODEL_TO_APP, _TOKEN_FILE +import requests +from .brainstem_api_client import ( + AuthenticationError, BrainstemClient, _MODEL_TO_APP, _TOKEN_FILE, +) + + +def _common_flags(suppress_defaults: bool = False) -> argparse.ArgumentParser: + """Shared flags available before and after the subcommand. + + The subcommand copies use ``argparse.SUPPRESS`` defaults so they do not + overwrite a flag given before the subcommand (``brainstem --url X load``). + """ + def default(value): + return argparse.SUPPRESS if suppress_defaults else value -def _build_parser() -> argparse.ArgumentParser: - # Shared flags available on every subcommand common = argparse.ArgumentParser(add_help=False) common.add_argument( "--token", - default=os.environ.get("BRAINSTEM_API_TOKEN"), + default=default(os.environ.get("BRAINSTEM_API_TOKEN")), help="API token. Defaults to $BRAINSTEM_API_TOKEN or the cached token.", ) common.add_argument( "--headless", action="store_true", + default=default(False), help="Print verification URL + code instead of opening a browser.", ) common.add_argument( "--url", - default=None, - help="Base URL of the BrainSTEM server (default: https://www.brainstem.org/).", + default=default(os.environ.get("BRAINSTEM_API_URL")), + help="Base URL of the BrainSTEM server. Defaults to $BRAINSTEM_API_URL " + "or https://www.brainstem.org/.", ) + return common + + +def _build_parser() -> argparse.ArgumentParser: + common = _common_flags(suppress_defaults=True) parser = argparse.ArgumentParser( prog="brainstem", description="BrainSTEM command-line API client.", - parents=[common], + parents=[_common_flags()], ) sub = parser.add_subparsers(dest="command", required=True) @@ -87,8 +107,17 @@ def _build_parser() -> argparse.ArgumentParser: metavar="RELATION", help="Related models to embed.", ) - p_load.add_argument("--limit", type=int, help="Max records (API max: 100).") - p_load.add_argument("--offset", type=int, help="Records to skip (pagination).") + p_load.add_argument("--limit", type=int, + help="Records per page (sent as per_page; default 100).") + p_load.add_argument("--offset", type=int, + help="Records to skip; must be a multiple of the page size.") + p_load.add_argument("--per-page", type=int, help="Records per page (default 100).") + p_load.add_argument("--page", type=int, help="Page number, starting at 1.") + p_load.add_argument( + "--all", + action="store_true", + help="Follow pagination and print all records merged into one JSON object.", + ) # ---- save ------------------------------------------------------- p_save = sub.add_parser("save", parents=[common], help="Create or update a record.") @@ -129,8 +158,11 @@ def main(): ) if args.command == "login": - print(f"Token: {client._token}") - print(f"Cached at: {_TOKEN_FILE}") + if args.token: + print("Logged in with the token from --token / $BRAINSTEM_API_TOKEN " + "(not cached).") + else: + print(f"Logged in. Token cached at {_TOKEN_FILE}") return if args.command == "load": @@ -141,16 +173,39 @@ def main(): k, v = expr.split("=", 1) filters[k] = v - resp = client.load( - args.model, + limit, offset = args.limit, args.offset + if args.per_page is not None or args.page is not None: + if limit is not None or offset is not None: + parser.error("Use either --per-page/--page or --limit/--offset, not both.") + if (args.per_page is not None and args.per_page < 1) or \ + (args.page is not None and args.page < 1): + parser.error("--per-page and --page must be at least 1.") + limit = args.per_page + if args.page is not None: + offset = (args.page - 1) * (args.per_page or 100) + + load_kwargs = dict( portal=args.portal, id=args.id, filters=filters or None, sort=args.sort, include=args.include, - limit=args.limit, - offset=args.offset, + limit=limit, + offset=offset, ) + if args.all: + try: + result = client.load(args.model, load_all=True, **load_kwargs) + except (ValueError, AuthenticationError, requests.HTTPError) as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1) + print(json.dumps(result, indent=2)) + return + + try: + resp = client.load(args.model, **load_kwargs) + except ValueError as exc: + parser.error(str(exc)) elif args.command == "save": try: diff --git a/tests/test_client.py b/tests/test_client.py index ae04d99..196cfe5 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -15,6 +15,7 @@ from brainstem_api_tools.brainstem_api_client import ( BrainstemClient, AuthenticationError, + ModelType, ExportError, _MODEL_TO_APP, _resolve_model, @@ -29,6 +30,13 @@ TOKEN = "test-token-abc123" +@pytest.fixture(autouse=True) +def _clear_brainstem_env(monkeypatch): + """Keep the developer's BRAINSTEM_* environment out of the tests.""" + monkeypatch.delenv("BRAINSTEM_API_TOKEN", raising=False) + monkeypatch.delenv("BRAINSTEM_API_URL", raising=False) + + def make_client(token=TOKEN): """Return a BrainstemClient with a pre-set token (no auth flow).""" return BrainstemClient(token=token) @@ -75,6 +83,16 @@ def test_auth_header_on_session(self): client = make_client() assert client._session.headers["Authorization"] == f"Bearer {TOKEN}" + def test_anonymous_sends_no_token_and_skips_login(self, monkeypatch): + monkeypatch.setenv("BRAINSTEM_API_TOKEN", "env-token") + with patch.object(BrainstemClient, "_load_cached_token") as cached, \ + patch.object(BrainstemClient, "_device_auth_flow") as flow: + client = BrainstemClient(anonymous=True) + cached.assert_not_called() + flow.assert_not_called() + assert client._token is None + assert "Authorization" not in client._session.headers + def test_cached_token_used_on_init(self, tmp_path, monkeypatch): token_file = tmp_path / "token" token_file.write_text("cached-token") @@ -96,6 +114,34 @@ def test_device_flow_triggered_when_no_cache(self, tmp_path, monkeypatch): mock_save.assert_called_once_with("new-token") assert client._token == "new-token" + def test_token_from_env(self, tmp_path, monkeypatch): + token_file = tmp_path / "token" + token_file.write_text("cached-token") + monkeypatch.setattr( + "brainstem_api_tools.brainstem_api_client._TOKEN_FILE", token_file + ) + monkeypatch.setenv("BRAINSTEM_API_TOKEN", "env-token") + client = BrainstemClient() + assert client._token == "env-token" + assert client._session.headers["Authorization"] == "Bearer env-token" + + def test_explicit_token_beats_env(self, monkeypatch): + monkeypatch.setenv("BRAINSTEM_API_TOKEN", "env-token") + assert make_client()._token == TOKEN + + def test_url_from_env(self, monkeypatch): + monkeypatch.setenv("BRAINSTEM_API_URL", "http://127.0.0.1:8000") + client = make_client() + assert client._address == "http://127.0.0.1:8000/api/" + + def test_explicit_url_beats_env(self, monkeypatch): + monkeypatch.setenv("BRAINSTEM_API_URL", "http://127.0.0.1:8000") + client = BrainstemClient(token=TOKEN, url="https://example.org") + assert client._address == "https://example.org/api/" + + def test_default_url_without_env(self): + assert make_client()._address == "https://www.brainstem.org/api/" + # --------------------------------------------------------------------------- # load() @@ -144,10 +190,45 @@ def test_load_include_appends_wildcard(self): def test_load_pagination_params(self): self._mock_get() - self.client.load("session", limit=50, offset=20) + self.client.load("session", limit=20, offset=40) params = self.client._session.get.call_args[1]["params"] - assert params["limit"] == 50 - assert params["offset"] == 20 + assert params["per_page"] == 20 + assert params["page"] == 3 + assert "limit" not in params and "offset" not in params + + def test_load_limit_only_sets_per_page(self): + self._mock_get() + self.client.load("session", limit=50) + params = self.client._session.get.call_args[1]["params"] + assert params["per_page"] == 50 + assert params["page"] == 1 + + def test_load_offset_without_limit_uses_page_size_100(self): + self._mock_get() + self.client.load("session", offset=200) + params = self.client._session.get.call_args[1]["params"] + assert params["per_page"] == 100 + assert params["page"] == 3 + + def test_load_offset_not_multiple_of_page_size_raises(self): + self._mock_get() + with pytest.raises(ValueError, match="multiple of limit"): + self.client.load("session", limit=50, offset=20) + with pytest.raises(ValueError, match="page size, 100"): + self.client.load("session", offset=30) + self.client._session.get.assert_not_called() + + def test_load_without_paging_sends_no_page_params(self): + self._mock_get() + self.client.load("session") + params = self.client._session.get.call_args[1]["params"] + assert "per_page" not in params and "page" not in params + + def test_load_with_id_ignores_paging(self): + self._mock_get() + self.client.load("session", id="00000000-0000-0000-0000-000000000001", + limit=10, offset=5) + assert self.client._session.get.call_args[1]["params"] == {} def test_load_public_portal(self): self._mock_get() @@ -159,16 +240,83 @@ def test_load_invalid_model_raises(self): with pytest.raises(ValueError, match="Unknown model"): self.client.load("notamodel") + @pytest.mark.parametrize("model,expected_app", [ + *[(m, "stem") for m in [ + "specimenset", "specimen", "epoch", "sessiondatastorage", + "collectionsession", "cohortsubject"]], + *[(m, "modules") for m in [ + "experimentalinterface", "experimentalinterfacetemplate", "materialusage", + "materialusagetemplate", "specimenprocedure", "specimenproceduretemplate", + "specimenlog", "dataacquisitioninterface", "manipulationinterface", + "dataacquisitionprocedure", "proceduretemplate", "dataacquisitiontemplate", + "manipulationtemplate", "procedureepisode", "procedureepisodetemplate", + "episodepersonnel"]], + *[(m, "taxonomies") for m in [ + "anatomicalregion", "anatomicalreference", "coordinatetransform"]], + ("pendinginvitation", "users"), + ]) + def test_new_models_routed(self, model, expected_app): + assert _MODEL_TO_APP[model] == expected_app + self._mock_get() + self.client.load(model) + assert f"/api/private/{expected_app}/{model}/" in self.client._session.get.call_args[0][0] + def test_load_invalid_portal_raises(self): with pytest.raises(ValueError, match="Unknown portal"): self.client.load("session", portal="badportal") + @pytest.mark.parametrize("old,new,app", [ + ("project_membership_invitation", "projectmembershipinvitation", "stem"), + ("project_group_membership_invitation", "projectgroupmembershipinvitation", "stem"), + ("group_membership_invitation", "groupmembershipinvitation", "users"), + ("group_membership_request", "groupmembershiprequest", "users"), + ]) + def test_old_underscored_names_resolve_with_warning(self, old, new, app): + self._mock_get() + with pytest.warns(FutureWarning, match=f"use '{new}'"): + self.client.load(old) + assert f"/api/private/{app}/{new}/" in self.client._session.get.call_args[0][0] + with pytest.warns(FutureWarning): + self.client.load(ModelType[old]) + + @pytest.mark.parametrize("model,app", [ + ("admingroup", "users"), + ("speciesapproval", "taxonomies"), + ("supplierapproval", "resources"), + ("publicationapproval", "dissemination"), + ("setupmembershipinvitation", "personal_attributes"), + ("datastoragegroupmembershipinvitation", "personal_attributes"), + ]) + def test_approval_and_invitation_resources_routed(self, model, app): + self._mock_get() + self.client.load(model) + assert f"/api/private/{app}/{model}/" in self.client._session.get.call_args[0][0] + + def test_brainregion_resolves_to_anatomicalregion(self): + self._mock_get() + with pytest.warns(FutureWarning, match="use 'anatomicalregion'"): + self.client.load("brainregion") + assert "/taxonomies/anatomicalregion/" in self.client._session.get.call_args[0][0] + + def test_limit_above_page_size_maximum_raises(self): + self._mock_get() + with pytest.raises(ValueError, match="between 1 and 500"): + self.client.load("session", limit=501) + with pytest.raises(ValueError, match="between 1 and 500"): + self.client.load("session", limit=0) + self.client._session.get.assert_not_called() + + def test_limit_at_page_size_maximum(self): + self._mock_get() + self.client.load("session", limit=500) + assert self.client._session.get.call_args[1]["params"]["per_page"] == 500 + def test_model_to_app_routing(self): """Spot-check a few app assignments.""" cases = [ ("session", "stem"), ("breeding", "stem"), - ("project_membership_invitation", "stem"), + ("projectmembershipinvitation", "stem"), ("procedure", "modules"), ("behavioralassay", "personal_attributes"), ("license", "personal_attributes"), @@ -177,11 +325,14 @@ def test_model_to_app_routing(self): ("behavioralparadigm", "taxonomies"), ("behavioralcategory", "taxonomies"), ("regulatoryauthority", "taxonomies"), - ("brainregion", "taxonomies"), + ("anatomicalregion", "taxonomies"), + ("specimenset", "stem"), + ("experimentalinterface", "modules"), + ("pendinginvitation", "users"), ("publication", "dissemination"), ("laboratory", "users"), - ("group_membership_invitation", "users"), - ("group_membership_request", "users"), + ("groupmembershipinvitation", "users"), + ("groupmembershiprequest", "users"), ] for model, expected_app in cases: self._mock_get() @@ -361,6 +512,11 @@ def test_offset_translated_to_page(self): self.client.load("session", limit=2, offset=2, load_all=True) assert self.client._session.get.call_args[1]["params"]["page"] == 2 + def test_load_all_reduces_page_size_to_maximum(self): + self._mock_pages([{"sessions": [{"id": "1"}], "meta": {"total_pages": 1}}]) + self.client.load("session", limit=1000, load_all=True) + assert self.client._session.get.call_args[1]["params"]["per_page"] == 500 + def test_offset_not_multiple_of_limit_raises(self): with pytest.raises(ValueError, match="multiple of limit"): self.client.load("session", limit=10, offset=5, load_all=True) @@ -589,7 +745,7 @@ def test_load_subject_sex_filter(self): def test_load_subject_strain_filter(self): uuid = "aaaaaaaa-0000-0000-0000-000000000000" self.client.load_subject(strain=uuid) - assert self._get_params()["filter{strain.id}"] == uuid + assert self._get_params()["filter{strain}"] == uuid def test_load_session_default_include(self): self.client.load_session() @@ -601,7 +757,7 @@ def test_load_session_default_include(self): def test_load_session_project_filter(self): uuid = "bbbbbbbb-0000-0000-0000-000000000000" self.client.load_session(projects=uuid) - assert self._get_params()["filter{projects.id}"] == uuid + assert self._get_params()["filter{projects}"] == uuid def test_load_collection_default_include(self): self.client.load_collection() @@ -614,22 +770,46 @@ def test_load_cohort_default_include(self): def test_load_behavior_session_filter(self): uuid = "cccccccc-0000-0000-0000-000000000000" self.client.load_behavior(session=uuid) - assert self._get_params()["filter{session.id}"] == uuid + assert self._get_params()["filter{session}"] == uuid def test_load_dataacquisition_session_filter(self): uuid = "dddddddd-0000-0000-0000-000000000000" self.client.load_dataacquisition(session=uuid) - assert self._get_params()["filter{session.id}"] == uuid + assert self._get_params()["filter{session}"] == uuid def test_load_manipulation_session_filter(self): uuid = "eeeeeeee-0000-0000-0000-000000000000" self.client.load_manipulation(session=uuid) - assert self._get_params()["filter{session.id}"] == uuid + assert self._get_params()["filter{session}"] == uuid def test_load_procedure_subject_filter(self): uuid = "ffffffff-0000-0000-0000-000000000000" self.client.load_procedure(subject=uuid) - assert self._get_params()["filter{subject.id}"] == uuid + assert self._get_params()["filter{subject}"] == uuid + + @pytest.mark.parametrize("loader,kwarg", [ + ("load_procedure", "subject"), + ("load_behavior", "session"), + ("load_dataacquisition", "session"), + ("load_manipulation", "session"), + ("load_procedurelog", "procedure"), + ("load_subjectlog", "subject"), + ("load_equipment", "setup"), + ("load_strain", "species"), + ]) + def test_non_nestable_relation_filters_use_bare_field(self, loader, kwarg): + # The server rejects filter{.id} for plain PK relations. + uuid = "12345678-0000-0000-0000-000000000000" + getattr(self.client, loader)(**{kwarg: uuid}) + params = self._get_params() + assert params[f"filter{{{kwarg}}}"] == uuid + assert f"filter{{{kwarg}.id}}" not in params + + def test_convenience_loader_paging_translated(self): + self.client.load_session(limit=25, offset=50) + params = self._get_params() + assert params["per_page"] == 25 + assert params["page"] == 3 def test_custom_include_overrides_default(self): self.client.load_session(include=["behaviors"]) @@ -673,6 +853,35 @@ def test_logout_no_token_file(self, tmp_path, capsys): assert "No cached token found" in capsys.readouterr().out +# --------------------------------------------------------------------------- +# CLI login +# --------------------------------------------------------------------------- + +class TestCLILogin: + def _run_login(self, argv, token): + import brainstem_api_tools.cli as cli_module + client = MagicMock() + client._token = token + with patch("brainstem_api_tools.cli.BrainstemClient", return_value=client) as cls, \ + patch("sys.argv", argv): + cli_module.main() + return cls + + def test_login_does_not_print_token(self, capsys): + import brainstem_api_tools.cli as cli_module + self._run_login(["brainstem", "login"], "secret-device-token") + out = capsys.readouterr().out + assert "secret-device-token" not in out + assert "Logged in" in out + assert str(cli_module._TOKEN_FILE) in out + + def test_login_with_explicit_token_does_not_print_it(self, capsys): + self._run_login(["brainstem", "login", "--token", TOKEN], TOKEN) + out = capsys.readouterr().out + assert TOKEN not in out + assert "Logged in" in out + + # --------------------------------------------------------------------------- # Timeouts # --------------------------------------------------------------------------- @@ -818,19 +1027,19 @@ def _get_url(self): def test_load_procedurelog_procedure_filter(self): uuid = "11111111-0000-0000-0000-000000000000" self.client.load_procedurelog(procedure=uuid) - assert self._get_params()["filter{procedure.id}"] == uuid + assert self._get_params()["filter{procedure}"] == uuid assert "/modules/procedurelog/" in self._get_url() def test_load_subjectlog_subject_filter(self): uuid = "22222222-0000-0000-0000-000000000000" self.client.load_subjectlog(subject=uuid) - assert self._get_params()["filter{subject.id}"] == uuid + assert self._get_params()["filter{subject}"] == uuid assert "/modules/subjectlog/" in self._get_url() def test_load_equipment_setup_filter(self): uuid = "33333333-0000-0000-0000-000000000000" self.client.load_equipment(setup=uuid) - assert self._get_params()["filter{setup.id}"] == uuid + assert self._get_params()["filter{setup}"] == uuid assert "/modules/equipment/" in self._get_url() def test_load_equipment_name_filter(self): @@ -858,10 +1067,11 @@ def test_load_hardwaredevice_url(self): self.client.load_hardwaredevice() assert "/resources/hardwaredevice/" in self._get_url() - def test_load_brainregion_name_filter(self): - self.client.load_brainregion(name="CA1") + def test_load_brainregion_is_deprecated_alias(self): + with pytest.warns(FutureWarning, match="load_anatomicalregion"): + self.client.load_brainregion(name="CA1") assert self._get_params()["filter{name.icontains}"] == "CA1" - assert "/taxonomies/brainregion/" in self._get_url() + assert "/taxonomies/anatomicalregion/" in self._get_url() def test_load_species_url(self): self.client.load_species() @@ -870,9 +1080,59 @@ def test_load_species_url(self): def test_load_strain_species_filter(self): uuid = "44444444-0000-0000-0000-000000000000" self.client.load_strain(species=uuid) - assert self._get_params()["filter{species.id}"] == uuid + assert self._get_params()["filter{species}"] == uuid assert "/taxonomies/strain/" in self._get_url() + def test_load_consumablestock_inventory_filter(self): + uuid = "77777777-0000-0000-0000-000000000000" + self.client.load_consumablestock(inventory=uuid) + assert self._get_params()["filter{inventory}"] == uuid + assert "/modules/consumablestock/" in self._get_url() + + @pytest.mark.parametrize("loader", [ + "load_behavior", "load_dataacquisition", "load_manipulation", "load_procedure", + "load_procedurelog", "load_subjectlog", "load_equipment", "load_consumablestock", + "load_behavioralassay", "load_datastorage", "load_setup", "load_hardwaredevice", + "load_brainregion", "load_species", "load_strain", "load_publication", "load_laboratory", + ]) + def test_untagged_models_have_no_tags_filter(self, loader): + # The server answers 400 "Invalid filter field: tags" for these models. + with pytest.raises(TypeError): + getattr(self.client, loader)(tags="x") + + def test_load_species_popular_name_filter(self): + self.client.load_species(popular_name="mouse") + assert self._get_params()["filter{popular_name.icontains}"] == "mouse" + assert "/taxonomies/species/" in self._get_url() + + def test_load_strain_popular_name_filter(self): + self.client.load_strain(popular_name="black 6") + assert self._get_params()["filter{popular_name.icontains}"] == "black 6" + + def test_popular_name_not_sent_unless_given(self): + self.client.load_species(name="Mus") + assert "filter{popular_name.icontains}" not in self._get_params() + + def test_load_anatomicalregion_filters(self): + ref = "55555555-0000-0000-0000-000000000000" + self.client.load_anatomicalregion(acronym="CA1", name="field", reference=ref) + params = self._get_params() + assert params["filter{acronym.iexact}"] == "CA1" + assert params["filter{name.icontains}"] == "field" + assert params["filter{reference}"] == ref + assert "/taxonomies/anatomicalregion/" in self._get_url() + + def test_load_anatomicalregion_descendants_url(self): + uuid = "66666666-0000-0000-0000-000000000000" + self.client.load_anatomicalregion_descendants(uuid, portal="public") + assert self._get_url() == ( + f"https://www.brainstem.org/api/public/taxonomies/anatomicalregion/{uuid}/descendants/" + ) + + def test_load_anatomicalregion_descendants_requires_id(self): + with pytest.raises(ValueError, match="'id' is required"): + self.client.load_anatomicalregion_descendants(None) + def test_load_publication_url(self): self.client.load_publication() assert "/dissemination/publication/" in self._get_url() @@ -947,3 +1207,89 @@ def test_cli_invalid_filter_exits(self): with pytest.raises(SystemExit): cli_module.main() + def _load_kwargs(self, argv): + client = MagicMock() + client.load.return_value = mock_response(200, {"sessions": []}) + self._run_cli(["brainstem", "load", "session", *argv], client) + return client.load.call_args[1] + + def test_cli_limit_offset_passed_through(self): + kwargs = self._load_kwargs(["--limit", "20", "--offset", "40"]) + assert kwargs["limit"] == 20 and kwargs["offset"] == 40 + + def test_cli_per_page_and_page(self): + kwargs = self._load_kwargs(["--per-page", "20", "--page", "3"]) + assert kwargs["limit"] == 20 + assert kwargs["offset"] == 40 + assert "load_all" not in kwargs + + def test_cli_page_without_per_page_uses_100(self): + kwargs = self._load_kwargs(["--page", "2"]) + assert kwargs["limit"] is None + assert kwargs["offset"] == 100 + + def test_cli_per_page_without_page(self): + kwargs = self._load_kwargs(["--per-page", "10"]) + assert kwargs["limit"] == 10 + assert kwargs["offset"] is None + + def test_cli_page_and_limit_conflict_exits(self): + with pytest.raises(SystemExit): + self._load_kwargs(["--page", "2", "--limit", "10"]) + + def test_cli_page_below_one_exits(self): + with pytest.raises(SystemExit): + self._load_kwargs(["--page", "0"]) + + def test_cli_all_prints_merged_json(self, capsys): + merged = {"sessions": [{"id": "1"}, {"id": "2"}], "meta": {"total_pages": 2}} + client = MagicMock() + client.load.return_value = merged + self._run_cli(["brainstem", "load", "session", "--all", "--per-page", "50"], client) + kwargs = client.load.call_args[1] + assert kwargs["load_all"] is True + assert kwargs["limit"] == 50 + assert json.loads(capsys.readouterr().out) == merged + + def test_cli_all_error_exits_nonzero(self, capsys): + client = MagicMock() + client.load.side_effect = AuthenticationError("API token is invalid or expired.") + with pytest.raises(SystemExit) as exc: + self._run_cli(["brainstem", "load", "session", "--all"], client) + assert exc.value.code == 1 + assert "invalid or expired" in capsys.readouterr().err + + def test_cli_global_flags_before_subcommand_kept(self): + client = MagicMock() + client.load.return_value = mock_response(200, {}) + with patch("brainstem_api_tools.cli.BrainstemClient", return_value=client) as cls, \ + patch("sys.argv", ["brainstem", "--token", TOKEN, "--url", "http://x/", + "load", "session"]): + import brainstem_api_tools.cli as cli_module + cli_module.main() + assert cls.call_args[1]["token"] == TOKEN + assert cls.call_args[1]["url"] == "http://x/" + + def test_cli_url_defaults_to_env(self, monkeypatch): + monkeypatch.setenv("BRAINSTEM_API_URL", "http://127.0.0.1:8000/") + monkeypatch.setenv("BRAINSTEM_API_TOKEN", "env-token") + client = MagicMock() + client.load.return_value = mock_response(200, {}) + with patch("brainstem_api_tools.cli.BrainstemClient", return_value=client) as cls, \ + patch("sys.argv", ["brainstem", "load", "session"]): + import brainstem_api_tools.cli as cli_module + cli_module.main() + assert cls.call_args[1]["url"] == "http://127.0.0.1:8000/" + assert cls.call_args[1]["token"] == "env-token" + + def test_cli_offset_not_multiple_of_page_size_exits(self, capsys): + import brainstem_api_tools.cli as cli_module + client = BrainstemClient(token=TOKEN) + client._session.get = MagicMock() + with patch("brainstem_api_tools.cli.BrainstemClient", return_value=client), \ + patch("sys.argv", ["brainstem", "load", "session", + "--limit", "20", "--offset", "30"]): + with pytest.raises(SystemExit): + cli_module.main() + assert "multiple of limit" in capsys.readouterr().err + client._session.get.assert_not_called() From 2b72987b21d8a85ffcc13bf85d6ad1c77d2b0ff2 Mon Sep 17 00:00:00 2001 From: "Peter C. Petersen" Date: Fri, 16 Oct 2026 06:39:36 +0200 Subject: [PATCH 4/6] Add agent reference client method and CLI command The server publishes a public, AI-agent-oriented description of its API and data model at /reference/. This adds BrainstemClient.agent_reference(page) to fetch a page as text (defaulting to index.md) and raises a ValueError on 404 for unknown pages or older servers. A new `brainstem reference [page]` CLI command prints a page using an anonymous client, so it never starts a login. Includes README docs and tests for the client method and CLI. --- README.md | 15 +++++++ brainstem_api_tools/brainstem_api_client.py | 26 +++++++++++ brainstem_api_tools/cli.py | 20 +++++++++ tests/test_client.py | 49 +++++++++++++++++++++ 4 files changed, 110 insertions(+) diff --git a/README.md b/README.md index f55d988..073c2cc 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,21 @@ Both exports have this structure: Problems raise `ExportError` with a readable message, for example when the project or session doesn't exist or you can't read it, a background job fails, or the server doesn't support exports yet. +## Agent reference + +The server describes its own API and data model (resources, fields, relationships, JSON schemas, guides) at +`/reference/`. It is written for AI agents, public, and needs no token: + +```python +print(client.agent_reference()) # index of all pages +print(client.agent_reference("resources/session.md")) # one resource +print(client.agent_reference("guides/query-planning.md")) # a guide +``` + +```bash +brainstem reference rules.md +``` + ## Command-line Interface After installation a `brainstem` command is available in your shell. diff --git a/brainstem_api_tools/brainstem_api_client.py b/brainstem_api_tools/brainstem_api_client.py index 7b6c913..0c40836 100644 --- a/brainstem_api_tools/brainstem_api_client.py +++ b/brainstem_api_tools/brainstem_api_client.py @@ -657,6 +657,32 @@ def _check_export_response(self, resp: Response, model: str) -> None: raise ExportError(resp.json().get("detail", resp.reason)) resp.raise_for_status() + # ------------------------------------------------------------------ + # Agent reference + # ------------------------------------------------------------------ + + def agent_reference(self, page: str = "index.md") -> str: + """Return one page of the server's agent reference as text. + + The server describes its own API and data model at ``/reference/`` (Markdown + pages such as ``rules.md``, ``resources/session.md`` or + ``guides/query-planning.md``, and JSON documents under ``data/``). It is + public, so this works with an anonymous client too. + + Parameters + ---------- + page : Page path relative to the reference root. Defaults to the index. + """ + url = self._base_url + "reference/" + page.lstrip("/") + resp = self._session.get(url, timeout=self.DEFAULT_TIMEOUT) + if resp.status_code == 404: + raise ValueError( + f"No agent reference page '{page}' on this server. The page does not exist, " + "or the server predates the agent reference." + ) + resp.raise_for_status() + return resp.text + def __enter__(self): return self diff --git a/brainstem_api_tools/cli.py b/brainstem_api_tools/cli.py index 6eb4d8c..088a5ac 100644 --- a/brainstem_api_tools/cli.py +++ b/brainstem_api_tools/cli.py @@ -136,6 +136,16 @@ def _build_parser() -> argparse.ArgumentParser: p_delete.add_argument("--portal", default="private") p_delete.add_argument("--id", required=True, help="UUID of the record to delete.") + # ---- reference -------------------------------------------------- + p_reference = sub.add_parser( + "reference", parents=[common], + help="Print a page of the server's agent reference (API and data model description).", + ) + p_reference.add_argument( + "page", nargs="?", default="index.md", + help="Page path, e.g. rules.md, resources/session.md, guides/query-planning.md (default: index.md).", + ) + return parser @@ -151,6 +161,16 @@ def main(): print("No cached token found.") return + if args.command == "reference": + # The reference is public: never start a login for it. + client = BrainstemClient(anonymous=True, url=getattr(args, "url", None)) + try: + print(client.agent_reference(args.page)) + except (ValueError, requests.HTTPError) as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1) + return + client = BrainstemClient( token=args.token, headless=getattr(args, "headless", False), diff --git a/tests/test_client.py b/tests/test_client.py index 196cfe5..b45acd7 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1154,6 +1154,23 @@ def _run_cli(self, argv, client_mock): patch("sys.argv", argv): cli_module.main() + def test_cli_reference_prints_page(self, capsys): + client = MagicMock() + client.agent_reference.return_value = "# rules" + with patch("brainstem_api_tools.cli.BrainstemClient", return_value=client) as factory, \ + patch("sys.argv", ["brainstem", "reference", "rules.md"]): + import brainstem_api_tools.cli as cli_module + cli_module.main() + assert factory.call_args.kwargs["anonymous"] is True + client.agent_reference.assert_called_once_with("rules.md") + assert "# rules" in capsys.readouterr().out + + def test_cli_reference_defaults_to_index(self, capsys): + client = MagicMock() + client.agent_reference.return_value = "# index" + self._run_cli(["brainstem", "--token", TOKEN, "reference"], client) + client.agent_reference.assert_called_once_with("index.md") + def test_cli_load_calls_load(self, capsys): resp = mock_response(200, {"sessions": []}) client = MagicMock() @@ -1293,3 +1310,35 @@ def test_cli_offset_not_multiple_of_page_size_exits(self, capsys): cli_module.main() assert "multiple of limit" in capsys.readouterr().err client._session.get.assert_not_called() + + +# --------------------------------------------------------------------------- +# Agent reference +# --------------------------------------------------------------------------- + +class TestAgentReference: + def test_fetches_page(self): + client = make_client() + resp = MagicMock(status_code=200, text="# rules") + with patch.object(client._session, "get", return_value=resp) as get: + assert client.agent_reference("rules.md") == "# rules" + assert get.call_args[0][0] == "https://www.brainstem.org/reference/rules.md" + + def test_default_page_is_index(self): + client = make_client() + with patch.object(client._session, "get", return_value=MagicMock(status_code=200, text="x")) as get: + client.agent_reference() + assert get.call_args[0][0].endswith("/reference/index.md") + + def test_unknown_page_raises_value_error(self): + client = make_client() + with patch.object(client._session, "get", return_value=MagicMock(status_code=404)): + with pytest.raises(ValueError, match="No agent reference page"): + client.agent_reference("nope.md") + + def test_anonymous_client_reads_the_public_reference(self): + client = BrainstemClient(anonymous=True) + with patch.object(client._session, "get", return_value=MagicMock(status_code=200, text="# index")) as get: + assert client.agent_reference() == "# index" + assert "Authorization" not in client._session.headers + get.assert_called_once() From 8883a60165661664bf91d4f5445c953c82ec39b5 Mon Sep 17 00:00:00 2001 From: "Peter C. Petersen" Date: Fri, 16 Oct 2026 09:18:02 +0200 Subject: [PATCH 5/6] Prepare 2.1.0 and improve device auth errors Report device-flow HTTP responses containing API errors as AuthenticationError. Bump the version to 2.1.0, add a changelog and tutorial examples for anonymous access, exports, and agent references, and ignore local scratch directories. --- .gitignore | 4 ++ CHANGELOG.md | 27 ++++++++++ brainstem_api_tools/__init__.py | 2 +- brainstem_api_tools/brainstem_api_client.py | 19 +++++-- brainstem_api_tutorial.ipynb | 59 +++++++++++++++++++++ brainstem_api_tutorial.py | 25 +++++++++ pyproject.toml | 2 +- tests/test_client.py | 25 +++++++++ 8 files changed, 157 insertions(+), 6 deletions(-) create mode 100644 CHANGELOG.md diff --git a/.gitignore b/.gitignore index 9c5ae03..a123d81 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,7 @@ dmypy.json stem_credentials_encoded.mat .DS_Store + +# Local scratch and stale sdist unpacks +state/ +brainstem_python_api_tools-*/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ea7036d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,27 @@ +# Changelog + +## 2.1.0 (unreleased) + +Server requirements: BrainSTEM with the `specimen-1` data model, the project/session export API +(format 2.1) and the public agent reference (`/reference/`, format 1.x). + +### Added +- `export_project()` and `export_session()`: download a project or session export (format 2.1), + directly or as a background job (`background=True`) that the client polls and downloads. + Problems raise `ExportError`. +- `agent_reference(page)` and the `brainstem reference ` command: read the server's public + agent reference without a token. +- `BrainstemClient(anonymous=True)`: read public data without logging in. +- Routes for the specimen data model (`specimenset`, `specimen`, `specimenprocedure`, + `specimenproceduretemplate`, `specimenlog`), interfaces, materials, episodes, coordinates and + anatomical references; `brainregion` remains as a deprecated alias of `anatomicalregion`. +- `limit`/`offset` and `--per-page`/`--page` paging, `load_all=True` merging of sideloaded lists. + +### Changed +- Device-flow errors returned by the server with a non-2xx status (`invalid_device_code`, + `token_error`) raise `AuthenticationError` instead of `requests.HTTPError`. + +## 2.0.0 + +- Browser-based device authorization with 2FA; token cached at `~/.config/brainstem/token`. +- `brainstem` command-line interface (`login`, `logout`, `load`, `save`, `delete`). diff --git a/brainstem_api_tools/__init__.py b/brainstem_api_tools/__init__.py index da50ef7..4b339fd 100644 --- a/brainstem_api_tools/__init__.py +++ b/brainstem_api_tools/__init__.py @@ -1,3 +1,3 @@ from .brainstem_api_client import BrainstemClient, ModelType, PortalType, AuthenticationError, ExportError -__version__ = "2.0.0" +__version__ = "2.1.0" diff --git a/brainstem_api_tools/brainstem_api_client.py b/brainstem_api_tools/brainstem_api_client.py index 0c40836..09a88c3 100644 --- a/brainstem_api_tools/brainstem_api_client.py +++ b/brainstem_api_tools/brainstem_api_client.py @@ -307,8 +307,18 @@ def _device_auth_flow(self, headless: bool = False, max_wait: int = 900) -> str: json={"device_code": device_code}, timeout=10, ) - poll.raise_for_status() - result = poll.json() + # The server answers 400 (invalid_device_code) or 500 (token_error) + # with a JSON "error" key; report those as authentication failures + # rather than bare HTTP errors. + try: + result = poll.json() + except ValueError: + result = {} + if not poll.ok: + error = result.get("error") if isinstance(result, dict) else None + if error: + raise AuthenticationError(f"Authorization failed: {error}") + poll.raise_for_status() if result.get("status") == "success": return result["token"] @@ -391,8 +401,9 @@ def load(self, if load_all and id is not None: raise ValueError("load_all=True cannot be used together with id.") - # The API paginates with page/per_page (limit/offset are ignored by - # the server), so translate limit/offset into those parameters. + # The API paginates with page/per_page (it also accepts limit/offset + # as aliases under the same multiple-of-page-size rule); translate + # limit/offset here so the request is explicit and validated early. page_size = 100 if limit is None else limit if load_all: page_size = min(page_size, self.MAX_PAGE_SIZE) diff --git a/brainstem_api_tutorial.ipynb b/brainstem_api_tutorial.ipynb index fa521e1..7606c0a 100644 --- a/brainstem_api_tutorial.ipynb +++ b/brainstem_api_tutorial.ipynb @@ -379,6 +379,65 @@ "\n", "print(\"Replace the placeholder UUID above with your own ID before running.\")" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Public data without logging in\n", + "\n", + "An anonymous client sends no token; only the public portal works with it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "public_client = BrainstemClient(anonymous=True)\n", + "public_projects = public_client.load(\"project\", portal=\"public\", load_all=True)\n", + "print(len(public_projects[\"project\"]), \"public projects\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Exporting a project or a session\n", + "\n", + "An export is one JSON document (format 2.1) with every record of the project or session you may read. Pass `background=True` for large projects: the server builds the file as a job and the client waits for it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "export = client.export_project(\"\")\n", + "print(export[\"export_info\"][\"record_count\"], \"records\")\n", + "\n", + "# session_export = client.export_session(\"\", background=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The agent reference\n", + "\n", + "The server describes its own API and data model in Markdown pages, written for AI agents and readable without a token." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(client.agent_reference(\"index.md\"))" + ] } ], "metadata": { diff --git a/brainstem_api_tutorial.py b/brainstem_api_tutorial.py index e959f8e..f831083 100644 --- a/brainstem_api_tutorial.py +++ b/brainstem_api_tutorial.py @@ -134,3 +134,28 @@ ## Request the public data by defining the portal to be public output4 = client.load("project", portal="public").json() + + +# Public data without logging in + +## An anonymous client sends no token; only the public portal works with it. +public_client = BrainstemClient(anonymous=True) +public_projects = public_client.load("project", portal="public", load_all=True) + + +# Exporting a project or a session + +## An export is one JSON document (format 2.1) with every record of the project +## or session you may read. Pass background=True for large projects: the server +## builds the file as a job and the client waits for it. +export = client.export_project("") +print(export["export_info"]["record_count"], "records") +session_export = client.export_session("", background=True) + + +# The agent reference + +## The server describes its own API and data model in Markdown pages, written +## for AI agents and readable without a token. +print(client.agent_reference("index.md")) +print(client.agent_reference("resources/session.md")) diff --git a/pyproject.toml b/pyproject.toml index 6e85b8c..b145ba5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "brainstem_python_api_tools" -version = "2.0.0" +version = "2.1.0" description = "A Python toolset for interacting with the BrainSTEM API." readme = "README.md" license = { text = "MIT" } diff --git a/tests/test_client.py b/tests/test_client.py index b45acd7..1f1ad7d 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -447,6 +447,31 @@ def test_access_denied_raises(self): with pytest.raises(AuthenticationError, match="Authorization failed"): client._device_auth_flow() + def test_http_error_with_json_error_raises_authentication_error(self): + """A 400 invalid_device_code (or 500 token_error) is reported as an + AuthenticationError, not as a bare requests.HTTPError.""" + client = make_client() + + device_resp = MagicMock() + device_resp.raise_for_status = MagicMock() + device_resp.json.return_value = { + "device_code": "dev123", + "user_code": "XYZ", + "verification_uri": "https://brainstem.org/activate", + "verification_uri_complete": "https://brainstem.org/activate?code=XYZ", + "interval": 0, + } + bad_resp = MagicMock() + bad_resp.ok = False + bad_resp.status_code = 400 + bad_resp.raise_for_status.side_effect = requests.HTTPError("400") + bad_resp.json.return_value = {"error": "invalid_device_code"} + + with patch("requests.post", side_effect=[device_resp, bad_resp]), \ + patch("webbrowser.open"): + with pytest.raises(AuthenticationError, match="invalid_device_code"): + client._device_auth_flow() + # --------------------------------------------------------------------------- # load_all auto-pagination From 730295c15e2f0475309766cdbac060776c3f4ea4 Mon Sep 17 00:00:00 2001 From: "Peter C. Petersen" Date: Fri, 16 Oct 2026 10:48:12 +0200 Subject: [PATCH 6/6] Filter records by tag name instead of tag ID Use filter{tags.name} for project, subject, session, collection, and cohort loaders, matching server support and the tag names returned by the API. Update docstrings and the changelog. --- CHANGELOG.md | 3 +++ brainstem_api_tools/brainstem_api_client.py | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea7036d..0bd3fb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ Server requirements: BrainSTEM with the `specimen-1` data model, the project/ses - `limit`/`offset` and `--per-page`/`--page` paging, `load_all=True` merging of sideloaded lists. ### Changed +- The `tags` filter of `load_project`, `load_subject`, `load_session`, `load_collection` and + `load_cohort` filters by tag name + (`filter{tags.name}`), which the server now supports; tag ids were never returned by the API. - Device-flow errors returned by the server with a non-2xx status (`invalid_device_code`, `token_error`) raise `AuthenticationError` instead of `requests.HTTPError`. diff --git a/brainstem_api_tools/brainstem_api_client.py b/brainstem_api_tools/brainstem_api_client.py index 09a88c3..077929b 100644 --- a/brainstem_api_tools/brainstem_api_client.py +++ b/brainstem_api_tools/brainstem_api_client.py @@ -743,13 +743,13 @@ def load_project(self, portal="private", id: str = None, name : Filter by project name (case-insensitive contains). sessions : Filter by session UUID. subjects : Filter by subject UUID. - tags : Filter by numeric tag id (the server does not filter by tag name). + tags : Filter by tag name (sent as ``filter{tags.name}``; records return tags as names). """ return self._convenience_load( "project", default_include=["sessions", "subjects", "collections", "cohorts"], filter_map={"name": "name.icontains", "sessions": "sessions", - "subjects": "subjects", "tags": "tags"}, + "subjects": "subjects", "tags": "tags.name"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, name=name, sessions=sessions, subjects=subjects, tags=tags, @@ -769,13 +769,13 @@ def load_subject(self, portal="private", id: str = None, projects : Filter by project UUID. strain : Filter by strain UUID. sex : Filter by sex (``'M'``, ``'F'``, or ``'U'``). - tags : Filter by numeric tag id (the server does not filter by tag name). + tags : Filter by tag name (sent as ``filter{tags.name}``; records return tags as names). """ return self._convenience_load( "subject", default_include=["procedures", "subjectlogs"], filter_map={"name": "name.icontains", "projects": "projects", - "strain": "strain", "sex": "sex", "tags": "tags"}, + "strain": "strain", "sex": "sex", "tags": "tags.name"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, name=name, projects=projects, strain=strain, sex=sex, tags=tags, @@ -794,13 +794,13 @@ def load_session(self, portal="private", id: str = None, name : Filter by session name (case-insensitive contains). projects : Filter by project UUID. datastorage : Filter by data storage UUID. - tags : Filter by numeric tag id (the server does not filter by tag name). + tags : Filter by tag name (sent as ``filter{tags.name}``; records return tags as names). """ return self._convenience_load( "session", default_include=["dataacquisition", "behaviors", "manipulations", "epochs"], filter_map={"name": "name.icontains", "projects": "projects", - "datastorage": "datastorage", "tags": "tags"}, + "datastorage": "datastorage", "tags": "tags.name"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, name=name, projects=projects, datastorage=datastorage, tags=tags, @@ -816,12 +816,12 @@ def load_collection(self, portal="private", id: str = None, Parameters ---------- name : Filter by collection name (case-insensitive contains). - tags : Filter by numeric tag id (the server does not filter by tag name). + tags : Filter by tag name (sent as ``filter{tags.name}``; records return tags as names). """ return self._convenience_load( "collection", default_include=["sessions"], - filter_map={"name": "name.icontains", "tags": "tags"}, + filter_map={"name": "name.icontains", "tags": "tags.name"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, name=name, tags=tags, @@ -837,12 +837,12 @@ def load_cohort(self, portal="private", id: str = None, Parameters ---------- name : Filter by cohort name (case-insensitive contains). - tags : Filter by numeric tag id (the server does not filter by tag name). + tags : Filter by tag name (sent as ``filter{tags.name}``; records return tags as names). """ return self._convenience_load( "cohort", default_include=["subjects"], - filter_map={"name": "name.icontains", "tags": "tags"}, + filter_map={"name": "name.icontains", "tags": "tags.name"}, portal=portal, id=id, filters=filters, sort=sort, include=include, limit=limit, offset=offset, load_all=load_all, name=name, tags=tags,